From 3093732654591e47b08f53add17ecf4ce0b0ea54 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 2 Feb 2025 15:58:11 -0800 Subject: [PATCH 01/62] feat: s3 transfer manager v2 This is an initial phase for the s3 transfer manager v2, which includes: - Progress Tracker with a default Console Progres Bar. - Dedicated Multipart Download Listener for listen to events specificly to multipart download. - Generic Transfer Listener that will be used in either a multipart upload or a multipart download. The progress tracker is dependant on the Generic Transfer Listener, and when enabled it uses the same parameter to be provided as the progress tracker. This is important because if there is a need for listening to transfer specific events and also track the progress then, a custom implementation must be done that incorporate those two needs together, otherwise one of each other must be used. - Single Object Download - Multipart Objet Download This initial implementation misses the test cases. --- .../S3Transfer/ConsoleProgressBar.php | 130 ++++++ .../S3Transfer/DefaultProgressTracker.php | 193 +++++++++ .../S3Transfer/GetMultipartDownloader.php | 50 +++ .../Features/S3Transfer/ListenerNotifier.php | 8 + .../S3Transfer/MultipartDownloadListener.php | 190 ++++++++ .../S3Transfer/MultipartDownloadType.php | 51 +++ .../S3Transfer/MultipartDownloader.php | 408 ++++++++++++++++++ .../S3Transfer/ObjectProgressTracker.php | 179 ++++++++ src/S3/Features/S3Transfer/ProgressBar.php | 14 + .../S3Transfer/ProgressBarFactory.php | 8 + .../S3Transfer/ProgressListenerHelper.php | 45 ++ .../S3Transfer/RangeMultipartDownloader.php | 64 +++ .../Features/S3Transfer/S3TransferManager.php | 143 ++++++ .../S3Transfer/S3TransferManagerTrait.php | 88 ++++ .../Features/S3Transfer/TransferListener.php | 252 +++++++++++ .../S3Transfer/TransferListenerFactory.php | 8 + 16 files changed, 1831 insertions(+) create mode 100644 src/S3/Features/S3Transfer/ConsoleProgressBar.php create mode 100644 src/S3/Features/S3Transfer/DefaultProgressTracker.php create mode 100644 src/S3/Features/S3Transfer/GetMultipartDownloader.php create mode 100644 src/S3/Features/S3Transfer/ListenerNotifier.php create mode 100644 src/S3/Features/S3Transfer/MultipartDownloadListener.php create mode 100644 src/S3/Features/S3Transfer/MultipartDownloadType.php create mode 100644 src/S3/Features/S3Transfer/MultipartDownloader.php create mode 100644 src/S3/Features/S3Transfer/ObjectProgressTracker.php create mode 100644 src/S3/Features/S3Transfer/ProgressBar.php create mode 100644 src/S3/Features/S3Transfer/ProgressBarFactory.php create mode 100644 src/S3/Features/S3Transfer/ProgressListenerHelper.php create mode 100644 src/S3/Features/S3Transfer/RangeMultipartDownloader.php create mode 100644 src/S3/Features/S3Transfer/S3TransferManager.php create mode 100644 src/S3/Features/S3Transfer/S3TransferManagerTrait.php create mode 100644 src/S3/Features/S3Transfer/TransferListener.php create mode 100644 src/S3/Features/S3Transfer/TransferListenerFactory.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php new file mode 100644 index 0000000000..3f4312bfa7 --- /dev/null +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -0,0 +1,130 @@ + [ + 'format' => "[|progress_bar|] |percent|%", + 'parameters' => [] + ], + 'transfer_format' => [ + 'format' => "[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|", + 'parameters' => [ + 'transferred', + 'tobe_transferred', + 'unit' + ] + ], + 'colored_transfer_format' => [ + 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|\033[0m", + 'parameters' => [ + 'transferred', + 'tobe_transferred', + 'unit', + 'color_code' + ] + ], + ]; + + /** @var string */ + private string $progressBarChar; + + /** @var int */ + private int $progressBarWidth; + + /** @var int */ + private int $percentCompleted; + + /** @var ?array */ + private ?array $format; + + private ?array $args; + + /** + * @param ?string $progressBarChar + * @param ?int $progressBarWidth + * @param ?int $percentCompleted + * @param array|null $format + */ + public function __construct( + ?string $progressBarChar = null, + ?int $progressBarWidth = null, + ?int $percentCompleted = null, + ?array $format = null, + ?array $args = null + ) { + $this->progressBarChar = $progressBarChar ?? '#'; + $this->progressBarWidth = $progressBarWidth ?? 25; + $this->percentCompleted = $percentCompleted ?? 0; + $this->format = $format ?: self::$formats['transfer_format']; + $this->args = $args ?: []; + } + + /** + * Set current progress percent. + * + * @param int $percent + * + * @return void + */ + public function setPercentCompleted(int $percent): void { + $this->percentCompleted = max(0, min(100, $percent)); + } + + /** + * @param array $args + * + * @return void + */ + public function setArgs(array $args): void + { + $this->args = $args; + } + + public function setArg(string $key, mixed $value): void + { + if (array_key_exists($key, $this->args)) { + $this->args[$key] = $value; + } + } + + private function renderProgressBar(): string { + $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); + return str_repeat($this->progressBarChar, $filledWidth) + . str_repeat(' ', $this->progressBarWidth - $filledWidth); + } + + /** + * + * @return string + */ + public function getPaintedProgress(): string { + foreach ($this->format['parameters'] as $param) { + if (!array_key_exists($param, $this->args)) { + throw new \InvalidArgumentException("Missing '{$param}' parameter for progress bar."); + } + } + + $replacements = [ + '|progress_bar|' => $this->renderProgressBar(), + '|percent|' => $this->percentCompleted + ]; + + foreach ($this->format['parameters'] as $param) { + $replacements["|$param|"] = $this->args[$param] ?? ''; + } + + return strtr($this->format['format'], $replacements); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php new file mode 100644 index 0000000000..ae47baab92 --- /dev/null +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -0,0 +1,193 @@ +clear(); + $this->initializeListener(); + $this->progressBarFactory = $progressBarFactory ?? $this->defaultProgressBarFactory(); + } + + private function initializeListener(): void { + $this->transferListener = new TransferListener(); + // Object transfer initialized + $this->transferListener->onObjectTransferInitiated = $this->objectTransferInitiated(); + // Object transfer made progress + $this->transferListener->onObjectTransferProgress = $this->objectTransferProgress(); + $this->transferListener->onObjectTransferFailed = $this->objectTransferFailed(); + $this->transferListener->onObjectTransferCompleted = $this->objectTransferCompleted(); + } + + /** + * @return TransferListener + */ + public function getTransferListener(): TransferListener { + return $this->transferListener; + } + + /** + * + * @return Closure + */ + private function objectTransferInitiated(): Closure + { + return function (string $objectKey, array &$requestArgs) { + $progressBarFactoryFn = $this->progressBarFactory; + $this->objects[$objectKey] = new ObjectProgressTracker( + objectKey: $objectKey, + objectBytesTransferred: 0, + objectSizeInBytes: 0, + status: 'initiated', + progressBar: $progressBarFactoryFn() + ); + $this->objectsInProgress++; + $this->objectsCount++; + + $this->showProgress(); + }; + } + + /** + * @return Closure + */ + private function objectTransferProgress(): Closure + { + return function ( + string $objectKey, + int $objectBytesTransferred, + int $objectSizeInBytes + ): void { + $objectProgressTracker = $this->objects[$objectKey]; + if ($objectProgressTracker->getObjectSizeInBytes() === 0) { + $objectProgressTracker->setObjectSizeInBytes($objectSizeInBytes); + // Increment objectsTotalSizeInBytes just the first time we set + // the object total size. + $this->objectsTotalSizeInBytes = + $this->objectsTotalSizeInBytes + $objectSizeInBytes; + } + $objectProgressTracker->incrementTotalBytesTransferred( + $objectBytesTransferred + ); + $objectProgressTracker->setStatus('progress'); + + $this->increaseBytesTransferred($objectBytesTransferred); + + $this->showProgress(); + }; + } + + public function objectTransferFailed(): Closure + { + return function ( + string $objectKey, + int $totalObjectBytesTransferred, + \Throwable | string $reason + ): void { + $objectProgressTracker = $this->objects[$objectKey]; + $objectProgressTracker->setStatus('failed'); + + $this->objectsInProgress--; + + $this->showProgress(); + }; + } + + public function objectTransferCompleted(): Closure + { + return function ( + string $objectKey, + int $objectBytesTransferred, + ): void { + $objectProgressTracker = $this->objects[$objectKey]; + $objectProgressTracker->setStatus('completed'); + $this->showProgress(); + }; + } + + /** + * Clear the internal state holders. + * + * @return void + */ + public function clear(): void + { + $this->objects = []; + $this->totalBytesTransferred = 0; + $this->objectsTotalSizeInBytes = 0; + $this->objectsInProgress = 0; + $this->objectsCount = 0; + $this->transferPercentCompleted = 0; + } + + private function increaseBytesTransferred(int $bytesTransferred): void { + $this->totalBytesTransferred += $bytesTransferred; + if ($this->objectsTotalSizeInBytes !== 0) { + $this->transferPercentCompleted = floor(($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100); + } + } + + private function showProgress(): void { + // Clear screen + fwrite(STDOUT, "\033[2J\033[H"); + + // Display progress header + echo sprintf( + "\r%d%% [%s/%s]\n", + $this->transferPercentCompleted, + $this->objectsInProgress, + $this->objectsCount + ); + + foreach ($this->objects as $name => $object) { + echo sprintf("\r%s:\n%s\n", $name, $object->getProgressBar()->getPaintedProgress()); + } + } + + private function defaultProgressBarFactory(): Closure| ProgressBarFactory { + return function () { + return new ConsoleProgressBar( + format: ConsoleProgressBar::$formats[ + ConsoleProgressBar::COLORED_TRANSFER_FORMAT + ], + args: [ + 'transferred' => 0, + 'tobe_transferred' => 0, + 'unit' => 'MB', + 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, + ] + ); + }; + } + +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/GetMultipartDownloader.php b/src/S3/Features/S3Transfer/GetMultipartDownloader.php new file mode 100644 index 0000000000..9548d95ded --- /dev/null +++ b/src/S3/Features/S3Transfer/GetMultipartDownloader.php @@ -0,0 +1,50 @@ +currentPartNo === 0) { + $this->currentPartNo = 1; + } else { + $this->currentPartNo++; + } + + $nextRequestArgs = array_slice($this->requestArgs, 0); + $nextRequestArgs['PartNumber'] = $this->currentPartNo; + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextRequestArgs + ); + } + + /** + * @inheritDoc + * + * @param Result $result + * + * @return void + */ + protected function computeObjectDimensions(ResultInterface $result): void + { + if (!empty($result['PartsCount'])) { + $this->objectPartsCount = $result['PartsCount']; + } + + $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ListenerNotifier.php b/src/S3/Features/S3Transfer/ListenerNotifier.php new file mode 100644 index 0000000000..836dfcd66e --- /dev/null +++ b/src/S3/Features/S3Transfer/ListenerNotifier.php @@ -0,0 +1,8 @@ +notify('onDownloadInitiated', [&$commandArgs, $initialPart]); + } + + /** + * Event for when a download fails. + * Warning: If this method is overridden, it is recommended + * to call parent::downloadFailed() in order to + * keep the states maintained in this implementation. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + public function downloadFailed(\Throwable $reason, int $totalPartsTransferred, int $totalBytesTransferred, int $lastPartTransferred): void { + $this->notify('onDownloadFailed', [$reason, $totalPartsTransferred, $totalBytesTransferred, $lastPartTransferred]); + } + + /** + * Event for when a download completes. + * Warning: If this method is overridden, it is recommended + * to call parent::onDownloadCompleted() in order to + * keep the states maintained in this implementation. + * + * @param resource $stream + * @param int $totalPartsDownloaded + * @param int $totalBytesDownloaded + * + * @return void + */ + public function downloadCompleted($stream, int $totalPartsDownloaded, int $totalBytesDownloaded): void { + $this->notify('onDownloadCompleted', [$stream, $totalPartsDownloaded, $totalBytesDownloaded]); + } + + /** + * Event for when a part download is initiated. + * Warning: If this method is overridden, it is recommended + * to call parent::partDownloadInitiated() in order to + * keep the states maintained in this implementation. + * + * @param mixed $partDownloadCommand + * @param int $partNo + * + * @return void + */ + public function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { + $this->notify('onPartDownloadInitiated', [$partDownloadCommand, $partNo]); + } + + /** + * Event for when a part download completes. + * Warning: If this method is overridden, it is recommended + * to call parent::onPartDownloadCompleted() in order to + * keep the states maintained in this implementation. + * + * @param ResultInterface $result + * @param int $partNo + * @param int $partTotalBytes + * @param int $totalParts + * @param int $objectBytesTransferred + * @param int $objectSizeInBytes + * @return void + */ + public function partDownloadCompleted( + ResultInterface $result, + int $partNo, + int $partTotalBytes, + int $totalParts, + int $objectBytesTransferred, + int $objectSizeInBytes + ): void + { + $this->notify('onPartDownloadCompleted', [ + $result, + $partNo, + $partTotalBytes, + $totalParts, + $objectBytesTransferred, + $objectSizeInBytes + ]); + } + + /** + * Event for when a part download fails. + * Warning: If this method is overridden, it is recommended + * to call parent::onPartDownloadFailed() in order to + * keep the states maintained in this implementation. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + public function partDownloadFailed(CommandInterface $partDownloadCommand, \Throwable $reason, int $partNo): void { + $this->notify('onPartDownloadFailed', [$partDownloadCommand, $reason, $partNo]); + } + + protected function notify(string $event, array $params = []): void + { + $listener = match ($event) { + 'onDownloadInitiated' => $this->onDownloadInitiated, + 'onDownloadFailed' => $this->onDownloadFailed, + 'onDownloadCompleted' => $this->onDownloadCompleted, + 'onPartDownloadInitiated' => $this->onPartDownloadInitiated, + 'onPartDownloadCompleted' => $this->onPartDownloadCompleted, + 'onPartDownloadFailed' => $this->onPartDownloadFailed, + default => null, + }; + + if ($listener instanceof Closure) { + $listener(...$params); + } + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloadType.php b/src/S3/Features/S3Transfer/MultipartDownloadType.php new file mode 100644 index 0000000000..6bbbfd6144 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloadType.php @@ -0,0 +1,51 @@ +value = $value; + } + + /** + * @return string + */ + public function __toString(): string { + return $this->value; + } + + /** + * @param MultipartDownloadType $type + * + * @return bool + */ + public function equals(MultipartDownloadType $type): bool + { + return $this->value === $type->value; + } + + /** + * @return MultipartDownloadType + */ + public static function rangedGet(): MultipartDownloadType { + return new static(self::$rangedGetType); + } + + /** + * @return MultipartDownloadType + */ + public static function partGet(): MultipartDownloadType { + return new static(self::$partGetType); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php new file mode 100644 index 0000000000..03b0e6430a --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -0,0 +1,408 @@ +clear(); + $this->s3Client = $s3Client; + $this->requestArgs = $requestArgs; + $this->config = $config; + $this->currentPartNo = $currentPartNo; + $this->listener = $listener; + $this->progressListener = $progressListener; + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } + + + /** + * Returns that resolves a multipart download operation, + * or to a rejection in case of any failures. + * + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + $this->downloadInitiated($this->requestArgs, $this->currentPartNo); + $initialCommand = $this->nextCommand(); + $this->partDownloadInitiated($initialCommand, $this->currentPartNo); + try { + yield $this->s3Client->executeAsync($initialCommand) + ->then(function (ResultInterface $result) { + // Calculate object size and parts count. + $this->computeObjectDimensions($result); + // Trigger first part completed + $this->partDownloadCompleted($result, $this->currentPartNo); + })->otherwise(function ($reason) use ($initialCommand) { + $this->partDownloadFailed($initialCommand, $reason, $this->currentPartNo); + + throw $reason; + }); + } catch (\Throwable $e) { + $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + // TODO: yield transfer exception modeled with a transfer failed response. + yield Create::rejectionFor($e); + } + + while ($this->currentPartNo < $this->objectPartsCount) { + $nextCommand = $this->nextCommand(); + $this->partDownloadInitiated($nextCommand, $this->currentPartNo); + try { + yield $this->s3Client->executeAsync($nextCommand) + ->then(function ($result) { + $this->partDownloadCompleted($result, $this->currentPartNo); + + return $result; + })->otherwise(function ($reason) use ($nextCommand) { + $this->partDownloadFailed($nextCommand, $reason, $this->currentPartNo); + + return $reason; + }); + } catch (\Throwable $e) { + $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + // TODO: yield transfer exception modeled with a transfer failed response. + yield Create::rejectionFor($e); + } + } + + // Transfer completed + $this->objectDownloadCompleted(); + + // TODO: yield the stream wrapped in a modeled transfer success response. + yield Create::promiseFor($this->stream); + }); + } + + /** + * Main purpose of this method is to propagate + * the download-initiated event to listeners, but + * also it does some computation regarding internal states + * that need to be maintained. + * + * @param array $commandArgs + * @param int|null $currentPartNo + * + * @return void + */ + private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void + { + $this->objectKey = $commandArgs['Key']; + $this->progressListener?->objectTransferInitiated( + $this->objectKey, + $commandArgs + ); + $this->_notifyMultipartDownloadListeners('downloadInitiated', [ + &$commandArgs, + $currentPartNo + ]); + } + + /** + * Propagates download-failed event to listeners. + * It may also do some computation in order to maintain internal states. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + private function downloadFailed( + \Throwable $reason, + int $totalPartsTransferred, + int $totalBytesTransferred, + int $lastPartTransferred + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $totalBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners('downloadFailed', [ + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ]); + } + + /** + * Propagates part-download-initiated event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param int $partNo + * + * @return void + */ + private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { + $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * @param int $partNo + * + * @return void + */ + private function partDownloadCompleted(ResultInterface $result, int $partNo): void { + $this->objectCompletedPartsCount++; + $partDownloadBytes = $result['ContentLength']; + $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + + $this->progressListener?->objectTransferProgress( + $this->objectKey, + $partDownloadBytes, + $this->objectSizeInBytes + ); + + $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ + $result, + $partNo, + $partDownloadBytes, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred, + $this->objectSizeInBytes + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + private function partDownloadFailed( + CommandInterface $partDownloadCommand, + \Throwable $reason, + int $partNo + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners( + 'partDownloadFailed', + [$partDownloadCommand, $reason, $partNo]); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function objectDownloadCompleted(): void + { + $this->stream->rewind(); + $this->progressListener?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred + ); + $this->_notifyMultipartDownloadListeners('downloadCompleted', [ + $this->stream, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred + ]); + } + + /** + * Internal helper method for notifying listeners of specific events. + * + * @param string $listenerMethod + * @param array $args + * + * @return void + */ + private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void + { + $this->listener?->{$listenerMethod}(...$args); + } + + /** + * Returns the next command for fetching the next object part. + * + * @return CommandInterface + */ + abstract protected function nextCommand() : CommandInterface; + + /** + * Compute the object dimensions, such as size and parts count. + * + * @param ResultInterface $result + * + * @return void + */ + abstract protected function computeObjectDimensions(ResultInterface $result): void; + + /** + * Calculates the object size dynamically. + * + * @param $sizeSource + * + * @return int + */ + protected function computeObjectSize($sizeSource): int { + if (gettype($sizeSource) === "integer") { + return (int) $sizeSource; + } + + if (empty($sizeSource)) { + throw new \RuntimeException('Range must not be empty'); + } + + if (preg_match("/\/(\d+)$/", $sizeSource, $matches)) { + return $matches[1]; + } + + throw new \RuntimeException('Invalid range format'); + } + + private function clear(): void { + $this->currentPartNo = 0; + $this->objectPartsCount = 0; + $this->objectCompletedPartsCount = 0; + $this->objectSizeInBytes = 0; + $this->objectBytesTransferred = 0; + $this->eTag = ""; + $this->objectKey = ""; + } + + /** + * MultipartDownloader factory method to return an instance + * of MultipartDownloader based on the multipart download type. + * + * @param S3ClientInterface $s3Client + * @param string $multipartDownloadType + * @param array $requestArgs + * @param array $config + * @param MultipartDownloadListener|null $listener + * @param TransferListener|null $progressTracker + * + * @return MultipartDownloader + */ + public static function chooseDownloader( + S3ClientInterface $s3Client, + string $multipartDownloadType, + array $requestArgs, + array $config, + ?MultipartDownloadListener $listener = null, + ?TransferListener $progressTracker = null + ) : MultipartDownloader + { + if ($multipartDownloadType === self::PART_GET_MULTIPART_DOWNLOADER) { + return new GetMultipartDownloader( + $s3Client, + $requestArgs, + $config, + 0, + $listener, + $progressTracker + ); + } elseif ($multipartDownloadType === self::RANGE_GET_MULTIPART_DOWNLOADER) { + return new RangeMultipartDownloader( + $s3Client, + $requestArgs, + $config, + 0, + $listener, + $progressTracker + ); + } + + throw new \RuntimeException("Unsupported download type $multipartDownloadType"); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ObjectProgressTracker.php b/src/S3/Features/S3Transfer/ObjectProgressTracker.php new file mode 100644 index 0000000000..9c876e363a --- /dev/null +++ b/src/S3/Features/S3Transfer/ObjectProgressTracker.php @@ -0,0 +1,179 @@ +objectKey = $objectKey; + $this->objectBytesTransferred = $objectBytesTransferred; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->status = $status; + $this->progressBar = $progressBar ?? $this->defaultProgressBar(); + } + + /** + * @return string + */ + public function getObjectKey(): string + { + return $this->objectKey; + } + + /** + * @param string $objectKey + * + * @return void + */ + public function setObjectKey(string $objectKey): void + { + $this->objectKey = $objectKey; + } + + /** + * @return int + */ + public function getObjectBytesTransferred(): int + { + return $this->objectBytesTransferred; + } + + /** + * @param int $objectBytesTransferred + * + * @return void + */ + public function setObjectBytesTransferred(int $objectBytesTransferred): void + { + $this->objectBytesTransferred = $objectBytesTransferred; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @param int $objectSizeInBytes + * + * @return void + */ + public function setObjectSizeInBytes(int $objectSizeInBytes): void + { + $this->objectSizeInBytes = $objectSizeInBytes; + // Update progress bar + $this->progressBar->setArg('tobe_transferred', $objectSizeInBytes); + } + + /** + * @return string + */ + public function getStatus(): string + { + return $this->status; + } + + /** + * @param string $status + * + * @return void + */ + public function setStatus(string $status): void + { + $this->status = $status; + $this->setProgressColor(); + } + + private function setProgressColor(): void { + if ($this->status === 'progress') { + $this->progressBar->setArg('color_code', ConsoleProgressBar::BLUE_COLOR_CODE); + } elseif ($this->status === 'completed') { + $this->progressBar->setArg('color_code', ConsoleProgressBar::GREEN_COLOR_CODE); + } elseif ($this->status === 'failed') { + $this->progressBar->setArg('color_code', ConsoleProgressBar::RED_COLOR_CODE); + } + } + + /** + * Increments the object bytes transferred. + * + * @param int $objectBytesTransferred + * + * @return void + */ + public function incrementTotalBytesTransferred( + int $objectBytesTransferred + ): void + { + $this->objectBytesTransferred += $objectBytesTransferred; + $progressPercent = (int) floor(($this->objectBytesTransferred / $this->objectSizeInBytes) * 100); + // Update progress bar + $this->progressBar->setPercentCompleted($progressPercent); + $this->progressBar->setArg('transferred', $this->objectBytesTransferred); + } + + /** + * @return ProgressBar|null + */ + public function getProgressBar(): ?ProgressBar + { + return $this->progressBar; + } + + /** + * @return ProgressBar + */ + private function defaultProgressBar(): ProgressBar { + return new ConsoleProgressBar( + format: ConsoleProgressBar::$formats[ + ConsoleProgressBar::COLORED_TRANSFER_FORMAT + ], + args: [ + 'transferred' => 0, + 'tobe_transferred' => 0, + 'unit' => 'B', + 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, + ] + ); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ProgressBar.php b/src/S3/Features/S3Transfer/ProgressBar.php new file mode 100644 index 0000000000..af38858d11 --- /dev/null +++ b/src/S3/Features/S3Transfer/ProgressBar.php @@ -0,0 +1,14 @@ + 'bytesToBytes', + 'KB' => 'bytesToKB', + 'MB' => 'bytesToMB', + ]; + + public static function getUnitValue(string $displayUnit, float $bytes): float { + $displayUnit = self::validateDisplayUnit($displayUnit); + if (isset(self::$displayUnitMapping[$displayUnit])) { + return number_format(call_user_func([__CLASS__, self::$displayUnitMapping[$displayUnit]], $bytes)); + } + + throw new \RuntimeException("Unknown display unit {$displayUnit}"); + } + + private static function validateDisplayUnit(string $displayUnit): string { + if (!isset(self::$displayUnitMapping[$displayUnit])) { + throw new \InvalidArgumentException("Invalid display unit specified: $displayUnit"); + } + + return $displayUnit; + } + + private static function bytesToBytes(float $bytes): float { + return $bytes; + } + + private static function bytesToKB(float $bytes): float { + return $bytes / 1024; + } + + private static function bytesToMB(float $bytes): float { + return $bytes / 1024 / 1024; + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php new file mode 100644 index 0000000000..5a76a39408 --- /dev/null +++ b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php @@ -0,0 +1,64 @@ +objectSizeInBytes !== 0) { + $this->computeObjectDimensions(new Result(['ContentRange' => $this->totalBytes])); + } + + if ($this->currentPartNo === 0) { + $this->currentPartNo = 1; + $this->partSize = $this->config['targetPartSizeBytes']; + } else { + $this->currentPartNo++; + } + + $nextRequestArgs = array_slice($this->requestArgs, 0); + $from = ($this->currentPartNo - 1) * ($this->partSize + 1); + $to = $this->currentPartNo * $this->partSize; + $nextRequestArgs['Range'] = "bytes=$from-$to"; + if (!empty($this->eTag)) { + $nextRequestArgs['IfMatch'] = $this->eTag; + } + + return $this->s3Client->getCommand( + self::GET_OBJECT_COMMAND, + $nextRequestArgs + ); + } + + /** + * @inheritDoc + * + * @param Result $result + * + * @return void + */ + protected function computeObjectDimensions(ResultInterface $result): void + { + $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + if ($this->objectSizeInBytes > $this->partSize) { + $this->objectPartsCount = intval(ceil($this->objectSizeInBytes / $this->partSize)); + } else { + $this->partSize = $this->objectSizeInBytes; + $this->currentPartNo = 1; + } + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php new file mode 100644 index 0000000000..f9c56e6b7c --- /dev/null +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -0,0 +1,143 @@ +s3Client = $this->defaultS3Client(); + } else { + $this->s3Client = $s3Client; + } + + $this->config = $config + self::$defaultConfig; + } + + /** + * @param string|array $source The object to be downloaded from S3. + * It can be either a string with a S3 URI or an array with a Bucket and Key + * properties set. + * @param array $downloadArgs The request arguments to be provided as part + * of the service client operation. + * @param array $config The configuration to be used for this operation. + * - listener: (null|MultipartDownloadListener) \ + * A listener to be notified in every stage of a multipart download operation. + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If not + * transfer tracker factory is provided and trackProgress is true then, + * the default progress listener implementation will be used. + * + * @return PromiseInterface + */ + public function download( + string | array $source, + array $downloadArgs, + array $config = [] + ): PromiseInterface { + if (is_string($source)) { + $sourceArgs = $this->s3UriAsBucketAndKey($source); + } elseif (is_array($source)) { + $sourceArgs = [ + 'Bucket' => $this->requireNonEmpty($source['Bucket'], "A valid bucket must be provided."), + 'Key' => $this->requireNonEmpty($source['Key'], "A valid key must be provided."), + ]; + } else { + throw new \InvalidArgumentException('Source must be a string or an array of strings'); + } + + $requestArgs = $sourceArgs + $downloadArgs; + if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { + return $this->tryMultipartDownload($requestArgs, $config); + } + + return $this->trySingleDownload($requestArgs); + } + + /** + * Tries an object multipart download. + * + * @param array $requestArgs + * @param array $config + * - listener: (?MultipartDownloadListener) \ + * A multipart download listener for watching every multipart download + * stage. + * + * @return PromiseInterface + */ + private function tryMultipartDownload( + array $requestArgs, + array $config + ): PromiseInterface { + $trackProgress = $config['trackProgress'] + ?? $this->config['trackProgress'] + ?? false; + $progressListenerFactory = $this->config['progressListenerFactory'] ?? null; + $progressListener = null; + if ($trackProgress) { + if ($progressListenerFactory !== null) { + $progressListener = $progressListenerFactory(); + } else { + $progressListener = new DefaultProgressTracker(); + } + } + $multipartDownloader = MultipartDownloader::chooseDownloader( + $this->s3Client, + $this->config['multipartDownloadType'], + $requestArgs, + $this->config, + $config['listener'] ?? null, + $progressListener?->getTransferListener() + ); + + return $multipartDownloader->promise(); + } + + /** + * Does a single object download. + * + * @param $requestArgs + * + * @return PromiseInterface + */ + private function trySingleDownload($requestArgs): PromiseInterface { + $command = $this->s3Client->getCommand(MultipartDownloader::GET_OBJECT_COMMAND, $requestArgs); + + return $this->s3Client->executeAsync($requestArgs); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php new file mode 100644 index 0000000000..d924280b46 --- /dev/null +++ b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php @@ -0,0 +1,88 @@ + 8 * 1024 * 1024, + 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, + 'multipartDownloadThresholdBytes' => 16 * 1024 * 1024, + 'checksumValidationEnabled' => true, + 'checksumAlgorithm' => 'crc32', + 'multipartDownloadType' => 'partGet', + 'concurrency' => 5, + ]; + + /** + * Returns a default instance of S3Client. + * + * @return S3Client + */ + private function defaultS3Client(): S3ClientInterface + { + return new S3Client([]); + } + + /** + * Validates a provided value is not empty, and if so then + * it throws an exception with the provided message. + * @param mixed $value + * + * @return mixed + */ + private function requireNonEmpty(mixed $value, string $message): mixed { + if (empty($value)) { + throw new \InvalidArgumentException($message); + } + + return $value; + } + + /** + * Validates a string value is a valid S3 URI. + * Valid S3 URI Example: S3://mybucket.dev/myobject.txt + * + * @param string $uri + * + * @return bool + */ + private function isValidS3URI(string $uri): bool + { + // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. + return str_starts_with(strtolower($uri), 's3://') + && count(explode('/', substr($uri, 5))) > 1; + } + + /** + * Converts a S3 URI into an array with a Bucket and Key + * properties set. + * + * @param string $uri: The S3 URI. + * + * @return array + */ + private function s3UriAsBucketAndKey(string $uri): array { + $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; + if (!$this->isValidS3URI($uri)) { + throw new \InvalidArgumentException($errorMessage); + } + + $path = substr($uri, 5); // without s3:// + $parts = explode('/', $path, 2); + + if (count($parts) < 2) { + throw new \InvalidArgumentException($errorMessage); + } + + return [ + 'Bucket' => $parts[0], + 'Key' => $parts[1], + ]; + } + +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php new file mode 100644 index 0000000000..9f3f308978 --- /dev/null +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -0,0 +1,252 @@ +notify('onTransferInitiated', []); + } + + /** + * Event for when an object transfer initiated. + * + * @param string $objectKey + * @param array $requestArgs + * + * @return void + */ + public function objectTransferInitiated(string $objectKey, array &$requestArgs): void { + $this->objectsToBeTransferred++; + if ($this->objectsToBeTransferred === 1) { + $this->transferInitiated(); + } + + $this->notify('onObjectTransferInitiated', [$objectKey, &$requestArgs]); + } + + /** + * Event for when an object transfer made some progress. + * + * @param string $objectKey + * @param int $objectBytesTransferred + * @param int $objectSizeInBytes + * + * @return void + */ + public function objectTransferProgress( + string $objectKey, + int $objectBytesTransferred, + int $objectSizeInBytes + ): void { + $this->objectsBytesTransferred += $objectBytesTransferred; + $this->notify('onObjectTransferProgress', [ + $objectKey, + $objectBytesTransferred, + $objectSizeInBytes + ]); + // Needs state management + $this->notify('onTransferProgress', [ + $this->objectsTransferCompleted, + $this->objectsBytesTransferred, + $this->objectsToBeTransferred + ]); + } + + /** + * Event for when an object transfer failed. + * + * @param string $objectKey + * @param int $objectBytesTransferred + * @param \Throwable|string $reason + * + * @return void + */ + public function objectTransferFailed( + string $objectKey, + int $objectBytesTransferred, + \Throwable | string $reason + ): void { + $this->objectsTransferFailed++; + $this->notify('onObjectTransferFailed', [ + $objectKey, + $objectBytesTransferred, + $reason + ]); + } + + /** + * Event for when an object transfer is completed. + * + * @param string $objectKey + * @param int $objectBytesCompleted + * + * @return void + */ + public function objectTransferCompleted ( + string $objectKey, + int $objectBytesCompleted + ): void { + $this->objectsTransferCompleted++; + $this->validateTransferComplete(); + $this->notify('onObjectTransferCompleted', [ + $objectKey, + $objectBytesCompleted + ]); + } + + /** + * Event for when a transfer is completed. + * + * @param int $objectsTransferCompleted + * @param int $objectsBytesTransferred + * + * @return void + */ + public function transferCompleted ( + int $objectsTransferCompleted, + int $objectsBytesTransferred, + ): void { + $this->notify('onTransferCompleted', [ + $objectsTransferCompleted, + $objectsBytesTransferred + ]); + } + + /** + * Event for when a transfer is completed. + * + * @param int $objectsTransferCompleted + * @param int $objectsBytesTransferred + * @param int $objectsTransferFailed + * + * @return void + */ + public function transferFailed ( + int $objectsTransferCompleted, + int $objectsBytesTransferred, + int $objectsTransferFailed, + Throwable | string $reason + ): void { + $this->notify('onTransferFailed', [ + $objectsTransferCompleted, + $objectsBytesTransferred, + $objectsTransferFailed, + $reason + ]); + } + + /** + * Validates if a transfer is completed, and if so then the event is propagated + * to the subscribed listeners. + * + * @return void + */ + private function validateTransferComplete(): void { + if ($this->objectsToBeTransferred === ($this->objectsTransferCompleted + $this->objectsTransferFailed)) { + if ($this->objectsTransferFailed > 0) { + $this->transferFailed( + $this->objectsTransferCompleted, + $this->objectsBytesTransferred, + $this->objectsTransferFailed, + "Transfer could not have been completed successfully." + ); + } else { + $this->transferCompleted( + $this->objectsTransferCompleted, + $this->objectsBytesTransferred + ); + } + } + } + + protected function notify(string $event, array $params = []): void + { + $listener = match ($event) { + 'onTransferInitiated' => $this->onTransferInitiated, + 'onObjectTransferInitiated' => $this->onObjectTransferInitiated, + 'onObjectTransferProgress' => $this->onObjectTransferProgress, + 'onObjectTransferFailed' => $this->onObjectTransferFailed, + 'onObjectTransferCompleted' => $this->onObjectTransferCompleted, + 'onTransferProgress' => $this->onTransferProgress, + 'onTransferCompleted' => $this->onTransferCompleted, + 'onTransferFailed' => $this->onTransferFailed, + default => null, + }; + + if ($listener instanceof Closure) { + $listener(...$params); + } + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListenerFactory.php b/src/S3/Features/S3Transfer/TransferListenerFactory.php new file mode 100644 index 0000000000..dd8c6422de --- /dev/null +++ b/src/S3/Features/S3Transfer/TransferListenerFactory.php @@ -0,0 +1,8 @@ + Date: Wed, 5 Feb 2025 17:27:10 -0800 Subject: [PATCH 02/62] chore: add tests cases and refactor - Refactor set a single argument, even when not exists, in the console progress bar. - Add a specific parameter for showing the progress rendering defaulted to STDOUT. - Add test cases for ConsoleProgressBar. - Add test cases for DefaultProgressTracker. - Add test cases for ObjectProgressTracker. - Add test cases for TransferListener. --- .../S3Transfer/ConsoleProgressBar.php | 14 +- .../S3Transfer/DefaultProgressTracker.php | 88 ++++++- .../S3Transfer/ObjectProgressTracker.php | 40 +-- .../Features/S3Transfer/TransferListener.php | 31 +++ .../S3Transfer/ConsoleProgressBarTest.php | 228 ++++++++++++++++++ .../S3Transfer/DefaultProgressTrackerTest.php | 189 +++++++++++++++ .../S3Transfer/MultipartDownloaderTest.php | 12 + .../S3Transfer/ObjectProgressTrackerTest.php | 127 ++++++++++ .../S3Transfer/TransferListenerTest.php | 216 +++++++++++++++++ 9 files changed, 903 insertions(+), 42 deletions(-) create mode 100644 tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php create mode 100644 tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php create mode 100644 tests/S3/Features/S3Transfer/MultipartDownloaderTest.php create mode 100644 tests/S3/Features/S3Transfer/ObjectProgressTrackerTest.php create mode 100644 tests/S3/Features/S3Transfer/TransferListenerTest.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php index 3f4312bfa7..23f41f0e57 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -92,11 +92,17 @@ public function setArgs(array $args): void $this->args = $args; } + /** + * Sets an argument. + * + * @param string $key + * @param mixed $value + * + * @return void + */ public function setArg(string $key, mixed $value): void { - if (array_key_exists($key, $this->args)) { - $this->args[$key] = $value; - } + $this->args[$key] = $value; } private function renderProgressBar(): string { @@ -112,7 +118,7 @@ private function renderProgressBar(): string { public function getPaintedProgress(): string { foreach ($this->format['parameters'] as $param) { if (!array_key_exists($param, $this->args)) { - throw new \InvalidArgumentException("Missing '{$param}' parameter for progress bar."); + throw new \InvalidArgumentException("Missing `$param` parameter for progress bar."); } } diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php index ae47baab92..3a8cd92c68 100644 --- a/src/S3/Features/S3Transfer/DefaultProgressTracker.php +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -27,16 +27,27 @@ class DefaultProgressTracker /** @var TransferListener */ private TransferListener $transferListener; - private Closure| ProgressBarFactory | null $progressBarFactory; + /** @var Closure|ProgressBarFactory|null */ + private Closure|ProgressBarFactory|null $progressBarFactory; + + /** @var resource */ + private $output; /** * @param Closure|ProgressBarFactory|null $progressBarFactory */ - public function __construct(Closure | ProgressBarFactory | null $progressBarFactory = null) - { + public function __construct( + Closure | ProgressBarFactory | null $progressBarFactory = null, + $output = STDOUT + ) { $this->clear(); $this->initializeListener(); $this->progressBarFactory = $progressBarFactory ?? $this->defaultProgressBarFactory(); + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + + $this->output = $output; } private function initializeListener(): void { @@ -56,6 +67,46 @@ public function getTransferListener(): TransferListener { return $this->transferListener; } + /** + * @return int + */ + public function getTotalBytesTransferred(): int + { + return $this->totalBytesTransferred; + } + + /** + * @return int + */ + public function getObjectsTotalSizeInBytes(): int + { + return $this->objectsTotalSizeInBytes; + } + + /** + * @return int + */ + public function getObjectsInProgress(): int + { + return $this->objectsInProgress; + } + + /** + * @return int + */ + public function getObjectsCount(): int + { + return $this->objectsCount; + } + + /** + * @return int + */ + public function getTransferPercentCompleted(): int + { + return $this->transferPercentCompleted; + } + /** * * @return Closure @@ -107,6 +158,9 @@ private function objectTransferProgress(): Closure }; } + /** + * @return Closure + */ public function objectTransferFailed(): Closure { return function ( @@ -123,6 +177,9 @@ public function objectTransferFailed(): Closure }; } + /** + * @return Closure + */ public function objectTransferCompleted(): Closure { return function ( @@ -150,6 +207,11 @@ public function clear(): void $this->transferPercentCompleted = 0; } + /** + * @param int $bytesTransferred + * + * @return void + */ private function increaseBytesTransferred(int $bytesTransferred): void { $this->totalBytesTransferred += $bytesTransferred; if ($this->objectsTotalSizeInBytes !== 0) { @@ -157,23 +219,33 @@ private function increaseBytesTransferred(int $bytesTransferred): void { } } + /** + * @return void + */ private function showProgress(): void { // Clear screen - fwrite(STDOUT, "\033[2J\033[H"); + fwrite($this->output, "\033[2J\033[H"); // Display progress header - echo sprintf( + fwrite($this->output, sprintf( "\r%d%% [%s/%s]\n", $this->transferPercentCompleted, $this->objectsInProgress, $this->objectsCount - ); + )); foreach ($this->objects as $name => $object) { - echo sprintf("\r%s:\n%s\n", $name, $object->getProgressBar()->getPaintedProgress()); + fwrite($this->output, sprintf( + "\r%s:\n%s\n", + $name, + $object->getProgressBar()->getPaintedProgress() + )); } } + /** + * @return Closure|ProgressBarFactory + */ private function defaultProgressBarFactory(): Closure| ProgressBarFactory { return function () { return new ConsoleProgressBar( @@ -183,7 +255,7 @@ private function defaultProgressBarFactory(): Closure| ProgressBarFactory { args: [ 'transferred' => 0, 'tobe_transferred' => 0, - 'unit' => 'MB', + 'unit' => 'B', 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, ] ); diff --git a/src/S3/Features/S3Transfer/ObjectProgressTracker.php b/src/S3/Features/S3Transfer/ObjectProgressTracker.php index 9c876e363a..5eadecfdc4 100644 --- a/src/S3/Features/S3Transfer/ObjectProgressTracker.php +++ b/src/S3/Features/S3Transfer/ObjectProgressTracker.php @@ -7,45 +7,25 @@ */ class ObjectProgressTracker { - /** @var string */ - private string $objectKey; - - /** @var int */ - private int $objectBytesTransferred; - - /** @var int */ - private int $objectSizeInBytes; - - /** @var ?ProgressBar */ - private ?ProgressBar $progressBar; - - /** - * @var string - * - initiated - * - progress - * - failed - * - completed - */ - private string $status; - /** * @param string $objectKey * @param int $objectBytesTransferred * @param int $objectSizeInBytes * @param string $status + * Possible values are: + * - initiated + * - progress + * - failed + * - completed * @param ?ProgressBar $progressBar */ public function __construct( - string $objectKey, - int $objectBytesTransferred, - int $objectSizeInBytes, - string $status, - ?ProgressBar $progressBar = null + private string $objectKey, + private int $objectBytesTransferred, + private int $objectSizeInBytes, + private string $status, + private ?ProgressBar $progressBar = null ) { - $this->objectKey = $objectKey; - $this->objectBytesTransferred = $objectBytesTransferred; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->status = $status; $this->progressBar = $progressBar ?? $this->defaultProgressBar(); } diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php index 9f3f308978..1fb5a3f74a 100644 --- a/src/S3/Features/S3Transfer/TransferListener.php +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -71,6 +71,37 @@ public function __construct( private int $objectsToBeTransferred = 0 ) {} + /** + * @return int + */ + public function getObjectsTransferCompleted(): int + { + return $this->objectsTransferCompleted; + } + + /** + * @return int + */ + public function getObjectsBytesTransferred(): int + { + return $this->objectsBytesTransferred; + } + + /** + * @return int + */ + public function getObjectsTransferFailed(): int + { + return $this->objectsTransferFailed; + } + + /** + * @return int + */ + public function getObjectsToBeTransferred(): int + { + return $this->objectsToBeTransferred; + } /** * Transfer initiated event. diff --git a/tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php b/tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php new file mode 100644 index 0000000000..8f59c5ac4c --- /dev/null +++ b/tests/S3/Features/S3Transfer/ConsoleProgressBarTest.php @@ -0,0 +1,228 @@ +setPercentCompleted($percent); + $progressBar->setArgs([ + 'transferred' => $transferred, + 'tobe_transferred' => $toBeTransferred, + 'unit' => $unit + ]); + + $output = $progressBar->getPaintedProgress(); + $this->assertEquals($expectedProgress, $output); + } + + /** + * Data provider for testing progress bar rendering. + * + * @return array + */ + public function progressBarPercentProvider(): array { + return [ + [ + 'percent' => 25, + 'transferred' => 25, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[###### ] 25% 25/100 B' + ], + [ + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[############# ] 50% 50/100 B' + ], + [ + 'percent' => 75, + 'transferred' => 75, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[################### ] 75% 75/100 B' + ], + [ + 'percent' => 100, + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'expected' => '[#########################] 100% 100/100 B' + ], + ]; + } + + /** + * Tests progress with custom char. + * + * @return void + */ + public function testProgressBarWithCustomChar() + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 30 + ); + $progressBar->setPercentCompleted(30); + $progressBar->setArgs([ + 'transferred' => '10', + 'tobe_transferred' => '100', + 'unit' => 'B' + ]); + + $output = $progressBar->getPaintedProgress(); + $this->assertStringContainsString('10/100 B', $output); + $this->assertStringContainsString(str_repeat('*', 9), $output); + } + + /** + * Tests progress with custom char. + * + * @return void + */ + public function testProgressBarWithCustomWidth() + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 100 + ); + $progressBar->setPercentCompleted(10); + $progressBar->setArgs([ + 'transferred' => '10', + 'tobe_transferred' => '100', + 'unit' => 'B' + ]); + + $output = $progressBar->getPaintedProgress(); + $this->assertStringContainsString('10/100 B', $output); + $this->assertStringContainsString(str_repeat('*', 10), $output); + } + + /** + * Tests missing parameters. + * + * @dataProvider progressBarMissingArgsProvider + * + * @return void + */ + public function testProgressBarMissingArgsThrowsException( + string $formatName, + string $parameter + ) + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Missing `$parameter` parameter for progress bar."); + + $format = ConsoleProgressBar::$formats[$formatName]; + $progressBar = new ConsoleProgressBar( + format: $format, + ); + foreach ($format['parameters'] as $param) { + if ($param === $parameter) { + continue; + } + + $progressBar->setArg($param, 'foo'); + } + + $progressBar->setPercentCompleted(20); + $progressBar->getPaintedProgress(); + } + + + /** + * Data provider for testing exception when arguments are missing. + * + * @return array + */ + public function progressBarMissingArgsProvider(): array + { + return [ + [ + 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, + 'parameter' => 'transferred', + ], + [ + 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, + 'parameter' => 'tobe_transferred', + ], + [ + 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, + 'parameter' => 'unit', + ] + ]; + } + + /** + * Tests the progress bar does not overflow when the percent is over 100. + * + * @return void + */ + public function testProgressBarDoesNotOverflowAfter100Percent() + { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 10, + ); + $progressBar->setPercentCompleted(110); + $progressBar->setArgs([ + 'transferred' => 'foo', + 'tobe_transferred' => 'foo', + 'unit' => 'MB' + ]); + $output = $progressBar->getPaintedProgress(); + $this->assertStringContainsString('100%', $output); + $this->assertStringContainsString('[**********]', $output); + } + + /** + * Tests the progress bar sets the arguments. + * + * @return void + */ + public function testProgressBarSetsArguments() { + $progressBar = new ConsoleProgressBar( + progressBarChar: '*', + progressBarWidth: 25, + format: ConsoleProgressBar::$formats[ConsoleProgressBar::TRANSFER_FORMAT] + ); + $progressBar->setArgs([ + 'transferred' => 'fooTransferred', + 'tobe_transferred' => 'fooToBeTransferred', + 'unit' => 'fooUnit', + ]); + $output = $progressBar->getPaintedProgress(); + $progressBar->setPercentCompleted(100); + $this->assertStringContainsString('fooTransferred', $output); + $this->assertStringContainsString('fooToBeTransferred', $output); + $this->assertStringContainsString('fooUnit', $output); + } +} diff --git a/tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php b/tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php new file mode 100644 index 0000000000..fd82cd78f5 --- /dev/null +++ b/tests/S3/Features/S3Transfer/DefaultProgressTrackerTest.php @@ -0,0 +1,189 @@ +progressTracker = new DefaultProgressTracker( + output: $this->output = fopen('php://temp', 'r+') + ); + } + + protected function tearDown(): void { + fclose($this->output); + } + + /** + * Tests initialization is clean. + * + * @return void + */ + public function testInitialization(): void + { + $this->assertInstanceOf(TransferListener::class, $this->progressTracker->getTransferListener()); + $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); + $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); + $this->assertEquals(0, $this->progressTracker->getObjectsCount()); + $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); + } + + /** + * Tests object transfer is initiated when the event is triggered. + * + * @return void + */ + public function testObjectTransferInitiated(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + + $this->assertEquals(1, $this->progressTracker->getObjectsInProgress()); + $this->assertEquals(1, $this->progressTracker->getObjectsCount()); + } + + /** + * Tests object transfer progress is propagated correctly. + * + * @dataProvider objectTransferProgressProvider + * + * @param string $objectKey + * @param int $objectSize + * @param array $progressList + * + * @return void + */ + public function testObjectTransferProgress( + string $objectKey, + int $objectSize, + array $progressList, + ): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)($objectKey, $fakeRequestArgs); + $totalProgress = 0; + foreach ($progressList as $progress) { + ($listener->onObjectTransferProgress)($objectKey, $progress, $objectSize); + $totalProgress += $progress; + } + + $this->assertEquals($totalProgress, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals($objectSize, $this->progressTracker->getObjectsTotalSizeInBytes()); + $percentCompleted = (int) floor($totalProgress / $objectSize) * 100; + $this->assertEquals($percentCompleted, $this->progressTracker->getTransferPercentCompleted()); + + rewind($this->output); + $this->assertStringContainsString("$percentCompleted% $totalProgress/$objectSize B", stream_get_contents($this->output)); + } + + /** + * Data provider for testing object progress tracker. + * + * @return array[] + */ + public function objectTransferProgressProvider(): array + { + return [ + [ + 'objectKey' => 'FooObjectKey', + 'objectSize' => 250, + 'progressList' => [ + 50, 100, 72, 28 + ] + ], + [ + 'objectKey' => 'FooObjectKey', + 'objectSize' => 10_000, + 'progressList' => [ + 100, 500, 1_000, 2_000, 5_000, 400, 700, 300 + ] + ], + [ + 'objectKey' => 'FooObjectKey', + 'objectSize' => 10_000, + 'progressList' => [ + 5_000, 5_000 + ] + ] + ]; + } + + /** + * Tests object transfer is completed. + * + * @return void + */ + public function testObjectTransferCompleted(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); + ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); + ($listener->onObjectTransferCompleted)('FooObjectKey', 100); + + $this->assertEquals(100, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(100, $this->progressTracker->getTransferPercentCompleted()); + + // Validate it completed 100% at the progress bar side. + rewind($this->output); + $this->assertStringContainsString("[#########################] 100% 100/100 B", stream_get_contents($this->output)); + } + + /** + * Tests object transfer failed. + * + * @return void + */ + public function testObjectTransferFailed(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + ($listener->onObjectTransferProgress)('FooObjectKey', 27, 100); + ($listener->onObjectTransferFailed)('FooObjectKey', 27, 'Transfer error'); + + $this->assertEquals(27, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(27, $this->progressTracker->getTransferPercentCompleted()); + $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); + + rewind($this->output); + $this->assertStringContainsString("27% 27/100 B", stream_get_contents($this->output)); + } + + /** + * Tests state are cleared. + * + * @return void + */ + public function testClearState(): void + { + $listener = $this->progressTracker->getTransferListener(); + $fakeRequestArgs = []; + ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); + ($listener->onObjectTransferProgress)('FooObjectKey', 10, 100); + + $this->progressTracker->clear(); + + $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); + $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); + $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); + $this->assertEquals(0, $this->progressTracker->getObjectsCount()); + $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); + } +} + diff --git a/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php new file mode 100644 index 0000000000..6be2e5cf4c --- /dev/null +++ b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php @@ -0,0 +1,12 @@ +mockProgressBar = $this->createMock(ProgressBar::class); + } + + /** + * Tests getter and setters. + * + * @return void + */ + public function testGettersAndSetters(): void + { + $tracker = new ObjectProgressTracker( + '', + 0, + 0, + '' + ); + $tracker->setObjectKey('FooKey'); + $this->assertEquals('FooKey', $tracker->getObjectKey()); + + $tracker->setObjectBytesTransferred(100); + $this->assertEquals(100, $tracker->getObjectBytesTransferred()); + + $tracker->setObjectSizeInBytes(100); + $this->assertEquals(100, $tracker->getObjectSizeInBytes()); + + $tracker->setStatus('initiated'); + $this->assertEquals('initiated', $tracker->getStatus()); + } + + /** + * Tests bytes transferred increments. + * + * @return void + */ + public function testIncrementTotalBytesTransferred(): void + { + $percentProgress = 0; + $this->mockProgressBar->expects($this->atLeast(4)) + ->method('setPercentCompleted') + ->willReturnCallback(function ($percent) use (&$percentProgress) { + $this->assertEquals($percentProgress +=25, $percent); + }); + + $tracker = new ObjectProgressTracker( + objectKey: 'FooKey', + objectBytesTransferred: 0, + objectSizeInBytes: 100, + status: 'initiated', + progressBar: $this->mockProgressBar + ); + + $tracker->incrementTotalBytesTransferred(25); + $tracker->incrementTotalBytesTransferred(25); + $tracker->incrementTotalBytesTransferred(25); + $tracker->incrementTotalBytesTransferred(25); + + $this->assertEquals(100, $tracker->getObjectBytesTransferred()); + } + + + /** + * Tests progress status color based on states. + * + * @return void + */ + public function testSetStatusUpdatesProgressBarColor() + { + $statusColorMapping = [ + 'progress' => ConsoleProgressBar::BLUE_COLOR_CODE, + 'completed' => ConsoleProgressBar::GREEN_COLOR_CODE, + 'failed' => ConsoleProgressBar::RED_COLOR_CODE, + ]; + $values = array_values($statusColorMapping); + $valueIndex = 0; + $this->mockProgressBar->expects($this->exactly(3)) + ->method('setArg') + ->willReturnCallback(function ($_, $argValue) use ($values, &$valueIndex) { + $this->assertEquals($argValue, $values[$valueIndex++]); + }); + + $tracker = new ObjectProgressTracker( + objectKey: 'FooKey', + objectBytesTransferred: 0, + objectSizeInBytes: 100, + status: 'initiated', + progressBar: $this->mockProgressBar + ); + + foreach ($statusColorMapping as $status => $value) { + $tracker->setStatus($status); + } + } + + /** + * Tests the default progress bar is initialized when not provided. + * + * @return void + */ + public function testDefaultProgressBarIsInitialized() + { + $tracker = new ObjectProgressTracker( + objectKey: 'FooKey', + objectBytesTransferred: 0, + objectSizeInBytes: 100, + status: 'initiated' + ); + $this->assertInstanceOf(ProgressBar::class, $tracker->getProgressBar()); + } +} diff --git a/tests/S3/Features/S3Transfer/TransferListenerTest.php b/tests/S3/Features/S3Transfer/TransferListenerTest.php new file mode 100644 index 0000000000..ab079ee97c --- /dev/null +++ b/tests/S3/Features/S3Transfer/TransferListenerTest.php @@ -0,0 +1,216 @@ +objectTransferInitiated('FooObjectKey', $requestArgs); + $this->assertEquals(1, $listener->getObjectsToBeTransferred()); + + $this->assertTrue($called); + } + + /** + * Tests object transfer is initiated. + * + * @return void + */ + public function testObjectTransferIsInitiated(): void + { + $called = false; + $listener = new TransferListener( + onObjectTransferInitiated: function () use (&$called) { + $called = true; + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey', $requestArgs); + $this->assertEquals(1, $listener->getObjectsToBeTransferred()); + + $this->assertTrue($called); + } + + /** + * Tests object transfer progress. + * + * @dataProvider objectTransferProgressProvider + * + * @param array $objects + * + * @return void + */ + public function testObjectTransferProgress( + array $objects + ): void { + $called = 0; + $listener = new TransferListener( + onObjectTransferProgress: function () use (&$called) { + $called++; + } + ); + $totalTransferred = 0; + foreach ($objects as $objectKey => $transferDetails) { + $requestArgs = []; + $listener->objectTransferInitiated( + $objectKey, + $requestArgs, + ); + $listener->objectTransferProgress( + $objectKey, + $transferDetails['transferredInBytes'], + $transferDetails['sizeInBytes'] + ); + $totalTransferred += $transferDetails['transferredInBytes']; + } + + $this->assertEquals(count($objects), $called); + $this->assertEquals(count($objects), $listener->getObjectsToBeTransferred()); + $this->assertEquals($totalTransferred, $listener->getObjectsBytesTransferred()); + } + + /** + * @return array + */ + public function objectTransferProgressProvider(): array + { + return [ + [ + [ + 'FooObjectKey1' => [ + 'sizeInBytes' => 100, + 'transferredInBytes' => 95, + ], + 'FooObjectKey2' => [ + 'sizeInBytes' => 500, + 'transferredInBytes' => 345, + ], + 'FooObjectKey3' => [ + 'sizeInBytes' => 1024, + 'transferredInBytes' => 256, + ], + ] + ] + ]; + } + + /** + * Tests object transfer failed. + * + * @return void + */ + public function testObjectTransferFailed(): void + { + $expectedBytesTransferred = 45; + $expectedReason = "Transfer failed!"; + $listener = new TransferListener( + onObjectTransferFailed: function ( + string $objectKey, + int $objectBytesTransferred, + string $reason + ) use ($expectedBytesTransferred, $expectedReason) { + $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); + $this->assertEquals($expectedReason, $reason); + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey', $requestArgs); + $listener->objectTransferFailed( + 'FooObjectKey', + $expectedBytesTransferred, + $expectedReason + ); + + $this->assertEquals(1, $listener->getObjectsTransferFailed()); + $this->assertEquals(0, $listener->getObjectsTransferCompleted()); + } + + /** + * Tests object transfer completed. + * + * @return void + */ + public function testObjectTransferCompleted(): void + { + $expectedBytesTransferred = 100; + $listener = new TransferListener( + onObjectTransferCompleted: function ($objectKey, $objectBytesTransferred) + use ($expectedBytesTransferred) { + $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey', $requestArgs); + $listener->objectTransferProgress( + 'FooObjectKey', + $expectedBytesTransferred, + $expectedBytesTransferred + ); + $listener->objectTransferCompleted('FooObjectKey', $expectedBytesTransferred); + + $this->assertEquals(1, $listener->getObjectsTransferCompleted()); + $this->assertEquals($expectedBytesTransferred, $listener->getObjectsBytesTransferred()); + } + + /** + * Tests transfer is completed once all the objects in progress are completed. + * + * @return void + */ + public function testTransferCompleted(): void + { + $expectedObjectsTransferred = 2; + $expectedObjectBytesTransferred = 200; + $listener = new TransferListener( + onTransferCompleted: function(int $objectsTransferredCompleted, int $objectsBytesTransferred) + use ($expectedObjectsTransferred, $expectedObjectBytesTransferred) { + $this->assertEquals($expectedObjectsTransferred, $objectsTransferredCompleted); + $this->assertEquals($expectedObjectBytesTransferred, $objectsBytesTransferred); + } + ); + $requestArgs = []; + $listener->objectTransferInitiated('FooObjectKey_1', $requestArgs); + $listener->objectTransferInitiated('FooObjectKey_2', $requestArgs); + $listener->objectTransferProgress( + 'FooObjectKey_1', + 100, + 100 + ); + $listener->objectTransferProgress( + 'FooObjectKey_2', + 100, + 100 + ); + $listener->objectTransferCompleted( + 'FooObjectKey_1', + 100, + ); + $listener->objectTransferCompleted( + 'FooObjectKey_2', + 100, + ); + + $this->assertEquals($expectedObjectsTransferred, $listener->getObjectsTransferCompleted()); + $this->assertEquals($expectedObjectBytesTransferred, $listener->getObjectsBytesTransferred()); + } +} From 1c82ab5aab901af01ef48b6e1c979d2f4154a911 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 5 Feb 2025 18:35:10 -0800 Subject: [PATCH 03/62] chore: add multipart download listener tests - Add test cases for multipart download listener. --- .../S3Transfer/MultipartDownloadListener.php | 34 +-- .../MultipartDownloadListenerTest.php | 207 ++++++++++++++++++ 2 files changed, 224 insertions(+), 17 deletions(-) create mode 100644 tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php diff --git a/src/S3/Features/S3Transfer/MultipartDownloadListener.php b/src/S3/Features/S3Transfer/MultipartDownloadListener.php index 4ffdff1559..69936cb2a5 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloadListener.php +++ b/src/S3/Features/S3Transfer/MultipartDownloadListener.php @@ -16,17 +16,17 @@ class MultipartDownloadListener extends ListenerNotifier * * @param Closure|null $onDownloadFailed * Parameters that will be passed when invoked: - * - $reason: The throwable with the reason why the transfer failed. - * - $totalPartsTransferred: The total of parts transferred before failure. - * - $totalBytesTransferred: The total of bytes transferred before failure. - * - $lastPartTransferred: The number of the last part that was transferred + * - $reason: The throwable with the reason why the download failed. + * - $totalPartsDownloaded: The total of parts downloaded before failure. + * - $totalBytesDownloaded: The total of bytes downloaded before failure. + * - $lastPartDownloaded: The number of the last part that was downloaded * before failure. * * @param Closure|null $onDownloadCompleted * Parameters that will be passed when invoked: * - $stream: The stream which holds the bytes for the file downloaded. - * - $totalPartsDownloaded: The number of objects that were transferred. - * - $totalBytesDownloaded: The total of bytes that were transferred. + * - $totalPartsDownloaded: The number of parts that were downloaded. + * - $totalBytesDownloaded: The total of bytes that were downloaded. * * @param Closure|null $onPartDownloadInitiated * Parameters that will be passed when invoked: @@ -39,7 +39,7 @@ class MultipartDownloadListener extends ListenerNotifier * - $partNo: The part number just downloaded. * - $partTotalBytes: The size of the part just downloaded. * - $totalParts: The total parts for the full object to be downloaded. - * - $objectBytesTransferred: The total in bytes already downloaded. + * - $objectBytesDownloaded: The total in bytes already downloaded. * - $objectSizeInBytes: The total in bytes for the full object to be downloaded. * * @param Closure|null $onPartDownloadFailed @@ -66,11 +66,11 @@ public function __construct( * keep the states maintained in this implementation. * * @param array &$commandArgs - * @param ?int $initialPart + * @param int $initialPart * * @return void */ - public function downloadInitiated(array &$commandArgs, ?int $initialPart): void { + public function downloadInitiated(array &$commandArgs, int $initialPart): void { $this->notify('onDownloadInitiated', [&$commandArgs, $initialPart]); } @@ -81,14 +81,14 @@ public function downloadInitiated(array &$commandArgs, ?int $initialPart): void * keep the states maintained in this implementation. * * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred + * @param int $totalPartsDownloaded + * @param int $totalBytesDownloaded + * @param int $lastPartDownloaded * * @return void */ - public function downloadFailed(\Throwable $reason, int $totalPartsTransferred, int $totalBytesTransferred, int $lastPartTransferred): void { - $this->notify('onDownloadFailed', [$reason, $totalPartsTransferred, $totalBytesTransferred, $lastPartTransferred]); + public function downloadFailed(\Throwable $reason, int $totalPartsDownloaded, int $totalBytesDownloaded, int $lastPartDownloaded): void { + $this->notify('onDownloadFailed', [$reason, $totalPartsDownloaded, $totalBytesDownloaded, $lastPartDownloaded]); } /** @@ -132,7 +132,7 @@ public function partDownloadInitiated(CommandInterface $partDownloadCommand, int * @param int $partNo * @param int $partTotalBytes * @param int $totalParts - * @param int $objectBytesTransferred + * @param int $objectBytesDownloaded * @param int $objectSizeInBytes * @return void */ @@ -141,7 +141,7 @@ public function partDownloadCompleted( int $partNo, int $partTotalBytes, int $totalParts, - int $objectBytesTransferred, + int $objectBytesDownloaded, int $objectSizeInBytes ): void { @@ -150,7 +150,7 @@ public function partDownloadCompleted( $partNo, $partTotalBytes, $totalParts, - $objectBytesTransferred, + $objectBytesDownloaded, $objectSizeInBytes ]); } diff --git a/tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php b/tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php new file mode 100644 index 0000000000..b615d3ad04 --- /dev/null +++ b/tests/S3/Features/S3Transfer/MultipartDownloadListenerTest.php @@ -0,0 +1,207 @@ +assertIsArray($commandArgs); + $this->assertIsInt($initialPart); + }; + + $listener = new MultipartDownloadListener(onDownloadInitiated: $callback); + + $commandArgs = ['Foo' => 'Buzz']; + $listener->downloadInitiated($commandArgs, 1); + + $this->assertTrue($called, "Expected onDownloadInitiated to be called."); + } + + /** + * Tests download failed event is propagated. + * + * @return void + */ + public function testDownloadFailed(): void + { + $called = false; + $expectedError = new Exception('Download failed'); + $expectedTotalPartsTransferred = 5; + $expectedTotalBytesTransferred = 1024; + $expectedLastPartTransferred = 4; + $callback = function ( + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ) use ( + &$called, + $expectedError, + $expectedTotalPartsTransferred, + $expectedTotalBytesTransferred, + $expectedLastPartTransferred + ) { + $called = true; + $this->assertEquals($reason, $expectedError); + $this->assertEquals($expectedTotalPartsTransferred, $totalPartsTransferred); + $this->assertEquals($expectedTotalBytesTransferred, $totalBytesTransferred); + $this->assertEquals($expectedLastPartTransferred, $lastPartTransferred); + + }; + $listener = new MultipartDownloadListener(onDownloadFailed: $callback); + $listener->downloadFailed( + $expectedError, + $expectedTotalPartsTransferred, + $expectedTotalBytesTransferred, + $expectedLastPartTransferred + ); + $this->assertTrue($called, "Expected onDownloadFailed to be called."); + } + + /** + * Tests download completed event is propagated. + * + * @return void + */ + public function testDownloadCompleted(): void + { + $called = false; + $expectedStream = fopen('php://temp', 'r+'); + $expectedTotalPartsDownloaded = 10; + $expectedTotalBytesDownloaded = 2048; + $callback = function ( + $stream, + $totalPartsDownloaded, + $totalBytesDownloaded + ) use ( + &$called, + $expectedStream, + $expectedTotalPartsDownloaded, + $expectedTotalBytesDownloaded + ) { + $called = true; + $this->assertIsResource($stream); + $this->assertEquals($expectedStream, $stream); + $this->assertEquals($expectedTotalPartsDownloaded, $totalPartsDownloaded); + $this->assertEquals($expectedTotalBytesDownloaded, $totalBytesDownloaded); + }; + + $listener = new MultipartDownloadListener(onDownloadCompleted: $callback); + $listener->downloadCompleted( + $expectedStream, + $expectedTotalPartsDownloaded, + $expectedTotalBytesDownloaded + ); + $this->assertTrue($called, "Expected onDownloadCompleted to be called."); + } + + /** + * Tests part downloaded initiated event is propagated. + * + * @return void + */ + public function testPartDownloadInitiated(): void + { + $called = false; + $mockCommand = $this->createMock(CommandInterface::class); + $expectedPartNo = 3; + $callable = function ($command, $partNo) + use (&$called, $mockCommand, $expectedPartNo) { + $called = true; + $this->assertEquals($expectedPartNo, $partNo); + $this->assertEquals($mockCommand, $command); + }; + $listener = new MultipartDownloadListener(onPartDownloadInitiated: $callable); + $listener->partDownloadInitiated($mockCommand, $expectedPartNo); + $this->assertTrue($called, "Expected onPartDownloadInitiated to be called."); + } + + /** + * Tests part download completed event is propagated. + * + * @return void + */ + public function testPartDownloadCompleted(): void + { + $called = false; + $mockResult = $this->createMock(ResultInterface::class); + $expectedPartNo = 3; + $expectedPartTotalBytes = 512; + $expectedTotalParts = 5; + $expectedObjectBytesTransferred = 1024; + $expectedObjectSizeInBytes = 2048; + $callback = function ( + $result, + $partNo, + $partTotalBytes, + $totalParts, + $objectBytesDownloaded, + $objectSizeInBytes + ) use ( + &$called, + $mockResult, + $expectedPartNo, + $expectedPartTotalBytes, + $expectedTotalParts, + $expectedObjectBytesTransferred, + $expectedObjectSizeInBytes + ) { + $called = true; + $this->assertEquals($mockResult, $result); + $this->assertEquals($expectedPartNo, $partNo); + $this->assertEquals($expectedPartTotalBytes, $partTotalBytes); + $this->assertEquals($expectedTotalParts, $totalParts); + $this->assertEquals($expectedObjectBytesTransferred, $objectBytesDownloaded); + $this->assertEquals($expectedObjectSizeInBytes, $objectSizeInBytes); + }; + $listener = new MultipartDownloadListener(onPartDownloadCompleted: $callback); + $listener->partDownloadCompleted( + $mockResult, + $expectedPartNo, + $expectedPartTotalBytes, + $expectedTotalParts, + $expectedObjectBytesTransferred, + $expectedObjectSizeInBytes + ); + $this->assertTrue($called, "Expected onPartDownloadCompleted to be called."); + } + + /** + * Tests part download failed event is propagated. + * + * @return void + */ + public function testPartDownloadFailed() + { + $called = false; + $mockCommand = $this->createMock(CommandInterface::class); + $expectedReason = new Exception('Part download failed'); + $expectedPartNo = 2; + $callable = function ($command, $reason, $partNo) + use (&$called, $mockCommand, $expectedReason, $expectedPartNo) { + $called = true; + $this->assertEquals($expectedReason, $reason); + $this->assertEquals($expectedPartNo, $partNo); + $this->assertEquals($mockCommand, $command); + }; + + $listener = new MultipartDownloadListener(onPartDownloadFailed: $callable); + $listener->partDownloadFailed($mockCommand, $expectedReason, $expectedPartNo); + $this->assertTrue($called, "Expected onPartDownloadFailed to be called."); + } +} \ No newline at end of file From 237fc7b1f34c5386b839682fc63029598ba8566a Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 6 Feb 2025 09:03:16 -0800 Subject: [PATCH 04/62] chore: refactor multipart downloaders and add tests - Add a trait to the MultipartDownloader implementation to keep the main implementatio cleaner. - Add test cases for multipart downloader, in specific testing part and range get multipart downloader. --- .../S3Transfer/MultipartDownloader.php | 387 ++++++------------ .../S3Transfer/MultipartDownloaderTrait.php | 174 ++++++++ .../S3Transfer/RangeMultipartDownloader.php | 82 +++- .../Features/S3Transfer/S3TransferManager.php | 30 +- .../S3Transfer/S3TransferManagerTrait.php | 1 - .../S3Transfer/MultipartDownloaderTest.php | 150 ++++++- 6 files changed, 548 insertions(+), 276 deletions(-) create mode 100644 src/S3/Features/S3Transfer/MultipartDownloaderTrait.php diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php index 03b0e6430a..50b575c455 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloader.php +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -3,7 +3,6 @@ namespace Aws\S3\Features\S3Transfer; use Aws\CommandInterface; -use Aws\Result; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; use GuzzleHttp\Promise\Coroutine; @@ -15,79 +14,97 @@ abstract class MultipartDownloader implements PromisorInterface { + use MultipartDownloaderTrait; public const GET_OBJECT_COMMAND = "GetObject"; public const PART_GET_MULTIPART_DOWNLOADER = "partGet"; public const RANGE_GET_MULTIPART_DOWNLOADER = "rangeGet"; - /** @var S3ClientInterface */ - protected S3ClientInterface $s3Client; - - /** @var array */ - protected array $requestArgs; - - /** @var array */ - protected array $config; - - /** @var int */ - protected int $currentPartNo; - - /** @var int */ - protected int $objectPartsCount; - - /** @var int */ - protected int $objectCompletedPartsCount; - - /** @var int */ - protected int $objectSizeInBytes; - - /** @var int */ - protected int $objectBytesTransferred; - - /** @var ?MultipartDownloadListener */ - protected ?MultipartDownloadListener $listener; - - /** @var ?TransferListener */ - protected ?TransferListener $progressListener; - - /** @var StreamInterface */ - private StreamInterface $stream; - - /** @var string */ - protected string $eTag; - - /** @var string */ - protected string $objectKey; - /** * @param S3ClientInterface $s3Client * @param array $requestArgs * @param array $config - * - targetPartSizeBytes: The minimum part size for a multipart download - * using range get. + * - minimumPartSize: The minimum part size for a multipart download + * using range get. This option MUST be set when using range get. * @param int $currentPartNo - * @param ?MultipartDownloadListener $listener - * @param ?TransferListener $transferListener + * @param int $objectPartsCount + * @param int $objectCompletedPartsCount + * @param int $objectSizeInBytes + * @param int $objectBytesTransferred + * @param string $eTag + * @param string $objectKey + * @param MultipartDownloadListener|null $listener + * @param TransferListener|null $progressListener + * @param StreamInterface|null $stream */ public function __construct( - S3ClientInterface $s3Client, - array $requestArgs, - array $config, - int $currentPartNo = 0, - ?MultipartDownloadListener $listener = null, - ?TransferListener $progressListener = null + protected readonly S3ClientInterface $s3Client, + protected array $requestArgs, + protected readonly array $config = [], + protected int $currentPartNo = 0, + protected int $objectPartsCount = 0, + protected int $objectCompletedPartsCount = 0, + protected int $objectSizeInBytes = 0, + protected int $objectBytesTransferred = 0, + protected string $eTag = "", + protected string $objectKey = "", + private readonly ?MultipartDownloadListener $listener = null, + private readonly ?TransferListener $progressListener = null, + private ?StreamInterface $stream = null ) { - $this->clear(); - $this->s3Client = $s3Client; - $this->requestArgs = $requestArgs; - $this->config = $config; - $this->currentPartNo = $currentPartNo; - $this->listener = $listener; - $this->progressListener = $progressListener; - $this->stream = Utils::streamFor( - fopen('php://temp', 'w+') - ); + if ($stream === null) { + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } } + /** + * @return int + */ + public function getCurrentPartNo(): int + { + return $this->currentPartNo; + } + + /** + * @return int + */ + public function getObjectPartsCount(): int + { + return $this->objectPartsCount; + } + + /** + * @return int + */ + public function getObjectCompletedPartsCount(): int + { + return $this->objectCompletedPartsCount; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return int + */ + public function getObjectBytesTransferred(): int + { + return $this->objectBytesTransferred; + } + + /** + * @return string + */ + public function getObjectKey(): string + { + return $this->objectKey; + } /** * Returns that resolves a multipart download operation, @@ -138,6 +155,8 @@ public function promise(): PromiseInterface // TODO: yield transfer exception modeled with a transfer failed response. yield Create::rejectionFor($e); } + + $lastPartIncrement = $this->currentPartNo; } // Transfer completed @@ -148,170 +167,6 @@ public function promise(): PromiseInterface }); } - /** - * Main purpose of this method is to propagate - * the download-initiated event to listeners, but - * also it does some computation regarding internal states - * that need to be maintained. - * - * @param array $commandArgs - * @param int|null $currentPartNo - * - * @return void - */ - private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void - { - $this->objectKey = $commandArgs['Key']; - $this->progressListener?->objectTransferInitiated( - $this->objectKey, - $commandArgs - ); - $this->_notifyMultipartDownloadListeners('downloadInitiated', [ - &$commandArgs, - $currentPartNo - ]); - } - - /** - * Propagates download-failed event to listeners. - * It may also do some computation in order to maintain internal states. - * - * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred - * - * @return void - */ - private function downloadFailed( - \Throwable $reason, - int $totalPartsTransferred, - int $totalBytesTransferred, - int $lastPartTransferred - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $totalBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners('downloadFailed', [ - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ]); - } - - /** - * Propagates part-download-initiated event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param int $partNo - * - * @return void - */ - private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { - $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ - $partDownloadCommand, - $partNo - ]); - } - - /** - * Propagates part-download-completed to listeners. - * It also does some computation in order to maintain internal states. - * In this specific method we move each part content into an accumulative - * stream, which is meant to hold the full object content once the download - * is completed. - * - * @param ResultInterface $result - * @param int $partNo - * - * @return void - */ - private function partDownloadCompleted(ResultInterface $result, int $partNo): void { - $this->objectCompletedPartsCount++; - $partDownloadBytes = $result['ContentLength']; - $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - Utils::copyToStream($result['Body'], $this->stream); - - $this->progressListener?->objectTransferProgress( - $this->objectKey, - $partDownloadBytes, - $this->objectSizeInBytes - ); - - $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ - $result, - $partNo, - $partDownloadBytes, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred, - $this->objectSizeInBytes - ]); - } - - /** - * Propagates part-download-failed event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param \Throwable $reason - * @param int $partNo - * - * @return void - */ - private function partDownloadFailed( - CommandInterface $partDownloadCommand, - \Throwable $reason, - int $partNo - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners( - 'partDownloadFailed', - [$partDownloadCommand, $reason, $partNo]); - } - - /** - * Propagates object-download-completed event to listeners. - * It also resets the pointer of the stream to the first position, - * so that the stream is ready to be consumed once returned. - * - * @return void - */ - private function objectDownloadCompleted(): void - { - $this->stream->rewind(); - $this->progressListener?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred - ); - $this->_notifyMultipartDownloadListeners('downloadCompleted', [ - $this->stream, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred - ]); - } - - /** - * Internal helper method for notifying listeners of specific events. - * - * @param string $listenerMethod - * @param array $args - * - * @return void - */ - private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void - { - $this->listener?->{$listenerMethod}(...$args); - } - /** * Returns the next command for fetching the next object part. * @@ -336,7 +191,7 @@ abstract protected function computeObjectDimensions(ResultInterface $result): vo * @return int */ protected function computeObjectSize($sizeSource): int { - if (gettype($sizeSource) === "integer") { + if (is_int($sizeSource)) { return (int) $sizeSource; } @@ -344,21 +199,12 @@ protected function computeObjectSize($sizeSource): int { throw new \RuntimeException('Range must not be empty'); } + // For extracting the object size from the ContentRange header value. if (preg_match("/\/(\d+)$/", $sizeSource, $matches)) { return $matches[1]; } - throw new \RuntimeException('Invalid range format'); - } - - private function clear(): void { - $this->currentPartNo = 0; - $this->objectPartsCount = 0; - $this->objectCompletedPartsCount = 0; - $this->objectSizeInBytes = 0; - $this->objectBytesTransferred = 0; - $this->eTag = ""; - $this->objectKey = ""; + throw new \RuntimeException('Invalid source size format'); } /** @@ -369,6 +215,13 @@ private function clear(): void { * @param string $multipartDownloadType * @param array $requestArgs * @param array $config + * @param int $currentPartNo + * @param int $objectPartsCount + * @param int $objectCompletedPartsCount + * @param int $objectSizeInBytes + * @param int $objectBytesTransferred + * @param string $eTag + * @param string $objectKey * @param MultipartDownloadListener|null $listener * @param TransferListener|null $progressTracker * @@ -379,30 +232,50 @@ public static function chooseDownloader( string $multipartDownloadType, array $requestArgs, array $config, + int $currentPartNo = 0, + int $objectPartsCount = 0, + int $objectCompletedPartsCount = 0, + int $objectSizeInBytes = 0, + int $objectBytesTransferred = 0, + string $eTag = "", + string $objectKey = "", ?MultipartDownloadListener $listener = null, ?TransferListener $progressTracker = null ) : MultipartDownloader { - if ($multipartDownloadType === self::PART_GET_MULTIPART_DOWNLOADER) { - return new GetMultipartDownloader( - $s3Client, - $requestArgs, - $config, - 0, - $listener, - $progressTracker - ); - } elseif ($multipartDownloadType === self::RANGE_GET_MULTIPART_DOWNLOADER) { - return new RangeMultipartDownloader( - $s3Client, - $requestArgs, - $config, - 0, - $listener, - $progressTracker - ); - } - - throw new \RuntimeException("Unsupported download type $multipartDownloadType"); + return match ($multipartDownloadType) { + self::PART_GET_MULTIPART_DOWNLOADER => new GetMultipartDownloader( + s3Client: $s3Client, + requestArgs: $requestArgs, + config: $config, + currentPartNo: $currentPartNo, + objectPartsCount: $objectPartsCount, + objectCompletedPartsCount: $objectCompletedPartsCount, + objectSizeInBytes: $objectSizeInBytes, + objectBytesTransferred: $objectBytesTransferred, + eTag: $eTag, + objectKey: $objectKey, + listener: $listener, + progressListener: $progressTracker + ), + self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeMultipartDownloader( + s3Client: $s3Client, + requestArgs: $requestArgs, + config: $config, + currentPartNo: 0, + objectPartsCount: 0, + objectCompletedPartsCount: 0, + objectSizeInBytes: 0, + objectBytesTransferred: 0, + eTag: "", + objectKey: "", + listener: $listener, + progressListener: $progressTracker + ), + default => throw new \RuntimeException( + "Unsupported download type $multipartDownloadType." + ."It should be either " . self::PART_GET_MULTIPART_DOWNLOADER . + " or " . self::RANGE_GET_MULTIPART_DOWNLOADER . ".") + }; } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php b/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php new file mode 100644 index 0000000000..c722da4314 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php @@ -0,0 +1,174 @@ +objectKey = $commandArgs['Key']; + $this->progressListener?->objectTransferInitiated( + $this->objectKey, + $commandArgs + ); + $this->_notifyMultipartDownloadListeners('downloadInitiated', [ + &$commandArgs, + $currentPartNo + ]); + } + + /** + * Propagates download-failed event to listeners. + * It may also do some computation in order to maintain internal states. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + private function downloadFailed( + \Throwable $reason, + int $totalPartsTransferred, + int $totalBytesTransferred, + int $lastPartTransferred + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $totalBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners('downloadFailed', [ + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ]); + } + + /** + * Propagates part-download-initiated event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param int $partNo + * + * @return void + */ + private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { + $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * @param int $partNo + * + * @return void + */ + private function partDownloadCompleted(ResultInterface $result, int $partNo): void { + $this->objectCompletedPartsCount++; + $partDownloadBytes = $result['ContentLength']; + $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + + $this->progressListener?->objectTransferProgress( + $this->objectKey, + $partDownloadBytes, + $this->objectSizeInBytes + ); + + $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ + $result, + $partNo, + $partDownloadBytes, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred, + $this->objectSizeInBytes + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + private function partDownloadFailed( + CommandInterface $partDownloadCommand, + \Throwable $reason, + int $partNo + ): void { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners( + 'partDownloadFailed', + [$partDownloadCommand, $reason, $partNo]); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function objectDownloadCompleted(): void + { + $this->stream->rewind(); + $this->progressListener?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred + ); + $this->_notifyMultipartDownloadListeners('downloadCompleted', [ + $this->stream, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred + ]); + } + + /** + * Internal helper method for notifying listeners of specific events. + * + * @param string $listenerMethod + * @param array $args + * + * @return void + */ + private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void + { + $this->listener?->{$listenerMethod}(...$args); + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php index 5a76a39408..ca26120642 100644 --- a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php +++ b/src/S3/Features/S3Transfer/RangeMultipartDownloader.php @@ -5,6 +5,8 @@ use Aws\CommandInterface; use Aws\Result; use Aws\ResultInterface; +use Aws\S3\S3ClientInterface; +use Psr\Http\Message\StreamInterface; class RangeMultipartDownloader extends MultipartDownloader { @@ -12,6 +14,63 @@ class RangeMultipartDownloader extends MultipartDownloader /** @var int */ private int $partSize; + /** + * @param S3ClientInterface $s3Client + * @param array $requestArgs + * @param array $config + * @param int $currentPartNo + * @param int $objectPartsCount + * @param int $objectCompletedPartsCount + * @param int $objectSizeInBytes + * @param int $objectBytesTransferred + * @param string $eTag + * @param string $objectKey + * @param MultipartDownloadListener|null $listener + * @param TransferListener|null $progressListener + * @param StreamInterface|null $stream + */ + public function __construct( + S3ClientInterface $s3Client, + array $requestArgs = [], + array $config = [], + int $currentPartNo = 0, + int $objectPartsCount = 0, + int $objectCompletedPartsCount = 0, + int $objectSizeInBytes = 0, + int $objectBytesTransferred = 0, + string $eTag = "", + string $objectKey = "", + ?MultipartDownloadListener $listener = null, + ?TransferListener $progressListener = null, + ?StreamInterface $stream = null + ) { + parent::__construct( + $s3Client, + $requestArgs, + $config, + $currentPartNo, + $objectPartsCount, + $objectCompletedPartsCount, + $objectSizeInBytes, + $objectBytesTransferred, + $eTag, + $objectKey, + $listener, + $progressListener, + $stream + ); + if (empty($config['minimumPartSize'])) { + throw new \RuntimeException('You must provide a valid minimum part size in bytes'); + } + $this->partSize = $config['minimumPartSize']; + // If object size is known at instantiation time then, we can compute + // the object dimensions. + if ($this->objectSizeInBytes !== 0) { + $this->computeObjectDimensions(new Result(['ContentRange' => $this->objectSizeInBytes])); + } + } + + /** * @inheritDoc * @@ -19,20 +78,21 @@ class RangeMultipartDownloader extends MultipartDownloader */ protected function nextCommand(): CommandInterface { - if ($this->objectSizeInBytes !== 0) { - $this->computeObjectDimensions(new Result(['ContentRange' => $this->totalBytes])); - } - + // If currentPartNo is not know then lets initialize it to 1 + // otherwise just increment it. if ($this->currentPartNo === 0) { $this->currentPartNo = 1; - $this->partSize = $this->config['targetPartSizeBytes']; } else { $this->currentPartNo++; } $nextRequestArgs = array_slice($this->requestArgs, 0); - $from = ($this->currentPartNo - 1) * ($this->partSize + 1); - $to = $this->currentPartNo * $this->partSize; + $from = ($this->currentPartNo - 1) * $this->partSize; + $to = ($this->currentPartNo * $this->partSize) - 1; + if ($this->objectSizeInBytes !== 0) { + $to = min($this->objectSizeInBytes, $to); + } + $nextRequestArgs['Range'] = "bytes=$from-$to"; if (!empty($this->eTag)) { $nextRequestArgs['IfMatch'] = $this->eTag; @@ -53,11 +113,17 @@ protected function nextCommand(): CommandInterface */ protected function computeObjectDimensions(ResultInterface $result): void { - $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + // Assign object size just if needed. + if ($this->objectSizeInBytes === 0) { + $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + } + if ($this->objectSizeInBytes > $this->partSize) { $this->objectPartsCount = intval(ceil($this->objectSizeInBytes / $this->partSize)); } else { + // Single download since partSize will be set to full object size. $this->partSize = $this->objectSizeInBytes; + $this->objectPartsCount = 1; $this->currentPartNo = 1; } } diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index f9c56e6b7c..aef8921881 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -21,8 +21,6 @@ class S3TransferManager * The minimum part size to be used in a multipart upload/download. * - multipartUploadThresholdBytes: (int, default=(16777216 `16 MB`)) \ * The threshold to decided whether a multipart upload is needed. - * - multipartDownloadThresholdBytes: (int, default=(16777216 `16 MB`)) \ - * The threshold to decided whether a multipart download is needed. * - checksumValidationEnabled: (bool, default=true) \ * To decide whether a checksum validation will be applied to the response. * - checksumAlgorithm: (string, default='crc32') \ @@ -62,6 +60,8 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) { * to decide whether transfer progress should be tracked. If not * transfer tracker factory is provided and trackProgress is true then, * the default progress listener implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. * * @return PromiseInterface */ @@ -83,7 +83,10 @@ public function download( $requestArgs = $sourceArgs + $downloadArgs; if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { - return $this->tryMultipartDownload($requestArgs, $config); + return $this->tryMultipartDownload( + $requestArgs, + $config + ); } return $this->trySingleDownload($requestArgs); @@ -97,6 +100,10 @@ public function download( * - listener: (?MultipartDownloadListener) \ * A multipart download listener for watching every multipart download * stage. + * - minimumPartSize: (int) \ + * The minimum part size in bytes for a range multipart download. If + * this parameter is not provided then it fallbacks to the transfer + * manager `targetPartSizeBytes` config value. * * @return PromiseInterface */ @@ -117,12 +124,17 @@ private function tryMultipartDownload( } } $multipartDownloader = MultipartDownloader::chooseDownloader( - $this->s3Client, - $this->config['multipartDownloadType'], - $requestArgs, - $this->config, - $config['listener'] ?? null, - $progressListener?->getTransferListener() + s3Client: $this->s3Client, + multipartDownloadType: $this->config['multipartDownloadType'], + requestArgs: $requestArgs, + config: [ + 'minimumPartSize' => max( + $config['minimumPartSize'] ?? 0, + $this->config['targetPartSizeBytes'] + ) + ], + listener: $config['listener'] ?? null, + progressTracker: $progressListener?->getTransferListener() ); return $multipartDownloader->promise(); diff --git a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php index d924280b46..6d2542cb04 100644 --- a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php +++ b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php @@ -11,7 +11,6 @@ trait S3TransferManagerTrait private static array $defaultConfig = [ 'targetPartSizeBytes' => 8 * 1024 * 1024, 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, - 'multipartDownloadThresholdBytes' => 16 * 1024 * 1024, 'checksumValidationEnabled' => true, 'checksumAlgorithm' => 'crc32', 'multipartDownloadType' => 'partGet', diff --git a/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php index 6be2e5cf4c..dbe3625479 100644 --- a/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/Features/S3Transfer/MultipartDownloaderTest.php @@ -2,11 +2,159 @@ namespace Aws\Test\S3\Features\S3Transfer; +use Aws\Command; +use Aws\Result; +use Aws\S3\Features\S3Transfer\MultipartDownloader; +use Aws\S3\S3Client; +use GuzzleHttp\Promise\Create; +use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; +use Psr\Http\Message\StreamInterface; +/** + * Tests multipart download implementation. + */ class MultipartDownloaderTest extends TestCase { - public function testMultipartDownloader(): void { + /** + * Tests part and range get multipart downloader. + * + * @param string $multipartDownloadType + * @param string $objectKey + * @param int $objectSizeInBytes + * @param int $targetPartSize + * + * @return void + * @dataProvider partGetMultipartDownloaderProvider + * + */ + public function testMultipartDownloader( + string $multipartDownloadType, + string $objectKey, + int $objectSizeInBytes, + int $targetPartSize + ): void { + $partsCount = (int) ceil($objectSizeInBytes / $targetPartSize); + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $remainingToTransfer = $objectSizeInBytes; + $mockClient->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'], + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength + ])); + }); + $mockClient->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $downloader = MultipartDownloader::chooseDownloader( + $mockClient, + $multipartDownloadType, + [ + 'Bucket' => 'FooBucket', + 'Key' => $objectKey, + ], + [ + 'minimumPartSize' => $targetPartSize, + ] + ); + $stream = $downloader->promise()->wait(); + + $this->assertInstanceOf(StreamInterface::class, $stream); + $this->assertEquals($objectKey, $downloader->getObjectKey()); + $this->assertEquals($objectSizeInBytes, $downloader->getObjectSizeInBytes()); + $this->assertEquals($objectSizeInBytes, $downloader->getObjectBytesTransferred()); + $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); + $this->assertEquals($partsCount, $downloader->getObjectCompletedPartsCount()); + } + + /** + * Part get multipart downloader data provider. + * + * @return array[] + */ + public function partGetMultipartDownloaderProvider(): array { + return [ + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ] + ]; } } \ No newline at end of file From c28c1657473b403756a75f9d6f91a2298ed9185b Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Feb 2025 08:57:10 -0800 Subject: [PATCH 05/62] feat: add download directory and refactor code Refactor: - Moves opening braces into a new line. - Make requestArgs an optional argument. - Remove unnecessary traits. - Use traditional declarations. Adds: - Download directory feature. --- .../S3Transfer/ConsoleProgressBar.php | 14 +- .../S3Transfer/DefaultProgressTracker.php | 42 +- src/S3/Features/S3Transfer/DownloadResult.php | 23 + .../Exceptions/S3TransferException.php | 7 + .../S3Transfer/MultipartDownloadListener.php | 62 ++- .../S3Transfer/MultipartDownloadType.php | 51 --- .../S3Transfer/MultipartDownloader.php | 246 ++++++++++- .../S3Transfer/MultipartDownloaderTrait.php | 174 -------- .../S3Transfer/ObjectProgressTracker.php | 6 +- ...der.php => PartGetMultipartDownloader.php} | 9 +- ...er.php => RangeGetMultipartDownloader.php} | 19 +- .../Features/S3Transfer/S3TransferManager.php | 396 ++++++++++++++++-- .../S3Transfer/S3TransferManagerTrait.php | 87 ---- .../Features/S3Transfer/TransferListener.php | 24 +- 14 files changed, 749 insertions(+), 411 deletions(-) create mode 100644 src/S3/Features/S3Transfer/DownloadResult.php create mode 100644 src/S3/Features/S3Transfer/Exceptions/S3TransferException.php delete mode 100644 src/S3/Features/S3Transfer/MultipartDownloadType.php delete mode 100644 src/S3/Features/S3Transfer/MultipartDownloaderTrait.php rename src/S3/Features/S3Transfer/{GetMultipartDownloader.php => PartGetMultipartDownloader.php} (80%) rename src/S3/Features/S3Transfer/{RangeMultipartDownloader.php => RangeGetMultipartDownloader.php} (86%) delete mode 100644 src/S3/Features/S3Transfer/S3TransferManagerTrait.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php index 23f41f0e57..2478812b63 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -49,7 +49,8 @@ class ConsoleProgressBar implements ProgressBar /** @var ?array */ private ?array $format; - private ?array $args; + /** @var array */ + private array $args; /** * @param ?string $progressBarChar @@ -62,7 +63,7 @@ public function __construct( ?int $progressBarWidth = null, ?int $percentCompleted = null, ?array $format = null, - ?array $args = null + array $args = [], ) { $this->progressBarChar = $progressBarChar ?? '#'; $this->progressBarWidth = $progressBarWidth ?? 25; @@ -78,7 +79,8 @@ public function __construct( * * @return void */ - public function setPercentCompleted(int $percent): void { + public function setPercentCompleted(int $percent): void + { $this->percentCompleted = max(0, min(100, $percent)); } @@ -105,7 +107,8 @@ public function setArg(string $key, mixed $value): void $this->args[$key] = $value; } - private function renderProgressBar(): string { + private function renderProgressBar(): string + { $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); return str_repeat($this->progressBarChar, $filledWidth) . str_repeat(' ', $this->progressBarWidth - $filledWidth); @@ -115,7 +118,8 @@ private function renderProgressBar(): string { * * @return string */ - public function getPaintedProgress(): string { + public function getPaintedProgress(): string + { foreach ($this->format['parameters'] as $param) { if (!array_key_exists($param, $this->args)) { throw new \InvalidArgumentException("Missing `$param` parameter for progress bar."); diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php index 3a8cd92c68..e91d7c4d77 100644 --- a/src/S3/Features/S3Transfer/DefaultProgressTracker.php +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -6,6 +6,9 @@ class DefaultProgressTracker { + public const TRACKING_OPERATION_DOWNLOADING = 'Downloading'; + public const TRACKING_OPERATION_UPLOADING = 'Uploading'; + /** @var ObjectProgressTracker[] */ private array $objects; @@ -33,12 +36,21 @@ class DefaultProgressTracker /** @var resource */ private $output; + /** @var string */ + private string $trackingOperation; + /** * @param Closure|ProgressBarFactory|null $progressBarFactory + * @param false|resource $output + * @param string $trackingOperation + * Valid values should be: + * - Downloading + * - Uploading */ public function __construct( Closure | ProgressBarFactory | null $progressBarFactory = null, - $output = STDOUT + mixed $output = STDOUT, + string $trackingOperation = '', ) { $this->clear(); $this->initializeListener(); @@ -48,9 +60,15 @@ public function __construct( } $this->output = $output; + if (!in_array(strtolower($trackingOperation), ['downloading', 'Uploading'], true)) { + throw new \InvalidArgumentException("Tracking operation '$trackingOperation' should be one of 'Downloading', 'Uploading'"); + } + + $this->trackingOperation = $trackingOperation; } - private function initializeListener(): void { + private function initializeListener(): void + { $this->transferListener = new TransferListener(); // Object transfer initialized $this->transferListener->onObjectTransferInitiated = $this->objectTransferInitiated(); @@ -63,7 +81,8 @@ private function initializeListener(): void { /** * @return TransferListener */ - public function getTransferListener(): TransferListener { + public function getTransferListener(): TransferListener + { return $this->transferListener; } @@ -212,7 +231,8 @@ public function clear(): void * * @return void */ - private function increaseBytesTransferred(int $bytesTransferred): void { + private function increaseBytesTransferred(int $bytesTransferred): void + { $this->totalBytesTransferred += $bytesTransferred; if ($this->objectsTotalSizeInBytes !== 0) { $this->transferPercentCompleted = floor(($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100); @@ -222,16 +242,18 @@ private function increaseBytesTransferred(int $bytesTransferred): void { /** * @return void */ - private function showProgress(): void { + private function showProgress(): void + { // Clear screen fwrite($this->output, "\033[2J\033[H"); // Display progress header fwrite($this->output, sprintf( - "\r%d%% [%s/%s]\n", - $this->transferPercentCompleted, + "\r%s [%d/%d] %d%%\n", + $this->trackingOperation, $this->objectsInProgress, - $this->objectsCount + $this->objectsCount, + $this->transferPercentCompleted )); foreach ($this->objects as $name => $object) { @@ -246,7 +268,8 @@ private function showProgress(): void { /** * @return Closure|ProgressBarFactory */ - private function defaultProgressBarFactory(): Closure| ProgressBarFactory { + private function defaultProgressBarFactory(): Closure| ProgressBarFactory + { return function () { return new ConsoleProgressBar( format: ConsoleProgressBar::$formats[ @@ -261,5 +284,4 @@ private function defaultProgressBarFactory(): Closure| ProgressBarFactory { ); }; } - } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/DownloadResult.php b/src/S3/Features/S3Transfer/DownloadResult.php new file mode 100644 index 0000000000..4f40ca1623 --- /dev/null +++ b/src/S3/Features/S3Transfer/DownloadResult.php @@ -0,0 +1,23 @@ +content; + } + + public function getMetadata(): array + { + return $this->metadata; + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/Exceptions/S3TransferException.php b/src/S3/Features/S3Transfer/Exceptions/S3TransferException.php new file mode 100644 index 0000000000..d1de07840b --- /dev/null +++ b/src/S3/Features/S3Transfer/Exceptions/S3TransferException.php @@ -0,0 +1,7 @@ +notify('onDownloadInitiated', [&$commandArgs, $initialPart]); } @@ -80,15 +83,25 @@ public function downloadInitiated(array &$commandArgs, int $initialPart): void { * to call parent::downloadFailed() in order to * keep the states maintained in this implementation. * - * @param \Throwable $reason + * @param Throwable $reason * @param int $totalPartsDownloaded * @param int $totalBytesDownloaded * @param int $lastPartDownloaded * * @return void */ - public function downloadFailed(\Throwable $reason, int $totalPartsDownloaded, int $totalBytesDownloaded, int $lastPartDownloaded): void { - $this->notify('onDownloadFailed', [$reason, $totalPartsDownloaded, $totalBytesDownloaded, $lastPartDownloaded]); + public function downloadFailed( + Throwable $reason, + int $totalPartsDownloaded, + int $totalBytesDownloaded, + int $lastPartDownloaded): void + { + $this->notify('onDownloadFailed', [ + $reason, + $totalPartsDownloaded, + $totalBytesDownloaded, + $lastPartDownloaded + ]); } /** @@ -97,14 +110,23 @@ public function downloadFailed(\Throwable $reason, int $totalPartsDownloaded, in * to call parent::onDownloadCompleted() in order to * keep the states maintained in this implementation. * - * @param resource $stream + * @param StreamInterface $stream * @param int $totalPartsDownloaded * @param int $totalBytesDownloaded * * @return void */ - public function downloadCompleted($stream, int $totalPartsDownloaded, int $totalBytesDownloaded): void { - $this->notify('onDownloadCompleted', [$stream, $totalPartsDownloaded, $totalBytesDownloaded]); + public function downloadCompleted( + StreamInterface $stream, + int $totalPartsDownloaded, + int $totalBytesDownloaded + ): void + { + $this->notify('onDownloadCompleted', [ + $stream, + $totalPartsDownloaded, + $totalBytesDownloaded + ]); } /** @@ -118,8 +140,15 @@ public function downloadCompleted($stream, int $totalPartsDownloaded, int $total * * @return void */ - public function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { - $this->notify('onPartDownloadInitiated', [$partDownloadCommand, $partNo]); + public function partDownloadInitiated( + CommandInterface $partDownloadCommand, + int $partNo + ): void + { + $this->notify('onPartDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); } /** @@ -162,13 +191,22 @@ public function partDownloadCompleted( * keep the states maintained in this implementation. * * @param CommandInterface $partDownloadCommand - * @param \Throwable $reason + * @param Throwable $reason * @param int $partNo * * @return void */ - public function partDownloadFailed(CommandInterface $partDownloadCommand, \Throwable $reason, int $partNo): void { - $this->notify('onPartDownloadFailed', [$partDownloadCommand, $reason, $partNo]); + public function partDownloadFailed( + CommandInterface $partDownloadCommand, + Throwable $reason, + int $partNo + ): void + { + $this->notify('onPartDownloadFailed', [ + $partDownloadCommand, + $reason, + $partNo + ]); } protected function notify(string $event, array $params = []): void diff --git a/src/S3/Features/S3Transfer/MultipartDownloadType.php b/src/S3/Features/S3Transfer/MultipartDownloadType.php deleted file mode 100644 index 6bbbfd6144..0000000000 --- a/src/S3/Features/S3Transfer/MultipartDownloadType.php +++ /dev/null @@ -1,51 +0,0 @@ -value = $value; - } - - /** - * @return string - */ - public function __toString(): string { - return $this->value; - } - - /** - * @param MultipartDownloadType $type - * - * @return bool - */ - public function equals(MultipartDownloadType $type): bool - { - return $this->value === $type->value; - } - - /** - * @return MultipartDownloadType - */ - public static function rangedGet(): MultipartDownloadType { - return new static(self::$rangedGetType); - } - - /** - * @return MultipartDownloadType - */ - public static function partGet(): MultipartDownloadType { - return new static(self::$partGetType); - } -} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php index 50b575c455..e3c1143da1 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloader.php +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -14,11 +14,37 @@ abstract class MultipartDownloader implements PromisorInterface { - use MultipartDownloaderTrait; public const GET_OBJECT_COMMAND = "GetObject"; public const PART_GET_MULTIPART_DOWNLOADER = "partGet"; public const RANGE_GET_MULTIPART_DOWNLOADER = "rangeGet"; + /** @var array */ + protected array $requestArgs; + + /** @var int */ + protected int $currentPartNo; + + /** @var int */ + protected int $objectPartsCount; + + /** @var int */ + protected int $objectCompletedPartsCount; + + /** @var int */ + protected int $objectSizeInBytes; + + /** @var int */ + protected int $objectBytesTransferred; + + /** @var string */ + protected string $eTag; + + /** @var string */ + protected string $objectKey; + + /** @var StreamInterface */ + private StreamInterface $stream; + /** * @param S3ClientInterface $s3Client * @param array $requestArgs @@ -38,23 +64,33 @@ abstract class MultipartDownloader implements PromisorInterface */ public function __construct( protected readonly S3ClientInterface $s3Client, - protected array $requestArgs, + array $requestArgs, protected readonly array $config = [], - protected int $currentPartNo = 0, - protected int $objectPartsCount = 0, - protected int $objectCompletedPartsCount = 0, - protected int $objectSizeInBytes = 0, - protected int $objectBytesTransferred = 0, - protected string $eTag = "", - protected string $objectKey = "", + int $currentPartNo = 0, + int $objectPartsCount = 0, + int $objectCompletedPartsCount = 0, + int $objectSizeInBytes = 0, + int $objectBytesTransferred = 0, + string $eTag = "", + string $objectKey = "", private readonly ?MultipartDownloadListener $listener = null, private readonly ?TransferListener $progressListener = null, - private ?StreamInterface $stream = null + ?StreamInterface $stream = null ) { + $this->requestArgs = $requestArgs; + $this->currentPartNo = $currentPartNo; + $this->objectPartsCount = $objectPartsCount; + $this->objectCompletedPartsCount = $objectCompletedPartsCount; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->objectBytesTransferred = $objectBytesTransferred; + $this->eTag = $eTag; + $this->objectKey = $objectKey; if ($stream === null) { $this->stream = Utils::streamFor( fopen('php://temp', 'w+') ); + } else { + $this->stream = $stream; } } @@ -156,14 +192,16 @@ public function promise(): PromiseInterface yield Create::rejectionFor($e); } - $lastPartIncrement = $this->currentPartNo; } // Transfer completed $this->objectDownloadCompleted(); // TODO: yield the stream wrapped in a modeled transfer success response. - yield Create::promiseFor($this->stream); + yield Create::promiseFor(new DownloadResult( + $this->stream, + [] + )); }); } @@ -190,7 +228,8 @@ abstract protected function computeObjectDimensions(ResultInterface $result): vo * * @return int */ - protected function computeObjectSize($sizeSource): int { + protected function computeObjectSize($sizeSource): int + { if (is_int($sizeSource)) { return (int) $sizeSource; } @@ -244,7 +283,7 @@ public static function chooseDownloader( ) : MultipartDownloader { return match ($multipartDownloadType) { - self::PART_GET_MULTIPART_DOWNLOADER => new GetMultipartDownloader( + self::PART_GET_MULTIPART_DOWNLOADER => new PartGetMultipartDownloader( s3Client: $s3Client, requestArgs: $requestArgs, config: $config, @@ -258,7 +297,7 @@ public static function chooseDownloader( listener: $listener, progressListener: $progressTracker ), - self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeMultipartDownloader( + self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeGetMultipartDownloader( s3Client: $s3Client, requestArgs: $requestArgs, config: $config, @@ -278,4 +317,181 @@ public static function chooseDownloader( " or " . self::RANGE_GET_MULTIPART_DOWNLOADER . ".") }; } + + /** + * Main purpose of this method is to propagate + * the download-initiated event to listeners, but + * also it does some computation regarding internal states + * that need to be maintained. + * + * @param array $commandArgs + * @param int|null $currentPartNo + * + * @return void + */ + private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void + { + $this->objectKey = $commandArgs['Key']; + $this->progressListener?->objectTransferInitiated( + $this->objectKey, + $commandArgs + ); + $this->_notifyMultipartDownloadListeners('downloadInitiated', [ + &$commandArgs, + $currentPartNo + ]); + } + + /** + * Propagates download-failed event to listeners. + * It may also do some computation in order to maintain internal states. + * + * @param \Throwable $reason + * @param int $totalPartsTransferred + * @param int $totalBytesTransferred + * @param int $lastPartTransferred + * + * @return void + */ + private function downloadFailed( + \Throwable $reason, + int $totalPartsTransferred, + int $totalBytesTransferred, + int $lastPartTransferred + ): void + { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $totalBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners('downloadFailed', [ + $reason, + $totalPartsTransferred, + $totalBytesTransferred, + $lastPartTransferred + ]); + } + + /** + * Propagates part-download-initiated event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param int $partNo + * + * @return void + */ + private function partDownloadInitiated( + CommandInterface $partDownloadCommand, + int $partNo + ): void + { + $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ + $partDownloadCommand, + $partNo + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * @param int $partNo + * + * @return void + */ + private function partDownloadCompleted( + ResultInterface $result, + int $partNo + ): void + { + $this->objectCompletedPartsCount++; + $partDownloadBytes = $result['ContentLength']; + $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + + $this->progressListener?->objectTransferProgress( + $this->objectKey, + $partDownloadBytes, + $this->objectSizeInBytes + ); + + $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ + $result, + $partNo, + $partDownloadBytes, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred, + $this->objectSizeInBytes + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param CommandInterface $partDownloadCommand + * @param \Throwable $reason + * @param int $partNo + * + * @return void + */ + private function partDownloadFailed( + CommandInterface $partDownloadCommand, + \Throwable $reason, + int $partNo + ): void + { + $this->progressListener?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + $this->_notifyMultipartDownloadListeners( + 'partDownloadFailed', + [$partDownloadCommand, $reason, $partNo]); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function objectDownloadCompleted(): void + { + $this->stream->rewind(); + $this->progressListener?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred + ); + $this->_notifyMultipartDownloadListeners('downloadCompleted', [ + $this->stream, + $this->objectCompletedPartsCount, + $this->objectBytesTransferred + ]); + } + + /** + * Internal helper method for notifying listeners of specific events. + * + * @param string $listenerMethod + * @param array $args + * + * @return void + */ + private function _notifyMultipartDownloadListeners( + string $listenerMethod, + array $args + ): void + { + $this->listener?->{$listenerMethod}(...$args); + } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php b/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php deleted file mode 100644 index c722da4314..0000000000 --- a/src/S3/Features/S3Transfer/MultipartDownloaderTrait.php +++ /dev/null @@ -1,174 +0,0 @@ -objectKey = $commandArgs['Key']; - $this->progressListener?->objectTransferInitiated( - $this->objectKey, - $commandArgs - ); - $this->_notifyMultipartDownloadListeners('downloadInitiated', [ - &$commandArgs, - $currentPartNo - ]); - } - - /** - * Propagates download-failed event to listeners. - * It may also do some computation in order to maintain internal states. - * - * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred - * - * @return void - */ - private function downloadFailed( - \Throwable $reason, - int $totalPartsTransferred, - int $totalBytesTransferred, - int $lastPartTransferred - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $totalBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners('downloadFailed', [ - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ]); - } - - /** - * Propagates part-download-initiated event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param int $partNo - * - * @return void - */ - private function partDownloadInitiated(CommandInterface $partDownloadCommand, int $partNo): void { - $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ - $partDownloadCommand, - $partNo - ]); - } - - /** - * Propagates part-download-completed to listeners. - * It also does some computation in order to maintain internal states. - * In this specific method we move each part content into an accumulative - * stream, which is meant to hold the full object content once the download - * is completed. - * - * @param ResultInterface $result - * @param int $partNo - * - * @return void - */ - private function partDownloadCompleted(ResultInterface $result, int $partNo): void { - $this->objectCompletedPartsCount++; - $partDownloadBytes = $result['ContentLength']; - $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - Utils::copyToStream($result['Body'], $this->stream); - - $this->progressListener?->objectTransferProgress( - $this->objectKey, - $partDownloadBytes, - $this->objectSizeInBytes - ); - - $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ - $result, - $partNo, - $partDownloadBytes, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred, - $this->objectSizeInBytes - ]); - } - - /** - * Propagates part-download-failed event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param \Throwable $reason - * @param int $partNo - * - * @return void - */ - private function partDownloadFailed( - CommandInterface $partDownloadCommand, - \Throwable $reason, - int $partNo - ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners( - 'partDownloadFailed', - [$partDownloadCommand, $reason, $partNo]); - } - - /** - * Propagates object-download-completed event to listeners. - * It also resets the pointer of the stream to the first position, - * so that the stream is ready to be consumed once returned. - * - * @return void - */ - private function objectDownloadCompleted(): void - { - $this->stream->rewind(); - $this->progressListener?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred - ); - $this->_notifyMultipartDownloadListeners('downloadCompleted', [ - $this->stream, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred - ]); - } - - /** - * Internal helper method for notifying listeners of specific events. - * - * @param string $listenerMethod - * @param array $args - * - * @return void - */ - private function _notifyMultipartDownloadListeners(string $listenerMethod, array $args): void - { - $this->listener?->{$listenerMethod}(...$args); - } -} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/ObjectProgressTracker.php b/src/S3/Features/S3Transfer/ObjectProgressTracker.php index 5eadecfdc4..ea6ac4f0d3 100644 --- a/src/S3/Features/S3Transfer/ObjectProgressTracker.php +++ b/src/S3/Features/S3Transfer/ObjectProgressTracker.php @@ -104,7 +104,8 @@ public function setStatus(string $status): void $this->setProgressColor(); } - private function setProgressColor(): void { + private function setProgressColor(): void + { if ($this->status === 'progress') { $this->progressBar->setArg('color_code', ConsoleProgressBar::BLUE_COLOR_CODE); } elseif ($this->status === 'completed') { @@ -143,7 +144,8 @@ public function getProgressBar(): ?ProgressBar /** * @return ProgressBar */ - private function defaultProgressBar(): ProgressBar { + private function defaultProgressBar(): ProgressBar + { return new ConsoleProgressBar( format: ConsoleProgressBar::$formats[ ConsoleProgressBar::COLORED_TRANSFER_FORMAT diff --git a/src/S3/Features/S3Transfer/GetMultipartDownloader.php b/src/S3/Features/S3Transfer/PartGetMultipartDownloader.php similarity index 80% rename from src/S3/Features/S3Transfer/GetMultipartDownloader.php rename to src/S3/Features/S3Transfer/PartGetMultipartDownloader.php index 9548d95ded..4a9ce76bad 100644 --- a/src/S3/Features/S3Transfer/GetMultipartDownloader.php +++ b/src/S3/Features/S3Transfer/PartGetMultipartDownloader.php @@ -9,14 +9,15 @@ /** * Multipart downloader using the part get approach. */ -class GetMultipartDownloader extends MultipartDownloader +class PartGetMultipartDownloader extends MultipartDownloader { /** * @inheritDoc * * @return CommandInterface */ - protected function nextCommand() : CommandInterface { + protected function nextCommand() : CommandInterface + { if ($this->currentPartNo === 0) { $this->currentPartNo = 1; } else { @@ -45,6 +46,8 @@ protected function computeObjectDimensions(ResultInterface $result): void $this->objectPartsCount = $result['PartsCount']; } - $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + $this->objectSizeInBytes = $this->computeObjectSize( + $result['ContentRange'] ?? "" + ); } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php b/src/S3/Features/S3Transfer/RangeGetMultipartDownloader.php similarity index 86% rename from src/S3/Features/S3Transfer/RangeMultipartDownloader.php rename to src/S3/Features/S3Transfer/RangeGetMultipartDownloader.php index ca26120642..615a947b46 100644 --- a/src/S3/Features/S3Transfer/RangeMultipartDownloader.php +++ b/src/S3/Features/S3Transfer/RangeGetMultipartDownloader.php @@ -5,10 +5,11 @@ use Aws\CommandInterface; use Aws\Result; use Aws\ResultInterface; +use Aws\S3\Features\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3ClientInterface; use Psr\Http\Message\StreamInterface; -class RangeMultipartDownloader extends MultipartDownloader +class RangeGetMultipartDownloader extends MultipartDownloader { /** @var int */ @@ -60,13 +61,17 @@ public function __construct( $stream ); if (empty($config['minimumPartSize'])) { - throw new \RuntimeException('You must provide a valid minimum part size in bytes'); + throw new S3TransferException( + 'You must provide a valid minimum part size in bytes' + ); } $this->partSize = $config['minimumPartSize']; // If object size is known at instantiation time then, we can compute // the object dimensions. if ($this->objectSizeInBytes !== 0) { - $this->computeObjectDimensions(new Result(['ContentRange' => $this->objectSizeInBytes])); + $this->computeObjectDimensions( + new Result(['ContentRange' => $this->objectSizeInBytes]) + ); } } @@ -115,11 +120,15 @@ protected function computeObjectDimensions(ResultInterface $result): void { // Assign object size just if needed. if ($this->objectSizeInBytes === 0) { - $this->objectSizeInBytes = $this->computeObjectSize($result['ContentRange'] ?? ""); + $this->objectSizeInBytes = $this->computeObjectSize( + $result['ContentRange'] ?? "" + ); } if ($this->objectSizeInBytes > $this->partSize) { - $this->objectPartsCount = intval(ceil($this->objectSizeInBytes / $this->partSize)); + $this->objectPartsCount = intval( + ceil($this->objectSizeInBytes / $this->partSize) + ); } else { // Single download since partSize will be set to full object size. $this->partSize = $this->objectSizeInBytes; diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index aef8921881..3a199839ca 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -2,15 +2,31 @@ namespace Aws\S3\Features\S3Transfer; +use Aws\Command; +use Aws\S3\Features\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; +use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use function Aws\filter; +use function Aws\map; class S3TransferManager { - use S3TransferManagerTrait; + private static array $defaultConfig = [ + 'targetPartSizeBytes' => 8 * 1024 * 1024, + 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, + 'checksumValidationEnabled' => true, + 'checksumAlgorithm' => 'crc32', + 'multipartDownloadType' => 'partGet', + 'concurrency' => 5, + 'trackProgress' => false, + 'region' => 'us-east-1', + ]; + /** @var S3Client */ private S3ClientInterface $s3Client; + /** @var array */ private array $config; @@ -30,13 +46,14 @@ class S3TransferManager * - concurrency: (int, default=5) \ * Maximum number of concurrent operations allowed during a multipart * upload/download. - * - trackProgress: (bool, default=true) \ + * - trackProgress: (bool, default=false) \ * To enable progress tracker in a multipart upload/download. - * - progressListenerFactory: (callable|TransferListenerFactory) + * - progressTrackerFactory: (callable|TransferListenerFactory) \ * A factory to create the listener which will receive notifications - * based in the different stages in an upload/download operation. + * based in the different stages an upload/download is. */ - public function __construct(?S3ClientInterface $s3Client, array $config = []) { + public function __construct(?S3ClientInterface $s3Client, array $config = []) + { if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -50,16 +67,21 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) { * @param string|array $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key * properties set. - * @param array $downloadArgs The request arguments to be provided as part - * of the service client operation. + * @param array $downloadArgs The getObject request arguments to be provided as part + * of each get object operation, except for the bucket and key, which + * are already provided as the source. + * @param MultipartDownloadListener|null $downloadListener A multipart download + * specific listener of the different states a multipart download can be. + * @param TransferListener|null $progressTracker A transfer listener implementation + * aimed to track the progress of a transfer. If not provided and trackProgress + * is resolved as true then, the default progressTrackerFactory will be used. * @param array $config The configuration to be used for this operation. - * - listener: (null|MultipartDownloadListener) \ - * A listener to be notified in every stage of a multipart download operation. * - trackProgress: (bool) \ * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If not - * transfer tracker factory is provided and trackProgress is true then, - * the default progress listener implementation will be used. + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. * - minimumPartSize: (int) \ * The minimum part size in bytes to be used in a range multipart download. * @@ -67,9 +89,12 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) { */ public function download( string | array $source, - array $downloadArgs, + array $downloadArgs = [], + ?MultipartDownloadListener $downloadListener = null, + ?TransferListener $progressTracker = null, array $config = [] - ): PromiseInterface { + ): PromiseInterface + { if (is_string($source)) { $sourceArgs = $this->s3UriAsBucketAndKey($source); } elseif (is_array($source)) { @@ -81,25 +106,149 @@ public function download( throw new \InvalidArgumentException('Source must be a string or an array of strings'); } + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING + ); + } + $requestArgs = $sourceArgs + $downloadArgs; if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, - $config + $downloadListener, + $progressTracker, + [ + 'minimumPartSize' => $config['minimumPartSize'] + ?? 0 + ] ); } - return $this->trySingleDownload($requestArgs); + return $this->trySingleDownload($requestArgs, $progressTracker); + } + + /** + * @param string $bucket The bucket from where the files are going to be + * downloaded from. + * @param string $destinationDirectory The destination path where the downloaded + * files will be placed in. + * @param array $downloadArgs The getObject request arguments to be provided + * as part of each get object request sent to the service, except for the + * bucket and key which will be resolved internally. + * @param MultipartDownloadListenerFactory|null $downloadListenerFactory + * A factory of multipart download listeners `MultipartDownloadListenerFactory` + * for listening to multipart download events. + * @param TransferListener|null $progressTracker + * @param array $config The config options for this download directory operation. \ + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. + * - listObjectV2Args: (array) \ + * The arguments to be included as part of the listObjectV2 request in + * order to fetch the objects to be downloaded. The most common arguments + * would be: + * - MaxKeys: (int) Sets the maximum number of keys returned in the response. + * - Prefix: (string) To limit the response to keys that begin with the + * specified prefix. + * - filter: (Closure) \ + * A callable which will receive an object key as parameter and should return + * true or false in order to determine whether the object should be downloaded. + * @return PromiseInterface + */ + public function downloadDirectory( + string $bucket, + string $destinationDirectory, + array $downloadArgs, + ?MultipartDownloadListenerFactory $downloadListenerFactory = null, + ?TransferListener $progressTracker = null, + array $config = [] + ): PromiseInterface + { + if (!file_exists($destinationDirectory)) { + throw new \InvalidArgumentException( + "Destination directory '$destinationDirectory' MUST exists." + ); + } + + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING + ); + } + + $listArgs = [ + 'Bucket' => $bucket + ] + ($config['listObjectV2Args'] ?? []); + $objects = $this->s3Client + ->getPaginator('ListObjectsV2', $listArgs) + ->search('Contents[].Key'); + $objects = map($objects, function (string $key) use ($bucket) { + return "s3://$bucket/$key"; + }); + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + } + + $filter = $config['filter']; + $objects = filter($objects, function (string $key) use ($filter) { + return call_user_func($filter, $key); + }); + } + + $promises = []; + foreach ($objects as $object) { + $objectKey = $this->s3UriAsBucketAndKey($object)['Key']; + $destinationFile = $destinationDirectory . '/' . $objectKey; + if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { + throw new S3TransferException( + "Cannot download key ' . $objectKey + . ', its relative path resolves outside the' + . ' parent directory" + ); + } + + $downloadListener = null; + if ($downloadListenerFactory !== null) { + $downloadListener = $downloadListenerFactory(); + } + + $promises[] = $this->download( + $object, + $downloadArgs, + $downloadListener, + $progressTracker, + [ + 'minimumPartSize' => $config['minimumPartSize'] ?? 0, + ] + )->then(function (DownloadResult $result) use ($destinationFile) { + $directory = dirname($destinationFile); + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + file_put_contents($destinationFile, $result->getContent()); + }); + } + + return Each::ofLimitAll($promises, $this->config['concurrency']); } /** * Tries an object multipart download. * * @param array $requestArgs + * @param MultipartDownloadListener|null $downloadListener + * @param TransferListener|null $progressTracker * @param array $config - * - listener: (?MultipartDownloadListener) \ - * A multipart download listener for watching every multipart download - * stage. * - minimumPartSize: (int) \ * The minimum part size in bytes for a range multipart download. If * this parameter is not provided then it fallbacks to the transfer @@ -109,20 +258,11 @@ public function download( */ private function tryMultipartDownload( array $requestArgs, - array $config - ): PromiseInterface { - $trackProgress = $config['trackProgress'] - ?? $this->config['trackProgress'] - ?? false; - $progressListenerFactory = $this->config['progressListenerFactory'] ?? null; - $progressListener = null; - if ($trackProgress) { - if ($progressListenerFactory !== null) { - $progressListener = $progressListenerFactory(); - } else { - $progressListener = new DefaultProgressTracker(); - } - } + ?MultipartDownloadListener $downloadListener = null, + ?TransferListener $progressTracker = null, + array $config = [] + ): PromiseInterface + { $multipartDownloader = MultipartDownloader::chooseDownloader( s3Client: $this->s3Client, multipartDownloadType: $this->config['multipartDownloadType'], @@ -133,8 +273,8 @@ private function tryMultipartDownload( $this->config['targetPartSizeBytes'] ) ], - listener: $config['listener'] ?? null, - progressTracker: $progressListener?->getTransferListener() + listener: $downloadListener, + progressTracker: $progressTracker, ); return $multipartDownloader->promise(); @@ -143,13 +283,191 @@ private function tryMultipartDownload( /** * Does a single object download. * - * @param $requestArgs + * @param array $requestArgs + * @param TransferListener|null $progressTracker * * @return PromiseInterface */ - private function trySingleDownload($requestArgs): PromiseInterface { - $command = $this->s3Client->getCommand(MultipartDownloader::GET_OBJECT_COMMAND, $requestArgs); + private function trySingleDownload( + array $requestArgs, + ?TransferListener $progressTracker + ): PromiseInterface + { + if ($progressTracker !== null) { + $progressTracker->objectTransferInitiated($requestArgs['Key'], $requestArgs); + $command = $this->s3Client->getCommand( + MultipartDownloader::GET_OBJECT_COMMAND, + $requestArgs + ); + + return $this->s3Client->executeAsync($command)->then( + function ($result) use ($progressTracker, $requestArgs) { + // Notify progress + $progressTracker->objectTransferProgress( + $requestArgs['Key'], + $result['Content-Length'] ?? 0, + $result['Content-Length'] ?? 0, + ); + + // Notify Completion + $progressTracker->objectTransferCompleted( + $requestArgs['Key'], + $result['Content-Length'] ?? 0, + ); + + return new DownloadResult( + content: $result['Body'], + metadata: $result['@metadata'], + ); + } + )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { + $progressTracker->objectTransferFailed( + $requestArgs['Key'], + 0, + $reason->getMessage(), + ); + + return $reason; + }); + } + + $command = $this->s3Client->getCommand( + MultipartDownloader::GET_OBJECT_COMMAND, + $requestArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function ($result) { + return new DownloadResult( + content: $result['Body'], + metadata: $result['@metadata'], + ); + }); + } + + /** + * Returns a default instance of S3Client. + * + * @return S3Client + */ + private function defaultS3Client(): S3ClientInterface + { + return new S3Client([ + 'region' => $this->config['region'], + ]); + } + + /** + * Validates a provided value is not empty, and if so then + * it throws an exception with the provided message. + * @param mixed $value + * @param string $message + * + * @return mixed + */ + private function requireNonEmpty(mixed $value, string $message): mixed + { + if (empty($value)) { + throw new \InvalidArgumentException($message); + } + + return $value; + } + + /** + * Validates a string value is a valid S3 URI. + * Valid S3 URI Example: S3://mybucket.dev/myobject.txt + * + * @param string $uri + * + * @return bool + */ + private function isValidS3URI(string $uri): bool + { + // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. + return str_starts_with(strtolower($uri), 's3://') + && count(explode('/', substr($uri, 5))) > 1; + } + + /** + * Converts a S3 URI into an array with a Bucket and Key + * properties set. + * + * @param string $uri: The S3 URI. + * + * @return array + */ + private function s3UriAsBucketAndKey(string $uri): array + { + $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; + if (!$this->isValidS3URI($uri)) { + throw new \InvalidArgumentException($errorMessage); + } + + $path = substr($uri, 5); // without s3:// + $parts = explode('/', $path, 2); + + if (count($parts) < 2) { + throw new \InvalidArgumentException($errorMessage); + } + + return [ + 'Bucket' => $parts[0], + 'Key' => $parts[1], + ]; + } + + /** + * Resolves the progress tracker to be used in the + * transfer operation if `$trackProgress` is true. + * + * @param string $trackingOperation + * + * @return TransferListener|null + */ + private function resolveDefaultProgressTracker( + string $trackingOperation + ): ?TransferListener + { + $progressTrackerFactory = $this->config['progressTrackerFactory'] ?? null; + if ($progressTrackerFactory === null) { + return (new DefaultProgressTracker(trackingOperation: $trackingOperation))->getTransferListener(); + } + + return $progressTrackerFactory([]); + } + + /** + * @param string $sink + * @param string $objectKey + * + * @return bool + */ + private function resolvesOutsideTargetDirectory( + string $sink, + string $objectKey + ): bool + { + $resolved = []; + $sections = explode('/', $sink); + $targetSectionsLength = count(explode('/', $objectKey)); + $targetSections = array_slice($sections, -($targetSectionsLength + 1)); + $targetDirectory = $targetSections[0]; + + foreach ($targetSections as $section) { + if ($section === '.' || $section === '') { + continue; + } + if ($section === '..') { + array_pop($resolved); + if (empty($resolved) || $resolved[0] !== $targetDirectory) { + return true; + } + } else { + $resolved []= $section; + } + } - return $this->s3Client->executeAsync($requestArgs); + return false; } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php b/src/S3/Features/S3Transfer/S3TransferManagerTrait.php deleted file mode 100644 index 6d2542cb04..0000000000 --- a/src/S3/Features/S3Transfer/S3TransferManagerTrait.php +++ /dev/null @@ -1,87 +0,0 @@ - 8 * 1024 * 1024, - 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, - 'checksumValidationEnabled' => true, - 'checksumAlgorithm' => 'crc32', - 'multipartDownloadType' => 'partGet', - 'concurrency' => 5, - ]; - - /** - * Returns a default instance of S3Client. - * - * @return S3Client - */ - private function defaultS3Client(): S3ClientInterface - { - return new S3Client([]); - } - - /** - * Validates a provided value is not empty, and if so then - * it throws an exception with the provided message. - * @param mixed $value - * - * @return mixed - */ - private function requireNonEmpty(mixed $value, string $message): mixed { - if (empty($value)) { - throw new \InvalidArgumentException($message); - } - - return $value; - } - - /** - * Validates a string value is a valid S3 URI. - * Valid S3 URI Example: S3://mybucket.dev/myobject.txt - * - * @param string $uri - * - * @return bool - */ - private function isValidS3URI(string $uri): bool - { - // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. - return str_starts_with(strtolower($uri), 's3://') - && count(explode('/', substr($uri, 5))) > 1; - } - - /** - * Converts a S3 URI into an array with a Bucket and Key - * properties set. - * - * @param string $uri: The S3 URI. - * - * @return array - */ - private function s3UriAsBucketAndKey(string $uri): array { - $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; - if (!$this->isValidS3URI($uri)) { - throw new \InvalidArgumentException($errorMessage); - } - - $path = substr($uri, 5); // without s3:// - $parts = explode('/', $path, 2); - - if (count($parts) < 2) { - throw new \InvalidArgumentException($errorMessage); - } - - return [ - 'Bucket' => $parts[0], - 'Key' => $parts[1], - ]; - } - -} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php index 1fb5a3f74a..892f159085 100644 --- a/src/S3/Features/S3Transfer/TransferListener.php +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -106,7 +106,8 @@ public function getObjectsToBeTransferred(): int /** * Transfer initiated event. */ - public function transferInitiated(): void { + public function transferInitiated(): void + { $this->notify('onTransferInitiated', []); } @@ -118,7 +119,8 @@ public function transferInitiated(): void { * * @return void */ - public function objectTransferInitiated(string $objectKey, array &$requestArgs): void { + public function objectTransferInitiated(string $objectKey, array &$requestArgs): void + { $this->objectsToBeTransferred++; if ($this->objectsToBeTransferred === 1) { $this->transferInitiated(); @@ -140,7 +142,8 @@ public function objectTransferProgress( string $objectKey, int $objectBytesTransferred, int $objectSizeInBytes - ): void { + ): void + { $this->objectsBytesTransferred += $objectBytesTransferred; $this->notify('onObjectTransferProgress', [ $objectKey, @@ -168,7 +171,8 @@ public function objectTransferFailed( string $objectKey, int $objectBytesTransferred, \Throwable | string $reason - ): void { + ): void + { $this->objectsTransferFailed++; $this->notify('onObjectTransferFailed', [ $objectKey, @@ -188,7 +192,8 @@ public function objectTransferFailed( public function objectTransferCompleted ( string $objectKey, int $objectBytesCompleted - ): void { + ): void + { $this->objectsTransferCompleted++; $this->validateTransferComplete(); $this->notify('onObjectTransferCompleted', [ @@ -208,7 +213,8 @@ public function objectTransferCompleted ( public function transferCompleted ( int $objectsTransferCompleted, int $objectsBytesTransferred, - ): void { + ): void + { $this->notify('onTransferCompleted', [ $objectsTransferCompleted, $objectsBytesTransferred @@ -229,7 +235,8 @@ public function transferFailed ( int $objectsBytesTransferred, int $objectsTransferFailed, Throwable | string $reason - ): void { + ): void + { $this->notify('onTransferFailed', [ $objectsTransferCompleted, $objectsBytesTransferred, @@ -244,7 +251,8 @@ public function transferFailed ( * * @return void */ - private function validateTransferComplete(): void { + private function validateTransferComplete(): void + { if ($this->objectsToBeTransferred === ($this->objectsTransferCompleted + $this->objectsTransferFailed)) { if ($this->objectsTransferFailed > 0) { $this->transferFailed( From 34321b29e521327bf778a50a70374c3b9b93ccb9 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Feb 2025 17:39:07 -0800 Subject: [PATCH 06/62] chore: add upload and refactor code Refactor: - Add a message placeholder for progress status. For example in case of errors. Adds: - Upload feature, missing multipart functionality. --- .../S3Transfer/ConsoleProgressBar.php | 14 +- .../S3Transfer/DefaultProgressTracker.php | 11 +- .../MultipartDownloadListenerFactory.php | 11 + .../S3Transfer/MultipartUploadListener.php | 58 +++++ .../S3Transfer/ObjectProgressTracker.php | 40 +++- .../Features/S3Transfer/S3TransferManager.php | 208 ++++++++++++++---- 6 files changed, 285 insertions(+), 57 deletions(-) create mode 100644 src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php create mode 100644 src/S3/Features/S3Transfer/MultipartUploadListener.php diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/Features/S3Transfer/ConsoleProgressBar.php index 2478812b63..e370362651 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/Features/S3Transfer/ConsoleProgressBar.php @@ -27,12 +27,13 @@ class ConsoleProgressBar implements ProgressBar ] ], 'colored_transfer_format' => [ - 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|\033[0m", + 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m", 'parameters' => [ 'transferred', 'tobe_transferred', 'unit', - 'color_code' + 'color_code', + 'message' ] ], ]; @@ -57,6 +58,7 @@ class ConsoleProgressBar implements ProgressBar * @param ?int $progressBarWidth * @param ?int $percentCompleted * @param array|null $format + * @param array $args */ public function __construct( ?string $progressBarChar = null, @@ -68,8 +70,8 @@ public function __construct( $this->progressBarChar = $progressBarChar ?? '#'; $this->progressBarWidth = $progressBarWidth ?? 25; $this->percentCompleted = $percentCompleted ?? 0; - $this->format = $format ?: self::$formats['transfer_format']; - $this->args = $args ?: []; + $this->format = $format ?? self::$formats['transfer_format']; + $this->args = $args ?? []; } /** @@ -122,13 +124,13 @@ public function getPaintedProgress(): string { foreach ($this->format['parameters'] as $param) { if (!array_key_exists($param, $this->args)) { - throw new \InvalidArgumentException("Missing `$param` parameter for progress bar."); + $this->args[$param] = ''; } } $replacements = [ '|progress_bar|' => $this->renderProgressBar(), - '|percent|' => $this->percentCompleted + '|percent|' => $this->percentCompleted, ]; foreach ($this->format['parameters'] as $param) { diff --git a/src/S3/Features/S3Transfer/DefaultProgressTracker.php b/src/S3/Features/S3Transfer/DefaultProgressTracker.php index e91d7c4d77..c09e302a6d 100644 --- a/src/S3/Features/S3Transfer/DefaultProgressTracker.php +++ b/src/S3/Features/S3Transfer/DefaultProgressTracker.php @@ -60,7 +60,10 @@ public function __construct( } $this->output = $output; - if (!in_array(strtolower($trackingOperation), ['downloading', 'Uploading'], true)) { + if (!in_array(strtolower($trackingOperation), [ + strtolower(self::TRACKING_OPERATION_DOWNLOADING), + strtolower(self::TRACKING_OPERATION_UPLOADING), + ], true)) { throw new \InvalidArgumentException("Tracking operation '$trackingOperation' should be one of 'Downloading', 'Uploading'"); } @@ -188,7 +191,7 @@ public function objectTransferFailed(): Closure \Throwable | string $reason ): void { $objectProgressTracker = $this->objects[$objectKey]; - $objectProgressTracker->setStatus('failed'); + $objectProgressTracker->setStatus('failed', $reason); $this->objectsInProgress--; @@ -235,7 +238,9 @@ private function increaseBytesTransferred(int $bytesTransferred): void { $this->totalBytesTransferred += $bytesTransferred; if ($this->objectsTotalSizeInBytes !== 0) { - $this->transferPercentCompleted = floor(($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100); + $this->transferPercentCompleted = floor( + ($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100 + ); } } diff --git a/src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php b/src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php new file mode 100644 index 0000000000..5516ae8a46 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartDownloadListenerFactory.php @@ -0,0 +1,11 @@ +objectKey = $objectKey; + $this->objectBytesTransferred = $objectBytesTransferred; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->status = $status; $this->progressBar = $progressBar ?? $this->defaultProgressBar(); } @@ -95,13 +117,18 @@ public function getStatus(): string /** * @param string $status + * @param string|null $message * * @return void */ - public function setStatus(string $status): void + public function setStatus(string $status, ?string $message = null): void { $this->status = $status; $this->setProgressColor(); + // To show specific messages for specific status. + if (!empty($message)) { + $this->progressBar->setArg('message', "$status: $message"); + } } private function setProgressColor(): void @@ -155,6 +182,7 @@ private function defaultProgressBar(): ProgressBar 'tobe_transferred' => 0, 'unit' => 'B', 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, + 'message' => '' ] ); } diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index 3a199839ca..ec5052b525 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -8,6 +8,8 @@ use Aws\S3\S3ClientInterface; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Psr7\Utils; +use Psr\Http\Message\StreamInterface; use function Aws\filter; use function Aws\map; @@ -24,6 +26,8 @@ class S3TransferManager 'region' => 'us-east-1', ]; + private const MIN_PART_SIZE = 5 * 1024 * 1024; + /** @var S3Client */ private S3ClientInterface $s3Client; @@ -70,29 +74,29 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) * @param array $downloadArgs The getObject request arguments to be provided as part * of each get object operation, except for the bucket and key, which * are already provided as the source. + * @param array $config The configuration to be used for this operation. + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. * @param MultipartDownloadListener|null $downloadListener A multipart download * specific listener of the different states a multipart download can be. * @param TransferListener|null $progressTracker A transfer listener implementation * aimed to track the progress of a transfer. If not provided and trackProgress * is resolved as true then, the default progressTrackerFactory will be used. - * @param array $config The configuration to be used for this operation. - * - trackProgress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener - * implementation will be used. - * - minimumPartSize: (int) \ - * The minimum part size in bytes to be used in a range multipart download. * * @return PromiseInterface */ public function download( string | array $source, array $downloadArgs = [], + array $config = [], ?MultipartDownloadListener $downloadListener = null, ?TransferListener $progressTracker = null, - array $config = [] ): PromiseInterface { if (is_string($source)) { @@ -117,12 +121,12 @@ public function download( if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, - $downloadListener, - $progressTracker, [ 'minimumPartSize' => $config['minimumPartSize'] ?? 0 - ] + ], + $downloadListener, + $progressTracker, ); } @@ -137,38 +141,39 @@ public function download( * @param array $downloadArgs The getObject request arguments to be provided * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. + * @param array $config The config options for this download directory operation. \ + * - trackProgress: (bool) \ + * Overrides the config option set in the transfer manager instantiation + * to decide whether transfer progress should be tracked. If a `progressListenerFactory` + * was not provided when the transfer manager instance was created + * and trackProgress resolved as true then, a default progress listener + * implementation will be used. + * - minimumPartSize: (int) \ + * The minimum part size in bytes to be used in a range multipart download. + * - listObjectV2Args: (array) \ + * The arguments to be included as part of the listObjectV2 request in + * order to fetch the objects to be downloaded. The most common arguments + * would be: + * - MaxKeys: (int) Sets the maximum number of keys returned in the response. + * - Prefix: (string) To limit the response to keys that begin with the + * specified prefix. + * - filter: (Closure) \ + * A callable which will receive an object key as parameter and should return + * true or false in order to determine whether the object should be downloaded. * @param MultipartDownloadListenerFactory|null $downloadListenerFactory * A factory of multipart download listeners `MultipartDownloadListenerFactory` * for listening to multipart download events. * @param TransferListener|null $progressTracker - * @param array $config The config options for this download directory operation. \ - * - trackProgress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener - * implementation will be used. - * - minimumPartSize: (int) \ - * The minimum part size in bytes to be used in a range multipart download. - * - listObjectV2Args: (array) \ - * The arguments to be included as part of the listObjectV2 request in - * order to fetch the objects to be downloaded. The most common arguments - * would be: - * - MaxKeys: (int) Sets the maximum number of keys returned in the response. - * - Prefix: (string) To limit the response to keys that begin with the - * specified prefix. - * - filter: (Closure) \ - * A callable which will receive an object key as parameter and should return - * true or false in order to determine whether the object should be downloaded. + * * @return PromiseInterface */ public function downloadDirectory( string $bucket, string $destinationDirectory, array $downloadArgs, + array $config = [], ?MultipartDownloadListenerFactory $downloadListenerFactory = null, ?TransferListener $progressTracker = null, - array $config = [] ): PromiseInterface { if (!file_exists($destinationDirectory)) { @@ -224,11 +229,11 @@ public function downloadDirectory( $promises[] = $this->download( $object, $downloadArgs, - $downloadListener, - $progressTracker, [ 'minimumPartSize' => $config['minimumPartSize'] ?? 0, - ] + ], + $downloadListener, + $progressTracker, )->then(function (DownloadResult $result) use ($destinationFile) { $directory = dirname($destinationFile); if (!is_dir($directory)) { @@ -242,25 +247,97 @@ public function downloadDirectory( return Each::ofLimitAll($promises, $this->config['concurrency']); } + /** + * @param string|StreamInterface $source + * @param string $bucketTo + * @param string $key + * @param array $requestArgs + * @param array $config The config options for this upload operation. + * - mup_threshold: (int, optional) To override the default threshold + * for when to use multipart upload. + * - trackProgress: (bool, optional) To override the + * + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function upload( + string | StreamInterface $source, + string $bucketTo, + string $key, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + if (is_string($source) && !is_readable($source)) { + throw new \InvalidArgumentException("Please provide a valid readable file path or a valid stream."); + } + + $mupThreshold = $config['mup_threshold'] ?? $this->config['multipartUploadThresholdBytes']; + if ($mupThreshold < self::MIN_PART_SIZE) { + throw new \InvalidArgumentException("\$config['mup_threshold'] must be greater than or equal to " . self::MIN_PART_SIZE); + } + + if ($source instanceof StreamInterface) { + $sourceSize = $source->getSize(); + $requestArgs['Body'] = $source; + } else { + $sourceSize = filesize($source); + $requestArgs['SourceFile'] = $source; + } + + $requestArgs['Bucket'] = $bucketTo; + $requestArgs['Key'] = $key; + $requestArgs['Size'] = $sourceSize; + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + if ($sourceSize < $mupThreshold) { + return $this->trySingleUpload( + $requestArgs, + $progressTracker + )->then(function ($result) { + $streams = get_resources("stream"); + + echo "Open file handles:\n"; + foreach ($streams as $stream) { + $metadata = stream_get_meta_data($stream); + echo "\nFile: " . ($metadata['uri'] ?? "") . "\n"; + } + + return $result; + }); + } + + throw new S3TransferException("Not implemented yet."); + } + /** * Tries an object multipart download. * * @param array $requestArgs + * @param array $config + * - minimumPartSize: (int) \ + * The minimum part size in bytes for a range multipart download. If + * this parameter is not provided then it fallbacks to the transfer + * manager `targetPartSizeBytes` config value. * @param MultipartDownloadListener|null $downloadListener * @param TransferListener|null $progressTracker - * @param array $config - * - minimumPartSize: (int) \ - * The minimum part size in bytes for a range multipart download. If - * this parameter is not provided then it fallbacks to the transfer - * manager `targetPartSizeBytes` config value. * * @return PromiseInterface */ private function tryMultipartDownload( array $requestArgs, + array $config = [], ?MultipartDownloadListener $downloadListener = null, ?TransferListener $progressTracker = null, - array $config = [] ): PromiseInterface { $multipartDownloader = MultipartDownloader::chooseDownloader( @@ -345,6 +422,53 @@ function ($result) use ($progressTracker, $requestArgs) { }); } + /** + * @param array $requestArgs + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + private function trySingleUpload( + array $requestArgs, + ?TransferListener $progressTracker = null + ): PromiseInterface { + if ($progressTracker !== null) { + $progressTracker->objectTransferInitiated( + $requestArgs['Key'], + $requestArgs + ); + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + return $this->s3Client->executeAsync($command)->then( + function ($result) use ($progressTracker, $requestArgs) { + $progressTracker->objectTransferProgress( + $requestArgs['Key'], + $requestArgs['Size'], + $requestArgs['Size'], + ); + + $progressTracker->objectTransferCompleted( + $requestArgs['Key'], + $requestArgs['Size'], + ); + + return $result; + } + )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { + $progressTracker->objectTransferFailed( + $requestArgs['Key'], + 0, + $reason->getMessage() + ); + + return $reason; + }); + } + + $command = $this->s3Client->getCommand('PutObject', $requestArgs); + + return $this->s3Client->executeAsync($command); + } + /** * Returns a default instance of S3Client. * From 5289f7ce4db523d59f01e17448e65a4682b1564c Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 14 Feb 2025 10:00:46 -0800 Subject: [PATCH 07/62] feat: add upload directory feature - Add upload directory feature --- .../Features/S3Transfer/S3TransferManager.php | 109 ++++++++++++++++-- 1 file changed, 99 insertions(+), 10 deletions(-) diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index ec5052b525..51ed767101 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -8,10 +8,10 @@ use Aws\S3\S3ClientInterface; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; -use GuzzleHttp\Psr7\Utils; use Psr\Http\Message\StreamInterface; use function Aws\filter; use function Aws\map; +use function Aws\recursive_dir_iterator; class S3TransferManager { @@ -303,20 +303,85 @@ public function upload( return $this->trySingleUpload( $requestArgs, $progressTracker - )->then(function ($result) { - $streams = get_resources("stream"); + ); + } + + return $this->tryMultipartUpload( + $source, + $requestArgs, + $uploadListener, + $progressTracker, + ); + } + + /** + * @param string $directory + * @param string $bucketTo + * @param array $requestArgs + * @param array $config + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function uploadDirectory( + string $directory, + string $bucketTo, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + if (!file_exists($directory)) { + throw new \InvalidArgumentException( + "Source directory '$directory' MUST exists." + ); + } + + if ($progressTracker === null + && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + $filter = null; + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + } + + $filter = $config['filter']; + } - echo "Open file handles:\n"; - foreach ($streams as $stream) { - $metadata = stream_get_meta_data($stream); - echo "\nFile: " . ($metadata['uri'] ?? "") . "\n"; + $files = filter( + recursive_dir_iterator($directory), + function ($file) use ($filter) { + if ($filter !== null) { + return !is_dir($file) && $filter($file); } - return $result; - }); + return !is_dir($file); + } + ); + + $promises = []; + foreach ($files as $file) { + $fileParts = explode("/", $file); + $key = end($fileParts); + $promises[] = $this->upload( + $file, + $bucketTo, + $key, + $requestArgs, + $config, + $uploadListener, + $progressTracker, + ); } - throw new S3TransferException("Not implemented yet."); + return Each::ofLimitAll($promises, $this->config['concurrency']); } /** @@ -469,6 +534,30 @@ function ($result) use ($progressTracker, $requestArgs) { return $this->s3Client->executeAsync($command); } + /** + * @param array $requestArgs + * + * @return PromiseInterface + */ + private function tryMultipartUpload( + string | StreamInterface $source, + array $requestArgs, + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface { + return (new MultipartUploaderV2( + $this->s3Client, + $source, + $requestArgs, + [ + 'target_part_size_bytes' => $this->config['targetPartSizeBytes'], + 'concurrency' => $this->config['concurrency'], + ], + $uploadListener, + $progressTracker, + ))->promise(); + } + /** * Returns a default instance of S3Client. * From c6c07800620f3e1d73c38b03f588f3d6be65ebae Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 17 Feb 2025 08:02:47 -0800 Subject: [PATCH 08/62] feat: multipart upload and some refactor - Add a dedicated multipart upload implementation - Add transfer progress to multipart upload - Add upload directory with the required options. - Create specific response models for upload, and upload directory. - Add multipart upload test cases. - Fix transfer listener completation eval. --- ...ownloadResult.php => DownloadResponse.php} | 2 +- .../S3Transfer/MultipartDownloader.php | 2 +- .../Features/S3Transfer/MultipartUploader.php | 427 +++++++++++++++ .../Features/S3Transfer/S3TransferManager.php | 513 +++++++++++------- .../Features/S3Transfer/TransferListener.php | 1 + .../S3Transfer/UploadDirectoryResponse.php | 38 ++ src/S3/Features/S3Transfer/UploadResponse.php | 21 + .../S3Transfer/MultipartUploaderTest.php | 214 ++++++++ 8 files changed, 1029 insertions(+), 189 deletions(-) rename src/S3/Features/S3Transfer/{DownloadResult.php => DownloadResponse.php} (94%) create mode 100644 src/S3/Features/S3Transfer/MultipartUploader.php create mode 100644 src/S3/Features/S3Transfer/UploadDirectoryResponse.php create mode 100644 src/S3/Features/S3Transfer/UploadResponse.php create mode 100644 tests/S3/Features/S3Transfer/MultipartUploaderTest.php diff --git a/src/S3/Features/S3Transfer/DownloadResult.php b/src/S3/Features/S3Transfer/DownloadResponse.php similarity index 94% rename from src/S3/Features/S3Transfer/DownloadResult.php rename to src/S3/Features/S3Transfer/DownloadResponse.php index 4f40ca1623..acd0365559 100644 --- a/src/S3/Features/S3Transfer/DownloadResult.php +++ b/src/S3/Features/S3Transfer/DownloadResponse.php @@ -4,7 +4,7 @@ use Psr\Http\Message\StreamInterface; -class DownloadResult +class DownloadResponse { public function __construct( private readonly StreamInterface $content, diff --git a/src/S3/Features/S3Transfer/MultipartDownloader.php b/src/S3/Features/S3Transfer/MultipartDownloader.php index e3c1143da1..98de586c02 100644 --- a/src/S3/Features/S3Transfer/MultipartDownloader.php +++ b/src/S3/Features/S3Transfer/MultipartDownloader.php @@ -198,7 +198,7 @@ public function promise(): PromiseInterface $this->objectDownloadCompleted(); // TODO: yield the stream wrapped in a modeled transfer success response. - yield Create::promiseFor(new DownloadResult( + yield Create::promiseFor(new DownloadResponse( $this->stream, [] )); diff --git a/src/S3/Features/S3Transfer/MultipartUploader.php b/src/S3/Features/S3Transfer/MultipartUploader.php new file mode 100644 index 0000000000..47e8c72ad2 --- /dev/null +++ b/src/S3/Features/S3Transfer/MultipartUploader.php @@ -0,0 +1,427 @@ +s3Client = $s3Client; + $this->createMultipartArgs = $createMultipartArgs; + $this->uploadPartArgs = $uploadPartArgs; + $this->completeMultipartArgs = $completeMultipartArgs; + $this->config = $config; + $this->body = $this->parseBody($source); + $this->objectSizeInBytes = $objectSizeInBytes; + $this->uploadId = $uploadId; + $this->parts = $parts; + $this->progressTracker = $progressTracker; + } + + /** + * @return string|null + */ + public function getUploadId(): ?string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getParts(): array + { + return $this->parts; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return int + */ + public function getObjectBytesTransferred(): int + { + return $this->objectBytesTransferred; + } + + /** + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + try { + yield $this->createMultipartUpload(); + yield $this->uploadParts(); + $result = yield $this->completeMultipartUpload(); + yield Create::promiseFor( + new UploadResponse($result->toArray()) + ); + } catch (Throwable $e) { + $this->uploadFailed($e); + throw $e; + } finally { + $this->callDeferredFns(); + } + }); + } + + /** + * @return PromiseInterface + */ + public function createMultipartUpload(): PromiseInterface { + $requestArgs = [...$this->createMultipartArgs]; + $this->uploadInitiated($requestArgs); + $command = $this->s3Client->getCommand( + 'CreateMultipartUpload', + $requestArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadId = $result['UploadId']; + + return $result; + }); + } + + /** + * @return PromiseInterface + */ + public function uploadParts(): PromiseInterface + { + $this->objectSizeInBytes = 0; // To repopulate + $isSeekable = $this->body->isSeekable(); + $partSize = $this->config['part_size'] ?? self::PART_MIN_SIZE; + if ($partSize > self::PART_MAX_SIZE) { + return Create::rejectionFor( + "The part size should not exceed " . self::PART_MAX_SIZE . " bytes." + ); + } + + $commands = []; + for ($partNo = 1; + $isSeekable + ? $this->body->tell() < $this->body->getSize() + : !$this->body->eof(); + $partNo++ + ) { + if ($isSeekable) { + $readSize = min($partSize, $this->body->getSize() - $this->body->tell()); + } else { + $readSize = $partSize; + } + + $partBody = Utils::streamFor( + $this->body->read($readSize) + ); + // To make sure we do not create an empty part when + // we already reached end of file. + if (!$isSeekable && $this->body->eof() && $partBody->getSize() === 0) { + break; + } + + $uploadPartCommandArgs = [ + 'UploadId' => $this->uploadId, + 'PartNumber' => $partNo, + 'Body' => $partBody, + 'ContentLength' => $partBody->getSize(), + ] + $this->uploadPartArgs; + + $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); + $commands[] = $command; + $this->objectSizeInBytes += $partBody->getSize(); + + if ($partNo > self::PART_MAX_NUM) { + return Create::rejectionFor( + "The max number of parts has been exceeded. " . + "Max = " . self::PART_MAX_NUM + ); + } + } + + return (new CommandPool( + $this->s3Client, + $commands, + [ + 'concurrency' => $this->config['concurrency'], + 'fulfilled' => function (ResultInterface $result, $index) + use ($commands) { + $command = $commands[$index]; + $this->collectPart( + $result, + $command + ); + + // Part Upload Completed Event + $this->partUploadCompleted($result, $command['ContentLength']); + }, + 'rejected' => function (Throwable $e) { + $this->partUploadFailed($e); + } + ] + ))->promise(); + } + + /** + * @return PromiseInterface + */ + public function completeMultipartUpload(): PromiseInterface + { + $this->sortParts(); + $command = $this->s3Client->getCommand('CompleteMultipartUpload', [ + 'UploadId' => $this->uploadId, + 'MpuObjectSize' => $this->objectSizeInBytes, + 'MultipartUpload' => [ + 'Parts' => $this->parts, + ] + ] + $this->completeMultipartArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadCompleted($result); + + return $result; + }); + } + + /** + * @return PromiseInterface + */ + public function abortMultipartUpload(): PromiseInterface { + $command = $this->s3Client->getCommand('AbortMultipartUpload', [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + ]); + + return $this->s3Client->executeAsync($command); + } + + /** + * @param ResultInterface $result + * @param CommandInterface $command + * + * @return void + */ + private function collectPart( + ResultInterface $result, + CommandInterface $command, + ): void + { + $checksumResult = $command->getName() === 'UploadPart' + ? $result + : $result[$command->getName() . 'Result']; + $partData = [ + 'PartNumber' => $command['PartNumber'], + 'ETag' => $result['ETag'], + ]; + if (isset($command['ChecksumAlgorithm'])) { + $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); + $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; + } + + $this->parts[] = $partData; + } + + /** + * @return void + */ + private function sortParts(): void + { + usort($this->parts, function($partOne, $partTwo) { + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; // Ascending order by age + }); + } + + /** + * @param string|StreamInterface $source + * @return StreamInterface + */ + private function parseBody(string | StreamInterface $source): StreamInterface + { + if (is_string($source)) { + // Make sure the files exists + if (!is_readable($source)) { + throw new \InvalidArgumentException( + "The source for this upload must be either a readable file or a valid stream." + ); + } + $file = Utils::tryFopen($source, 'r'); + // To make sure the resource is closed. + $this->deferFns[] = function () use ($file) { + fclose($file); + }; + $body = Utils::streamFor($file); + } elseif ($source instanceof StreamInterface) { + $body = $source; + } else { + throw new \InvalidArgumentException( + "The source must be a string or a StreamInterface." + ); + } + + return $body; + } + + /** + * @return void + */ + private function uploadInitiated(array &$requestArgs): void { + $this->objectKey = $this->createMultipartArgs['Key']; + $this->progressTracker?->objectTransferInitiated( + $this->objectKey, + $requestArgs + ); + } + + /** + * @param Throwable $reason + * + * @return void + */ + private function uploadFailed(Throwable $reason): void { + if (!empty($this->uploadId)) { + $this->abortMultipartUpload()->wait(); + } + $this->progressTracker?->objectTransferFailed( + $this->objectKey, + $this->objectBytesTransferred, + $reason + ); + } + + /** + * @param ResultInterface $result + * + * @return void + */ + private function uploadCompleted(ResultInterface $result): void { + $this->progressTracker?->objectTransferCompleted( + $this->objectKey, + $this->objectBytesTransferred, + ); + } + + /** + * @param ResultInterface $result + * @param int $partSize + * + * @return void + */ + private function partUploadCompleted(ResultInterface $result, int $partSize): void { + $this->objectBytesTransferred = $this->objectBytesTransferred + $partSize; + $this->progressTracker?->objectTransferProgress( + $this->objectKey, + $partSize, + $this->objectSizeInBytes + ); + } + + /** + * @param Throwable $reason + * + * @return void + */ + private function partUploadFailed(Throwable $reason): void + { + } + + /** + * @return void + */ + private function callDeferredFns(): void + { + foreach ($this->deferFns as $fn) { + $fn(); + } + + $this->deferFns = []; + } +} diff --git a/src/S3/Features/S3Transfer/S3TransferManager.php b/src/S3/Features/S3Transfer/S3TransferManager.php index 51ed767101..75a5b5124e 100644 --- a/src/S3/Features/S3Transfer/S3TransferManager.php +++ b/src/S3/Features/S3Transfer/S3TransferManager.php @@ -2,7 +2,6 @@ namespace Aws\S3\Features\S3Transfer; -use Aws\Command; use Aws\S3\Features\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; @@ -16,13 +15,13 @@ class S3TransferManager { private static array $defaultConfig = [ - 'targetPartSizeBytes' => 8 * 1024 * 1024, - 'multipartUploadThresholdBytes' => 16 * 1024 * 1024, - 'checksumValidationEnabled' => true, - 'checksumAlgorithm' => 'crc32', - 'multipartDownloadType' => 'partGet', + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'multipart_upload_threshold_bytes' => 16 * 1024 * 1024, + 'checksum_validation_enabled' => true, + 'checksum_algorithm' => 'crc32', + 'multipart_download_type' => 'partGet', 'concurrency' => 5, - 'trackProgress' => false, + 'track_progress' => false, 'region' => 'us-east-1', ]; @@ -35,29 +34,34 @@ class S3TransferManager private array $config; /** - * @param ?S3ClientInterface $s3Client + * @param S3ClientInterface | null $s3Client If provided as null then, + * a default client will be created where its region will be the one + * resolved from either the default from the config or the provided. * @param array $config - * - targetPartSizeBytes: (int, default=(8388608 `8MB`)) \ + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) \ * The minimum part size to be used in a multipart upload/download. - * - multipartUploadThresholdBytes: (int, default=(16777216 `16 MB`)) \ + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) \ * The threshold to decided whether a multipart upload is needed. - * - checksumValidationEnabled: (bool, default=true) \ + * - checksum_validation_enabled: (bool, default=true) \ * To decide whether a checksum validation will be applied to the response. - * - checksumAlgorithm: (string, default='crc32') \ + * - checksum_algorithm: (string, default='crc32') \ * The checksum algorithm to be used in an upload request. - * - multipartDownloadType: (string, default='partGet') + * - multipart_download_type: (string, default='partGet') * The download type to be used in a multipart download. * - concurrency: (int, default=5) \ * Maximum number of concurrent operations allowed during a multipart * upload/download. - * - trackProgress: (bool, default=false) \ + * - track_progress: (bool, default=false) \ * To enable progress tracker in a multipart upload/download. - * - progressTrackerFactory: (callable|TransferListenerFactory) \ + * - region: (string, default="us-east-2") + * - progress_tracker_factory: (callable|TransferListenerFactory) \ * A factory to create the listener which will receive notifications * based in the different stages an upload/download is. */ - public function __construct(?S3ClientInterface $s3Client, array $config = []) - { + public function __construct( + ?S3ClientInterface $s3Client = null, + array $config = [] + ) { if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -67,6 +71,196 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) $this->config = $config + self::$defaultConfig; } + /** + * @param string|StreamInterface $source + * @param array $requestArgs The putObject request arguments. + * Required parameters would be: + * - Bucket: (string, required) + * - Key: (string, required) + * @param array $config The config options for this upload operation. + * - multipart_upload_threshold_bytes: (int, optional) + * To override the default threshold for when to use multipart upload. + * - target_part_size_bytes: (int, optional) To override the default + * target part size in bytes. + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function upload( + string | StreamInterface $source, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + // Make sure the source is what is expected + if (!is_string($source) && !$source instanceof StreamInterface) { + throw new \InvalidArgumentException( + '`source` must be a string or a StreamInterface' + ); + } + // Make sure it is a valid in path in case of a string + if (is_string($source) && !is_readable($source)) { + throw new \InvalidArgumentException( + "Please provide a valid readable file path or a valid stream as source." + ); + } + // Valid required parameters + foreach (['Bucket', 'Key'] as $reqParam) { + $this->requireNonEmpty( + $requestArgs[$reqParam] ?? null, + "The `$reqParam` parameter must be provided as part of the request arguments." + ); + } + + $mupThreshold = $config['multipart_upload_threshold_bytes'] + ?? $this->config['multipart_upload_threshold_bytes']; + if ($mupThreshold < self::MIN_PART_SIZE) { + throw new \InvalidArgumentException( + "The provided config `multipart_upload_threshold_bytes`" + ."must be greater than or equal to " . self::MIN_PART_SIZE + ); + } + + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config['track_progress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + if ($this->requiresMultipartUpload($source, $mupThreshold)) { + return $this->tryMultipartUpload( + $source, + $requestArgs, + $config['target_part_size_bytes'] + ?? $this->config['target_part_size_bytes'], + $uploadListener, + $progressTracker, + ); + } + + return $this->trySingleUpload( + $source, + $requestArgs, + $progressTracker + ); + } + + + /** + * @param string $directory + * @param string $bucketTo + * @param array $requestArgs + * @param array $config + * - follow_symbolic_links: (bool, optional, defaulted to false) + * - recursive: (bool, optional, defaulted to false) + * - s3_prefix: (string, optional, defaulted to null) + * - filter: (Closure(string), optional) + * - s3_delimiter: (string, optional, defaulted to `/`) + * - put_object_request_callback: (Closure, optional) + * - failure_policy: (Closure, optional) + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker + * + * @return PromiseInterface + */ + public function uploadDirectory( + string $directory, + string $bucketTo, + array $requestArgs = [], + array $config = [], + ?MultipartUploadListener $uploadListener = null, + ?TransferListener $progressTracker = null, + ): PromiseInterface + { + if (!is_dir($directory)) { + throw new \InvalidArgumentException( + "Please provide a valid directory path. " + . "Provided = " . $directory + ); + } + + if ($progressTracker === null + && ($config['track_progress'] ?? $this->config['track_progress'])) { + $progressTracker = $this->resolveDefaultProgressTracker( + DefaultProgressTracker::TRACKING_OPERATION_UPLOADING + ); + } + + $filter = null; + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + } + + $filter = $config['filter']; + } + + $files = filter( + recursive_dir_iterator($directory), + function ($file) use ($filter) { + if ($filter !== null) { + return !is_dir($file) && $filter($file); + } + + return !is_dir($file); + } + ); + + $prefix = $config['s3_prefix'] ?? ''; + if ($prefix !== '' && !str_ends_with($prefix, '/')) { + $prefix .= '/'; + } + $delimiter = $config['s3_delimiter'] ?? '/'; + $promises = []; + $objectsUploaded = 0; + $objectsFailed = 0; + foreach ($files as $file) { + $baseDir = rtrim($directory, '/') . '/'; + $relativePath = substr($file, strlen($baseDir)); + if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { + throw new S3TransferException( + "The filename must not contain the provided delimiter `". $delimiter ."`" + ); + } + $objectKey = $prefix.$relativePath; + $objectKey = str_replace( + DIRECTORY_SEPARATOR, + $delimiter, + $objectKey + ); + $promises[] = $this->upload( + $file, + [ + ...$requestArgs, + 'Bucket' => $bucketTo, + 'Key' => $objectKey, + ], + $config, + $uploadListener, + $progressTracker, + )->then(function ($result) use (&$objectsUploaded) { + $objectsUploaded++; + + return $result; + })->otherwise(function ($reason) use (&$objectsFailed) { + $objectsFailed++; + + return $reason; + }); + } + + return Each::ofLimitAll($promises, $this->config['concurrency']) + ->then(function ($results) use ($objectsUploaded, $objectsFailed) { + return new UploadDirectoryResponse($objectsUploaded, $objectsFailed); + }); + } + /** * @param string|array $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key @@ -75,19 +269,19 @@ public function __construct(?S3ClientInterface $s3Client, array $config = []) * of each get object operation, except for the bucket and key, which * are already provided as the source. * @param array $config The configuration to be used for this operation. - * - trackProgress: (bool) \ + * - track_progress: (bool) \ * Overrides the config option set in the transfer manager instantiation * to decide whether transfer progress should be tracked. If a `progressListenerFactory` * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener + * and track_progress resolved as true then, a default progress listener * implementation will be used. * - minimumPartSize: (int) \ * The minimum part size in bytes to be used in a range multipart download. * @param MultipartDownloadListener|null $downloadListener A multipart download * specific listener of the different states a multipart download can be. * @param TransferListener|null $progressTracker A transfer listener implementation - * aimed to track the progress of a transfer. If not provided and trackProgress - * is resolved as true then, the default progressTrackerFactory will be used. + * aimed to track the progress of a transfer. If not provided and track_progress + * is resolved as true then, the default progress_tracker_factory will be used. * * @return PromiseInterface */ @@ -111,7 +305,7 @@ public function download( } if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = $this->resolveDefaultProgressTracker( DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING ); @@ -142,11 +336,11 @@ public function download( * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. * @param array $config The config options for this download directory operation. \ - * - trackProgress: (bool) \ + * - track_progress: (bool) \ * Overrides the config option set in the transfer manager instantiation * to decide whether transfer progress should be tracked. If a `progressListenerFactory` * was not provided when the transfer manager instance was created - * and trackProgress resolved as true then, a default progress listener + * and track_progress resolved as true then, a default progress listener * implementation will be used. * - minimumPartSize: (int) \ * The minimum part size in bytes to be used in a range multipart download. @@ -183,7 +377,7 @@ public function downloadDirectory( } if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { + && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = $this->resolveDefaultProgressTracker( DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING ); @@ -234,7 +428,7 @@ public function downloadDirectory( ], $downloadListener, $progressTracker, - )->then(function (DownloadResult $result) use ($destinationFile) { + )->then(function (DownloadResponse $result) use ($destinationFile) { $directory = dirname($destinationFile); if (!is_dir($directory)) { mkdir($directory, 0777, true); @@ -247,143 +441,6 @@ public function downloadDirectory( return Each::ofLimitAll($promises, $this->config['concurrency']); } - /** - * @param string|StreamInterface $source - * @param string $bucketTo - * @param string $key - * @param array $requestArgs - * @param array $config The config options for this upload operation. - * - mup_threshold: (int, optional) To override the default threshold - * for when to use multipart upload. - * - trackProgress: (bool, optional) To override the - * - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker - * - * @return PromiseInterface - */ - public function upload( - string | StreamInterface $source, - string $bucketTo, - string $key, - array $requestArgs = [], - array $config = [], - ?MultipartUploadListener $uploadListener = null, - ?TransferListener $progressTracker = null, - ): PromiseInterface - { - if (is_string($source) && !is_readable($source)) { - throw new \InvalidArgumentException("Please provide a valid readable file path or a valid stream."); - } - - $mupThreshold = $config['mup_threshold'] ?? $this->config['multipartUploadThresholdBytes']; - if ($mupThreshold < self::MIN_PART_SIZE) { - throw new \InvalidArgumentException("\$config['mup_threshold'] must be greater than or equal to " . self::MIN_PART_SIZE); - } - - if ($source instanceof StreamInterface) { - $sourceSize = $source->getSize(); - $requestArgs['Body'] = $source; - } else { - $sourceSize = filesize($source); - $requestArgs['SourceFile'] = $source; - } - - $requestArgs['Bucket'] = $bucketTo; - $requestArgs['Key'] = $key; - $requestArgs['Size'] = $sourceSize; - if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); - } - - if ($sourceSize < $mupThreshold) { - return $this->trySingleUpload( - $requestArgs, - $progressTracker - ); - } - - return $this->tryMultipartUpload( - $source, - $requestArgs, - $uploadListener, - $progressTracker, - ); - } - - /** - * @param string $directory - * @param string $bucketTo - * @param array $requestArgs - * @param array $config - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker - * - * @return PromiseInterface - */ - public function uploadDirectory( - string $directory, - string $bucketTo, - array $requestArgs = [], - array $config = [], - ?MultipartUploadListener $uploadListener = null, - ?TransferListener $progressTracker = null, - ): PromiseInterface - { - if (!file_exists($directory)) { - throw new \InvalidArgumentException( - "Source directory '$directory' MUST exists." - ); - } - - if ($progressTracker === null - && ($config['trackProgress'] ?? $this->config['trackProgress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); - } - - $filter = null; - if (isset($config['filter'])) { - if (!is_callable($config['filter'])) { - throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); - } - - $filter = $config['filter']; - } - - $files = filter( - recursive_dir_iterator($directory), - function ($file) use ($filter) { - if ($filter !== null) { - return !is_dir($file) && $filter($file); - } - - return !is_dir($file); - } - ); - - $promises = []; - foreach ($files as $file) { - $fileParts = explode("/", $file); - $key = end($fileParts); - $promises[] = $this->upload( - $file, - $bucketTo, - $key, - $requestArgs, - $config, - $uploadListener, - $progressTracker, - ); - } - - return Each::ofLimitAll($promises, $this->config['concurrency']); - } - /** * Tries an object multipart download. * @@ -392,7 +449,7 @@ function ($file) use ($filter) { * - minimumPartSize: (int) \ * The minimum part size in bytes for a range multipart download. If * this parameter is not provided then it fallbacks to the transfer - * manager `targetPartSizeBytes` config value. + * manager `target_part_size_bytes` config value. * @param MultipartDownloadListener|null $downloadListener * @param TransferListener|null $progressTracker * @@ -407,13 +464,10 @@ private function tryMultipartDownload( { $multipartDownloader = MultipartDownloader::chooseDownloader( s3Client: $this->s3Client, - multipartDownloadType: $this->config['multipartDownloadType'], + multipartDownloadType: $this->config['multipart_download_type'], requestArgs: $requestArgs, config: [ - 'minimumPartSize' => max( - $config['minimumPartSize'] ?? 0, - $this->config['targetPartSizeBytes'] - ) + 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, ], listener: $downloadListener, progressTracker: $progressTracker, @@ -457,7 +511,7 @@ function ($result) use ($progressTracker, $requestArgs) { $result['Content-Length'] ?? 0, ); - return new DownloadResult( + return new DownloadResponse( content: $result['Body'], metadata: $result['@metadata'], ); @@ -480,7 +534,7 @@ function ($result) use ($progressTracker, $requestArgs) { return $this->s3Client->executeAsync($command) ->then(function ($result) { - return new DownloadResult( + return new DownloadResponse( content: $result['Body'], metadata: $result['@metadata'], ); @@ -488,35 +542,50 @@ function ($result) use ($progressTracker, $requestArgs) { } /** + * @param string|StreamInterface $source * @param array $requestArgs * @param TransferListener|null $progressTracker * * @return PromiseInterface */ private function trySingleUpload( + string | StreamInterface $source, array $requestArgs, ?TransferListener $progressTracker = null ): PromiseInterface { + if (is_string($source) && is_readable($source)) { + $requestArgs['SourceFile'] = $source; + $objectSize = filesize($source); + } elseif ($source instanceof StreamInterface && $source->isSeekable()) { + $requestArgs['Body'] = $source; + $objectSize = $source->getSize(); + } else { + throw new S3TransferException( + "Unable to process upload request due to the type of the source" + ); + } + if ($progressTracker !== null) { $progressTracker->objectTransferInitiated( $requestArgs['Key'], $requestArgs ); + $command = $this->s3Client->getCommand('PutObject', $requestArgs); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($progressTracker, $requestArgs) { + function ($result) use ($objectSize, $progressTracker, $requestArgs) { $progressTracker->objectTransferProgress( $requestArgs['Key'], - $requestArgs['Size'], - $requestArgs['Size'], + $objectSize, + $objectSize, ); $progressTracker->objectTransferCompleted( $requestArgs['Key'], - $requestArgs['Size'], + $objectSize, ); - return $result; + return new UploadResponse($result->toArray()); } )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { $progressTracker->objectTransferFailed( @@ -531,33 +600,78 @@ function ($result) use ($progressTracker, $requestArgs) { $command = $this->s3Client->getCommand('PutObject', $requestArgs); - return $this->s3Client->executeAsync($command); + return $this->s3Client->executeAsync($command) + ->then(function ($result) { + return new UploadResponse($result->toArray()); + }); } /** + * @param string|StreamInterface $source * @param array $requestArgs + * @param array $config + * @param MultipartUploadListener|null $uploadListener + * @param TransferListener|null $progressTracker * * @return PromiseInterface */ private function tryMultipartUpload( string | StreamInterface $source, array $requestArgs, + int $partSizeBytes, ?MultipartUploadListener $uploadListener = null, ?TransferListener $progressTracker = null, ): PromiseInterface { - return (new MultipartUploaderV2( + $createMultipartArgs = [...$requestArgs]; + $uploadPartArgs = [...$requestArgs]; + $completeMultipartArgs = [...$requestArgs]; + if ($this->containsChecksum($requestArgs)) { + $completeMultipartArgs['ChecksumType'] = 'FULL_OBJECT'; + } + + return (new MultipartUploader( $this->s3Client, - $source, - $requestArgs, + $createMultipartArgs, + $uploadPartArgs, + $completeMultipartArgs, [ - 'target_part_size_bytes' => $this->config['targetPartSizeBytes'], + 'part_size_bytes' => $partSizeBytes, 'concurrency' => $this->config['concurrency'], ], - $uploadListener, - $progressTracker, + $source, + progressTracker: $progressTracker, ))->promise(); } + /** + * @param string|StreamInterface $source + * @param int $mupThreshold + * + * @return bool + */ + private function requiresMultipartUpload( + string | StreamInterface $source, + int $mupThreshold + ): bool + { + if (is_string($source)) { + $sourceSize = filesize($source); + + return $sourceSize > $mupThreshold; + } elseif ($source instanceof StreamInterface) { + // When the stream's size is unknown then we could try a multipart upload. + if (empty($source->getSize())) { + return true; + } + + if (!empty($source->getSize())) { + return $source->getSize() > $mupThreshold; + } + } + + return false; + } + /** * Returns a default instance of S3Client. * @@ -632,7 +746,7 @@ private function s3UriAsBucketAndKey(string $uri): array /** * Resolves the progress tracker to be used in the - * transfer operation if `$trackProgress` is true. + * transfer operation if `$track_progress` is true. * * @param string $trackingOperation * @@ -642,12 +756,12 @@ private function resolveDefaultProgressTracker( string $trackingOperation ): ?TransferListener { - $progressTrackerFactory = $this->config['progressTrackerFactory'] ?? null; - if ($progressTrackerFactory === null) { + $progress_tracker_factory = $this->config['progress_tracker_factory'] ?? null; + if ($progress_tracker_factory === null) { return (new DefaultProgressTracker(trackingOperation: $trackingOperation))->getTransferListener(); } - return $progressTrackerFactory([]); + return $progress_tracker_factory([]); } /** @@ -683,4 +797,29 @@ private function resolvesOutsideTargetDirectory( return false; } + + /** + * Verifies if a checksum was provided. + * + * @param array $requestArgs + * + * @return bool + */ + private function containsChecksum(array $requestArgs): bool + { + $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return true; + } + } + + return false; + } } \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/TransferListener.php b/src/S3/Features/S3Transfer/TransferListener.php index 892f159085..30b620c799 100644 --- a/src/S3/Features/S3Transfer/TransferListener.php +++ b/src/S3/Features/S3Transfer/TransferListener.php @@ -174,6 +174,7 @@ public function objectTransferFailed( ): void { $this->objectsTransferFailed++; + $this->validateTransferComplete(); $this->notify('onObjectTransferFailed', [ $objectKey, $objectBytesTransferred, diff --git a/src/S3/Features/S3Transfer/UploadDirectoryResponse.php b/src/S3/Features/S3Transfer/UploadDirectoryResponse.php new file mode 100644 index 0000000000..df8d804cf8 --- /dev/null +++ b/src/S3/Features/S3Transfer/UploadDirectoryResponse.php @@ -0,0 +1,38 @@ +objectsUploaded = $objectsUploaded; + $this->objectsFailed = $objectsFailed; + } + + /** + * @return int + */ + public function getObjectsUploaded(): int + { + return $this->objectsUploaded; + } + + /** + * @return int + */ + public function getObjectsFailed(): int + { + return $this->objectsFailed; + } +} \ No newline at end of file diff --git a/src/S3/Features/S3Transfer/UploadResponse.php b/src/S3/Features/S3Transfer/UploadResponse.php new file mode 100644 index 0000000000..f39c16362d --- /dev/null +++ b/src/S3/Features/S3Transfer/UploadResponse.php @@ -0,0 +1,21 @@ +uploadResponse = $uploadResponse; + } + + public function getUploadResponse(): array + { + return $this->uploadResponse; + } +} \ No newline at end of file diff --git a/tests/S3/Features/S3Transfer/MultipartUploaderTest.php b/tests/S3/Features/S3Transfer/MultipartUploaderTest.php new file mode 100644 index 0000000000..0404393b9e --- /dev/null +++ b/tests/S3/Features/S3Transfer/MultipartUploaderTest.php @@ -0,0 +1,214 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) + { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'FooETag' + ])); + } + + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ]; + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $requestArgs, + $requestArgs, + $config + [ + 'concurrency' => 3, + ], + $stream + ); + $multipartUploader->promise()->wait(); + + $this->assertCount($expected['parts'], $multipartUploader->getParts()); + $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectBytesTransferred()); + $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectSizeInBytes()); + } + + /** + * @return array[] + */ + public function multipartUploadProvider(): array { + return [ + '5_parts_upload' => [ + 'stream' => Utils::streamFor( + str_repeat('*', 1024 * 5), + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 5, + 'bytesUploaded' => 1024 * 5, + ] + ], + '100_parts_upload' => [ + 'stream' => Utils::streamFor( + str_repeat('*', 1024 * 100), + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 1024 * 100, + ] + ], + '5_parts_no_seekable_stream' => [ + 'stream' => new NoSeekStream( + Utils::streamFor( + str_repeat('*', 1024 * 5) + ) + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 5, + 'bytesUploaded' => 1024 * 5, + ] + ], + '100_parts_no_seekable_stream' => [ + 'stream' => new NoSeekStream( + Utils::streamFor( + str_repeat('*', 1024 * 100) + ) + ), + 'config' => [ + 'part_size' => 1024 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 1024 * 100, + ] + ] + ]; + } + + /** + * @return S3ClientInterface + */ + private function multipartUploadS3Client(): S3ClientInterface + { + return new S3Client([ + 'region' => 'us-east-2', + 'http_handler' => function (RequestInterface $request) { + $uri = $request->getUri(); + // Create multipart upload + if ($uri->getQuery() === 'uploads') { + $body = << + + Foo + Test file + FooUploadId + +EOF; + return new Response(200, [], $body); + } + + // Parts upload + if (str_starts_with($request->getUri(), 'uploadId=') && str_contains($request->getUri(), 'partNumber=')) { + return new Response(200, ['ETag' => random_bytes(16)]); + } + + // Complete multipart upload + return new Response(200, [], null); + } + ]); + } + + /** + * @return void + */ + public function testInvalidSourceStringThrowsException(): void + { + $nonExistentFile = '/path/to/nonexistent/file.txt'; + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The source for this upload must be either a readable file or a valid stream." + ); + new MultipartUploader( + $this->multipartUploadS3Client(), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [], + [], + [], + $nonExistentFile + ); + } + + /** + * @return void + */ + public function testInvalidSourceTypeThrowsException(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The source for this upload must be either a readable file or a valid stream." + ); + new MultipartUploader( + $this->multipartUploadS3Client(), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [], + [], + [], + 12345 + ); + } +} \ No newline at end of file From 034b50d942daeeb8d234a59755e7a845b1228448 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 17 Feb 2025 08:07:26 -0800 Subject: [PATCH 09/62] chore: short namespace Short namespace from `Aws\S3\Features\S3Transfer` to `Aws\S3\S3Transfer`. --- src/S3/{Features => }/S3Transfer/ConsoleProgressBar.php | 2 +- .../{Features => }/S3Transfer/DefaultProgressTracker.php | 2 +- src/S3/{Features => }/S3Transfer/DownloadResponse.php | 2 +- .../S3Transfer/Exceptions/S3TransferException.php | 2 +- src/S3/{Features => }/S3Transfer/ListenerNotifier.php | 2 +- .../S3Transfer/MultipartDownloadListener.php | 2 +- .../S3Transfer/MultipartDownloadListenerFactory.php | 2 +- src/S3/{Features => }/S3Transfer/MultipartDownloader.php | 2 +- .../{Features => }/S3Transfer/MultipartUploadListener.php | 2 +- src/S3/{Features => }/S3Transfer/MultipartUploader.php | 2 +- .../{Features => }/S3Transfer/ObjectProgressTracker.php | 2 +- .../S3Transfer/PartGetMultipartDownloader.php | 2 +- src/S3/{Features => }/S3Transfer/ProgressBar.php | 2 +- src/S3/{Features => }/S3Transfer/ProgressBarFactory.php | 2 +- .../{Features => }/S3Transfer/ProgressListenerHelper.php | 2 +- .../S3Transfer/RangeGetMultipartDownloader.php | 4 ++-- src/S3/{Features => }/S3Transfer/S3TransferManager.php | 4 ++-- src/S3/{Features => }/S3Transfer/TransferListener.php | 2 +- .../{Features => }/S3Transfer/TransferListenerFactory.php | 2 +- .../{Features => }/S3Transfer/UploadDirectoryResponse.php | 2 +- src/S3/{Features => }/S3Transfer/UploadResponse.php | 2 +- .../{Features => }/S3Transfer/ConsoleProgressBarTest.php | 4 ++-- .../S3Transfer/DefaultProgressTrackerTest.php | 6 +++--- .../S3Transfer/MultipartDownloadListenerTest.php | 4 ++-- .../{Features => }/S3Transfer/MultipartDownloaderTest.php | 4 ++-- .../{Features => }/S3Transfer/MultipartUploaderTest.php | 4 ++-- .../S3Transfer/ObjectProgressTrackerTest.php | 8 ++++---- .../S3/{Features => }/S3Transfer/TransferListenerTest.php | 5 ++--- 28 files changed, 40 insertions(+), 41 deletions(-) rename src/S3/{Features => }/S3Transfer/ConsoleProgressBar.php (99%) rename src/S3/{Features => }/S3Transfer/DefaultProgressTracker.php (99%) rename src/S3/{Features => }/S3Transfer/DownloadResponse.php (91%) rename src/S3/{Features => }/S3Transfer/Exceptions/S3TransferException.php (56%) rename src/S3/{Features => }/S3Transfer/ListenerNotifier.php (76%) rename src/S3/{Features => }/S3Transfer/MultipartDownloadListener.php (99%) rename src/S3/{Features => }/S3Transfer/MultipartDownloadListenerFactory.php (82%) rename src/S3/{Features => }/S3Transfer/MultipartDownloader.php (99%) rename src/S3/{Features => }/S3Transfer/MultipartUploadListener.php (98%) rename src/S3/{Features => }/S3Transfer/MultipartUploader.php (99%) rename src/S3/{Features => }/S3Transfer/ObjectProgressTracker.php (99%) rename src/S3/{Features => }/S3Transfer/PartGetMultipartDownloader.php (96%) rename src/S3/{Features => }/S3Transfer/ProgressBar.php (86%) rename src/S3/{Features => }/S3Transfer/ProgressBarFactory.php (69%) rename src/S3/{Features => }/S3Transfer/ProgressListenerHelper.php (96%) rename src/S3/{Features => }/S3Transfer/RangeGetMultipartDownloader.php (97%) rename src/S3/{Features => }/S3Transfer/S3TransferManager.php (99%) rename src/S3/{Features => }/S3Transfer/TransferListener.php (99%) rename src/S3/{Features => }/S3Transfer/TransferListenerFactory.php (73%) rename src/S3/{Features => }/S3Transfer/UploadDirectoryResponse.php (94%) rename src/S3/{Features => }/S3Transfer/UploadResponse.php (89%) rename tests/S3/{Features => }/S3Transfer/ConsoleProgressBarTest.php (98%) rename tests/S3/{Features => }/S3Transfer/DefaultProgressTrackerTest.php (97%) rename tests/S3/{Features => }/S3Transfer/MultipartDownloadListenerTest.php (98%) rename tests/S3/{Features => }/S3Transfer/MultipartDownloaderTest.php (98%) rename tests/S3/{Features => }/S3Transfer/MultipartUploaderTest.php (98%) rename tests/S3/{Features => }/S3Transfer/ObjectProgressTrackerTest.php (94%) rename tests/S3/{Features => }/S3Transfer/TransferListenerTest.php (98%) diff --git a/src/S3/Features/S3Transfer/ConsoleProgressBar.php b/src/S3/S3Transfer/ConsoleProgressBar.php similarity index 99% rename from src/S3/Features/S3Transfer/ConsoleProgressBar.php rename to src/S3/S3Transfer/ConsoleProgressBar.php index e370362651..ebbadd771b 100644 --- a/src/S3/Features/S3Transfer/ConsoleProgressBar.php +++ b/src/S3/S3Transfer/ConsoleProgressBar.php @@ -1,6 +1,6 @@ Date: Fri, 21 Feb 2025 17:22:04 -0800 Subject: [PATCH 10/62] chore: refactor and address feedback - Implement progress tracker based on SEP spec. - Add a default progress bar implementation. - Add different progress tracker formats: -- Plain progress format: [|progress_bar|] |percent|% -- Transfer progress format: [|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| -- Colored progress format: |object_name|:\n\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m - Add a default single progress tracker implementation. - Add a default multi progress tracker implementation for tracking directory transfers. - Include tests unit just for console progress bar. --- src/S3/S3Transfer/ConsoleProgressBar.php | 142 ------- src/S3/S3Transfer/DefaultProgressTracker.php | 292 --------------- src/S3/S3Transfer/ListenerNotifier.php | 8 - .../S3Transfer/MultipartDownloadListener.php | 228 ------------ .../MultipartDownloadListenerFactory.php | 11 - src/S3/S3Transfer/MultipartDownloader.php | 313 ++++------------ src/S3/S3Transfer/MultipartUploadListener.php | 58 --- src/S3/S3Transfer/MultipartUploader.php | 217 +++++++---- src/S3/S3Transfer/ObjectProgressTracker.php | 189 ---------- .../ColoredTransferProgressBarFormat.php | 45 +++ .../Progress/ConsoleProgressBar.php | 103 ++++++ .../Progress/MultiProgressTracker.php | 169 +++++++++ .../Progress/PlainProgressBarFormat.php | 24 ++ .../Progress/ProgressBarColorEnum.php | 16 + .../S3Transfer/Progress/ProgressBarFormat.php | 88 +++++ .../Progress/ProgressBarInterface.php | 31 ++ .../Progress/ProgressTrackerInterface.php | 13 + .../Progress/SingleProgressTracker.php | 216 +++++++++++ .../Progress/TransferProgressBarFormat.php | 33 ++ .../Progress/TransferProgressSnapshot.php | 78 ++++ src/S3/S3Transfer/ProgressBar.php | 14 - src/S3/S3Transfer/ProgressBarFactory.php | 8 - src/S3/S3Transfer/ProgressListenerHelper.php | 45 --- .../RangeGetMultipartDownloader.php | 33 +- src/S3/S3Transfer/S3TransferManager.php | 345 +++++++++--------- src/S3/S3Transfer/TransferListener.php | 289 ++------------- src/S3/S3Transfer/TransferListenerFactory.php | 8 - .../S3Transfer/TransferListenerNotifier.php | 74 ++++ .../S3/S3Transfer/ConsoleProgressBarTest.php | 228 ------------ .../Progress/ConsoleProgressBarTest.php | 261 +++++++++++++ 30 files changed, 1570 insertions(+), 2009 deletions(-) delete mode 100644 src/S3/S3Transfer/ConsoleProgressBar.php delete mode 100644 src/S3/S3Transfer/DefaultProgressTracker.php delete mode 100644 src/S3/S3Transfer/ListenerNotifier.php delete mode 100644 src/S3/S3Transfer/MultipartDownloadListener.php delete mode 100644 src/S3/S3Transfer/MultipartDownloadListenerFactory.php delete mode 100644 src/S3/S3Transfer/MultipartUploadListener.php delete mode 100644 src/S3/S3Transfer/ObjectProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ConsoleProgressBar.php create mode 100644 src/S3/S3Transfer/Progress/MultiProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/PlainProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarColorEnum.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarInterface.php create mode 100644 src/S3/S3Transfer/Progress/ProgressTrackerInterface.php create mode 100644 src/S3/S3Transfer/Progress/SingleProgressTracker.php create mode 100644 src/S3/S3Transfer/Progress/TransferProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/TransferProgressSnapshot.php delete mode 100644 src/S3/S3Transfer/ProgressBar.php delete mode 100644 src/S3/S3Transfer/ProgressBarFactory.php delete mode 100644 src/S3/S3Transfer/ProgressListenerHelper.php delete mode 100644 src/S3/S3Transfer/TransferListenerFactory.php create mode 100644 src/S3/S3Transfer/TransferListenerNotifier.php delete mode 100644 tests/S3/S3Transfer/ConsoleProgressBarTest.php create mode 100644 tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php diff --git a/src/S3/S3Transfer/ConsoleProgressBar.php b/src/S3/S3Transfer/ConsoleProgressBar.php deleted file mode 100644 index ebbadd771b..0000000000 --- a/src/S3/S3Transfer/ConsoleProgressBar.php +++ /dev/null @@ -1,142 +0,0 @@ - [ - 'format' => "[|progress_bar|] |percent|%", - 'parameters' => [] - ], - 'transfer_format' => [ - 'format' => "[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|", - 'parameters' => [ - 'transferred', - 'tobe_transferred', - 'unit' - ] - ], - 'colored_transfer_format' => [ - 'format' => "\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m", - 'parameters' => [ - 'transferred', - 'tobe_transferred', - 'unit', - 'color_code', - 'message' - ] - ], - ]; - - /** @var string */ - private string $progressBarChar; - - /** @var int */ - private int $progressBarWidth; - - /** @var int */ - private int $percentCompleted; - - /** @var ?array */ - private ?array $format; - - /** @var array */ - private array $args; - - /** - * @param ?string $progressBarChar - * @param ?int $progressBarWidth - * @param ?int $percentCompleted - * @param array|null $format - * @param array $args - */ - public function __construct( - ?string $progressBarChar = null, - ?int $progressBarWidth = null, - ?int $percentCompleted = null, - ?array $format = null, - array $args = [], - ) { - $this->progressBarChar = $progressBarChar ?? '#'; - $this->progressBarWidth = $progressBarWidth ?? 25; - $this->percentCompleted = $percentCompleted ?? 0; - $this->format = $format ?? self::$formats['transfer_format']; - $this->args = $args ?? []; - } - - /** - * Set current progress percent. - * - * @param int $percent - * - * @return void - */ - public function setPercentCompleted(int $percent): void - { - $this->percentCompleted = max(0, min(100, $percent)); - } - - /** - * @param array $args - * - * @return void - */ - public function setArgs(array $args): void - { - $this->args = $args; - } - - /** - * Sets an argument. - * - * @param string $key - * @param mixed $value - * - * @return void - */ - public function setArg(string $key, mixed $value): void - { - $this->args[$key] = $value; - } - - private function renderProgressBar(): string - { - $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); - return str_repeat($this->progressBarChar, $filledWidth) - . str_repeat(' ', $this->progressBarWidth - $filledWidth); - } - - /** - * - * @return string - */ - public function getPaintedProgress(): string - { - foreach ($this->format['parameters'] as $param) { - if (!array_key_exists($param, $this->args)) { - $this->args[$param] = ''; - } - } - - $replacements = [ - '|progress_bar|' => $this->renderProgressBar(), - '|percent|' => $this->percentCompleted, - ]; - - foreach ($this->format['parameters'] as $param) { - $replacements["|$param|"] = $this->args[$param] ?? ''; - } - - return strtr($this->format['format'], $replacements); - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/DefaultProgressTracker.php b/src/S3/S3Transfer/DefaultProgressTracker.php deleted file mode 100644 index ed2d28683c..0000000000 --- a/src/S3/S3Transfer/DefaultProgressTracker.php +++ /dev/null @@ -1,292 +0,0 @@ -clear(); - $this->initializeListener(); - $this->progressBarFactory = $progressBarFactory ?? $this->defaultProgressBarFactory(); - if (get_resource_type($output) !== 'stream') { - throw new \InvalidArgumentException("The type for $output must be a stream"); - } - - $this->output = $output; - if (!in_array(strtolower($trackingOperation), [ - strtolower(self::TRACKING_OPERATION_DOWNLOADING), - strtolower(self::TRACKING_OPERATION_UPLOADING), - ], true)) { - throw new \InvalidArgumentException("Tracking operation '$trackingOperation' should be one of 'Downloading', 'Uploading'"); - } - - $this->trackingOperation = $trackingOperation; - } - - private function initializeListener(): void - { - $this->transferListener = new TransferListener(); - // Object transfer initialized - $this->transferListener->onObjectTransferInitiated = $this->objectTransferInitiated(); - // Object transfer made progress - $this->transferListener->onObjectTransferProgress = $this->objectTransferProgress(); - $this->transferListener->onObjectTransferFailed = $this->objectTransferFailed(); - $this->transferListener->onObjectTransferCompleted = $this->objectTransferCompleted(); - } - - /** - * @return TransferListener - */ - public function getTransferListener(): TransferListener - { - return $this->transferListener; - } - - /** - * @return int - */ - public function getTotalBytesTransferred(): int - { - return $this->totalBytesTransferred; - } - - /** - * @return int - */ - public function getObjectsTotalSizeInBytes(): int - { - return $this->objectsTotalSizeInBytes; - } - - /** - * @return int - */ - public function getObjectsInProgress(): int - { - return $this->objectsInProgress; - } - - /** - * @return int - */ - public function getObjectsCount(): int - { - return $this->objectsCount; - } - - /** - * @return int - */ - public function getTransferPercentCompleted(): int - { - return $this->transferPercentCompleted; - } - - /** - * - * @return Closure - */ - private function objectTransferInitiated(): Closure - { - return function (string $objectKey, array &$requestArgs) { - $progressBarFactoryFn = $this->progressBarFactory; - $this->objects[$objectKey] = new ObjectProgressTracker( - objectKey: $objectKey, - objectBytesTransferred: 0, - objectSizeInBytes: 0, - status: 'initiated', - progressBar: $progressBarFactoryFn() - ); - $this->objectsInProgress++; - $this->objectsCount++; - - $this->showProgress(); - }; - } - - /** - * @return Closure - */ - private function objectTransferProgress(): Closure - { - return function ( - string $objectKey, - int $objectBytesTransferred, - int $objectSizeInBytes - ): void { - $objectProgressTracker = $this->objects[$objectKey]; - if ($objectProgressTracker->getObjectSizeInBytes() === 0) { - $objectProgressTracker->setObjectSizeInBytes($objectSizeInBytes); - // Increment objectsTotalSizeInBytes just the first time we set - // the object total size. - $this->objectsTotalSizeInBytes = - $this->objectsTotalSizeInBytes + $objectSizeInBytes; - } - $objectProgressTracker->incrementTotalBytesTransferred( - $objectBytesTransferred - ); - $objectProgressTracker->setStatus('progress'); - - $this->increaseBytesTransferred($objectBytesTransferred); - - $this->showProgress(); - }; - } - - /** - * @return Closure - */ - public function objectTransferFailed(): Closure - { - return function ( - string $objectKey, - int $totalObjectBytesTransferred, - \Throwable | string $reason - ): void { - $objectProgressTracker = $this->objects[$objectKey]; - $objectProgressTracker->setStatus('failed', $reason); - - $this->objectsInProgress--; - - $this->showProgress(); - }; - } - - /** - * @return Closure - */ - public function objectTransferCompleted(): Closure - { - return function ( - string $objectKey, - int $objectBytesTransferred, - ): void { - $objectProgressTracker = $this->objects[$objectKey]; - $objectProgressTracker->setStatus('completed'); - $this->showProgress(); - }; - } - - /** - * Clear the internal state holders. - * - * @return void - */ - public function clear(): void - { - $this->objects = []; - $this->totalBytesTransferred = 0; - $this->objectsTotalSizeInBytes = 0; - $this->objectsInProgress = 0; - $this->objectsCount = 0; - $this->transferPercentCompleted = 0; - } - - /** - * @param int $bytesTransferred - * - * @return void - */ - private function increaseBytesTransferred(int $bytesTransferred): void - { - $this->totalBytesTransferred += $bytesTransferred; - if ($this->objectsTotalSizeInBytes !== 0) { - $this->transferPercentCompleted = floor( - ($this->totalBytesTransferred / $this->objectsTotalSizeInBytes) * 100 - ); - } - } - - /** - * @return void - */ - private function showProgress(): void - { - // Clear screen - fwrite($this->output, "\033[2J\033[H"); - - // Display progress header - fwrite($this->output, sprintf( - "\r%s [%d/%d] %d%%\n", - $this->trackingOperation, - $this->objectsInProgress, - $this->objectsCount, - $this->transferPercentCompleted - )); - - foreach ($this->objects as $name => $object) { - fwrite($this->output, sprintf( - "\r%s:\n%s\n", - $name, - $object->getProgressBar()->getPaintedProgress() - )); - } - } - - /** - * @return Closure|ProgressBarFactory - */ - private function defaultProgressBarFactory(): Closure| ProgressBarFactory - { - return function () { - return new ConsoleProgressBar( - format: ConsoleProgressBar::$formats[ - ConsoleProgressBar::COLORED_TRANSFER_FORMAT - ], - args: [ - 'transferred' => 0, - 'tobe_transferred' => 0, - 'unit' => 'B', - 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, - ] - ); - }; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/ListenerNotifier.php b/src/S3/S3Transfer/ListenerNotifier.php deleted file mode 100644 index c3ab9d8844..0000000000 --- a/src/S3/S3Transfer/ListenerNotifier.php +++ /dev/null @@ -1,8 +0,0 @@ -notify('onDownloadInitiated', [&$commandArgs, $initialPart]); - } - - /** - * Event for when a download fails. - * Warning: If this method is overridden, it is recommended - * to call parent::downloadFailed() in order to - * keep the states maintained in this implementation. - * - * @param Throwable $reason - * @param int $totalPartsDownloaded - * @param int $totalBytesDownloaded - * @param int $lastPartDownloaded - * - * @return void - */ - public function downloadFailed( - Throwable $reason, - int $totalPartsDownloaded, - int $totalBytesDownloaded, - int $lastPartDownloaded): void - { - $this->notify('onDownloadFailed', [ - $reason, - $totalPartsDownloaded, - $totalBytesDownloaded, - $lastPartDownloaded - ]); - } - - /** - * Event for when a download completes. - * Warning: If this method is overridden, it is recommended - * to call parent::onDownloadCompleted() in order to - * keep the states maintained in this implementation. - * - * @param StreamInterface $stream - * @param int $totalPartsDownloaded - * @param int $totalBytesDownloaded - * - * @return void - */ - public function downloadCompleted( - StreamInterface $stream, - int $totalPartsDownloaded, - int $totalBytesDownloaded - ): void - { - $this->notify('onDownloadCompleted', [ - $stream, - $totalPartsDownloaded, - $totalBytesDownloaded - ]); - } - - /** - * Event for when a part download is initiated. - * Warning: If this method is overridden, it is recommended - * to call parent::partDownloadInitiated() in order to - * keep the states maintained in this implementation. - * - * @param mixed $partDownloadCommand - * @param int $partNo - * - * @return void - */ - public function partDownloadInitiated( - CommandInterface $partDownloadCommand, - int $partNo - ): void - { - $this->notify('onPartDownloadInitiated', [ - $partDownloadCommand, - $partNo - ]); - } - - /** - * Event for when a part download completes. - * Warning: If this method is overridden, it is recommended - * to call parent::onPartDownloadCompleted() in order to - * keep the states maintained in this implementation. - * - * @param ResultInterface $result - * @param int $partNo - * @param int $partTotalBytes - * @param int $totalParts - * @param int $objectBytesDownloaded - * @param int $objectSizeInBytes - * @return void - */ - public function partDownloadCompleted( - ResultInterface $result, - int $partNo, - int $partTotalBytes, - int $totalParts, - int $objectBytesDownloaded, - int $objectSizeInBytes - ): void - { - $this->notify('onPartDownloadCompleted', [ - $result, - $partNo, - $partTotalBytes, - $totalParts, - $objectBytesDownloaded, - $objectSizeInBytes - ]); - } - - /** - * Event for when a part download fails. - * Warning: If this method is overridden, it is recommended - * to call parent::onPartDownloadFailed() in order to - * keep the states maintained in this implementation. - * - * @param CommandInterface $partDownloadCommand - * @param Throwable $reason - * @param int $partNo - * - * @return void - */ - public function partDownloadFailed( - CommandInterface $partDownloadCommand, - Throwable $reason, - int $partNo - ): void - { - $this->notify('onPartDownloadFailed', [ - $partDownloadCommand, - $reason, - $partNo - ]); - } - - protected function notify(string $event, array $params = []): void - { - $listener = match ($event) { - 'onDownloadInitiated' => $this->onDownloadInitiated, - 'onDownloadFailed' => $this->onDownloadFailed, - 'onDownloadCompleted' => $this->onDownloadCompleted, - 'onPartDownloadInitiated' => $this->onPartDownloadInitiated, - 'onPartDownloadCompleted' => $this->onPartDownloadCompleted, - 'onPartDownloadFailed' => $this->onPartDownloadFailed, - default => null, - }; - - if ($listener instanceof Closure) { - $listener(...$params); - } - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartDownloadListenerFactory.php b/src/S3/S3Transfer/MultipartDownloadListenerFactory.php deleted file mode 100644 index 0c5f38a2ce..0000000000 --- a/src/S3/S3Transfer/MultipartDownloadListenerFactory.php +++ /dev/null @@ -1,11 +0,0 @@ -requestArgs = $requestArgs; $this->currentPartNo = $currentPartNo; $this->objectPartsCount = $objectPartsCount; - $this->objectCompletedPartsCount = $objectCompletedPartsCount; $this->objectSizeInBytes = $objectSizeInBytes; - $this->objectBytesTransferred = $objectBytesTransferred; $this->eTag = $eTag; - $this->objectKey = $objectKey; if ($stream === null) { $this->stream = Utils::streamFor( fopen('php://temp', 'w+') @@ -92,6 +83,8 @@ public function __construct( } else { $this->stream = $stream; } + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; } /** @@ -110,14 +103,6 @@ public function getObjectPartsCount(): int return $this->objectPartsCount; } - /** - * @return int - */ - public function getObjectCompletedPartsCount(): int - { - return $this->objectCompletedPartsCount; - } - /** * @return int */ @@ -127,19 +112,11 @@ public function getObjectSizeInBytes(): int } /** - * @return int + * @return TransferProgressSnapshot */ - public function getObjectBytesTransferred(): int + public function getCurrentSnapshot(): TransferProgressSnapshot { - return $this->objectBytesTransferred; - } - - /** - * @return string - */ - public function getObjectKey(): string - { - return $this->objectKey; + return $this->currentSnapshot; } /** @@ -151,51 +128,47 @@ public function getObjectKey(): string public function promise(): PromiseInterface { return Coroutine::of(function () { - $this->downloadInitiated($this->requestArgs, $this->currentPartNo); - $initialCommand = $this->nextCommand(); - $this->partDownloadInitiated($initialCommand, $this->currentPartNo); + $this->downloadInitiated($this->requestArgs); try { - yield $this->s3Client->executeAsync($initialCommand) + yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function (ResultInterface $result) { // Calculate object size and parts count. $this->computeObjectDimensions($result); // Trigger first part completed - $this->partDownloadCompleted($result, $this->currentPartNo); - })->otherwise(function ($reason) use ($initialCommand) { - $this->partDownloadFailed($initialCommand, $reason, $this->currentPartNo); + $this->partDownloadCompleted($result); + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); throw $reason; }); } catch (\Throwable $e) { - $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + $this->downloadFailed($e); // TODO: yield transfer exception modeled with a transfer failed response. yield Create::rejectionFor($e); } while ($this->currentPartNo < $this->objectPartsCount) { - $nextCommand = $this->nextCommand(); - $this->partDownloadInitiated($nextCommand, $this->currentPartNo); try { - yield $this->s3Client->executeAsync($nextCommand) + yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function ($result) { - $this->partDownloadCompleted($result, $this->currentPartNo); + $this->partDownloadCompleted($result); return $result; - })->otherwise(function ($reason) use ($nextCommand) { - $this->partDownloadFailed($nextCommand, $reason, $this->currentPartNo); + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); return $reason; }); - } catch (\Throwable $e) { - $this->downloadFailed($e, $this->objectCompletedPartsCount, $this->objectBytesTransferred, $this->currentPartNo); + } catch (\Throwable $reason) { + $this->downloadFailed($reason); // TODO: yield transfer exception modeled with a transfer failed response. - yield Create::rejectionFor($e); + yield Create::rejectionFor($reason); } } // Transfer completed - $this->objectDownloadCompleted(); + $this->downloadComplete(); // TODO: yield the stream wrapped in a modeled transfer success response. yield Create::promiseFor(new DownloadResponse( @@ -235,7 +208,7 @@ protected function computeObjectSize($sizeSource): int } if (empty($sizeSource)) { - throw new \RuntimeException('Range must not be empty'); + return 0; } // For extracting the object size from the ContentRange header value. @@ -246,78 +219,6 @@ protected function computeObjectSize($sizeSource): int throw new \RuntimeException('Invalid source size format'); } - /** - * MultipartDownloader factory method to return an instance - * of MultipartDownloader based on the multipart download type. - * - * @param S3ClientInterface $s3Client - * @param string $multipartDownloadType - * @param array $requestArgs - * @param array $config - * @param int $currentPartNo - * @param int $objectPartsCount - * @param int $objectCompletedPartsCount - * @param int $objectSizeInBytes - * @param int $objectBytesTransferred - * @param string $eTag - * @param string $objectKey - * @param MultipartDownloadListener|null $listener - * @param TransferListener|null $progressTracker - * - * @return MultipartDownloader - */ - public static function chooseDownloader( - S3ClientInterface $s3Client, - string $multipartDownloadType, - array $requestArgs, - array $config, - int $currentPartNo = 0, - int $objectPartsCount = 0, - int $objectCompletedPartsCount = 0, - int $objectSizeInBytes = 0, - int $objectBytesTransferred = 0, - string $eTag = "", - string $objectKey = "", - ?MultipartDownloadListener $listener = null, - ?TransferListener $progressTracker = null - ) : MultipartDownloader - { - return match ($multipartDownloadType) { - self::PART_GET_MULTIPART_DOWNLOADER => new PartGetMultipartDownloader( - s3Client: $s3Client, - requestArgs: $requestArgs, - config: $config, - currentPartNo: $currentPartNo, - objectPartsCount: $objectPartsCount, - objectCompletedPartsCount: $objectCompletedPartsCount, - objectSizeInBytes: $objectSizeInBytes, - objectBytesTransferred: $objectBytesTransferred, - eTag: $eTag, - objectKey: $objectKey, - listener: $listener, - progressListener: $progressTracker - ), - self::RANGE_GET_MULTIPART_DOWNLOADER => new RangeGetMultipartDownloader( - s3Client: $s3Client, - requestArgs: $requestArgs, - config: $config, - currentPartNo: 0, - objectPartsCount: 0, - objectCompletedPartsCount: 0, - objectSizeInBytes: 0, - objectBytesTransferred: 0, - eTag: "", - objectKey: "", - listener: $listener, - progressListener: $progressTracker - ), - default => throw new \RuntimeException( - "Unsupported download type $multipartDownloadType." - ."It should be either " . self::PART_GET_MULTIPART_DOWNLOADER . - " or " . self::RANGE_GET_MULTIPART_DOWNLOADER . ".") - }; - } - /** * Main purpose of this method is to propagate * the download-initiated event to listeners, but @@ -325,70 +226,45 @@ public static function chooseDownloader( * that need to be maintained. * * @param array $commandArgs - * @param int|null $currentPartNo * * @return void */ - private function downloadInitiated(array &$commandArgs, ?int $currentPartNo): void + private function downloadInitiated(array $commandArgs): void { - $this->objectKey = $commandArgs['Key']; - $this->progressListener?->objectTransferInitiated( - $this->objectKey, - $commandArgs - ); - $this->_notifyMultipartDownloadListeners('downloadInitiated', [ - &$commandArgs, - $currentPartNo + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $commandArgs['Key'], + 0, + $this->objectSizeInBytes + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + $this->listenerNotifier?->transferInitiated([ + 'request_args' => $commandArgs, + 'progress_snapshot' => $this->currentSnapshot, ]); } /** * Propagates download-failed event to listeners. - * It may also do some computation in order to maintain internal states. * * @param \Throwable $reason - * @param int $totalPartsTransferred - * @param int $totalBytesTransferred - * @param int $lastPartTransferred * * @return void */ - private function downloadFailed( - \Throwable $reason, - int $totalPartsTransferred, - int $totalBytesTransferred, - int $lastPartTransferred - ): void + private function downloadFailed(\Throwable $reason): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $totalBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners('downloadFailed', [ - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ]); - } - - /** - * Propagates part-download-initiated event to listeners. - * - * @param CommandInterface $partDownloadCommand - * @param int $partNo - * - * @return void - */ - private function partDownloadInitiated( - CommandInterface $partDownloadCommand, - int $partNo - ): void - { - $this->_notifyMultipartDownloadListeners('partDownloadInitiated', [ - $partDownloadCommand, - $partNo + $this->listenerNotifier?->transferFail([ + 'request_args' => $this->requestArgs, + 'progress_snapshot' => $this->currentSnapshot, + 'reason' => $reason, ]); } @@ -400,62 +276,43 @@ private function partDownloadInitiated( * is completed. * * @param ResultInterface $result - * @param int $partNo * * @return void */ private function partDownloadCompleted( - ResultInterface $result, - int $partNo + ResultInterface $result ): void { - $this->objectCompletedPartsCount++; $partDownloadBytes = $result['ContentLength']; - $this->objectBytesTransferred = $this->objectBytesTransferred + $partDownloadBytes; if (isset($result['ETag'])) { $this->eTag = $result['ETag']; } Utils::copyToStream($result['Body'], $this->stream); - - $this->progressListener?->objectTransferProgress( - $this->objectKey, - $partDownloadBytes, - $this->objectSizeInBytes + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, + $this->objectSizeInBytes, + $result->toArray() ); - - $this->_notifyMultipartDownloadListeners('partDownloadCompleted', [ - $result, - $partNo, - $partDownloadBytes, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred, - $this->objectSizeInBytes + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + 'request_args' => $this->requestArgs, + 'progress_snapshot' => $this->currentSnapshot, ]); } /** * Propagates part-download-failed event to listeners. * - * @param CommandInterface $partDownloadCommand * @param \Throwable $reason - * @param int $partNo * * @return void */ private function partDownloadFailed( - CommandInterface $partDownloadCommand, \Throwable $reason, - int $partNo ): void { - $this->progressListener?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); - $this->_notifyMultipartDownloadListeners( - 'partDownloadFailed', - [$partDownloadCommand, $reason, $partNo]); + $this->downloadFailed($reason); } /** @@ -465,33 +322,19 @@ private function partDownloadFailed( * * @return void */ - private function objectDownloadCompleted(): void + private function downloadComplete(): void { $this->stream->rewind(); - $this->progressListener?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->objectSizeInBytes, + $this->currentSnapshot->getResponse() ); - $this->_notifyMultipartDownloadListeners('downloadCompleted', [ - $this->stream, - $this->objectCompletedPartsCount, - $this->objectBytesTransferred + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + 'request_args' => $this->requestArgs, + 'progress_snapshot' => $this->currentSnapshot, ]); } - - /** - * Internal helper method for notifying listeners of specific events. - * - * @param string $listenerMethod - * @param array $args - * - * @return void - */ - private function _notifyMultipartDownloadListeners( - string $listenerMethod, - array $args - ): void - { - $this->listener?->{$listenerMethod}(...$args); - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartUploadListener.php b/src/S3/S3Transfer/MultipartUploadListener.php deleted file mode 100644 index b69597a421..0000000000 --- a/src/S3/S3Transfer/MultipartUploadListener.php +++ /dev/null @@ -1,58 +0,0 @@ -s3Client = $s3Client; $this->createMultipartArgs = $createMultipartArgs; - $this->uploadPartArgs = $uploadPartArgs; - $this->completeMultipartArgs = $completeMultipartArgs; $this->config = $config; $this->body = $this->parseBody($source); - $this->objectSizeInBytes = $objectSizeInBytes; $this->uploadId = $uploadId; $this->parts = $parts; - $this->progressTracker = $progressTracker; + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; } /** @@ -117,17 +104,17 @@ public function getParts(): array /** * @return int */ - public function getObjectSizeInBytes(): int + public function getCalculatedObjectSize(): int { - return $this->objectSizeInBytes; + return $this->calculatedObjectSize; } /** - * @return int + * @return TransferProgressSnapshot|null */ - public function getObjectBytesTransferred(): int + public function getCurrentSnapshot(): ?TransferProgressSnapshot { - return $this->objectBytesTransferred; + return $this->currentSnapshot; } /** @@ -155,7 +142,8 @@ public function promise(): PromiseInterface /** * @return PromiseInterface */ - public function createMultipartUpload(): PromiseInterface { + public function createMultipartUpload(): PromiseInterface + { $requestArgs = [...$this->createMultipartArgs]; $this->uploadInitiated($requestArgs); $command = $this->s3Client->getCommand( @@ -176,7 +164,7 @@ public function createMultipartUpload(): PromiseInterface { */ public function uploadParts(): PromiseInterface { - $this->objectSizeInBytes = 0; // To repopulate + $this->calculatedObjectSize = 0; $isSeekable = $this->body->isSeekable(); $partSize = $this->config['part_size'] ?? self::PART_MIN_SIZE; if ($partSize > self::PART_MAX_SIZE) { @@ -202,22 +190,23 @@ public function uploadParts(): PromiseInterface $this->body->read($readSize) ); // To make sure we do not create an empty part when - // we already reached end of file. + // we already reached the end of file. if (!$isSeekable && $this->body->eof() && $partBody->getSize() === 0) { break; } $uploadPartCommandArgs = [ - 'UploadId' => $this->uploadId, - 'PartNumber' => $partNo, - 'Body' => $partBody, - 'ContentLength' => $partBody->getSize(), - ] + $this->uploadPartArgs; - + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + 'PartNumber' => $partNo, + 'Body' => $partBody, + 'ContentLength' => $partBody->getSize(), + ]; + // To get `requestArgs` when notifying the bytesTransfer listeners. + $uploadPartCommandArgs['requestArgs'] = $uploadPartCommandArgs; $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); $commands[] = $command; - $this->objectSizeInBytes += $partBody->getSize(); - + $this->calculatedObjectSize += $partBody->getSize(); if ($partNo > self::PART_MAX_NUM) { return Create::rejectionFor( "The max number of parts has been exceeded. " . @@ -238,9 +227,11 @@ public function uploadParts(): PromiseInterface $result, $command ); - // Part Upload Completed Event - $this->partUploadCompleted($result, $command['ContentLength']); + $this->partUploadCompleted( + $command['ContentLength'], + $command['requestArgs'] + ); }, 'rejected' => function (Throwable $e) { $this->partUploadFailed($e); @@ -255,13 +246,21 @@ public function uploadParts(): PromiseInterface public function completeMultipartUpload(): PromiseInterface { $this->sortParts(); - $command = $this->s3Client->getCommand('CompleteMultipartUpload', [ - 'UploadId' => $this->uploadId, - 'MpuObjectSize' => $this->objectSizeInBytes, - 'MultipartUpload' => [ - 'Parts' => $this->parts, - ] - ] + $this->completeMultipartArgs + $completeMultipartUploadArgs = [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + 'MpuObjectSize' => $this->calculatedObjectSize, + 'MultipartUpload' => [ + 'Parts' => $this->parts, + ] + ]; + if ($this->containsChecksum($this->createMultipartArgs)) { + $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + } + + $command = $this->s3Client->getCommand( + 'CompleteMultipartUpload', + $completeMultipartUploadArgs ); return $this->s3Client->executeAsync($command) @@ -275,7 +274,8 @@ public function completeMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - public function abortMultipartUpload(): PromiseInterface { + public function abortMultipartUpload(): PromiseInterface + { $command = $this->s3Client->getCommand('AbortMultipartUpload', [ ...$this->createMultipartArgs, 'UploadId' => $this->uploadId, @@ -316,12 +316,13 @@ private function collectPart( private function sortParts(): void { usort($this->parts, function($partOne, $partTwo) { - return $partOne['PartNumber'] <=> $partTwo['PartNumber']; // Ascending order by age + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; }); } /** * @param string|StreamInterface $source + * * @return StreamInterface */ private function parseBody(string | StreamInterface $source): StreamInterface @@ -333,12 +334,11 @@ private function parseBody(string | StreamInterface $source): StreamInterface "The source for this upload must be either a readable file or a valid stream." ); } - $file = Utils::tryFopen($source, 'r'); + $body = new LazyOpenStream($source, 'r'); // To make sure the resource is closed. - $this->deferFns[] = function () use ($file) { - fclose($file); + $this->deferFns[] = function () use ($body) { + $body->close(); }; - $body = Utils::streamFor($file); } elseif ($source instanceof StreamInterface) { $body = $source; } else { @@ -351,14 +351,31 @@ private function parseBody(string | StreamInterface $source): StreamInterface } /** + * @param array $requestArgs + * * @return void */ - private function uploadInitiated(array &$requestArgs): void { - $this->objectKey = $this->createMultipartArgs['Key']; - $this->progressTracker?->objectTransferInitiated( - $this->objectKey, - $requestArgs - ); + private function uploadInitiated(array $requestArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $this->body->getSize(), + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + $this->listenerNotifier?->transferInitiated([ + 'request_args' => $requestArgs, + 'progress_snapshot' => $this->currentSnapshot + ]); } /** @@ -370,11 +387,11 @@ private function uploadFailed(Throwable $reason): void { if (!empty($this->uploadId)) { $this->abortMultipartUpload()->wait(); } - $this->progressTracker?->objectTransferFailed( - $this->objectKey, - $this->objectBytesTransferred, - $reason - ); + $this->listenerNotifier?->transferFail([ + 'request_args' => $this->createMultipartArgs, + 'progress_snapshot' => $this->currentSnapshot, + 'reason' => $reason, + ]); } /** @@ -383,25 +400,41 @@ private function uploadFailed(Throwable $reason): void { * @return void */ private function uploadCompleted(ResultInterface $result): void { - $this->progressTracker?->objectTransferCompleted( - $this->objectKey, - $this->objectBytesTransferred, + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $result->toArray() ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + 'request_args' => $this->createMultipartArgs, + 'progress_snapshot' => $this->currentSnapshot, + ]); } /** - * @param ResultInterface $result - * @param int $partSize + * @param int $partCompletedBytes + * @param array $requestArgs * * @return void */ - private function partUploadCompleted(ResultInterface $result, int $partSize): void { - $this->objectBytesTransferred = $this->objectBytesTransferred + $partSize; - $this->progressTracker?->objectTransferProgress( - $this->objectKey, - $partSize, - $this->objectSizeInBytes + private function partUploadCompleted( + int $partCompletedBytes, + array $requestArgs + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partCompletedBytes, + $this->currentSnapshot->getTotalBytes() ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + 'request_args' => $requestArgs, + 'progress_snapshot' => $this->currentSnapshot, + $this->currentSnapshot + ]); } /** @@ -411,6 +444,7 @@ private function partUploadCompleted(ResultInterface $result, int $partSize): vo */ private function partUploadFailed(Throwable $reason): void { + $this->uploadFailed($reason); } /** @@ -424,4 +458,29 @@ private function callDeferredFns(): void $this->deferFns = []; } + + /** + * Verifies if a checksum was provided. + * + * @param array $requestArgs + * + * @return bool + */ + private function containsChecksum(array $requestArgs): bool + { + $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return true; + } + } + + return false; + } } diff --git a/src/S3/S3Transfer/ObjectProgressTracker.php b/src/S3/S3Transfer/ObjectProgressTracker.php deleted file mode 100644 index a6a30f1648..0000000000 --- a/src/S3/S3Transfer/ObjectProgressTracker.php +++ /dev/null @@ -1,189 +0,0 @@ -objectKey = $objectKey; - $this->objectBytesTransferred = $objectBytesTransferred; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->status = $status; - $this->progressBar = $progressBar ?? $this->defaultProgressBar(); - } - - /** - * @return string - */ - public function getObjectKey(): string - { - return $this->objectKey; - } - - /** - * @param string $objectKey - * - * @return void - */ - public function setObjectKey(string $objectKey): void - { - $this->objectKey = $objectKey; - } - - /** - * @return int - */ - public function getObjectBytesTransferred(): int - { - return $this->objectBytesTransferred; - } - - /** - * @param int $objectBytesTransferred - * - * @return void - */ - public function setObjectBytesTransferred(int $objectBytesTransferred): void - { - $this->objectBytesTransferred = $objectBytesTransferred; - } - - /** - * @return int - */ - public function getObjectSizeInBytes(): int - { - return $this->objectSizeInBytes; - } - - /** - * @param int $objectSizeInBytes - * - * @return void - */ - public function setObjectSizeInBytes(int $objectSizeInBytes): void - { - $this->objectSizeInBytes = $objectSizeInBytes; - // Update progress bar - $this->progressBar->setArg('tobe_transferred', $objectSizeInBytes); - } - - /** - * @return string - */ - public function getStatus(): string - { - return $this->status; - } - - /** - * @param string $status - * @param string|null $message - * - * @return void - */ - public function setStatus(string $status, ?string $message = null): void - { - $this->status = $status; - $this->setProgressColor(); - // To show specific messages for specific status. - if (!empty($message)) { - $this->progressBar->setArg('message', "$status: $message"); - } - } - - private function setProgressColor(): void - { - if ($this->status === 'progress') { - $this->progressBar->setArg('color_code', ConsoleProgressBar::BLUE_COLOR_CODE); - } elseif ($this->status === 'completed') { - $this->progressBar->setArg('color_code', ConsoleProgressBar::GREEN_COLOR_CODE); - } elseif ($this->status === 'failed') { - $this->progressBar->setArg('color_code', ConsoleProgressBar::RED_COLOR_CODE); - } - } - - /** - * Increments the object bytes transferred. - * - * @param int $objectBytesTransferred - * - * @return void - */ - public function incrementTotalBytesTransferred( - int $objectBytesTransferred - ): void - { - $this->objectBytesTransferred += $objectBytesTransferred; - $progressPercent = (int) floor(($this->objectBytesTransferred / $this->objectSizeInBytes) * 100); - // Update progress bar - $this->progressBar->setPercentCompleted($progressPercent); - $this->progressBar->setArg('transferred', $this->objectBytesTransferred); - } - - /** - * @return ProgressBar|null - */ - public function getProgressBar(): ?ProgressBar - { - return $this->progressBar; - } - - /** - * @return ProgressBar - */ - private function defaultProgressBar(): ProgressBar - { - return new ConsoleProgressBar( - format: ConsoleProgressBar::$formats[ - ConsoleProgressBar::COLORED_TRANSFER_FORMAT - ], - args: [ - 'transferred' => 0, - 'tobe_transferred' => 0, - 'unit' => 'B', - 'color_code' => ConsoleProgressBar::BLACK_COLOR_CODE, - 'message' => '' - ] - ); - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php new file mode 100644 index 0000000000..e5a3eb95f0 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -0,0 +1,45 @@ + ColoredTransferProgressBarFormat::BLACK_COLOR_CODE, + ]; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/ConsoleProgressBar.php b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php new file mode 100644 index 0000000000..6fe118b167 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php @@ -0,0 +1,103 @@ +progressBarChar = $progressBarChar; + $this->progressBarWidth = min( + $progressBarWidth, + self::MAX_PROGRESS_BAR_WIDTH + ); + $this->percentCompleted = $percentCompleted; + $this->progressBarFormat = $progressBarFormat; + } + + /** + * @return string + */ + public function getProgressBarChar(): string + { + return $this->progressBarChar; + } + + /** + * @return int + */ + public function getProgressBarWidth(): int + { + return $this->progressBarWidth; + } + + /** + * @return int + */ + public function getPercentCompleted(): int + { + return $this->percentCompleted; + } + + /** + * @return ProgressBarFormat + */ + public function getProgressBarFormat(): ProgressBarFormat + { + return $this->progressBarFormat; + } + + /** + * Set current progress percent. + * + * @param int $percent + * + * @return void + */ + public function setPercentCompleted(int $percent): void + { + $this->percentCompleted = max(0, min(100, $percent)); + } + + /** + * @inheritDoc + */ + public function render(): string + { + $filledWidth = (int) round(($this->progressBarWidth * $this->percentCompleted) / 100); + $progressBar = str_repeat($this->progressBarChar, $filledWidth) + . str_repeat(' ', $this->progressBarWidth - $filledWidth); + + // Common arguments + $this->progressBarFormat->setArg('progress_bar', $progressBar); + $this->progressBarFormat->setArg('percent', $this->percentCompleted); + + return $this->progressBarFormat->format(); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php new file mode 100644 index 0000000000..6dcba0053c --- /dev/null +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -0,0 +1,169 @@ +singleProgressTrackers = $singleProgressTrackers; + $this->output = $output; + $this->transferCount = $transferCount; + $this->completed = $completed; + $this->failed = $failed; + } + + /** + * @return array + */ + public function getSingleProgressTrackers(): array + { + return $this->singleProgressTrackers; + } + + /** + * @return mixed + */ + public function getOutput(): mixed + { + return $this->output; + } + + /** + * @return int + */ + public function getTransferCount(): int + { + return $this->transferCount; + } + + /** + * @return int + */ + public function getCompleted(): int + { + return $this->completed; + } + + /** + * @return int + */ + public function getFailed(): int + { + return $this->failed; + } + + /** + * @inheritDoc + */ + public function transferInitiated(array $context): void + { + $this->transferCount++; + $snapshot = $context['progress_snapshot']; + $progressTracker = new SingleProgressTracker( + clear: false, + ); + $progressTracker->transferInitiated($context); + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context['progress_snapshot']; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->bytesTransferred($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function transferComplete(array $context): void + { + $this->completed++; + $snapshot = $context['progress_snapshot']; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->transferComplete($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function transferFail(array $context): void + { + $this->failed++; + $snapshot = $context['progress_snapshot']; + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; + $progressTracker->transferFail($context); + $this->showProgress(); + } + + /** + * @inheritDoc + */ + public function showProgress(): void + { + fwrite($this->output, "\033[2J\033[H"); + $percentsSum = 0; + foreach ($this->singleProgressTrackers as $_ => $progressTracker) { + $progressTracker->showProgress(); + $percentsSum += $progressTracker->getProgressBar()->getPercentCompleted(); + } + + $percent = (int) floor($percentsSum / $this->transferCount); + $allTransferProgressBar = new ConsoleProgressBar( + percentCompleted: $percent, + progressBarFormat: new PlainProgressBarFormat() + ); + fwrite($this->output, "\n" . str_repeat( + '-', + $allTransferProgressBar->getProgressBarWidth()) + ); + fwrite( + $this->output, + sprintf( + "\n%s Completed: %d/%d, Failed: %d/%d\n", + $allTransferProgressBar->render(), + $this->completed, + $this->transferCount, + $this->failed, + $this->transferCount + ) + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php new file mode 100644 index 0000000000..55a2ec1cba --- /dev/null +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -0,0 +1,24 @@ +args = $args; + } + + public function getArgs(): array { + return $this->args; + } + + /** + * To set multiple arguments at once. + * It does not override all the values, instead + * it adds the arguments individually and if a value + * already exists then that value will be overridden. + * + * @param array $args + * + * @return void + */ + public function setArgs(array $args): void + { + foreach ($args as $key => $value) { + $this->args[$key] = $value; + } + } + + /** + * @param string $key + * @param mixed $value + * + * @return void + */ + public function setArg(string $key, mixed $value): void + { + $this->args[$key] = $value; + } + + /** + * @return string + */ + public function format(): string { + $parameters = $this->getFormatParameters(); + $defaultParameterValues = $this->getFormatDefaultParameterValues(); + foreach ($parameters as $param) { + if (!array_key_exists($param, $this->args)) { + $this->args[$param] = $defaultParameterValues[$param] ?? ''; + } + } + + $replacements = []; + foreach ($parameters as $param) { + $replacements["|$param|"] = $this->args[$param] ?? ''; + } + + return strtr($this->getFormatTemplate(), $replacements); + } + + /** + * @return string + */ + abstract public function getFormatTemplate(): string; + + /** + * @return array + */ + abstract public function getFormatParameters(): array; + + /** + * @return array + */ + abstract protected function getFormatDefaultParameterValues(): array; +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/ProgressBarInterface.php b/src/S3/S3Transfer/Progress/ProgressBarInterface.php new file mode 100644 index 0000000000..ed8de193c2 --- /dev/null +++ b/src/S3/S3Transfer/Progress/ProgressBarInterface.php @@ -0,0 +1,31 @@ +progressBar = $progressBar; + if (get_resource_type($output) !== 'stream') { + throw new \InvalidArgumentException("The type for $output must be a stream"); + } + $this->output = $output; + $this->objectName = $objectName; + $this->clear = $clear; + } + + /** + * @return ProgressBarInterface + */ + public function getProgressBar(): ProgressBarInterface + { + return $this->progressBar; + } + + /** + * @return mixed + */ + public function getOutput(): mixed + { + return $this->output; + } + + /** + * @return string + */ + public function getObjectName(): string + { + return $this->objectName; + } + + /** + * @return bool + */ + public function isClear(): bool { + return $this->clear; + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferInitiated(array $context): void + { + $snapshot = $context['progress_snapshot']; + $this->objectName = $snapshot->getIdentifier(); + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'object_name', + $this->objectName + ); + } + + $this->updateProgressBar($snapshot); + } + + /** + * @inheritDoc + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ); + } + + $this->updateProgressBar($context['progress_snapshot']); + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferComplete(array $context): void + { + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ); + } + + $snapshot = $context['progress_snapshot']; + $this->updateProgressBar( + $snapshot, + $snapshot->getTotalBytes() === 0 + ); + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferFail(array $context): void + { + $progressFormat = $this->progressBar->getProgressBarFormat(); + if ($progressFormat instanceof ColoredTransferProgressBarFormat) { + $progressFormat->setArg( + 'color_code', + ColoredTransferProgressBarFormat::RED_COLOR_CODE + ); + $progressFormat->setArg( + 'message', + $context['reason'] + ); + } + + $this->updateProgressBar($context['progress_snapshot']); + } + + /** + * Updates the progress bar with the transfer snapshot + * and also call showProgress. + * + * @param TransferProgressSnapshot $snapshot + * @param bool $forceCompletion To force the progress bar to be + * completed. This is useful for files where its size is zero, + * for which a ratio will return zero, and hence the percent + * will be zero. + * + * @return void + */ + private function updateProgressBar( + TransferProgressSnapshot $snapshot, + bool $forceCompletion = false + ): void + { + if (!$forceCompletion) { + $this->progressBar->setPercentCompleted( + ((int)floor($snapshot->ratioTransferred() * 100)) + ); + } else { + $this->progressBar->setPercentCompleted(100); + } + + $this->progressBar->getProgressBarFormat()->setArgs([ + 'transferred' => $snapshot->getTransferredBytes(), + 'tobe_transferred' => $snapshot->getTotalBytes(), + 'unit' => 'B', + ]); + // Display progress + $this->showProgress(); + } + + /** + * @inheritDoc + * + * @return void + */ + public function showProgress(): void + { + if (empty($this->objectName)) { + throw new \RuntimeException( + "Progress tracker requires an object name to be set." + ); + } + + if ($this->clear) { + fwrite($this->output, "\033[2J\033[H"); + } + + fwrite($this->output, sprintf( + "\r\n%s", + $this->progressBar->render() + )); + fflush($this->output); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php new file mode 100644 index 0000000000..0ac129c43c --- /dev/null +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -0,0 +1,33 @@ +identifier = $identifier; + $this->transferredBytes = $transferredBytes; + $this->totalBytes = $totalBytes; + $this->response = $response; + } + + public function getIdentifier(): string + { + return $this->identifier; + } + + /** + * @return int + */ + public function getTransferredBytes(): int + { + return $this->transferredBytes; + } + + /** + * @return int + */ + public function getTotalBytes(): int + { + return $this->totalBytes; + } + + /** + * @return array + */ + public function getResponse(): array + { + return $this->response; + } + + /** + * @return float + */ + public function ratioTransferred(): float + { + if ($this->totalBytes === 0) { + // Unable to calculate ratio + return 0; + } + + return $this->transferredBytes / $this->totalBytes; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/ProgressBar.php b/src/S3/S3Transfer/ProgressBar.php deleted file mode 100644 index 0421c5a039..0000000000 --- a/src/S3/S3Transfer/ProgressBar.php +++ /dev/null @@ -1,14 +0,0 @@ - 'bytesToBytes', - 'KB' => 'bytesToKB', - 'MB' => 'bytesToMB', - ]; - - public static function getUnitValue(string $displayUnit, float $bytes): float { - $displayUnit = self::validateDisplayUnit($displayUnit); - if (isset(self::$displayUnitMapping[$displayUnit])) { - return number_format(call_user_func([__CLASS__, self::$displayUnitMapping[$displayUnit]], $bytes)); - } - - throw new \RuntimeException("Unknown display unit {$displayUnit}"); - } - - private static function validateDisplayUnit(string $displayUnit): string { - if (!isset(self::$displayUnitMapping[$displayUnit])) { - throw new \InvalidArgumentException("Invalid display unit specified: $displayUnit"); - } - - return $displayUnit; - } - - private static function bytesToBytes(float $bytes): float { - return $bytes; - } - - private static function bytesToKB(float $bytes): float { - return $bytes / 1024; - } - - private static function bytesToMB(float $bytes): float { - return $bytes / 1024 / 1024; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index f2da868473..d9107ac5cd 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -7,6 +7,7 @@ use Aws\ResultInterface; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Psr\Http\Message\StreamInterface; class RangeGetMultipartDownloader extends MultipartDownloader @@ -19,31 +20,29 @@ class RangeGetMultipartDownloader extends MultipartDownloader * @param S3ClientInterface $s3Client * @param array $requestArgs * @param array $config + * - minimum_part_size: The minimum part size for a multipart download + * using range get. This option MUST be set when using range get. * @param int $currentPartNo * @param int $objectPartsCount * @param int $objectCompletedPartsCount * @param int $objectSizeInBytes * @param int $objectBytesTransferred * @param string $eTag - * @param string $objectKey - * @param MultipartDownloadListener|null $listener - * @param TransferListener|null $progressListener * @param StreamInterface|null $stream + * @param TransferProgressSnapshot|null $currentSnapshot + * @param TransferListenerNotifier|null $listenerNotifier */ public function __construct( S3ClientInterface $s3Client, - array $requestArgs = [], + array $requestArgs, array $config = [], int $currentPartNo = 0, int $objectPartsCount = 0, - int $objectCompletedPartsCount = 0, int $objectSizeInBytes = 0, - int $objectBytesTransferred = 0, string $eTag = "", - string $objectKey = "", - ?MultipartDownloadListener $listener = null, - ?TransferListener $progressListener = null, - ?StreamInterface $stream = null + ?StreamInterface $stream = null, + ?TransferProgressSnapshot $currentSnapshot = null, + ?TransferListenerNotifier $listenerNotifier = null, ) { parent::__construct( $s3Client, @@ -51,21 +50,18 @@ public function __construct( $config, $currentPartNo, $objectPartsCount, - $objectCompletedPartsCount, $objectSizeInBytes, - $objectBytesTransferred, $eTag, - $objectKey, - $listener, - $progressListener, - $stream + $stream, + $currentSnapshot, + $listenerNotifier, ); - if (empty($config['minimumPartSize'])) { + if (empty($config['minimum_part_size'])) { throw new S3TransferException( 'You must provide a valid minimum part size in bytes' ); } - $this->partSize = $config['minimumPartSize']; + $this->partSize = $config['minimum_part_size']; // If object size is known at instantiation time then, we can compute // the object dimensions. if ($this->objectSizeInBytes !== 0) { @@ -75,7 +71,6 @@ public function __construct( } } - /** * @inheritDoc * diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index e5e5a5eca3..c29ad896a9 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -5,6 +5,9 @@ use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Progress\MultiProgressTracker; +use Aws\S3\S3Transfer\Progress\SingleProgressTracker; +use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; use Psr\Http\Message\StreamInterface; @@ -25,8 +28,6 @@ class S3TransferManager 'region' => 'us-east-1', ]; - private const MIN_PART_SIZE = 5 * 1024 * 1024; - /** @var S3Client */ private S3ClientInterface $s3Client; @@ -52,11 +53,9 @@ class S3TransferManager * Maximum number of concurrent operations allowed during a multipart * upload/download. * - track_progress: (bool, default=false) \ - * To enable progress tracker in a multipart upload/download. + * To enable progress tracker in a multipart upload/download, and or + * a directory upload/download operation. * - region: (string, default="us-east-2") - * - progress_tracker_factory: (callable|TransferListenerFactory) \ - * A factory to create the listener which will receive notifications - * based in the different stages an upload/download is. */ public function __construct( ?S3ClientInterface $s3Client = null, @@ -80,11 +79,13 @@ public function __construct( * @param array $config The config options for this upload operation. * - multipart_upload_threshold_bytes: (int, optional) * To override the default threshold for when to use multipart upload. - * - target_part_size_bytes: (int, optional) To override the default + * - part_size: (int, optional) To override the default * target part size in bytes. * - track_progress: (bool, optional) To override the default option for - * enabling progress tracking. - * @param MultipartUploadListener|null $uploadListener + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. + * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * * @return PromiseInterface @@ -93,7 +94,7 @@ public function upload( string | StreamInterface $source, array $requestArgs = [], array $config = [], - ?MultipartUploadListener $uploadListener = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -119,35 +120,39 @@ public function upload( $mupThreshold = $config['multipart_upload_threshold_bytes'] ?? $this->config['multipart_upload_threshold_bytes']; - if ($mupThreshold < self::MIN_PART_SIZE) { + if ($mupThreshold < MultipartUploader::PART_MIN_SIZE) { throw new \InvalidArgumentException( "The provided config `multipart_upload_threshold_bytes`" - ."must be greater than or equal to " . self::MIN_PART_SIZE + ."must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE ); } if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); + $progressTracker = new SingleProgressTracker(); + } + + if ($progressTracker !== null) { + $listeners[] = $progressTracker; } + $listenerNotifier = new TransferListenerNotifier($listeners); if ($this->requiresMultipartUpload($source, $mupThreshold)) { return $this->tryMultipartUpload( $source, $requestArgs, - $config['target_part_size_bytes'] - ?? $this->config['target_part_size_bytes'], - $uploadListener, - $progressTracker, + [ + 'part_size' => $config['part_size'] ?? $this->config['target_part_size_bytes'], + 'concurrency' => $this->config['concurrency'], + ], + $listenerNotifier ); } return $this->trySingleUpload( $source, $requestArgs, - $progressTracker + $listenerNotifier ); } @@ -164,9 +169,16 @@ public function upload( * - s3_delimiter: (string, optional, defaulted to `/`) * - put_object_request_callback: (Closure, optional) * - failure_policy: (Closure, optional) - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker - * + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. + * @param TransferListener[]|null $listeners The listeners for watching + * transfer events. Each listener will be cloned per file upload. + * @param TransferListener|null $progressTracker Ideally the progress + * tracker implementation provided here should be able to track multiple + * transfers at once. Please see MultiProgressTracker implementation. + * * @return PromiseInterface */ public function uploadDirectory( @@ -174,7 +186,7 @@ public function uploadDirectory( string $bucketTo, array $requestArgs = [], array $config = [], - ?MultipartUploadListener $uploadListener = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -187,9 +199,7 @@ public function uploadDirectory( if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_UPLOADING - ); + $progressTracker = new MultiProgressTracker(); } $filter = null; @@ -242,7 +252,7 @@ function ($file) use ($filter) { 'Key' => $objectKey, ], $config, - $uploadListener, + array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, )->then(function ($result) use (&$objectsUploaded) { $objectsUploaded++; @@ -251,7 +261,7 @@ function ($file) use ($filter) { })->otherwise(function ($reason) use (&$objectsFailed) { $objectsFailed++; - return $reason; + throw $reason; }); } @@ -269,19 +279,20 @@ function ($file) use ($filter) { * of each get object operation, except for the bucket and key, which * are already provided as the source. * @param array $config The configuration to be used for this operation. + * - multipart_download_type: (string, optional) \ + * Overrides the resolved value from the transfer manager config. * - track_progress: (bool) \ * Overrides the config option set in the transfer manager instantiation * to decide whether transfer progress should be tracked. If a `progressListenerFactory` * was not provided when the transfer manager instance was created * and track_progress resolved as true then, a default progress listener * implementation will be used. - * - minimumPartSize: (int) \ + * - minimum_part_size: (int) \ * The minimum part size in bytes to be used in a range multipart download. - * @param MultipartDownloadListener|null $downloadListener A multipart download - * specific listener of the different states a multipart download can be. - * @param TransferListener|null $progressTracker A transfer listener implementation - * aimed to track the progress of a transfer. If not provided and track_progress - * is resolved as true then, the default progress_tracker_factory will be used. + * If this parameter is not provided then it fallbacks to the transfer + * manager `target_part_size_bytes` config value. + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker * * @return PromiseInterface */ @@ -289,7 +300,7 @@ public function download( string | array $source, array $downloadArgs = [], array $config = [], - ?MultipartDownloadListener $downloadListener = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -306,21 +317,25 @@ public function download( if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING - ); + $progressTracker = new SingleProgressTracker(); } + if ($progressTracker !== null) { + $listeners[] = $progressTracker; + } + + $listenerNotifier = new TransferListenerNotifier($listeners); $requestArgs = $sourceArgs + $downloadArgs; if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, [ - 'minimumPartSize' => $config['minimumPartSize'] - ?? 0 + 'minimum_part_size' => $config['minimum_part_size'] + ?? $this->config['target_part_size_bytes'], + 'multipart_download_type' => $config['multipart_download_type'] + ?? $this->config['multipart_download_type'], ], - $downloadListener, - $progressTracker, + $listenerNotifier, ); } @@ -354,11 +369,12 @@ public function download( * - filter: (Closure) \ * A callable which will receive an object key as parameter and should return * true or false in order to determine whether the object should be downloaded. - * @param MultipartDownloadListenerFactory|null $downloadListenerFactory - * A factory of multipart download listeners `MultipartDownloadListenerFactory` - * for listening to multipart download events. - * @param TransferListener|null $progressTracker - * + * @param TransferListener[] $listeners The listeners for watching + * transfer events. Each listener will be cloned per file upload. + * @param TransferListener|null $progressTracker Ideally the progress + * tracker implementation provided here should be able to track multiple + * transfers at once. Please see MultiProgressTracker implementation. + * * @return PromiseInterface */ public function downloadDirectory( @@ -366,7 +382,7 @@ public function downloadDirectory( string $destinationDirectory, array $downloadArgs, array $config = [], - ?MultipartDownloadListenerFactory $downloadListenerFactory = null, + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -378,9 +394,7 @@ public function downloadDirectory( if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { - $progressTracker = $this->resolveDefaultProgressTracker( - DefaultProgressTracker::TRACKING_OPERATION_DOWNLOADING - ); + $progressTracker = new MultiProgressTracker(); } $listArgs = [ @@ -415,18 +429,13 @@ public function downloadDirectory( ); } - $downloadListener = null; - if ($downloadListenerFactory !== null) { - $downloadListener = $downloadListenerFactory(); - } - $promises[] = $this->download( $object, $downloadArgs, [ 'minimumPartSize' => $config['minimumPartSize'] ?? 0, ], - $downloadListener, + array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, )->then(function (DownloadResponse $result) use ($destinationFile) { $directory = dirname($destinationFile); @@ -446,31 +455,33 @@ public function downloadDirectory( * * @param array $requestArgs * @param array $config - * - minimumPartSize: (int) \ - * The minimum part size in bytes for a range multipart download. If - * this parameter is not provided then it fallbacks to the transfer - * manager `target_part_size_bytes` config value. - * @param MultipartDownloadListener|null $downloadListener - * @param TransferListener|null $progressTracker + * - minimum_part_size: (int) \ + * The minimum part size in bytes for a range multipart download. + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartDownload( array $requestArgs, array $config = [], - ?MultipartDownloadListener $downloadListener = null, - ?TransferListener $progressTracker = null, + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $multipartDownloader = MultipartDownloader::chooseDownloader( - s3Client: $this->s3Client, - multipartDownloadType: $this->config['multipart_download_type'], - requestArgs: $requestArgs, - config: [ - 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, - ], - listener: $downloadListener, - progressTracker: $progressTracker, + $downloaderClassName = match ($config['multipart_download_type']) { + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\PartGetMultipartDownloader', + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', + default => throw new \InvalidArgumentException( + "The config value for `multipart_download_type` must be one of:\n" + . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER + ."\n" + . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER + ) + }; + $multipartDownloader = new $downloaderClassName( + $this->s3Client, + $requestArgs, + $config, + listenerNotifier: $listenerNotifier, ); return $multipartDownloader->promise(); @@ -480,48 +491,60 @@ private function tryMultipartDownload( * Does a single object download. * * @param array $requestArgs - * @param TransferListener|null $progressTracker + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function trySingleDownload( array $requestArgs, - ?TransferListener $progressTracker + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - if ($progressTracker !== null) { - $progressTracker->objectTransferInitiated($requestArgs['Key'], $requestArgs); + if ($listenerNotifier !== null) { + $listenerNotifier->transferInitiated([ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + 0 + ) + ]); $command = $this->s3Client->getCommand( MultipartDownloader::GET_OBJECT_COMMAND, $requestArgs ); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($progressTracker, $requestArgs) { + function ($result) use ($requestArgs, $listenerNotifier) { // Notify progress - $progressTracker->objectTransferProgress( - $requestArgs['Key'], - $result['Content-Length'] ?? 0, - $result['Content-Length'] ?? 0, - ); - + $progressContext = [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + $result['Content-Length'] ?? 0, + $result['Content-Length'] ?? 0, + $result->toArray() + ) + ]; + $listenerNotifier->bytesTransferred($progressContext); // Notify Completion - $progressTracker->objectTransferCompleted( - $requestArgs['Key'], - $result['Content-Length'] ?? 0, - ); + $listenerNotifier->transferComplete($progressContext); return new DownloadResponse( content: $result['Body'], metadata: $result['@metadata'], ); } - )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { - $progressTracker->objectTransferFailed( - $requestArgs['Key'], - 0, - $reason->getMessage(), - ); + )->otherwise(function ($reason) use ($requestArgs, $listenerNotifier) { + $listenerNotifier->transferFail([ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + 0, + ), + 'reason' => $reason + ]); return $reason; }); @@ -544,14 +567,14 @@ function ($result) use ($progressTracker, $requestArgs) { /** * @param string|StreamInterface $source * @param array $requestArgs - * @param TransferListener|null $progressTracker + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function trySingleUpload( string | StreamInterface $source, array $requestArgs, - ?TransferListener $progressTracker = null + ?TransferListenerNotifier $listenerNotifier = null ): PromiseInterface { if (is_string($source) && is_readable($source)) { $requestArgs['SourceFile'] = $source; @@ -565,33 +588,57 @@ private function trySingleUpload( ); } - if ($progressTracker !== null) { - $progressTracker->objectTransferInitiated( - $requestArgs['Key'], - $requestArgs + if (!empty($listenerNotifier)) { + $listenerNotifier->transferInitiated( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $objectSize, + ), + ] ); $command = $this->s3Client->getCommand('PutObject', $requestArgs); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($objectSize, $progressTracker, $requestArgs) { - $progressTracker->objectTransferProgress( - $requestArgs['Key'], - $objectSize, - $objectSize, + function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { + $listenerNotifier->bytesTransferred( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + $objectSize, + $objectSize, + ), + ] ); - $progressTracker->objectTransferCompleted( - $requestArgs['Key'], - $objectSize, + $listenerNotifier->transferComplete( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + $objectSize, + $objectSize, + $result->toArray() + ), + ] ); return new UploadResponse($result->toArray()); } - )->otherwise(function ($reason) use ($requestArgs, $progressTracker) { - $progressTracker->objectTransferFailed( - $requestArgs['Key'], - 0, - $reason->getMessage() + )->otherwise(function ($reason) use ($objectSize, $requestArgs, $listenerNotifier) { + $listenerNotifier->transferFail( + [ + 'request_args' => $requestArgs, + 'progress_snapshot' => new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $objectSize, + ), + 'reason' => $reason, + ] ); return $reason; @@ -610,36 +657,23 @@ function ($result) use ($objectSize, $progressTracker, $requestArgs) { * @param string|StreamInterface $source * @param array $requestArgs * @param array $config - * @param MultipartUploadListener|null $uploadListener - * @param TransferListener|null $progressTracker + * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartUpload( string | StreamInterface $source, array $requestArgs, - int $partSizeBytes, - ?MultipartUploadListener $uploadListener = null, - ?TransferListener $progressTracker = null, + array $config = [], + ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { $createMultipartArgs = [...$requestArgs]; - $uploadPartArgs = [...$requestArgs]; - $completeMultipartArgs = [...$requestArgs]; - if ($this->containsChecksum($requestArgs)) { - $completeMultipartArgs['ChecksumType'] = 'FULL_OBJECT'; - } - return (new MultipartUploader( $this->s3Client, $createMultipartArgs, - $uploadPartArgs, - $completeMultipartArgs, - [ - 'part_size_bytes' => $partSizeBytes, - 'concurrency' => $this->config['concurrency'], - ], + $config, $source, - progressTracker: $progressTracker, + listenerNotifier: $listenerNotifier, ))->promise(); } @@ -744,26 +778,6 @@ private function s3UriAsBucketAndKey(string $uri): array ]; } - /** - * Resolves the progress tracker to be used in the - * transfer operation if `$track_progress` is true. - * - * @param string $trackingOperation - * - * @return TransferListener|null - */ - private function resolveDefaultProgressTracker( - string $trackingOperation - ): ?TransferListener - { - $progress_tracker_factory = $this->config['progress_tracker_factory'] ?? null; - if ($progress_tracker_factory === null) { - return (new DefaultProgressTracker(trackingOperation: $trackingOperation))->getTransferListener(); - } - - return $progress_tracker_factory([]); - } - /** * @param string $sink * @param string $objectKey @@ -797,29 +811,4 @@ private function resolvesOutsideTargetDirectory( return false; } - - /** - * Verifies if a checksum was provided. - * - * @param array $requestArgs - * - * @return bool - */ - private function containsChecksum(array $requestArgs): bool - { - $algorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - foreach ($algorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return true; - } - } - - return false; - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/TransferListener.php b/src/S3/S3Transfer/TransferListener.php index 8b2c88635d..2a24707d17 100644 --- a/src/S3/S3Transfer/TransferListener.php +++ b/src/S3/S3Transfer/TransferListener.php @@ -2,291 +2,46 @@ namespace Aws\S3\S3Transfer; -use Closure; -use Throwable; - -class TransferListener extends ListenerNotifier +abstract class TransferListener { /** - * @param Closure|null $onTransferInitiated - * No parameters will be passed. - * - * @param Closure|null $onObjectTransferInitiated - * Parameters that will be passed when invoked: - * - $objectKey: The key that identifies the object being transferred. - * - $objectRequestArgs: The arguments that initiated the object transfer request. - * - * @param Closure|null $onObjectTransferProgress - * Parameters that will be passed when invoked: - * - $objectKey: The key that identifies the object being transferred. - * - $objectBytesTransferred: The total of bytes transferred for this object. - * - $objectSizeInBytes: The size in bytes of the object. - * - * @param Closure|null $onObjectTransferFailed - * Parameters that will be passed when invoked: - * - $objectKey: The object key for which the transfer has failed. - * - $objectBytesTransferred: The total of bytes transferred from - * this object. - * - $reason: The reason why the transfer failed for this object. - * - * @param Closure|null $onObjectTransferCompleted - * Parameters that will be passed when invoked: - * - $objectKey: The object key for which the transfer was completed. - * - $objectBytesCompleted: The total of bytes transferred for this object. - * - * @param Closure|null $onTransferProgress - * Parameters that will be passed when invoked: - * - $totalObjectsTransferred: The number of objects transferred. - * - $totalBytesTransferred: The total of bytes already transferred on this event. - * - $totalBytes: The total of bytes to be transferred. - * - * @param Closure|null $onTransferCompleted - * Parameters that will be passed when invoked: - * - $objectsTransferCompleted: The number of objects that were transferred. - * - $objectsBytesTransferred: The total of bytes that were transferred. - * - * @param Closure|null $onTransferFailed - * Parameters that will be passed when invoked: - * - $objectsTransferCompleted: The total of objects transferred before failure. - * - $objectsBytesTransferred: The total of bytes transferred before failure. - * - $objectsTransferFailed: The total of objects that failed in the transfer. - * - $reason: The throwable with the reason why the transfer failed. - * @param int $objectsTransferCompleted - * @param int $objectsBytesTransferred - * @param int $objectsTransferFailed - * @param int $objectsToBeTransferred - */ - public function __construct( - public ?Closure $onTransferInitiated = null, - public ?Closure $onObjectTransferInitiated = null, - public ?Closure $onObjectTransferProgress = null, - public ?Closure $onObjectTransferFailed = null, - public ?Closure $onObjectTransferCompleted = null, - public ?Closure $onTransferProgress = null, - public ?Closure $onTransferCompleted = null, - public ?Closure $onTransferFailed = null, - private int $objectsTransferCompleted = 0, - private int $objectsBytesTransferred = 0, - private int $objectsTransferFailed = 0, - private int $objectsToBeTransferred = 0 - ) {} - - /** - * @return int - */ - public function getObjectsTransferCompleted(): int - { - return $this->objectsTransferCompleted; - } - - /** - * @return int - */ - public function getObjectsBytesTransferred(): int - { - return $this->objectsBytesTransferred; - } - - /** - * @return int - */ - public function getObjectsTransferFailed(): int - { - return $this->objectsTransferFailed; - } - - /** - * @return int - */ - public function getObjectsToBeTransferred(): int - { - return $this->objectsToBeTransferred; - } - - /** - * Transfer initiated event. - */ - public function transferInitiated(): void - { - $this->notify('onTransferInitiated', []); - } - - /** - * Event for when an object transfer initiated. - * - * @param string $objectKey - * @param array $requestArgs + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the request initialization. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * * @return void */ - public function objectTransferInitiated(string $objectKey, array &$requestArgs): void - { - $this->objectsToBeTransferred++; - if ($this->objectsToBeTransferred === 1) { - $this->transferInitiated(); - } - - $this->notify('onObjectTransferInitiated', [$objectKey, &$requestArgs]); - } + public function transferInitiated(array $context): void {} /** - * Event for when an object transfer made some progress. - * - * @param string $objectKey - * @param int $objectBytesTransferred - * @param int $objectSizeInBytes + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the operation that originated the bytes transferred event. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * * @return void */ - public function objectTransferProgress( - string $objectKey, - int $objectBytesTransferred, - int $objectSizeInBytes - ): void - { - $this->objectsBytesTransferred += $objectBytesTransferred; - $this->notify('onObjectTransferProgress', [ - $objectKey, - $objectBytesTransferred, - $objectSizeInBytes - ]); - // Needs state management - $this->notify('onTransferProgress', [ - $this->objectsTransferCompleted, - $this->objectsBytesTransferred, - $this->objectsToBeTransferred - ]); - } + public function bytesTransferred(array $context): void {} /** - * Event for when an object transfer failed. - * - * @param string $objectKey - * @param int $objectBytesTransferred - * @param \Throwable|string $reason + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the operation that originated the bytes transferred event. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. * * @return void */ - public function objectTransferFailed( - string $objectKey, - int $objectBytesTransferred, - \Throwable | string $reason - ): void - { - $this->objectsTransferFailed++; - $this->validateTransferComplete(); - $this->notify('onObjectTransferFailed', [ - $objectKey, - $objectBytesTransferred, - $reason - ]); - } + public function transferComplete(array $context): void {} /** - * Event for when an object transfer is completed. - * - * @param string $objectKey - * @param int $objectBytesCompleted + * @param array $context + * - request_args: (array) The request arguments that will be provided + * as part of the operation that originated the bytes transferred event. + * - progress_snapshot: (TransferProgressSnapshot) The transfer snapshot holder. + * - reason: (Throwable) The exception originated by the transfer failure. * * @return void */ - public function objectTransferCompleted ( - string $objectKey, - int $objectBytesCompleted - ): void - { - $this->objectsTransferCompleted++; - $this->validateTransferComplete(); - $this->notify('onObjectTransferCompleted', [ - $objectKey, - $objectBytesCompleted - ]); - } - - /** - * Event for when a transfer is completed. - * - * @param int $objectsTransferCompleted - * @param int $objectsBytesTransferred - * - * @return void - */ - public function transferCompleted ( - int $objectsTransferCompleted, - int $objectsBytesTransferred, - ): void - { - $this->notify('onTransferCompleted', [ - $objectsTransferCompleted, - $objectsBytesTransferred - ]); - } - - /** - * Event for when a transfer is completed. - * - * @param int $objectsTransferCompleted - * @param int $objectsBytesTransferred - * @param int $objectsTransferFailed - * - * @return void - */ - public function transferFailed ( - int $objectsTransferCompleted, - int $objectsBytesTransferred, - int $objectsTransferFailed, - Throwable | string $reason - ): void - { - $this->notify('onTransferFailed', [ - $objectsTransferCompleted, - $objectsBytesTransferred, - $objectsTransferFailed, - $reason - ]); - } - - /** - * Validates if a transfer is completed, and if so then the event is propagated - * to the subscribed listeners. - * - * @return void - */ - private function validateTransferComplete(): void - { - if ($this->objectsToBeTransferred === ($this->objectsTransferCompleted + $this->objectsTransferFailed)) { - if ($this->objectsTransferFailed > 0) { - $this->transferFailed( - $this->objectsTransferCompleted, - $this->objectsBytesTransferred, - $this->objectsTransferFailed, - "Transfer could not have been completed successfully." - ); - } else { - $this->transferCompleted( - $this->objectsTransferCompleted, - $this->objectsBytesTransferred - ); - } - } - } - - protected function notify(string $event, array $params = []): void - { - $listener = match ($event) { - 'onTransferInitiated' => $this->onTransferInitiated, - 'onObjectTransferInitiated' => $this->onObjectTransferInitiated, - 'onObjectTransferProgress' => $this->onObjectTransferProgress, - 'onObjectTransferFailed' => $this->onObjectTransferFailed, - 'onObjectTransferCompleted' => $this->onObjectTransferCompleted, - 'onTransferProgress' => $this->onTransferProgress, - 'onTransferCompleted' => $this->onTransferCompleted, - 'onTransferFailed' => $this->onTransferFailed, - default => null, - }; - - if ($listener instanceof Closure) { - $listener(...$params); - } - } + public function transferFail(array $context): void {} } \ No newline at end of file diff --git a/src/S3/S3Transfer/TransferListenerFactory.php b/src/S3/S3Transfer/TransferListenerFactory.php deleted file mode 100644 index 40f9747cf2..0000000000 --- a/src/S3/S3Transfer/TransferListenerFactory.php +++ /dev/null @@ -1,8 +0,0 @@ -listeners = $listeners; + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferInitiated(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->transferInitiated($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function bytesTransferred(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->bytesTransferred($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferComplete(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->transferComplete($context); + } + } + + /** + * @inheritDoc + * + * @return void + */ + public function transferFail(array $context): void + { + foreach ($this->listeners as $name => $listener) { + $listener->transferFail($context); + } + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/ConsoleProgressBarTest.php deleted file mode 100644 index 31ea5ebd5d..0000000000 --- a/tests/S3/S3Transfer/ConsoleProgressBarTest.php +++ /dev/null @@ -1,228 +0,0 @@ -setPercentCompleted($percent); - $progressBar->setArgs([ - 'transferred' => $transferred, - 'tobe_transferred' => $toBeTransferred, - 'unit' => $unit - ]); - - $output = $progressBar->getPaintedProgress(); - $this->assertEquals($expectedProgress, $output); - } - - /** - * Data provider for testing progress bar rendering. - * - * @return array - */ - public function progressBarPercentProvider(): array { - return [ - [ - 'percent' => 25, - 'transferred' => 25, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[###### ] 25% 25/100 B' - ], - [ - 'percent' => 50, - 'transferred' => 50, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[############# ] 50% 50/100 B' - ], - [ - 'percent' => 75, - 'transferred' => 75, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[################### ] 75% 75/100 B' - ], - [ - 'percent' => 100, - 'transferred' => 100, - 'tobe_transferred' => 100, - 'unit' => 'B', - 'expected' => '[#########################] 100% 100/100 B' - ], - ]; - } - - /** - * Tests progress with custom char. - * - * @return void - */ - public function testProgressBarWithCustomChar() - { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 30 - ); - $progressBar->setPercentCompleted(30); - $progressBar->setArgs([ - 'transferred' => '10', - 'tobe_transferred' => '100', - 'unit' => 'B' - ]); - - $output = $progressBar->getPaintedProgress(); - $this->assertStringContainsString('10/100 B', $output); - $this->assertStringContainsString(str_repeat('*', 9), $output); - } - - /** - * Tests progress with custom char. - * - * @return void - */ - public function testProgressBarWithCustomWidth() - { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 100 - ); - $progressBar->setPercentCompleted(10); - $progressBar->setArgs([ - 'transferred' => '10', - 'tobe_transferred' => '100', - 'unit' => 'B' - ]); - - $output = $progressBar->getPaintedProgress(); - $this->assertStringContainsString('10/100 B', $output); - $this->assertStringContainsString(str_repeat('*', 10), $output); - } - - /** - * Tests missing parameters. - * - * @dataProvider progressBarMissingArgsProvider - * - * @return void - */ - public function testProgressBarMissingArgsThrowsException( - string $formatName, - string $parameter - ) - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("Missing `$parameter` parameter for progress bar."); - - $format = ConsoleProgressBar::$formats[$formatName]; - $progressBar = new ConsoleProgressBar( - format: $format, - ); - foreach ($format['parameters'] as $param) { - if ($param === $parameter) { - continue; - } - - $progressBar->setArg($param, 'foo'); - } - - $progressBar->setPercentCompleted(20); - $progressBar->getPaintedProgress(); - } - - - /** - * Data provider for testing exception when arguments are missing. - * - * @return array - */ - public function progressBarMissingArgsProvider(): array - { - return [ - [ - 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, - 'parameter' => 'transferred', - ], - [ - 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, - 'parameter' => 'tobe_transferred', - ], - [ - 'formatName' => ConsoleProgressBar::TRANSFER_FORMAT, - 'parameter' => 'unit', - ] - ]; - } - - /** - * Tests the progress bar does not overflow when the percent is over 100. - * - * @return void - */ - public function testProgressBarDoesNotOverflowAfter100Percent() - { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 10, - ); - $progressBar->setPercentCompleted(110); - $progressBar->setArgs([ - 'transferred' => 'foo', - 'tobe_transferred' => 'foo', - 'unit' => 'MB' - ]); - $output = $progressBar->getPaintedProgress(); - $this->assertStringContainsString('100%', $output); - $this->assertStringContainsString('[**********]', $output); - } - - /** - * Tests the progress bar sets the arguments. - * - * @return void - */ - public function testProgressBarSetsArguments() { - $progressBar = new ConsoleProgressBar( - progressBarChar: '*', - progressBarWidth: 25, - format: ConsoleProgressBar::$formats[ConsoleProgressBar::TRANSFER_FORMAT] - ); - $progressBar->setArgs([ - 'transferred' => 'fooTransferred', - 'tobe_transferred' => 'fooToBeTransferred', - 'unit' => 'fooUnit', - ]); - $output = $progressBar->getPaintedProgress(); - $progressBar->setPercentCompleted(100); - $this->assertStringContainsString('fooTransferred', $output); - $this->assertStringContainsString('fooToBeTransferred', $output); - $this->assertStringContainsString('fooUnit', $output); - } -} diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php new file mode 100644 index 0000000000..89a76d452b --- /dev/null +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -0,0 +1,261 @@ +assertEquals( + ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH, + $progressBar->getProgressBarWidth() + ); + $this->assertEquals( + ConsoleProgressBar::DEFAULT_PROGRESS_BAR_CHAR, + $progressBar->getProgressBarChar() + ); + $this->assertEquals( + 0, + $progressBar->getPercentCompleted() + ); + $this->assertInstanceOf( + ColoredTransferProgressBarFormat::class, + $progressBar->getProgressBarFormat() + ); + } + + /** + * Tests the percent is updated properly. + * + * @return void + */ + public function testSetPercentCompleted(): void { + $progressBar = new ConsoleProgressBar(); + $progressBar->setPercentCompleted(10); + $this->assertEquals(10, $progressBar->getPercentCompleted()); + $progressBar->setPercentCompleted(100); + $this->assertEquals(100, $progressBar->getPercentCompleted()); + } + + /** + * @return void + */ + public function testSetCustomValues(): void { + $progressBar = new ConsoleProgressBar( + progressBarChar: '-', + progressBarWidth: 10, + percentCompleted: 25, + progressBarFormat: new PlainProgressBarFormat() + ); + $this->assertEquals('-', $progressBar->getProgressBarChar()); + $this->assertEquals(10, $progressBar->getProgressBarWidth()); + $this->assertEquals(25, $progressBar->getPercentCompleted()); + $this->assertInstanceOf( + PlainProgressBarFormat::class, + $progressBar->getProgressBarFormat() + ); + } + + /** + * To make sure the percent is not over 100. + * + * @return void + */ + public function testPercentIsNotOverOneHundred(): void { + $progressBar = new ConsoleProgressBar(); + $progressBar->setPercentCompleted(150); + $this->assertEquals(100, $progressBar->getPercentCompleted()); + } + + /** + * @param string $progressBarChar + * @param int $progressBarWidth + * @param int $percentCompleted + * @param ProgressBarFormat $progressBarFormat + * @param array $progressBarFormatArgs + * @param string $expectedOutput + * + * @return void + * @dataProvider progressBarRenderingProvider + * + */ + public function testProgressBarRendering( + string $progressBarChar, + int $progressBarWidth, + int $percentCompleted, + ProgressBarFormat $progressBarFormat, + array $progressBarFormatArgs, + string $expectedOutput + ): void { + $progressBarFormat->setArgs($progressBarFormatArgs); + $progressBar = new ConsoleProgressBar( + $progressBarChar, + $progressBarWidth, + $percentCompleted, + $progressBarFormat, + ); + + $this->assertEquals($expectedOutput, $progressBar->render()); + } + + /** + * Data provider for testing progress bar rendering. + * + * @return array + */ + public function progressBarRenderingProvider(): array { + return [ + 'plain_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 15, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[######## ] 15%' + ], + 'plain_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 45, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[####################### ] 45%' + ], + 'plain_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 100, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[##################################################] 100%' + ], + 'plain_progress_bar_format_4' => [ + 'progress_bar_char' => '.', + 'progress_bar_width' => 50, + 'percent_completed' => 100, + 'progress_bar_format' => new PlainProgressBarFormat(), + 'progress_bar_format_args' => [], + 'expected_output' => '[..................................................] 100%' + ], + 'transfer_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 50, + 'percent_completed' => 23, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 23, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[############ ] 23% 23/100 B' + ], + 'transfer_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 25, + 'percent_completed' => 75, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 75, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[################### ] 75% 75/100 B' + ], + 'transfer_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 30, + 'percent_completed' => 100, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[##############################] 100% 100/100 B' + ], + 'transfer_progress_bar_format_4' => [ + 'progress_bar_char' => '*', + 'progress_bar_width' => 30, + 'percent_completed' => 100, + 'progress_bar_format' => new TransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => '[******************************] 100% 100/100 B' + ], + 'colored_progress_bar_format_1' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 20, + 'percent_completed' => 10, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_1', + 'transferred' => 10, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_output' => "ObjectName_1:\n\033[30m[## ] 10% 10/100 B \033[0m" + ], + 'colored_progress_bar_format_2' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 20, + 'percent_completed' => 50, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_2', + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_output' => "ObjectName_2:\n\033[34m[########## ] 50% 50/100 B \033[0m" + ], + 'colored_progress_bar_format_3' => [ + 'progress_bar_char' => '#', + 'progress_bar_width' => 25, + 'percent_completed' => 100, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_3', + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_output' => "ObjectName_3:\n\033[32m[#########################] 100% 100/100 B \033[0m" + ], + 'colored_progress_bar_format_4' => [ + 'progress_bar_char' => '=', + 'progress_bar_width' => 25, + 'percent_completed' => 100, + 'progress_bar_format' => new ColoredTransferProgressBarFormat(), + 'progress_bar_format_args' => [ + 'object_name' => 'ObjectName_3', + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_output' => "ObjectName_3:\n\033[32m[=========================] 100% 100/100 B \033[0m" + ] + ]; + } +} From e681d10705fa91f199c5f2b786decb98f0af4325 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 24 Feb 2025 07:07:53 -0800 Subject: [PATCH 11/62] chore: fix test cases - Fixes current test cases for: - MultipartUploader - MultipartDownloader - ProgressTracker --- src/S3/S3Transfer/MultipartDownloader.php | 23 +- src/S3/S3Transfer/MultipartUploader.php | 2 +- .../Progress/MultiProgressTracker.php | 12 +- src/S3/S3Transfer/S3TransferManager.php | 13 +- .../S3Transfer/DefaultProgressTrackerTest.php | 189 ---------------- .../MultipartDownloadListenerTest.php | 207 ------------------ .../S3/S3Transfer/MultipartDownloaderTest.php | 26 ++- tests/S3/S3Transfer/MultipartUploaderTest.php | 16 +- .../S3Transfer/ObjectProgressTrackerTest.php | 127 ----------- 9 files changed, 56 insertions(+), 559 deletions(-) delete mode 100644 tests/S3/S3Transfer/DefaultProgressTrackerTest.php delete mode 100644 tests/S3/S3Transfer/MultipartDownloadListenerTest.php delete mode 100644 tests/S3/S3Transfer/ObjectProgressTrackerTest.php diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index d110d01635..f447ac5815 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -51,9 +51,7 @@ abstract class MultipartDownloader implements PromisorInterface * using range get. This option MUST be set when using range get. * @param int $currentPartNo * @param int $objectPartsCount - * @param int $objectCompletedPartsCount * @param int $objectSizeInBytes - * @param int $objectBytesTransferred * @param string $eTag * @param StreamInterface|null $stream * @param TransferProgressSnapshot|null $currentSnapshot @@ -337,4 +335,25 @@ private function downloadComplete(): void 'progress_snapshot' => $this->currentSnapshot, ]); } + + /** + * @param mixed $multipartDownloadType + * + * @return string + */ + public static function chooseDownloaderClassName( + string $multipartDownloadType + ): string + { + return match ($multipartDownloadType) { + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\PartGetMultipartDownloader', + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', + default => throw new \InvalidArgumentException( + "The config value for `multipart_download_type` must be one of:\n" + . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER + ."\n" + . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER + ) + }; + } } \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 805eb5179e..c9885bb9c9 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -132,7 +132,7 @@ public function promise(): PromiseInterface ); } catch (Throwable $e) { $this->uploadFailed($e); - throw $e; + yield Create::rejectionFor($e); } finally { $this->callDeferredFns(); } diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php index 6dcba0053c..6b00ea440f 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressTracker.php +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -90,11 +90,15 @@ public function transferInitiated(array $context): void { $this->transferCount++; $snapshot = $context['progress_snapshot']; - $progressTracker = new SingleProgressTracker( - clear: false, - ); + if (isset($this->singleProgressTrackers[$snapshot['key']])) { + $progressTracker = $this->singleProgressTrackers[$snapshot['key']]; + } else { + $progressTracker = new SingleProgressTracker( + clear: false, + ); + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; + } $progressTracker->transferInitiated($context); - $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; $this->showProgress(); } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index c29ad896a9..e14c3037b6 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -467,16 +467,9 @@ private function tryMultipartDownload( ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $downloaderClassName = match ($config['multipart_download_type']) { - MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\PartGetMultipartDownloader', - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', - default => throw new \InvalidArgumentException( - "The config value for `multipart_download_type` must be one of:\n" - . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER - ."\n" - . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER - ) - }; + $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $config['multipart_download_type'] + ); $multipartDownloader = new $downloaderClassName( $this->s3Client, $requestArgs, diff --git a/tests/S3/S3Transfer/DefaultProgressTrackerTest.php b/tests/S3/S3Transfer/DefaultProgressTrackerTest.php deleted file mode 100644 index 0c9d8f0e53..0000000000 --- a/tests/S3/S3Transfer/DefaultProgressTrackerTest.php +++ /dev/null @@ -1,189 +0,0 @@ -progressTracker = new DefaultProgressTracker( - output: $this->output = fopen('php://temp', 'r+') - ); - } - - protected function tearDown(): void { - fclose($this->output); - } - - /** - * Tests initialization is clean. - * - * @return void - */ - public function testInitialization(): void - { - $this->assertInstanceOf(TransferListener::class, $this->progressTracker->getTransferListener()); - $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); - $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); - $this->assertEquals(0, $this->progressTracker->getObjectsCount()); - $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); - } - - /** - * Tests object transfer is initiated when the event is triggered. - * - * @return void - */ - public function testObjectTransferInitiated(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - - $this->assertEquals(1, $this->progressTracker->getObjectsInProgress()); - $this->assertEquals(1, $this->progressTracker->getObjectsCount()); - } - - /** - * Tests object transfer progress is propagated correctly. - * - * @dataProvider objectTransferProgressProvider - * - * @param string $objectKey - * @param int $objectSize - * @param array $progressList - * - * @return void - */ - public function testObjectTransferProgress( - string $objectKey, - int $objectSize, - array $progressList, - ): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)($objectKey, $fakeRequestArgs); - $totalProgress = 0; - foreach ($progressList as $progress) { - ($listener->onObjectTransferProgress)($objectKey, $progress, $objectSize); - $totalProgress += $progress; - } - - $this->assertEquals($totalProgress, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals($objectSize, $this->progressTracker->getObjectsTotalSizeInBytes()); - $percentCompleted = (int) floor($totalProgress / $objectSize) * 100; - $this->assertEquals($percentCompleted, $this->progressTracker->getTransferPercentCompleted()); - - rewind($this->output); - $this->assertStringContainsString("$percentCompleted% $totalProgress/$objectSize B", stream_get_contents($this->output)); - } - - /** - * Data provider for testing object progress tracker. - * - * @return array[] - */ - public function objectTransferProgressProvider(): array - { - return [ - [ - 'objectKey' => 'FooObjectKey', - 'objectSize' => 250, - 'progressList' => [ - 50, 100, 72, 28 - ] - ], - [ - 'objectKey' => 'FooObjectKey', - 'objectSize' => 10_000, - 'progressList' => [ - 100, 500, 1_000, 2_000, 5_000, 400, 700, 300 - ] - ], - [ - 'objectKey' => 'FooObjectKey', - 'objectSize' => 10_000, - 'progressList' => [ - 5_000, 5_000 - ] - ] - ]; - } - - /** - * Tests object transfer is completed. - * - * @return void - */ - public function testObjectTransferCompleted(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); - ($listener->onObjectTransferProgress)('FooObjectKey', 50, 100); - ($listener->onObjectTransferCompleted)('FooObjectKey', 100); - - $this->assertEquals(100, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(100, $this->progressTracker->getTransferPercentCompleted()); - - // Validate it completed 100% at the progress bar side. - rewind($this->output); - $this->assertStringContainsString("[#########################] 100% 100/100 B", stream_get_contents($this->output)); - } - - /** - * Tests object transfer failed. - * - * @return void - */ - public function testObjectTransferFailed(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - ($listener->onObjectTransferProgress)('FooObjectKey', 27, 100); - ($listener->onObjectTransferFailed)('FooObjectKey', 27, 'Transfer error'); - - $this->assertEquals(27, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(27, $this->progressTracker->getTransferPercentCompleted()); - $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); - - rewind($this->output); - $this->assertStringContainsString("27% 27/100 B", stream_get_contents($this->output)); - } - - /** - * Tests state are cleared. - * - * @return void - */ - public function testClearState(): void - { - $listener = $this->progressTracker->getTransferListener(); - $fakeRequestArgs = []; - ($listener->onObjectTransferInitiated)('FooObjectKey', $fakeRequestArgs); - ($listener->onObjectTransferProgress)('FooObjectKey', 10, 100); - - $this->progressTracker->clear(); - - $this->assertEquals(0, $this->progressTracker->getTotalBytesTransferred()); - $this->assertEquals(0, $this->progressTracker->getObjectsTotalSizeInBytes()); - $this->assertEquals(0, $this->progressTracker->getObjectsInProgress()); - $this->assertEquals(0, $this->progressTracker->getObjectsCount()); - $this->assertEquals(0, $this->progressTracker->getTransferPercentCompleted()); - } -} - diff --git a/tests/S3/S3Transfer/MultipartDownloadListenerTest.php b/tests/S3/S3Transfer/MultipartDownloadListenerTest.php deleted file mode 100644 index fc76b48e97..0000000000 --- a/tests/S3/S3Transfer/MultipartDownloadListenerTest.php +++ /dev/null @@ -1,207 +0,0 @@ -assertIsArray($commandArgs); - $this->assertIsInt($initialPart); - }; - - $listener = new MultipartDownloadListener(onDownloadInitiated: $callback); - - $commandArgs = ['Foo' => 'Buzz']; - $listener->downloadInitiated($commandArgs, 1); - - $this->assertTrue($called, "Expected onDownloadInitiated to be called."); - } - - /** - * Tests download failed event is propagated. - * - * @return void - */ - public function testDownloadFailed(): void - { - $called = false; - $expectedError = new Exception('Download failed'); - $expectedTotalPartsTransferred = 5; - $expectedTotalBytesTransferred = 1024; - $expectedLastPartTransferred = 4; - $callback = function ( - $reason, - $totalPartsTransferred, - $totalBytesTransferred, - $lastPartTransferred - ) use ( - &$called, - $expectedError, - $expectedTotalPartsTransferred, - $expectedTotalBytesTransferred, - $expectedLastPartTransferred - ) { - $called = true; - $this->assertEquals($reason, $expectedError); - $this->assertEquals($expectedTotalPartsTransferred, $totalPartsTransferred); - $this->assertEquals($expectedTotalBytesTransferred, $totalBytesTransferred); - $this->assertEquals($expectedLastPartTransferred, $lastPartTransferred); - - }; - $listener = new MultipartDownloadListener(onDownloadFailed: $callback); - $listener->downloadFailed( - $expectedError, - $expectedTotalPartsTransferred, - $expectedTotalBytesTransferred, - $expectedLastPartTransferred - ); - $this->assertTrue($called, "Expected onDownloadFailed to be called."); - } - - /** - * Tests download completed event is propagated. - * - * @return void - */ - public function testDownloadCompleted(): void - { - $called = false; - $expectedStream = fopen('php://temp', 'r+'); - $expectedTotalPartsDownloaded = 10; - $expectedTotalBytesDownloaded = 2048; - $callback = function ( - $stream, - $totalPartsDownloaded, - $totalBytesDownloaded - ) use ( - &$called, - $expectedStream, - $expectedTotalPartsDownloaded, - $expectedTotalBytesDownloaded - ) { - $called = true; - $this->assertIsResource($stream); - $this->assertEquals($expectedStream, $stream); - $this->assertEquals($expectedTotalPartsDownloaded, $totalPartsDownloaded); - $this->assertEquals($expectedTotalBytesDownloaded, $totalBytesDownloaded); - }; - - $listener = new MultipartDownloadListener(onDownloadCompleted: $callback); - $listener->downloadCompleted( - $expectedStream, - $expectedTotalPartsDownloaded, - $expectedTotalBytesDownloaded - ); - $this->assertTrue($called, "Expected onDownloadCompleted to be called."); - } - - /** - * Tests part downloaded initiated event is propagated. - * - * @return void - */ - public function testPartDownloadInitiated(): void - { - $called = false; - $mockCommand = $this->createMock(CommandInterface::class); - $expectedPartNo = 3; - $callable = function ($command, $partNo) - use (&$called, $mockCommand, $expectedPartNo) { - $called = true; - $this->assertEquals($expectedPartNo, $partNo); - $this->assertEquals($mockCommand, $command); - }; - $listener = new MultipartDownloadListener(onPartDownloadInitiated: $callable); - $listener->partDownloadInitiated($mockCommand, $expectedPartNo); - $this->assertTrue($called, "Expected onPartDownloadInitiated to be called."); - } - - /** - * Tests part download completed event is propagated. - * - * @return void - */ - public function testPartDownloadCompleted(): void - { - $called = false; - $mockResult = $this->createMock(ResultInterface::class); - $expectedPartNo = 3; - $expectedPartTotalBytes = 512; - $expectedTotalParts = 5; - $expectedObjectBytesTransferred = 1024; - $expectedObjectSizeInBytes = 2048; - $callback = function ( - $result, - $partNo, - $partTotalBytes, - $totalParts, - $objectBytesDownloaded, - $objectSizeInBytes - ) use ( - &$called, - $mockResult, - $expectedPartNo, - $expectedPartTotalBytes, - $expectedTotalParts, - $expectedObjectBytesTransferred, - $expectedObjectSizeInBytes - ) { - $called = true; - $this->assertEquals($mockResult, $result); - $this->assertEquals($expectedPartNo, $partNo); - $this->assertEquals($expectedPartTotalBytes, $partTotalBytes); - $this->assertEquals($expectedTotalParts, $totalParts); - $this->assertEquals($expectedObjectBytesTransferred, $objectBytesDownloaded); - $this->assertEquals($expectedObjectSizeInBytes, $objectSizeInBytes); - }; - $listener = new MultipartDownloadListener(onPartDownloadCompleted: $callback); - $listener->partDownloadCompleted( - $mockResult, - $expectedPartNo, - $expectedPartTotalBytes, - $expectedTotalParts, - $expectedObjectBytesTransferred, - $expectedObjectSizeInBytes - ); - $this->assertTrue($called, "Expected onPartDownloadCompleted to be called."); - } - - /** - * Tests part download failed event is propagated. - * - * @return void - */ - public function testPartDownloadFailed() - { - $called = false; - $mockCommand = $this->createMock(CommandInterface::class); - $expectedReason = new Exception('Part download failed'); - $expectedPartNo = 2; - $callable = function ($command, $reason, $partNo) - use (&$called, $mockCommand, $expectedReason, $expectedPartNo) { - $called = true; - $this->assertEquals($expectedReason, $reason); - $this->assertEquals($expectedPartNo, $partNo); - $this->assertEquals($mockCommand, $command); - }; - - $listener = new MultipartDownloadListener(onPartDownloadFailed: $callable); - $listener->partDownloadFailed($mockCommand, $expectedReason, $expectedPartNo); - $this->assertTrue($called, "Expected onPartDownloadFailed to be called."); - } -} \ No newline at end of file diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 0838f86ab5..e986ac4dd6 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -5,6 +5,7 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; +use Aws\S3\S3Transfer\DownloadResponse; use Aws\S3\S3Transfer\MultipartDownloader; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; @@ -25,9 +26,9 @@ class MultipartDownloaderTest extends TestCase * @param int $objectSizeInBytes * @param int $targetPartSize * - * @return void * @dataProvider partGetMultipartDownloaderProvider * + * @return void */ public function testMultipartDownloader( string $multipartDownloadType, @@ -67,25 +68,30 @@ public function testMultipartDownloader( -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - $downloader = MultipartDownloader::chooseDownloader( + $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $multipartDownloadType + ); + /** @var MultipartDownloader $downloader */ + $downloader = new $downloaderClassName( $mockClient, - $multipartDownloadType, [ 'Bucket' => 'FooBucket', 'Key' => $objectKey, ], [ - 'minimumPartSize' => $targetPartSize, + 'minimum_part_size' => $targetPartSize, ] ); - $stream = $downloader->promise()->wait(); + /** @var DownloadResponse $response */ + $response = $downloader->promise()->wait(); + $snapshot = $downloader->getCurrentSnapshot(); - $this->assertInstanceOf(StreamInterface::class, $stream); - $this->assertEquals($objectKey, $downloader->getObjectKey()); - $this->assertEquals($objectSizeInBytes, $downloader->getObjectSizeInBytes()); - $this->assertEquals($objectSizeInBytes, $downloader->getObjectBytesTransferred()); + $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertEquals($objectKey, $snapshot->getIdentifier()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); - $this->assertEquals($partsCount, $downloader->getObjectCompletedPartsCount()); + $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); } /** diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index b74dd3d6e3..d727595132 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -7,6 +7,7 @@ use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\MultipartUploader; +use Aws\S3\S3Transfer\UploadResponse; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; @@ -62,18 +63,19 @@ public function testMultipartUpload( $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, - $requestArgs, - $requestArgs, $config + [ 'concurrency' => 3, ], $stream ); - $multipartUploader->promise()->wait(); + /** @var UploadResponse $response */ + $response = $multipartUploader->promise()->wait(); + $snapshot = $multipartUploader->getCurrentSnapshot(); + $this->assertInstanceOf(UploadResponse::class, $response); $this->assertCount($expected['parts'], $multipartUploader->getParts()); - $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectBytesTransferred()); - $this->assertEquals($expected['bytesUploaded'], $multipartUploader->getObjectSizeInBytes()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); } /** @@ -187,8 +189,6 @@ public function testInvalidSourceStringThrowsException(): void $this->multipartUploadS3Client(), ['Bucket' => 'test-bucket', 'Key' => 'test-key'], [], - [], - [], $nonExistentFile ); } @@ -206,8 +206,6 @@ public function testInvalidSourceTypeThrowsException(): void $this->multipartUploadS3Client(), ['Bucket' => 'test-bucket', 'Key' => 'test-key'], [], - [], - [], 12345 ); } diff --git a/tests/S3/S3Transfer/ObjectProgressTrackerTest.php b/tests/S3/S3Transfer/ObjectProgressTrackerTest.php deleted file mode 100644 index 07795ad893..0000000000 --- a/tests/S3/S3Transfer/ObjectProgressTrackerTest.php +++ /dev/null @@ -1,127 +0,0 @@ -mockProgressBar = $this->createMock(ProgressBar::class); - } - - /** - * Tests getter and setters. - * - * @return void - */ - public function testGettersAndSetters(): void - { - $tracker = new ObjectProgressTracker( - '', - 0, - 0, - '' - ); - $tracker->setObjectKey('FooKey'); - $this->assertEquals('FooKey', $tracker->getObjectKey()); - - $tracker->setObjectBytesTransferred(100); - $this->assertEquals(100, $tracker->getObjectBytesTransferred()); - - $tracker->setObjectSizeInBytes(100); - $this->assertEquals(100, $tracker->getObjectSizeInBytes()); - - $tracker->setStatus('initiated'); - $this->assertEquals('initiated', $tracker->getStatus()); - } - - /** - * Tests bytes transferred increments. - * - * @return void - */ - public function testIncrementTotalBytesTransferred(): void - { - $percentProgress = 0; - $this->mockProgressBar->expects($this->atLeast(4)) - ->method('setPercentCompleted') - ->willReturnCallback(function ($percent) use (&$percentProgress) { - $this->assertEquals($percentProgress +=25, $percent); - }); - - $tracker = new ObjectProgressTracker( - objectKey: 'FooKey', - objectBytesTransferred: 0, - objectSizeInBytes: 100, - status: 'initiated', - progressBar: $this->mockProgressBar - ); - - $tracker->incrementTotalBytesTransferred(25); - $tracker->incrementTotalBytesTransferred(25); - $tracker->incrementTotalBytesTransferred(25); - $tracker->incrementTotalBytesTransferred(25); - - $this->assertEquals(100, $tracker->getObjectBytesTransferred()); - } - - - /** - * Tests progress status color based on states. - * - * @return void - */ - public function testSetStatusUpdatesProgressBarColor() - { - $statusColorMapping = [ - 'progress' => ConsoleProgressBar::BLUE_COLOR_CODE, - 'completed' => ConsoleProgressBar::GREEN_COLOR_CODE, - 'failed' => ConsoleProgressBar::RED_COLOR_CODE, - ]; - $values = array_values($statusColorMapping); - $valueIndex = 0; - $this->mockProgressBar->expects($this->exactly(3)) - ->method('setArg') - ->willReturnCallback(function ($_, $argValue) use ($values, &$valueIndex) { - $this->assertEquals($argValue, $values[$valueIndex++]); - }); - - $tracker = new ObjectProgressTracker( - objectKey: 'FooKey', - objectBytesTransferred: 0, - objectSizeInBytes: 100, - status: 'initiated', - progressBar: $this->mockProgressBar - ); - - foreach ($statusColorMapping as $status => $value) { - $tracker->setStatus($status); - } - } - - /** - * Tests the default progress bar is initialized when not provided. - * - * @return void - */ - public function testDefaultProgressBarIsInitialized() - { - $tracker = new ObjectProgressTracker( - objectKey: 'FooKey', - objectBytesTransferred: 0, - objectSizeInBytes: 100, - status: 'initiated' - ); - $this->assertInstanceOf(ProgressBar::class, $tracker->getProgressBar()); - } -} From 1f094cd8fc74dbd429047ca0b55831b2ef4886c8 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 24 Feb 2025 07:13:57 -0800 Subject: [PATCH 12/62] chore: remove unused implementation - Remove progress bar color enum since the colors were moved into the specific format that requires them. --- .../S3Transfer/Progress/ProgressBarColorEnum.php | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/S3/S3Transfer/Progress/ProgressBarColorEnum.php diff --git a/src/S3/S3Transfer/Progress/ProgressBarColorEnum.php b/src/S3/S3Transfer/Progress/ProgressBarColorEnum.php deleted file mode 100644 index a932e76432..0000000000 --- a/src/S3/S3Transfer/Progress/ProgressBarColorEnum.php +++ /dev/null @@ -1,16 +0,0 @@ - Date: Mon, 24 Feb 2025 07:19:43 -0800 Subject: [PATCH 13/62] chore: remove invalid test TransferListener must be tested from the implementations that extends and use this abstract class. --- tests/S3/S3Transfer/TransferListenerTest.php | 215 ------------------- 1 file changed, 215 deletions(-) delete mode 100644 tests/S3/S3Transfer/TransferListenerTest.php diff --git a/tests/S3/S3Transfer/TransferListenerTest.php b/tests/S3/S3Transfer/TransferListenerTest.php deleted file mode 100644 index c0d0870661..0000000000 --- a/tests/S3/S3Transfer/TransferListenerTest.php +++ /dev/null @@ -1,215 +0,0 @@ -objectTransferInitiated('FooObjectKey', $requestArgs); - $this->assertEquals(1, $listener->getObjectsToBeTransferred()); - - $this->assertTrue($called); - } - - /** - * Tests object transfer is initiated. - * - * @return void - */ - public function testObjectTransferIsInitiated(): void - { - $called = false; - $listener = new TransferListener( - onObjectTransferInitiated: function () use (&$called) { - $called = true; - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey', $requestArgs); - $this->assertEquals(1, $listener->getObjectsToBeTransferred()); - - $this->assertTrue($called); - } - - /** - * Tests object transfer progress. - * - * @dataProvider objectTransferProgressProvider - * - * @param array $objects - * - * @return void - */ - public function testObjectTransferProgress( - array $objects - ): void { - $called = 0; - $listener = new TransferListener( - onObjectTransferProgress: function () use (&$called) { - $called++; - } - ); - $totalTransferred = 0; - foreach ($objects as $objectKey => $transferDetails) { - $requestArgs = []; - $listener->objectTransferInitiated( - $objectKey, - $requestArgs, - ); - $listener->objectTransferProgress( - $objectKey, - $transferDetails['transferredInBytes'], - $transferDetails['sizeInBytes'] - ); - $totalTransferred += $transferDetails['transferredInBytes']; - } - - $this->assertEquals(count($objects), $called); - $this->assertEquals(count($objects), $listener->getObjectsToBeTransferred()); - $this->assertEquals($totalTransferred, $listener->getObjectsBytesTransferred()); - } - - /** - * @return array - */ - public function objectTransferProgressProvider(): array - { - return [ - [ - [ - 'FooObjectKey1' => [ - 'sizeInBytes' => 100, - 'transferredInBytes' => 95, - ], - 'FooObjectKey2' => [ - 'sizeInBytes' => 500, - 'transferredInBytes' => 345, - ], - 'FooObjectKey3' => [ - 'sizeInBytes' => 1024, - 'transferredInBytes' => 256, - ], - ] - ] - ]; - } - - /** - * Tests object transfer failed. - * - * @return void - */ - public function testObjectTransferFailed(): void - { - $expectedBytesTransferred = 45; - $expectedReason = "Transfer failed!"; - $listener = new TransferListener( - onObjectTransferFailed: function ( - string $objectKey, - int $objectBytesTransferred, - string $reason - ) use ($expectedBytesTransferred, $expectedReason) { - $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); - $this->assertEquals($expectedReason, $reason); - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey', $requestArgs); - $listener->objectTransferFailed( - 'FooObjectKey', - $expectedBytesTransferred, - $expectedReason - ); - - $this->assertEquals(1, $listener->getObjectsTransferFailed()); - $this->assertEquals(0, $listener->getObjectsTransferCompleted()); - } - - /** - * Tests object transfer completed. - * - * @return void - */ - public function testObjectTransferCompleted(): void - { - $expectedBytesTransferred = 100; - $listener = new TransferListener( - onObjectTransferCompleted: function ($objectKey, $objectBytesTransferred) - use ($expectedBytesTransferred) { - $this->assertEquals($expectedBytesTransferred, $objectBytesTransferred); - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey', $requestArgs); - $listener->objectTransferProgress( - 'FooObjectKey', - $expectedBytesTransferred, - $expectedBytesTransferred - ); - $listener->objectTransferCompleted('FooObjectKey', $expectedBytesTransferred); - - $this->assertEquals(1, $listener->getObjectsTransferCompleted()); - $this->assertEquals($expectedBytesTransferred, $listener->getObjectsBytesTransferred()); - } - - /** - * Tests transfer is completed once all the objects in progress are completed. - * - * @return void - */ - public function testTransferCompleted(): void - { - $expectedObjectsTransferred = 2; - $expectedObjectBytesTransferred = 200; - $listener = new TransferListener( - onTransferCompleted: function(int $objectsTransferredCompleted, int $objectsBytesTransferred) - use ($expectedObjectsTransferred, $expectedObjectBytesTransferred) { - $this->assertEquals($expectedObjectsTransferred, $objectsTransferredCompleted); - $this->assertEquals($expectedObjectBytesTransferred, $objectsBytesTransferred); - } - ); - $requestArgs = []; - $listener->objectTransferInitiated('FooObjectKey_1', $requestArgs); - $listener->objectTransferInitiated('FooObjectKey_2', $requestArgs); - $listener->objectTransferProgress( - 'FooObjectKey_1', - 100, - 100 - ); - $listener->objectTransferProgress( - 'FooObjectKey_2', - 100, - 100 - ); - $listener->objectTransferCompleted( - 'FooObjectKey_1', - 100, - ); - $listener->objectTransferCompleted( - 'FooObjectKey_2', - 100, - ); - - $this->assertEquals($expectedObjectsTransferred, $listener->getObjectsTransferCompleted()); - $this->assertEquals($expectedObjectBytesTransferred, $listener->getObjectsBytesTransferred()); - } -} From 09e493fce9e85131fd12acacecbbadda7e7c881f Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 24 Feb 2025 13:04:21 -0800 Subject: [PATCH 14/62] fix: add nullable type Add nullable type to listenerNotifier property in the MultipartUploader implementation. --- src/S3/S3Transfer/MultipartUploader.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index c9885bb9c9..7dcfde1b4a 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -49,7 +49,7 @@ class MultipartUploader implements PromisorInterface private array $deferFns = []; /** @var TransferListenerNotifier | null */ - private TransferListenerNotifier | null $listenerNotifier; + private ?TransferListenerNotifier $listenerNotifier; /** Tracking Members */ /** @var TransferProgressSnapshot|null */ @@ -73,7 +73,7 @@ public function __construct( ?string $uploadId = null, array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, - TransferListenerNotifier $listenerNotifier = null, + ?TransferListenerNotifier $listenerNotifier = null, ) { $this->s3Client = $s3Client; $this->createMultipartArgs = $createMultipartArgs; From f10522ba5339d41f482672ccb0d709ec349d858f Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 26 Feb 2025 09:02:09 -0800 Subject: [PATCH 15/62] chore: add more tests - Tests for MultiProgressTracker - Tests for SingleProgressTracker - Tests for ProgressBarFormat - Tests for TransferProgressSnapshot - Tests for TransferListenerNotifier --- .../Exceptions/ProgressTrackerException.php | 8 + src/S3/S3Transfer/MultipartDownloader.php | 5 +- src/S3/S3Transfer/MultipartUploader.php | 3 +- .../Progress/MultiProgressBarFormat.php | 36 + .../Progress/MultiProgressTracker.php | 78 +- .../Progress/PlainProgressBarFormat.php | 3 +- .../Progress/ProgressBarFactoryInterface.php | 8 + .../Progress/SingleProgressTracker.php | 83 +- .../{ => Progress}/TransferListener.php | 2 +- .../TransferListenerNotifier.php | 4 +- .../Progress/TransferProgressBarFormat.php | 3 +- .../RangeGetMultipartDownloader.php | 1 + src/S3/S3Transfer/S3TransferManager.php | 2 + .../Progress/ConsoleProgressBarTest.php | 56 +- .../Progress/MultiProgressTrackerTest.php | 732 ++++++++++++++++++ .../Progress/ProgressBarFormatTest.php | 153 ++++ .../Progress/SingleProgressTrackerTest.php | 299 +++++++ .../Progress/TransferListenerNotifierTest.php | 39 + .../Progress/TransferProgressSnapshotTest.php | 84 ++ 19 files changed, 1521 insertions(+), 78 deletions(-) create mode 100644 src/S3/S3Transfer/Exceptions/ProgressTrackerException.php create mode 100644 src/S3/S3Transfer/Progress/MultiProgressBarFormat.php create mode 100644 src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php rename src/S3/S3Transfer/{ => Progress}/TransferListener.php (97%) rename src/S3/S3Transfer/{ => Progress}/TransferListenerNotifier.php (96%) create mode 100644 tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php create mode 100644 tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php create mode 100644 tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php create mode 100644 tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php create mode 100644 tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php diff --git a/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php new file mode 100644 index 0000000000..66d2a90cbd --- /dev/null +++ b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php @@ -0,0 +1,8 @@ + 'Aws\S3\S3Transfer\PartGetMultipartDownloader', - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => 'Aws\S3\S3Transfer\RangeGetMultipartDownloader', + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, default => throw new \InvalidArgumentException( "The config value for `multipart_download_type` must be one of:\n" . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 7dcfde1b4a..2172e604e8 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -5,6 +5,7 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; use GuzzleHttp\Promise\Create; @@ -468,7 +469,7 @@ private function callDeferredFns(): void */ private function containsChecksum(array $requestArgs): bool { - $algorithms = [ + static $algorithms = [ 'ChecksumCRC32', 'ChecksumCRC32C', 'ChecksumCRC64NVME', diff --git a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php new file mode 100644 index 0000000000..16f65a220c --- /dev/null +++ b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php @@ -0,0 +1,36 @@ +singleProgressTrackers = $singleProgressTrackers; @@ -41,6 +46,7 @@ public function __construct( $this->transferCount = $transferCount; $this->completed = $completed; $this->failed = $failed; + $this->progressBarFactory = $progressBarFactory; } /** @@ -83,6 +89,14 @@ public function getFailed(): int return $this->failed; } + /** + * @return ProgressBarFactoryInterface|Closure|null + */ + public function getProgressBarFactory(): ProgressBarFactoryInterface | Closure | null + { + return $this->progressBarFactory; + } + /** * @inheritDoc */ @@ -90,14 +104,28 @@ public function transferInitiated(array $context): void { $this->transferCount++; $snapshot = $context['progress_snapshot']; - if (isset($this->singleProgressTrackers[$snapshot['key']])) { - $progressTracker = $this->singleProgressTrackers[$snapshot['key']]; + if (isset($this->singleProgressTrackers[$snapshot->getIdentifier()])) { + $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; } else { - $progressTracker = new SingleProgressTracker( - clear: false, - ); + if ($this->progressBarFactory === null) { + $progressTracker = new SingleProgressTracker( + output: $this->output, + clear: false, + showProgressOnUpdate: false, + ); + } else { + $progressBarFactoryFn = $this->progressBarFactory; + $progressTracker = new SingleProgressTracker( + progressBar: $progressBarFactoryFn(), + output: $this->output, + clear: false, + showProgressOnUpdate: false, + ); + } + $this->singleProgressTrackers[$snapshot->getIdentifier()] = $progressTracker; } + $progressTracker->transferInitiated($context); $this->showProgress(); } @@ -144,29 +172,49 @@ public function showProgress(): void { fwrite($this->output, "\033[2J\033[H"); $percentsSum = 0; + /** + * @var $_ + * @var SingleProgressTracker $progressTracker + */ foreach ($this->singleProgressTrackers as $_ => $progressTracker) { $progressTracker->showProgress(); $percentsSum += $progressTracker->getProgressBar()->getPercentCompleted(); } + $allProgressBarWidth = ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH; + if (count($this->singleProgressTrackers) !== 0) { + $firstKey = array_key_first($this->singleProgressTrackers); + $allProgressBarWidth = $this->singleProgressTrackers[$firstKey] + ->getProgressBar()->getProgressBarWidth(); + } + $percent = (int) floor($percentsSum / $this->transferCount); + $multiProgressBarFormat = new MultiProgressBarFormat(); + $multiProgressBarFormat->setArgs([ + 'completed' => $this->completed, + 'failed' => $this->failed, + 'total' => $this->transferCount, + ]); $allTransferProgressBar = new ConsoleProgressBar( + progressBarWidth: $allProgressBarWidth, percentCompleted: $percent, - progressBarFormat: new PlainProgressBarFormat() + progressBarFormat: $multiProgressBarFormat ); - fwrite($this->output, "\n" . str_repeat( - '-', - $allTransferProgressBar->getProgressBarWidth()) + fwrite( + $this->output, + sprintf( + "\n%s\n", + str_repeat( + '-', + $allTransferProgressBar->getProgressBarWidth() + ) + ) ); fwrite( $this->output, sprintf( - "\n%s Completed: %d/%d, Failed: %d/%d\n", + "%s\n", $allTransferProgressBar->render(), - $this->completed, - $this->transferCount, - $this->failed, - $this->transferCount ) ); } diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php index 55a2ec1cba..0e89093fb5 100644 --- a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -6,12 +6,13 @@ final class PlainProgressBarFormat extends ProgressBarFormat { public function getFormatTemplate(): string { - return '[|progress_bar|] |percent|%'; + return "|object_name|:\n[|progress_bar|] |percent|%"; } public function getFormatParameters(): array { return [ + 'object_name', 'progress_bar', 'percent', ]; diff --git a/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php b/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php new file mode 100644 index 0000000000..87f0fea51c --- /dev/null +++ b/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php @@ -0,0 +1,8 @@ +progressBar = $progressBar; @@ -39,8 +44,9 @@ public function __construct( throw new \InvalidArgumentException("The type for $output must be a stream"); } $this->output = $output; - $this->objectName = $objectName; $this->clear = $clear; + $this->currentSnapshot = $currentSnapshot; + $this->showProgressOnUpdate = $showProgressOnUpdate; } /** @@ -60,18 +66,26 @@ public function getOutput(): mixed } /** - * @return string + * @return bool + */ + public function isClear(): bool { + return $this->clear; + } + + /** + * @return TransferProgressSnapshot|null */ - public function getObjectName(): string + public function getCurrentSnapshot(): ?TransferProgressSnapshot { - return $this->objectName; + return $this->currentSnapshot; } /** * @return bool */ - public function isClear(): bool { - return $this->clear; + public function isShowProgressOnUpdate(): bool + { + return $this->showProgressOnUpdate; } /** @@ -81,17 +95,15 @@ public function isClear(): bool { */ public function transferInitiated(array $context): void { - $snapshot = $context['progress_snapshot']; - $this->objectName = $snapshot->getIdentifier(); + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); - if ($progressFormat instanceof ColoredTransferProgressBarFormat) { - $progressFormat->setArg( - 'object_name', - $this->objectName - ); - } + // Probably a common argument + $progressFormat->setArg( + 'object_name', + $this->currentSnapshot->getIdentifier() + ); - $this->updateProgressBar($snapshot); + $this->updateProgressBar(); } /** @@ -101,6 +113,7 @@ public function transferInitiated(array $context): void */ public function bytesTransferred(array $context): void { + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -109,7 +122,7 @@ public function bytesTransferred(array $context): void ); } - $this->updateProgressBar($context['progress_snapshot']); + $this->updateProgressBar(); } /** @@ -119,6 +132,7 @@ public function bytesTransferred(array $context): void */ public function transferComplete(array $context): void { + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -127,10 +141,8 @@ public function transferComplete(array $context): void ); } - $snapshot = $context['progress_snapshot']; $this->updateProgressBar( - $snapshot, - $snapshot->getTotalBytes() === 0 + $this->currentSnapshot->getTotalBytes() === 0 ); } @@ -141,6 +153,7 @@ public function transferComplete(array $context): void */ public function transferFail(array $context): void { + $this->currentSnapshot = $context['progress_snapshot']; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -153,14 +166,13 @@ public function transferFail(array $context): void ); } - $this->updateProgressBar($context['progress_snapshot']); + $this->updateProgressBar(); } /** * Updates the progress bar with the transfer snapshot * and also call showProgress. * - * @param TransferProgressSnapshot $snapshot * @param bool $forceCompletion To force the progress bar to be * completed. This is useful for files where its size is zero, * for which a ratio will return zero, and hence the percent @@ -169,25 +181,26 @@ public function transferFail(array $context): void * @return void */ private function updateProgressBar( - TransferProgressSnapshot $snapshot, bool $forceCompletion = false ): void { if (!$forceCompletion) { $this->progressBar->setPercentCompleted( - ((int)floor($snapshot->ratioTransferred() * 100)) + ((int)floor($this->currentSnapshot->ratioTransferred() * 100)) ); } else { $this->progressBar->setPercentCompleted(100); } $this->progressBar->getProgressBarFormat()->setArgs([ - 'transferred' => $snapshot->getTransferredBytes(), - 'tobe_transferred' => $snapshot->getTotalBytes(), + 'transferred' => $this->currentSnapshot->getTransferredBytes(), + 'tobe_transferred' => $this->currentSnapshot->getTotalBytes(), 'unit' => 'B', ]); // Display progress - $this->showProgress(); + if ($this->showProgressOnUpdate) { + $this->showProgress(); + } } /** @@ -197,9 +210,9 @@ private function updateProgressBar( */ public function showProgress(): void { - if (empty($this->objectName)) { - throw new \RuntimeException( - "Progress tracker requires an object name to be set." + if ($this->currentSnapshot === null) { + throw new ProgressTrackerException( + "There is not snapshot to show progress for." ); } diff --git a/src/S3/S3Transfer/TransferListener.php b/src/S3/S3Transfer/Progress/TransferListener.php similarity index 97% rename from src/S3/S3Transfer/TransferListener.php rename to src/S3/S3Transfer/Progress/TransferListener.php index 2a24707d17..811a3bd47d 100644 --- a/src/S3/S3Transfer/TransferListener.php +++ b/src/S3/S3Transfer/Progress/TransferListener.php @@ -1,6 +1,6 @@ assertEquals( ConsoleProgressBar::DEFAULT_PROGRESS_BAR_WIDTH, @@ -45,7 +46,8 @@ public function testDefaultValues(): void { * * @return void */ - public function testSetPercentCompleted(): void { + public function testSetPercentCompleted(): void + { $progressBar = new ConsoleProgressBar(); $progressBar->setPercentCompleted(10); $this->assertEquals(10, $progressBar->getPercentCompleted()); @@ -56,7 +58,8 @@ public function testSetPercentCompleted(): void { /** * @return void */ - public function testSetCustomValues(): void { + public function testSetCustomValues(): void + { $progressBar = new ConsoleProgressBar( progressBarChar: '-', progressBarWidth: 10, @@ -77,7 +80,8 @@ public function testSetCustomValues(): void { * * @return void */ - public function testPercentIsNotOverOneHundred(): void { + public function testPercentIsNotOverOneHundred(): void + { $progressBar = new ConsoleProgressBar(); $progressBar->setPercentCompleted(150); $this->assertEquals(100, $progressBar->getPercentCompleted()); @@ -87,7 +91,7 @@ public function testPercentIsNotOverOneHundred(): void { * @param string $progressBarChar * @param int $progressBarWidth * @param int $percentCompleted - * @param ProgressBarFormat $progressBarFormat + * @param ProgressBarFormatTest $progressBarFormat * @param array $progressBarFormatArgs * @param string $expectedOutput * @@ -102,7 +106,8 @@ public function testProgressBarRendering( ProgressBarFormat $progressBarFormat, array $progressBarFormatArgs, string $expectedOutput - ): void { + ): void + { $progressBarFormat->setArgs($progressBarFormatArgs); $progressBar = new ConsoleProgressBar( $progressBarChar, @@ -119,39 +124,48 @@ public function testProgressBarRendering( * * @return array */ - public function progressBarRenderingProvider(): array { + public function progressBarRenderingProvider(): array + { return [ 'plain_progress_bar_format_1' => [ 'progress_bar_char' => '#', 'progress_bar_width' => 50, 'percent_completed' => 15, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[######## ] 15%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[######## ] 15%" ], 'plain_progress_bar_format_2' => [ 'progress_bar_char' => '#', 'progress_bar_width' => 50, 'percent_completed' => 45, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[####################### ] 45%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[####################### ] 45%" ], 'plain_progress_bar_format_3' => [ 'progress_bar_char' => '#', 'progress_bar_width' => 50, 'percent_completed' => 100, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[##################################################] 100%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[##################################################] 100%" ], 'plain_progress_bar_format_4' => [ 'progress_bar_char' => '.', 'progress_bar_width' => 50, 'percent_completed' => 100, 'progress_bar_format' => new PlainProgressBarFormat(), - 'progress_bar_format_args' => [], - 'expected_output' => '[..................................................] 100%' + 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', + ], + 'expected_output' => "FooObject:\n[..................................................] 100%" ], 'transfer_progress_bar_format_1' => [ 'progress_bar_char' => '#', @@ -159,11 +173,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 23, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 23, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[############ ] 23% 23/100 B' + 'expected_output' => "FooObject:\n[############ ] 23% 23/100 B" ], 'transfer_progress_bar_format_2' => [ 'progress_bar_char' => '#', @@ -171,11 +186,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 75, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 75, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[################### ] 75% 75/100 B' + 'expected_output' => "FooObject:\n[################### ] 75% 75/100 B" ], 'transfer_progress_bar_format_3' => [ 'progress_bar_char' => '#', @@ -183,11 +199,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 100, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 100, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[##############################] 100% 100/100 B' + 'expected_output' => "FooObject:\n[##############################] 100% 100/100 B" ], 'transfer_progress_bar_format_4' => [ 'progress_bar_char' => '*', @@ -195,11 +212,12 @@ public function progressBarRenderingProvider(): array { 'percent_completed' => 100, 'progress_bar_format' => new TransferProgressBarFormat(), 'progress_bar_format_args' => [ + 'object_name' => 'FooObject', 'transferred' => 100, 'tobe_transferred' => 100, 'unit' => 'B' ], - 'expected_output' => '[******************************] 100% 100/100 B' + 'expected_output' => "FooObject:\n[******************************] 100% 100/100 B" ], 'colored_progress_bar_format_1' => [ 'progress_bar_char' => '#', diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php new file mode 100644 index 0000000000..6c10b44df5 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -0,0 +1,732 @@ +assertEquals([], $progressTracker->getSingleProgressTrackers()); + $this->assertEquals(STDOUT, $progressTracker->getOutput()); + $this->assertEquals(0, $progressTracker->getTransferCount()); + $this->assertEquals(0, $progressTracker->getCompleted()); + $this->assertEquals(0, $progressTracker->getFailed()); + } + + /** + * @dataProvider customInitializationProvider + * + * @param array $progressTrackers + * @param mixed $output + * @param int $transferCount + * @param int $completed + * @param int $failed + * + * @return void + */ + public function testCustomInitialization( + array $progressTrackers, + mixed $output, + int $transferCount, + int $completed, + int $failed + ): void + { + $progressTracker = new MultiProgressTracker( + $progressTrackers, + $output, + $transferCount, + $completed, + $failed + ); + $this->assertSame($output, $progressTracker->getOutput()); + $this->assertSame($transferCount, $progressTracker->getTransferCount()); + $this->assertSame($completed, $progressTracker->getCompleted()); + $this->assertSame($failed, $progressTracker->getFailed()); + } + + /** + * @param ProgressBarFactoryInterface $progressBarFactory + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @return void + * @dataProvider multiProgressTrackerProvider + * + */ + public function testMultiProgressTracker( + Closure $progressBarFactory, + callable $eventInvoker, + array $expectedOutputs, + ): void + { + $output = fopen("php://temp", "w+"); + $progressTracker = new MultiProgressTracker( + output: $output, + progressBarFactory: $progressBarFactory + ); + $eventInvoker($progressTracker); + + $this->assertEquals( + $expectedOutputs['transfer_count'], + $progressTracker->getTransferCount() + ); + $this->assertEquals( + $expectedOutputs['completed'], + $progressTracker->getCompleted() + ); + $this->assertEquals( + $expectedOutputs['failed'], + $progressTracker->getFailed() + ); + $progress = $expectedOutputs['progress']; + if (is_array($progress)) { + $progress = join('', $progress); + } + rewind($output); + $this->assertEquals( + $progress, + stream_get_contents($output), + ); + } + + /** + * @return array + */ + public function customInitializationProvider(): array + { + return [ + 'custom_initialization_1' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + new SingleProgressTracker(), + ], + 'output' => STDOUT, + 'transfer_count' => 20, + 'completed' => 20, + 'failed' => 0, + ], + 'custom_initialization_2' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + ], + 'output' => STDOUT, + 'transfer_count' => 25, + 'completed' => 20, + 'failed' => 5, + ], + 'custom_initialization_3' => [ + 'progress_trackers' => [ + new SingleProgressTracker(), + new SingleProgressTracker(), + new SingleProgressTracker(), + new SingleProgressTracker(), + ], + 'output' => fopen("php://temp", "w"), + 'transfer_count' => 50, + 'completed' => 35, + 'failed' => 15, + ] + ]; + } + + /** + * @return array + */ + public function multiProgressTrackerProvider(): array + { + return [ + 'multi_progress_tracker_1_single_tracking_object' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $tracker): void + { + $tracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $tracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'transfer_count' => 1, + 'completed' => 0, + 'failed' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "Foo:\n[########## ] 50%\n", + "--------------------\n", + "[########## ] 50% Completed: 0/1, Failed: 0/1\n" + ] + ], + ], + 'multi_progress_tracker_2' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $progressTracker): void + { + $events = [ + 'transfer_initiated' => [ + 'request_args' => [], + 'total_bytes' => 1024 + ], + 'transfer_progress_1' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 342, + ], + 'transfer_progress_2' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 684, + ], + 'transfer_progress_3' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_complete' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ] + ]; + foreach ($events as $eventName => $event) { + if ($eventName === 'transfer_initiated') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferInitiated([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ) + ]); + } + } elseif (str_starts_with($eventName, 'transfer_progress')) { + for ($i = 0; $i < 3; $i++) { + $progressTracker->bytesTransferred([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_complete') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferComplete([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } + } + }, + 'expected_outputs' => [ + 'transfer_count' => 3, + 'completed' => 3, + 'failed' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/2, Failed: 0/2\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[## ] 11% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[#### ] 22% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[####### ] 33% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[######### ] 44% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[####### ] 33%\n", + "--------------------\n", + "[########### ] 55% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[############# ] 66% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[############### ] 77% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[############# ] 66%\n", + "--------------------\n", + "[################## ] 88% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 1/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 2/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\n", + "--------------------\n", + "[####################] 100% Completed: 3/3, Failed: 0/3\n", + + ] + ], + ], + 'multi_progress_tracker_3' => [ + 'progress_bar_factory' => function() { + return new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat(), + ); + }, + 'event_invoker' => function (MultiProgressTracker $progressTracker): void + { + $events = [ + 'transfer_initiated' => [ + 'request_args' => [], + 'total_bytes' => 1024 + ], + 'transfer_progress_1' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 342, + ], + 'transfer_progress_2' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 684, + ], + 'transfer_progress_3' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_complete' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 1024, + ], + 'transfer_fail' => [ + 'request_args' => [], + 'total_bytes' => 1024, + 'bytes_transferred' => 0, + 'reason' => 'Transfer failed' + ] + ]; + foreach ($events as $eventName => $event) { + if ($eventName === 'transfer_initiated') { + for ($i = 0; $i < 5; $i++) { + $progressTracker->transferInitiated([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ) + ]); + } + } elseif (str_starts_with($eventName, 'transfer_progress')) { + for ($i = 0; $i < 3; $i++) { + $progressTracker->bytesTransferred([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_complete') { + for ($i = 0; $i < 3; $i++) { + $progressTracker->transferComplete([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + $event['bytes_transferred'], + $event['total_bytes'], + ) + ]); + } + } elseif ($eventName === 'transfer_fail') { + // Just two of them will fail + for ($i = 3; $i < 5; $i++) { + $progressTracker->transferFail([ + 'request_args' => $event['request_args'], + 'progress_snapshot' => new TransferProgressSnapshot( + "FooObject_$i", + 0, + $event['total_bytes'], + ), + 'reason' => $event['reason'] + ]); + } + } + } + }, + 'expected_outputs' => [ + 'transfer_count' => 5, + 'completed' => 3, + 'failed' => 2, + 'progress' => [ + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/1, Failed: 0/1\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/2, Failed: 0/2\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/3, Failed: 0/3\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/4, Failed: 0/4\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[ ] 0%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[ ] 0% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[ ] 0%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[# ] 6% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[ ] 0%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[### ] 13% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####### ] 33%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[#### ] 19% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[####### ] 33%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[##### ] 26% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[####### ] 33%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[####### ] 33% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[############# ] 66%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[######## ] 39% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[############# ] 66%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[######### ] 46% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[############# ] 66%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[########### ] 53% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 0/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 1/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 2/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 0/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 1/5\n", + "\033[2J\033[H\r\n", + "FooObject_0:\n", + "[####################] 100%\r\n", + "FooObject_1:\n", + "[####################] 100%\r\n", + "FooObject_2:\n", + "[####################] 100%\r\n", + "FooObject_3:\n", + "[ ] 0%\r\n", + "FooObject_4:\n", + "[ ] 0%\n", + "--------------------\n", + "[############ ] 60% Completed: 3/5, Failed: 2/5\n", + ] + ], + ] + ]; + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php new file mode 100644 index 0000000000..5e1c03a397 --- /dev/null +++ b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php @@ -0,0 +1,153 @@ +setArgs($args); + + $this->assertEquals($expectedFormat, $progressBarFormat->format()); + } + + /** + * @return array[] + */ + public function progressBarFormatProvider(): array + { + return [ + 'plain_progress_bar_format_1' => [ + 'implementation_class' => PlainProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..........', + 'percent' => 100, + ], + 'expected_format' => "foo:\n[..........] 100%", + ], + 'plain_progress_bar_format_2' => [ + 'implementation_class' => PlainProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..... ', + 'percent' => 50, + ], + 'expected_format' => "foo:\n[..... ] 50%", + ], + 'transfer_progress_bar_format_1' => [ + 'implementation_class' => TransferProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..........', + 'percent' => 100, + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_format' => "foo:\n[..........] 100% 100/100 B", + ], + 'transfer_progress_bar_format_2' => [ + 'implementation_class' => TransferProgressBarFormat::class, + 'args' => [ + 'object_name' => 'foo', + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B' + ], + 'expected_format' => "foo:\n[..... ] 50% 50/100 B", + ], + 'colored_transfer_progress_bar_format_1_color_code_black_defaulted' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject' + ], + 'expected_format' => "FooObject:\n\033[30m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_1' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[34m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_2' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[32m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_3' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..... ', + 'percent' => 50, + 'transferred' => 50, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::RED_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[31m[..... ] 50% 50/100 B \033[0m", + ], + 'colored_transfer_progress_bar_format_4' => [ + 'implementation_class' => ColoredTransferProgressBarFormat::class, + 'args' => [ + 'progress_bar' => '..........', + 'percent' => 100, + 'transferred' => 100, + 'tobe_transferred' => 100, + 'unit' => 'B', + 'object_name' => 'FooObject', + 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE + ], + 'expected_format' => "FooObject:\n\033[34m[..........] 100% 100/100 B \033[0m", + ], + ]; + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php new file mode 100644 index 0000000000..04f8be80fe --- /dev/null +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -0,0 +1,299 @@ +assertInstanceOf(ConsoleProgressBar::class, $progressTracker->getProgressBar()); + $this->assertEquals(STDOUT, $progressTracker->getOutput()); + $this->assertTrue($progressTracker->isClear()); + $this->assertNull($progressTracker->getCurrentSnapshot()); + } + + /** + * @param ProgressBarInterface $progressBar + * @param mixed $output + * @param bool $clear + * @param TransferProgressSnapshot $snapshot + * + * @dataProvider customInitializationProvider + * + * @return void + */ + public function testCustomInitialization( + ProgressBarInterface $progressBar, + mixed $output, + bool $clear, + TransferProgressSnapshot $snapshot + ): void + { + $progressTracker = new SingleProgressTracker( + $progressBar, + $output, + $clear, + $snapshot, + ); + $this->assertSame($progressBar, $progressTracker->getProgressBar()); + $this->assertSame($output, $progressTracker->getOutput()); + $this->assertSame($clear, $progressTracker->isClear()); + $this->assertSame($snapshot, $progressTracker->getCurrentSnapshot()); + } + + /** + * @return array[] + */ + public function customInitializationProvider(): array + { + return [ + 'initialization_1' => [ + 'progress_bar' => new ConsoleProgressBar(), + 'output' => STDOUT, + 'clear' => true, + 'snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 10 + ), + ], + 'initialization_2' => [ + 'progress_bar' => new ConsoleProgressBar(), + 'output' => fopen('php://temp', 'w'), + 'clear' => true, + 'snapshot' => new TransferProgressSnapshot( + 'FooTest', + 50, + 500 + ), + ], + ]; + } + + /** + * @param ProgressBarInterface $progressBar + * @param callable $eventInvoker + * @param array $expectedOutputs + * + * @dataProvider singleProgressTrackingProvider + * + * @return void + */ + public function testSingleProgressTracking( + ProgressBarInterface $progressBar, + callable $eventInvoker, + array $expectedOutputs, + ): void + { + $output = fopen('php://temp', 'w'); + $progressTracker = new SingleProgressTracker( + $progressBar, + $output, + ); + $eventInvoker($progressTracker); + $this->assertEquals( + $expectedOutputs['identifier'], + $progressTracker->getCurrentSnapshot()->getIdentifier() + ); + $this->assertEquals( + $expectedOutputs['transferred_bytes'], + $progressTracker->getCurrentSnapshot()->getTransferredBytes() + ); + $this->assertEquals( + $expectedOutputs['total_bytes'], + $progressTracker->getCurrentSnapshot()->getTotalBytes() + ); + + $progress = $expectedOutputs['progress']; + if (is_array($progress)) { + $progress = join('', $expectedOutputs['progress']); + } + rewind($output); + $this->assertEquals( + $progress, + stream_get_contents($output) + ); + } + + /** + * @return array[] + */ + public function singleProgressTrackingProvider(): array + { + return [ + 'progress_rendering_1_transfer_initiated' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 0, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%" + ] + ], + ], + 'progress_rendering_2_transfer_progress' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $progressTracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 256, + 1024 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 256, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[##### ] 25%" + ] + ], + ], + 'progress_rendering_3_transfer_force_completion_when_total_bytes_zero' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new PlainProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 0 + ) + ]); + $progressTracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 1024, + 0 + ) + ]); + $progressTracker->transferComplete([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 2048, + 0 + ) + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 2048, + 'total_bytes' => 0, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[ ] 0%", + "\033[2J\033[H\r\n", + "Foo:\n[####################] 100%" + ] + ], + ], + 'progress_rendering_4_transfer_fail_with_colored_transfer_format' => [ + 'progress_bar' => new ConsoleProgressBar( + progressBarWidth: 20, + progressBarFormat: new ColoredTransferProgressBarFormat() + ), + 'event_invoker' => function (singleProgressTracker $progressTracker): void + { + $progressTracker->transferInitiated([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 0, + 1024 + ) + ]); + $progressTracker->bytesTransferred([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ) + ]); + $progressTracker->transferFail([ + 'request_args' => [], + 'progress_snapshot' => new TransferProgressSnapshot( + 'Foo', + 512, + 1024 + ), + 'reason' => "Error transferring!" + ]); + }, + 'expected_outputs' => [ + 'identifier' => 'Foo', + 'transferred_bytes' => 512, + 'total_bytes' => 1024, + 'progress' => [ + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[30m[ ] 0% 0/1024 B ", + "\033[0m", + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[34m[########## ] 50% 512/1024 B ", + "\033[0m", + "\033[2J\033[H\r\n", + "Foo:\n", + "\033[31m[########## ] 50% 512/1024 B Error transferring!", + "\033[0m" + ] + ], + ] + ]; + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php new file mode 100644 index 0000000000..7ccd168ede --- /dev/null +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -0,0 +1,39 @@ +getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + $this->getMockBuilder(TransferListener::class) + ->getMock(), + ]; + foreach ($listeners as $listener) { + $listener->expects($this->once())->method('transferInitiated'); + $listener->expects($this->once())->method('bytesTransferred'); + $listener->expects($this->once())->method('transferComplete'); + $listener->expects($this->once())->method('transferFail'); + } + $listenerNotifier = new TransferListenerNotifier($listeners); + $listenerNotifier->transferInitiated([]); + $listenerNotifier->bytesTransferred([]); + $listenerNotifier->transferComplete([]); + $listenerNotifier->transferFail([]); + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php new file mode 100644 index 0000000000..c0033324bb --- /dev/null +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -0,0 +1,84 @@ + 'Bar'] + ); + + $this->assertEquals($snapshot->getIdentifier(), 'FooObject'); + $this->assertEquals($snapshot->getTransferredBytes(), 0); + $this->assertEquals($snapshot->getTotalBytes(), 10); + $this->assertEquals($snapshot->getResponse(), ['Foo' => 'Bar']); + } + + /** + * @param int $transferredBytes + * @param int $totalBytes + * @param float $expectedRatio + * + * @return void + * @dataProvider ratioTransferredProvider + * + */ + public function testRatioTransferred( + int $transferredBytes, + int $totalBytes, + float $expectedRatio + ): void + { + $snapshot = new TransferProgressSnapshot( + 'FooObject', + $transferredBytes, + $totalBytes + ); + $this->assertEquals($expectedRatio, $snapshot->ratioTransferred()); + } + + /** + * @return array + */ + public function ratioTransferredProvider(): array + { + return [ + 'ratio_1' => [ + 'transferred_bytes' => 10, + 'total_bytes' => 100, + 'expected_ratio' => 10 / 100, + ], + 'ratio_2_transferred_bytes_zero' => [ + 'transferred_bytes' => 0, + 'total_bytes' => 100, + 'expected_ratio' => 0, + ], + 'ratio_3_unknown_total_bytes' => [ + 'transferred_bytes' => 100, + 'total_bytes' => 0, + 'expected_ratio' => 0, + ], + 'ratio_4' => [ + 'transferred_bytes' => 50, + 'total_bytes' => 256, + 'expected_ratio' => 50 / 256, + ], + 'ratio_5' => [ + 'transferred_bytes' => 250, + 'total_bytes' => 256, + 'expected_ratio' => 250 / 256, + ], + ]; + } +} \ No newline at end of file From a55e1b3dc32e0414b54be15becc540ef496230ba Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Mar 2025 08:23:40 -0700 Subject: [PATCH 16/62] chore: add upload unit tests and refactor - Refactor code to address some styling related feedback. - Add upload and uploadDirectory unit tests. --- src/S3/S3Transfer/DownloadResponse.php | 23 - .../Models/DownloadDirectoryResponse.php | 47 + src/S3/S3Transfer/Models/DownloadResponse.php | 33 + .../{ => Models}/UploadDirectoryResponse.php | 2 +- .../{ => Models}/UploadResponse.php | 2 +- src/S3/S3Transfer/MultipartDownloader.php | 16 +- src/S3/S3Transfer/MultipartUploader.php | 72 +- .../S3Transfer/PartGetMultipartDownloader.php | 2 +- .../Progress/SingleProgressTracker.php | 3 +- .../RangeGetMultipartDownloader.php | 2 - src/S3/S3Transfer/S3TransferManager.php | 349 +++-- .../S3/S3Transfer/MultipartDownloaderTest.php | 22 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 2 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 1345 +++++++++++++++++ 14 files changed, 1773 insertions(+), 147 deletions(-) delete mode 100644 src/S3/S3Transfer/DownloadResponse.php create mode 100644 src/S3/S3Transfer/Models/DownloadDirectoryResponse.php create mode 100644 src/S3/S3Transfer/Models/DownloadResponse.php rename src/S3/S3Transfer/{ => Models}/UploadDirectoryResponse.php (94%) rename src/S3/S3Transfer/{ => Models}/UploadResponse.php (90%) create mode 100644 tests/S3/S3Transfer/S3TransferManagerTest.php diff --git a/src/S3/S3Transfer/DownloadResponse.php b/src/S3/S3Transfer/DownloadResponse.php deleted file mode 100644 index d501882c24..0000000000 --- a/src/S3/S3Transfer/DownloadResponse.php +++ /dev/null @@ -1,23 +0,0 @@ -content; - } - - public function getMetadata(): array - { - return $this->metadata; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php new file mode 100644 index 0000000000..d493630e16 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php @@ -0,0 +1,47 @@ +objectsDownloaded = $objectsUploaded; + $this->objectsFailed = $objectsFailed; + } + + /** + * @return int + */ + public function getObjectsDownloaded(): int + { + return $this->objectsDownloaded; + } + + /** + * @return int + */ + public function getObjectsFailed(): int + { + return $this->objectsFailed; + } + + public function __toString(): string + { + return sprintf( + "DownloadDirectoryResponse: %d objects downloaded, %d objects failed", + $this->objectsDownloaded, + $this->objectsFailed + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadResponse.php b/src/S3/S3Transfer/Models/DownloadResponse.php new file mode 100644 index 0000000000..836ba428bc --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadResponse.php @@ -0,0 +1,33 @@ +data; + } + + /** + * @return array + */ + public function getMetadata(): array + { + return $this->metadata; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/UploadDirectoryResponse.php b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php similarity index 94% rename from src/S3/S3Transfer/UploadDirectoryResponse.php rename to src/S3/S3Transfer/Models/UploadDirectoryResponse.php index b098be7ac0..867442986c 100644 --- a/src/S3/S3Transfer/UploadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php @@ -1,6 +1,6 @@ stream = $stream; + // Position at the end of the stream + $this->stream->seek($stream->getSize()); } $this->currentSnapshot = $currentSnapshot; $this->listenerNotifier = $listenerNotifier; @@ -128,13 +131,16 @@ public function promise(): PromiseInterface { return Coroutine::of(function () { $this->downloadInitiated($this->requestArgs); + $result = ['@metadata'=>[]]; try { - yield $this->s3Client->executeAsync($this->nextCommand()) + $result = yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function (ResultInterface $result) { // Calculate object size and parts count. $this->computeObjectDimensions($result); // Trigger first part completed $this->partDownloadCompleted($result); + + return $result; })->otherwise(function ($reason) { $this->partDownloadFailed($reason); @@ -156,7 +162,7 @@ public function promise(): PromiseInterface })->otherwise(function ($reason) { $this->partDownloadFailed($reason); - return $reason; + throw $reason; }); } catch (\Throwable $reason) { $this->downloadFailed($reason); @@ -169,10 +175,9 @@ public function promise(): PromiseInterface // Transfer completed $this->downloadComplete(); - // TODO: yield the stream wrapped in a modeled transfer success response. yield Create::promiseFor(new DownloadResponse( $this->stream, - [] + $result['@metadata'] )); }); } @@ -260,6 +265,7 @@ private function downloadInitiated(array $commandArgs): void */ private function downloadFailed(\Throwable $reason): void { + $this->stream->close(); $this->listenerNotifier?->transferFail([ 'request_args' => $this->requestArgs, 'progress_snapshot' => $this->currentSnapshot, @@ -342,7 +348,7 @@ private function downloadComplete(): void * * @return string */ - public static function chooseDownloaderClassName( + public static function chooseDownloaderClass( string $multipartDownloadType ): string { diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 2172e604e8..a99162d21e 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -3,8 +3,11 @@ use Aws\CommandInterface; use Aws\CommandPool; +use Aws\HashingStream; +use Aws\PhpHash; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; @@ -14,6 +17,7 @@ use GuzzleHttp\Psr7\LazyOpenStream; use GuzzleHttp\Psr7\Utils; use Psr\Http\Message\StreamInterface; +use Psr\Http\Message\StreamInterface as Stream; use Throwable; /** @@ -21,8 +25,8 @@ */ class MultipartUploader implements PromisorInterface { - const PART_MIN_SIZE = 5242880; - const PART_MAX_SIZE = 5368709120; + public const PART_MIN_SIZE = 5 * 1024 * 1024; // 5 MBs + public const PART_MAX_SIZE = 5 * 1024 * 1024 * 1024; // 5 GBs public const PART_MAX_NUM = 10000; /** @var S3ClientInterface */ @@ -60,6 +64,8 @@ class MultipartUploader implements PromisorInterface * @param S3ClientInterface $s3Client * @param array $createMultipartArgs * @param array $config + * - part_size: (int, optional) + * - concurrency: (int, required) * @param string | StreamInterface $source * @param string|null $uploadId * @param array $parts @@ -78,6 +84,7 @@ public function __construct( ) { $this->s3Client = $s3Client; $this->createMultipartArgs = $createMultipartArgs; + $this->validateConfig($config); $this->config = $config; $this->body = $this->parseBody($source); $this->uploadId = $uploadId; @@ -86,6 +93,25 @@ public function __construct( $this->listenerNotifier = $listenerNotifier; } + /** + * @param array $config + * + * @return void + */ + private function validateConfig(array &$config): void + { + if (isset($config['part_size'])) { + if ($config['part_size'] < self::PART_MIN_SIZE || $config['part_size'] > self::PART_MAX_SIZE) { + throw new \InvalidArgumentException( + "The config `part_size` value must be between " + . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE . "." + ); + } + } else { + $config['part_size'] = self::PART_MIN_SIZE; + } + } + /** * @return string|null */ @@ -143,7 +169,7 @@ public function promise(): PromiseInterface /** * @return PromiseInterface */ - public function createMultipartUpload(): PromiseInterface + private function createMultipartUpload(): PromiseInterface { $requestArgs = [...$this->createMultipartArgs]; $this->uploadInitiated($requestArgs); @@ -163,19 +189,13 @@ public function createMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - public function uploadParts(): PromiseInterface + private function uploadParts(): PromiseInterface { $this->calculatedObjectSize = 0; $isSeekable = $this->body->isSeekable(); - $partSize = $this->config['part_size'] ?? self::PART_MIN_SIZE; - if ($partSize > self::PART_MAX_SIZE) { - return Create::rejectionFor( - "The part size should not exceed " . self::PART_MAX_SIZE . " bytes." - ); - } - + $partSize = $this->config['part_size']; $commands = []; - for ($partNo = 1; + for ($partNo = count($this->parts) + 1; $isSeekable ? $this->body->tell() < $this->body->getSize() : !$this->body->eof(); @@ -196,15 +216,20 @@ public function uploadParts(): PromiseInterface break; } + $uploadPartCommandArgs = [ ...$this->createMultipartArgs, 'UploadId' => $this->uploadId, 'PartNumber' => $partNo, - 'Body' => $partBody, 'ContentLength' => $partBody->getSize(), ]; // To get `requestArgs` when notifying the bytesTransfer listeners. - $uploadPartCommandArgs['requestArgs'] = $uploadPartCommandArgs; + $uploadPartCommandArgs['requestArgs'] = [...$uploadPartCommandArgs]; + // Attach body + $uploadPartCommandArgs['Body'] = $this->decorateWithHashes( + $partBody, + $uploadPartCommandArgs + ); $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); $commands[] = $command; $this->calculatedObjectSize += $partBody->getSize(); @@ -244,7 +269,7 @@ public function uploadParts(): PromiseInterface /** * @return PromiseInterface */ - public function completeMultipartUpload(): PromiseInterface + private function completeMultipartUpload(): PromiseInterface { $this->sortParts(); $completeMultipartUploadArgs = [ @@ -275,7 +300,7 @@ public function completeMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - public function abortMultipartUpload(): PromiseInterface + private function abortMultipartUpload(): PromiseInterface { $command = $this->s3Client->getCommand('AbortMultipartUpload', [ ...$this->createMultipartArgs, @@ -484,4 +509,19 @@ private function containsChecksum(array $requestArgs): bool return false; } + + /** + * @param Stream $stream + * @param array $data + * + * @return Stream + */ + private function decorateWithHashes(Stream $stream, array &$data): StreamInterface + { + // Decorate source with a hashing stream + $hash = new PhpHash('sha256'); + return new HashingStream($stream, $hash, function ($result) use (&$data) { + $data['ContentSHA256'] = bin2hex($result); + }); + } } diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index 2ceb3987ab..bb30b3e952 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -16,7 +16,7 @@ class PartGetMultipartDownloader extends MultipartDownloader * * @return CommandInterface */ - protected function nextCommand() : CommandInterface + protected function nextCommand(): CommandInterface { if ($this->currentPartNo === 0) { $this->currentPartNo = 1; diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index fca06b00d1..1024068d4f 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -68,7 +68,8 @@ public function getOutput(): mixed /** * @return bool */ - public function isClear(): bool { + public function isClear(): bool + { return $this->clear; } diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index d6dfdc5549..5fa10cde99 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -25,9 +25,7 @@ class RangeGetMultipartDownloader extends MultipartDownloader * using range get. This option MUST be set when using range get. * @param int $currentPartNo * @param int $objectPartsCount - * @param int $objectCompletedPartsCount * @param int $objectSizeInBytes - * @param int $objectBytesTransferred * @param string $eTag * @param StreamInterface|null $stream * @param TransferProgressSnapshot|null $currentSnapshot diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index d2f9e81610..513908c264 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,20 +2,27 @@ namespace Aws\S3\S3Transfer; +use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; +use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\Progress\MultiProgressTracker; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use FilesystemIterator; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use InvalidArgumentException; use Psr\Http\Message\StreamInterface; +use Throwable; use function Aws\filter; use function Aws\map; -use function Aws\recursive_dir_iterator; class S3TransferManager { @@ -41,20 +48,20 @@ class S3TransferManager * a default client will be created where its region will be the one * resolved from either the default from the config or the provided. * @param array $config - * - target_part_size_bytes: (int, default=(8388608 `8MB`)) \ + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) * The minimum part size to be used in a multipart upload/download. - * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) \ + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) * The threshold to decided whether a multipart upload is needed. - * - checksum_validation_enabled: (bool, default=true) \ + * - checksum_validation_enabled: (bool, default=true) * To decide whether a checksum validation will be applied to the response. - * - checksum_algorithm: (string, default='crc32') \ + * - checksum_algorithm: (string, default='crc32') * The checksum algorithm to be used in an upload request. * - multipart_download_type: (string, default='partGet') * The download type to be used in a multipart download. - * - concurrency: (int, default=5) \ + * - concurrency: (int, default=5) * Maximum number of concurrent operations allowed during a multipart * upload/download. - * - track_progress: (bool, default=false) \ + * - track_progress: (bool, default=false) * To enable progress tracker in a multipart upload/download, and or * a directory upload/download operation. * - region: (string, default="us-east-2") @@ -63,13 +70,28 @@ public function __construct( ?S3ClientInterface $s3Client = null, array $config = [] ) { + $this->config = $config + self::$defaultConfig; if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { $this->s3Client = $s3Client; } + } - $this->config = $config + self::$defaultConfig; + /** + * @return S3ClientInterface + */ + public function getS3Client(): S3ClientInterface + { + return $this->s3Client; + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; } /** @@ -86,7 +108,10 @@ public function __construct( * - track_progress: (bool, optional) To override the default option for * enabling progress tracking. If this option is resolved as true and * a progressTracker parameter is not provided then, a default implementation - * will be resolved. + * will be resolved. This option is intended to make the operation to use + * a default progress tracker implementation when $progressTracker is null. + * - checksum_algorithm: (bool, optional) To override the default + * checksum algorithm. * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * @@ -100,15 +125,9 @@ public function upload( ?TransferListener $progressTracker = null, ): PromiseInterface { - // Make sure the source is what is expected - if (!is_string($source) && !$source instanceof StreamInterface) { - throw new \InvalidArgumentException( - '`source` must be a string or a StreamInterface' - ); - } // Make sure it is a valid in path in case of a string if (is_string($source) && !is_readable($source)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "Please provide a valid readable file path or a valid stream as source." ); } @@ -123,12 +142,18 @@ public function upload( $mupThreshold = $config['multipart_upload_threshold_bytes'] ?? $this->config['multipart_upload_threshold_bytes']; if ($mupThreshold < MultipartUploader::PART_MIN_SIZE) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "The provided config `multipart_upload_threshold_bytes`" ."must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE ); } + if (!isset($requestArgs['ChecksumAlgorithm'])) { + $algorithm = $config['checksum_algorithm'] + ?? $this->config['checksum_algorithm']; + $requestArgs['ChecksumAlgorithm'] = strtoupper($algorithm); + } + if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = new SingleProgressTracker(); @@ -158,19 +183,31 @@ public function upload( ); } - /** * @param string $directory * @param string $bucketTo * @param array $requestArgs - * @param array $config + * @param array $config The config options for this request that are: * - follow_symbolic_links: (bool, optional, defaulted to false) * - recursive: (bool, optional, defaulted to false) * - s3_prefix: (string, optional, defaulted to null) - * - filter: (Closure(string), optional) + * - filter: (Closure(SplFileInfo|string), optional) + * By default an instance of SplFileInfo will be provided, however + * you can annotate the parameter with a string type and by doing + * so you will get the full path of the file. * - s3_delimiter: (string, optional, defaulted to `/`) - * - put_object_request_callback: (Closure, optional) - * - failure_policy: (Closure, optional) + * - put_object_request_callback: (Closure, optional) A callback function + * to be invoked right before the request initiates and that will receive + * as parameter the request arguments for each upload request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on an upload failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $uploadDirectoryRequestArgs: (array) The arguments for the upload + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $uploadDirectoryResponse: (UploadDirectoryResponse) The upload response + * to that point in the upload process. * - track_progress: (bool, optional) To override the default option for * enabling progress tracking. If this option is resolved as true and * a progressTracker parameter is not provided then, a default implementation @@ -193,7 +230,7 @@ public function uploadDirectory( ): PromiseInterface { if (!is_dir($directory)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "Please provide a valid directory path. " . "Provided = " . $directory ); @@ -207,14 +244,44 @@ public function uploadDirectory( $filter = null; if (isset($config['filter'])) { if (!is_callable($config['filter'])) { - throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); } $filter = $config['filter']; } + $putObjectRequestCallback = null; + if (isset($config['put_object_request_callback'])) { + if (!is_callable($config['put_object_request_callback'])) { + throw new InvalidArgumentException( + "The parameter \$config['put_object_request_callback'] must be callable." + ); + } + + $putObjectRequestCallback = $config['put_object_request_callback']; + } + + $failurePolicyCallback = null; + if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { + throw new InvalidArgumentException( + "The parameter \$config['failure_policy'] must be callable." + ); + } elseif (isset($config['failure_policy'])) { + $failurePolicyCallback = $config['failure_policy']; + } + + $dirIterator = new \RecursiveDirectoryIterator($directory); + $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); + if (($config['follow_symbolic_links'] ?? false) === true) { + $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); + } + + if (($config['recursive'] ?? false) === true) { + $dirIterator = new \RecursiveIteratorIterator($dirIterator); + } + $files = filter( - recursive_dir_iterator($directory), + $dirIterator, function ($file) use ($filter) { if ($filter !== null) { return !is_dir($file) && $filter($file); @@ -237,7 +304,7 @@ function ($file) use ($filter) { $relativePath = substr($file, strlen($baseDir)); if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( - "The filename must not contain the provided delimiter `". $delimiter ."`" + "The filename `$relativePath` must not contain the provided delimiter `$delimiter`" ); } $objectKey = $prefix.$relativePath; @@ -246,29 +313,54 @@ function ($file) use ($filter) { $delimiter, $objectKey ); + $uploadRequestArgs = [ + ...$requestArgs, + 'Bucket' => $bucketTo, + 'Key' => $objectKey, + ]; + if ($putObjectRequestCallback !== null) { + $putObjectRequestCallback($uploadRequestArgs); + } + $promises[] = $this->upload( $file, - [ - ...$requestArgs, - 'Bucket' => $bucketTo, - 'Key' => $objectKey, - ], + $uploadRequestArgs, $config, array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, - )->then(function ($result) use (&$objectsUploaded) { + )->then(function (UploadResponse $response) use (&$objectsUploaded) { $objectsUploaded++; - return $result; - })->otherwise(function ($reason) use (&$objectsFailed) { + return $response; + })->otherwise(function ($reason) use ( + $failurePolicyCallback, + $uploadRequestArgs, + $requestArgs, + &$objectsUploaded, + &$objectsFailed + ) { $objectsFailed++; + if($failurePolicyCallback !== null) { + call_user_func( + $failurePolicyCallback, + $requestArgs, + $uploadRequestArgs, + $reason, + new UploadDirectoryResponse( + $objectsUploaded, + $objectsFailed + ) + ); + + return; + } throw $reason; }); } return Each::ofLimitAll($promises, $this->config['concurrency']) - ->then(function ($results) use ($objectsUploaded, $objectsFailed) { + ->then(function ($_) use ($objectsUploaded, $objectsFailed) { return new UploadDirectoryResponse($objectsUploaded, $objectsFailed); }); } @@ -280,19 +372,20 @@ function ($file) use ($filter) { * @param array $downloadArgs The getObject request arguments to be provided as part * of each get object operation, except for the bucket and key, which * are already provided as the source. - * @param array $config The configuration to be used for this operation. - * - multipart_download_type: (string, optional) \ + * @param array $config The configuration to be used for this operation: + * - multipart_download_type: (string, optional) * Overrides the resolved value from the transfer manager config. - * - track_progress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and track_progress resolved as true then, a default progress listener - * implementation will be used. - * - minimum_part_size: (int) \ - * The minimum part size in bytes to be used in a range multipart download. - * If this parameter is not provided then it fallbacks to the transfer - * manager `target_part_size_bytes` config value. + * - checksum_validation_enabled: (bool, optional) Overrides the resolved + * value from transfer manager config for whether checksum validation + * should be done. This option will be considered just if ChecksumMode + * is not present in the request args. + * - track_progress: (bool) Overrides the config option set in the transfer + * manager instantiation to decide whether transfer progress should be + * tracked. + * - minimum_part_size: (int) The minimum part size in bytes to be used + * in a range multipart download. If this parameter is not provided + * then it fallbacks to the transfer manager `target_part_size_bytes` + * config value. * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * @@ -314,7 +407,16 @@ public function download( 'Key' => $this->requireNonEmpty($source['Key'], "A valid key must be provided."), ]; } else { - throw new \InvalidArgumentException('Source must be a string or an array of strings'); + throw new InvalidArgumentException('Source must be a string or an array of strings'); + } + + if (!isset($requestArgs['ChecksumMode'])) { + $checksumEnabled = $config['checksum_validation_enabled'] + ?? $this->config['checksum_validation_enabled'] + ?? false; + if ($checksumEnabled) { + $requestArgs['ChecksumMode'] = 'enabled'; + } } if ($progressTracker === null @@ -352,31 +454,44 @@ public function download( * @param array $downloadArgs The getObject request arguments to be provided * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. - * @param array $config The config options for this download directory operation. \ - * - track_progress: (bool) \ - * Overrides the config option set in the transfer manager instantiation - * to decide whether transfer progress should be tracked. If a `progressListenerFactory` - * was not provided when the transfer manager instance was created - * and track_progress resolved as true then, a default progress listener - * implementation will be used. - * - minimumPartSize: (int) \ - * The minimum part size in bytes to be used in a range multipart download. - * - listObjectV2Args: (array) \ - * The arguments to be included as part of the listObjectV2 request in - * order to fetch the objects to be downloaded. The most common arguments - * would be: + * @param array $config The config options for this download directory operation. + * - s3_prefix: (string, optional) This parameter will be considered just if + * not provided as part of the list_object_v2_args config option. + * - s3_delimiter: (string, optional, defaulted to '/') This parameter will be + * considered just if not provided as part of the list_object_v2_args config + * option. + * - filter: (Closure, optional) A callable which will receive an object key as + * parameter and should return true or false in order to determine + * whether the object should be downloaded. + * - get_object_request_callback: (Closure, optional) A function that will + * be invoked right before the download request is performed and that will + * receive as parameter the request arguments for each request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on a download failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $downloadDirectoryRequestArgs: (array) The arguments for the download + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $downloadDirectoryResponse: (DownloadDirectoryResponse) The download response + * to that point in the upload process. + * - track_progress: (bool, optional) Overrides the config option set + * in the transfer manager instantiation to decide whether transfer + * progress should be tracked. + * - minimum_part_size: (int, optional) The minimum part size in bytes + * to be used in a range multipart download. + * - list_object_v2_args: (array, optional) The arguments to be included + * as part of the listObjectV2 request in order to fetch the objects to + * be downloaded. The most common arguments would be: * - MaxKeys: (int) Sets the maximum number of keys returned in the response. * - Prefix: (string) To limit the response to keys that begin with the * specified prefix. - * - filter: (Closure) \ - * A callable which will receive an object key as parameter and should return - * true or false in order to determine whether the object should be downloaded. * @param TransferListener[] $listeners The listeners for watching * transfer events. Each listener will be cloned per file upload. * @param TransferListener|null $progressTracker Ideally the progress * tracker implementation provided here should be able to track multiple * transfers at once. Please see MultiProgressTracker implementation. - * + * * @return PromiseInterface */ public function downloadDirectory( @@ -389,7 +504,7 @@ public function downloadDirectory( ): PromiseInterface { if (!file_exists($destinationDirectory)) { - throw new \InvalidArgumentException( + throw new InvalidArgumentException( "Destination directory '$destinationDirectory' MUST exists." ); } @@ -401,7 +516,15 @@ public function downloadDirectory( $listArgs = [ 'Bucket' => $bucket - ] + ($config['listObjectV2Args'] ?? []); + ] + ($config['list_object_v2_args'] ?? []); + if (isset($config['s3_prefix']) && !isset($listArgs['Prefix'])) { + $listArgs['Prefix'] = $config['s3_prefix']; + } + + if (isset($config['s3_delimiter']) && !isset($listArgs['Delimiter'])) { + $listArgs['Delimiter'] = $config['s3_delimiter']; + } + $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) ->search('Contents[].Key'); @@ -410,7 +533,7 @@ public function downloadDirectory( }); if (isset($config['filter'])) { if (!is_callable($config['filter'])) { - throw new \InvalidArgumentException("The parameter \$config['filter'] must be callable."); + throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); } $filter = $config['filter']; @@ -420,6 +543,8 @@ public function downloadDirectory( } $promises = []; + $objectsDownloaded = 0; + $objectsFailed = 0; foreach ($objects as $object) { $objectKey = $this->s3UriAsBucketAndKey($object)['Key']; $destinationFile = $destinationDirectory . '/' . $objectKey; @@ -431,25 +556,69 @@ public function downloadDirectory( ); } + $requestArgs = [...$downloadArgs]; + if (isset($config['get_object_request_callback'])) { + if (!is_callable($config['get_object_request_callback'])) { + throw new InvalidArgumentException( + "The parameter \$config['get_object_request_callback'] must be callable." + ); + } + + call_user_func($config['get_object_request_callback'], $requestArgs); + } + $promises[] = $this->download( $object, - $downloadArgs, + $requestArgs, [ - 'minimumPartSize' => $config['minimumPartSize'] ?? 0, + 'minimum_part_size' => $config['minimum_part_size'] ?? 0, ], array_map(function ($listener) { return clone $listener; }, $listeners), $progressTracker, - )->then(function (DownloadResponse $result) use ($destinationFile) { + )->then(function (DownloadResponse $result) use ( + &$objectsDownloaded, + $destinationFile + ) { $directory = dirname($destinationFile); if (!is_dir($directory)) { mkdir($directory, 0777, true); } - file_put_contents($destinationFile, $result->getContent()); + file_put_contents($destinationFile, $result->getData()); + // Close the stream + $result->getData()->close(); + $objectsDownloaded++; + })->otherwise(function ($reason) use ( + &$objectsDownloaded, + &$objectsFailed, + $downloadArgs, + $requestArgs + ) { + $objectsFailed++; + if (isset($config['failure_policy']) && is_callable($config['failure_policy'])) { + call_user_func( + $config['failure_policy'], + $requestArgs, + $downloadArgs, + $reason, + new DownloadDirectoryResponse( + $objectsDownloaded, + $objectsFailed + ) + ); + } + + throw $reason; }); } - return Each::ofLimitAll($promises, $this->config['concurrency']); + return Each::ofLimitAll($promises, $this->config['concurrency']) + ->then(function ($_) use (&$objectsFailed, &$objectsDownloaded) { + return new DownloadDirectoryResponse( + $objectsDownloaded, + $objectsFailed + ); + }); } /** @@ -457,8 +626,8 @@ public function downloadDirectory( * * @param array $requestArgs * @param array $config - * - minimum_part_size: (int) \ - * The minimum part size in bytes for a range multipart download. + * - minimum_part_size: (int) The minimum part size in bytes for a + * range multipart download. * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface @@ -469,7 +638,7 @@ private function tryMultipartDownload( ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $downloaderClassName = MultipartDownloader::chooseDownloaderClass( $config['multipart_download_type'] ); $multipartDownloader = new $downloaderClassName( @@ -526,7 +695,7 @@ function ($result) use ($requestArgs, $listenerNotifier) { $listenerNotifier->transferComplete($progressContext); return new DownloadResponse( - content: $result['Body'], + data: $result['Body'], metadata: $result['@metadata'], ); } @@ -541,7 +710,7 @@ function ($result) use ($requestArgs, $listenerNotifier) { 'reason' => $reason ]); - return $reason; + throw $reason; }); } @@ -553,7 +722,7 @@ function ($result) use ($requestArgs, $listenerNotifier) { return $this->s3Client->executeAsync($command) ->then(function ($result) { return new DownloadResponse( - content: $result['Body'], + data: $result['Body'], metadata: $result['@metadata'], ); }); @@ -636,7 +805,7 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { ] ); - return $reason; + throw $reason; }); } @@ -683,22 +852,18 @@ private function requiresMultipartUpload( int $mupThreshold ): bool { - if (is_string($source)) { - $sourceSize = filesize($source); - - return $sourceSize > $mupThreshold; + if (is_string($source) && is_readable($source)) { + return filesize($source) >= $mupThreshold; } elseif ($source instanceof StreamInterface) { // When the stream's size is unknown then we could try a multipart upload. if (empty($source->getSize())) { return true; } - if (!empty($source->getSize())) { - return $source->getSize() > $mupThreshold; - } + return $source->getSize() >= $mupThreshold; } - return false; + throw new S3TransferException("Unable to determine if a multipart is required"); } /** @@ -724,7 +889,7 @@ private function defaultS3Client(): S3ClientInterface private function requireNonEmpty(mixed $value, string $message): mixed { if (empty($value)) { - throw new \InvalidArgumentException($message); + throw new InvalidArgumentException($message); } return $value; @@ -757,14 +922,14 @@ private function s3UriAsBucketAndKey(string $uri): array { $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; if (!$this->isValidS3URI($uri)) { - throw new \InvalidArgumentException($errorMessage); + throw new InvalidArgumentException($errorMessage); } $path = substr($uri, 5); // without s3:// $parts = explode('/', $path, 2); if (count($parts) < 2) { - throw new \InvalidArgumentException($errorMessage); + throw new InvalidArgumentException($errorMessage); } return [ @@ -806,4 +971,4 @@ private function resolvesOutsideTargetDirectory( return false; } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index e986ac4dd6..1ab1dd8fc8 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -5,19 +5,19 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; -use Aws\S3\S3Transfer\DownloadResponse; +use Aws\S3\S3Transfer\Models\DownloadResponse; use Aws\S3\S3Transfer\MultipartDownloader; +use Aws\S3\S3Transfer\PartGetMultipartDownloader; +use Aws\S3\S3Transfer\RangeGetMultipartDownloader; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; -use Psr\Http\Message\StreamInterface; /** * Tests multipart download implementation. */ class MultipartDownloaderTest extends TestCase { - /** * Tests part and range get multipart downloader. * @@ -68,7 +68,7 @@ public function testMultipartDownloader( -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - $downloaderClassName = MultipartDownloader::chooseDownloaderClassName( + $downloaderClassName = MultipartDownloader::chooseDownloaderClass( $multipartDownloadType ); /** @var MultipartDownloader $downloader */ @@ -163,4 +163,18 @@ public function partGetMultipartDownloaderProvider(): array { ] ]; } + + /** + * @return void + */ + public function testChooseDownloaderClass(): void { + $multipartDownloadTypes = [ + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + ]; + foreach ($multipartDownloadTypes as $multipartDownloadType => $class) { + $resolvedClass = MultipartDownloader::chooseDownloaderClass($multipartDownloadType); + $this->assertEquals($class, $resolvedClass); + } + } } \ No newline at end of file diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index d727595132..3a52b33b0c 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -6,8 +6,8 @@ use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\MultipartUploader; -use Aws\S3\S3Transfer\UploadResponse; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php new file mode 100644 index 0000000000..cfea7c6c06 --- /dev/null +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -0,0 +1,1345 @@ +assertArrayHasKey( + 'target_part_size_bytes', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'multipart_upload_threshold_bytes', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'checksum_validation_enabled', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'checksum_algorithm', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'multipart_download_type', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'concurrency', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'track_progress', + $manager->getConfig() + ); + $this->assertArrayHasKey( + 'region', + $manager->getConfig() + ); + $this->assertInstanceOf( + S3Client::class, + $manager->getS3Client() + ); + } + + /** + * @return void + */ + public function testCustomConfigIsSet(): void + { + $manager = new S3TransferManager( + null, + [ + 'target_part_size_bytes' => 1024, + 'multipart_upload_threshold_bytes' => 1024, + 'checksum_validation_enabled' => false, + 'checksum_algorithm' => 'sha256', + 'multipart_download_type' => 'partGet', + 'concurrency' => 20, + 'track_progress' => true, + 'region' => 'us-west-1', + ] + ); + $config = $manager->getConfig(); + $this->assertEquals(1024, $config['target_part_size_bytes']); + $this->assertEquals(1024, $config['multipart_upload_threshold_bytes']); + $this->assertFalse($config['checksum_validation_enabled']); + $this->assertEquals('sha256', $config['checksum_algorithm']); + $this->assertEquals('partGet', $config['multipart_download_type']); + $this->assertEquals(20, $config['concurrency']); + $this->assertTrue($config['track_progress']); + $this->assertEquals('us-west-1', $config['region']); + } + + /** + * @return void + */ + public function testUploadExpectsAReadableSource(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Please provide a valid readable file path or a valid stream as source."); + $manager = new S3TransferManager(); + $manager->upload( + "noreadablefile", + )->wait(); + } + + /** + * @dataProvider uploadBucketAndKeyProvider + * + * @return void + */ + public function testUploadFailsWhenBucketAndKeyAreNotProvided( + array $bucketKeyArgs, + string $missingProperty + ): void + { + $manager = new S3TransferManager(); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The `$missingProperty` parameter must be provided as part of the request arguments."); + $manager->upload( + Utils::streamFor(), + $bucketKeyArgs + )->wait(); + } + + /** + * @return array[] + */ + public function uploadBucketAndKeyProvider(): array + { + return [ + 'bucket_missing' => [ + 'bucket_key_args' => [ + 'Key' => 'Key', + ], + 'missing_property' => 'Bucket', + ], + 'key_missing' => [ + 'bucket_key_args' => [ + 'Bucket' => 'Bucket', + ], + 'missing_property' => 'Key', + ], + ]; + } + + /** + * @return void + */ + public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The provided config `multipart_upload_threshold_bytes`" + . "must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE); + $manager = new S3TransferManager(); + $manager->upload( + Utils::streamFor(), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE - 1 + ] + )->wait(); + } + + /** + * This tests takes advantage of the transfer listeners to validate + * if a multipart upload was done. How?, it will check if bytesTransfer + * event happens more than once, which only will occur in a multipart upload. + * + * @return void + */ + public function testDoesMultipartUploadWhenApplicable(): void + { + $client = $this->getS3ClientMock();; + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $expectedPartCount = 2; + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'part_size' => MultipartUploader::PART_MIN_SIZE, + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testDoesSingleUploadWhenApplicable(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->once()) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE - 1) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", $manager->getConfig()['multipart_upload_threshold_bytes']) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'part_size' => intval( + $manager->getConfig()['multipart_upload_threshold_bytes'] / $expectedPartCount + ), + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * + * @param int $mupThreshold + * @param int $expectedPartCount + * @param int $expectedPartSize + * @param bool $isMultipartUpload + * + * @dataProvider uploadUsesCustomMupThresholdProvider + * + * @return void + */ + public function testUploadUsesCustomMupThreshold( + int $mupThreshold, + int $expectedPartCount, + int $expectedPartSize, + bool $isMultipartUpload + ): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $expectedIncrementalPartSize = $expectedPartSize; + $transferListener->method('bytesTransferred') + -> willReturnCallback(function ($context) use ($expectedPartSize, &$expectedIncrementalPartSize) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context['progress_snapshot']; + $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); + $expectedIncrementalPartSize += $expectedPartSize; + }); + $manager->upload( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $mupThreshold, + 'part_size' => $expectedPartSize, + ], + [ + $transferListener, + ] + )->wait(); + if ($isMultipartUpload) { + $this->assertGreaterThan(1, $expectedPartCount); + } + } + + /** + * @return array + */ + public function uploadUsesCustomMupThresholdProvider(): array + { + return [ + 'mup_threshold_multipart_upload' => [ + 'mup_threshold' => 1024 * 1024 * 7, + 'expected_part_count' => 3, + 'expected_part_size' => 1024 * 1024 * 7, + 'is_multipart_upload' => true, + ], + 'mup_threshold_single_upload' => [ + 'mup_threshold' => 1024 * 1024 * 7, + 'expected_part_count' => 1, + 'expected_part_size' => 1024 * 1024 * 5, + 'is_multipart_upload' => false, + ] + ]; + } + + /** + * @return void + */ + public function testUploadUsesTransferManagerConfigDefaultTargetPartSize(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $transferListener = $this->createMock(TransferListener::class); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + $manager->upload( + Utils::streamFor( + str_repeat("#", $manager->getConfig()['target_part_size_bytes'] * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $manager->getConfig()['target_part_size_bytes'], + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesCustomPartSize(): void + { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $expectedPartCount = 2; + $expectedPartSize = 6 * 1024 * 1024; // 6 MBs + $transferListener = $this->getMockBuilder(TransferListener::class) + ->onlyMethods(['bytesTransferred']) + ->getMock(); + $expectedIncrementalPartSize = $expectedPartSize; + $transferListener->method('bytesTransferred') + ->willReturnCallback(function ($context) use ( + $expectedPartSize, + &$expectedIncrementalPartSize + ) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context['progress_snapshot']; + $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); + $expectedIncrementalPartSize += $expectedPartSize; + }); + $transferListener->expects($this->exactly($expectedPartCount)) + ->method('bytesTransferred'); + + $manager->upload( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'part_size' => $expectedPartSize, + 'multipart_upload_threshold_bytes' => $expectedPartSize, + ], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @return void + */ + public function testUploadUsesDefaultChecksumAlgorithm(): void + { + $manager = new S3TransferManager(); + $this->testUploadResolvedChecksum( + [], // No checksum provided + $manager->getConfig()['checksum_algorithm'] // default checksum algo + ); + } + + /** + * @param string $checksumAlgorithm + * + * @dataProvider uploadUsesCustomChecksumAlgorithmProvider + * + * @return void + */ + public function testUploadUsesCustomChecksumAlgorithm( + string $checksumAlgorithm, + ): void + { + $this->testUploadResolvedChecksum( + ['checksum_algorithm' => $checksumAlgorithm], + $checksumAlgorithm + ); + } + + /** + * @return array[] + */ + public function uploadUsesCustomChecksumAlgorithmProvider(): array + { + return [ + 'checksum_sha256' => [ + 'checksum_algorithm' => 'sha256', + ], + 'checksum_sha1' => [ + 'checksum_algorithm' => 'sha1', + ], + 'checksum_crc32c' => [ + 'checksum_algorithm' => 'crc32c', + ], + 'checksum_crc32' => [ + 'checksum_algorithm' => 'crc32', + ] + ]; + } + + /** + * @param array $config + * @param string $expectedChecksum + * + * @return void + */ + private function testUploadResolvedChecksum( + array $config, + string $expectedChecksum + ): void { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $manager = new S3TransferManager( + $client, + ); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use ( + $expectedChecksum + ) { + $this->assertEquals( + strtoupper($expectedChecksum), + strtoupper($args['ChecksumAlgorithm']) + ); + + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + return Create::promiseFor(new Result([])); + }); + $manager->upload( + Utils::streamFor(), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + $config + )->wait(); + } + + /** + * @param string $directory + * @param bool $isDirectoryValid + * + * @dataProvider uploadDirectoryValidatesProvidedDirectoryProvider + * + * @return void + */ + public function testUploadDirectoryValidatesProvidedDirectory( + string $directory, + bool $isDirectoryValid + ): void + { + if (!$isDirectoryValid) { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "Please provide a valid directory path. " + . "Provided = " . $directory); + } else { + $this->assertTrue(true); + } + + $manager = new S3TransferManager( + $this->getS3ClientMock(), + ); + $manager->uploadDirectory( + $directory, + "Bucket", + )->wait(); + // Clean up resources + if ($isDirectoryValid) { + rmdir($directory); + } + } + + /** + * @return array[] + */ + public function uploadDirectoryValidatesProvidedDirectoryProvider(): array + { + $validDirectory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($validDirectory)) { + mkdir($validDirectory, 0777, true); + } + + return [ + 'valid_directory' => [ + 'directory' => $validDirectory, + 'is_valid_directory' => true, + ], + 'invalid_directory' => [ + 'directory' => 'invalid-directory', + 'is_valid_directory' => false, + ] + ]; + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidFilter(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'The parameter $config[\'filter\'] must be callable' + ); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'filter' => 'invalid_filter', + ] + )->wait(); + } finally { + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFileFilter(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + $filesCreated = []; + $validFilesCount = 0; + for ($i = 0; $i < 10; $i++) { + $fileName = "file-$i"; + if ($i % 2 === 0) { + $fileName .= "-valid"; + $validFilesCount++; + } + + $filePathName = $directory . "/" . $fileName . ".txt"; + file_put_contents($filePathName, "test"); + $filesCreated[] = $filePathName; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $calledTimes = 0; + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'filter' => function (string $objectKey) { + return str_ends_with($objectKey, "-valid.txt"); + }, + 'put_object_request_callback' => function ($requestArgs) use (&$calledTimes) { + $this->assertStringContainsString( + 'valid.txt', + $requestArgs["Key"] + ); + $calledTimes++; + } + ] + )->wait(); + $this->assertEquals($validFilesCount, $calledTimes); + } finally { + foreach ($filesCreated as $filePathName) { + unlink($filePathName); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryRecursive(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; + if (!is_dir($subDirectory)) { + mkdir($subDirectory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($subDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryNonRecursive(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $subDirectory = $directory . "/sub-directory"; + if (!is_dir($subDirectory)) { + mkdir($subDirectory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $subDirectory . "/subdir-file-1.txt", + $subDirectory . "/subdir-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => false, + ] + )->wait(); + $subDirPrefix = str_replace($directory . "/", "", $subDirectory); + foreach ($objectKeys as $key => $validated) { + if (str_starts_with($key, $subDirPrefix)) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($subDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFollowsSymbolicLink(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + $linkDirectory = sys_get_temp_dir() . "/link-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + if (!is_dir($linkDirectory)) { + mkdir($linkDirectory, 0777, true); + } + $symLinkDirectory = $directory . "/upload-directory-test-link"; + if (is_link($symLinkDirectory)) { + unlink($symLinkDirectory); + } + symlink($linkDirectory, $symLinkDirectory); + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $linkDirectory . "/symlink-file-1.txt", + $linkDirectory . "/symlink-file-2.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + // Remove the directory from the file path to leave + // just what will be the object key + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = str_replace($linkDirectory . "/", "", $objectKey); + if (str_contains($objectKey, 'symlink-file')) { + $objectKey = "upload-directory-test-link/" . $objectKey; + } + + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + // First lets make sure that when follows_symbolic_link is false + // the directory in the link will not be traversed. + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => false, + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + if (str_contains($key, "symlink")) { + // Files in subdirectory should have been ignored + $this->assertFalse($validated, "Key {$key} should have not been considered"); + } else { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } + // Now let's enable follow_symbolic_links and all files should have + // been considered, included the ones in the symlink directory. + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true, + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been considered"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + + unlink($symLinkDirectory); + rmdir($linkDirectory); + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesProvidedPrefix(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", + ]; + $s3Prefix = 'expenses-files/'; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$s3Prefix . $objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix + ] + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesProvidedDelimiter(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/dir-file-5.txt", + ]; + $s3Prefix = 'expenses-files/today/records/'; + $s3Delimiter = '|'; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKey = $s3Prefix . $objectKey; + $objectKey = str_replace("/", $s3Delimiter, $objectKey); + $objectKeys[$objectKey] = false; + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + $objectKeys[$args["Key"]] = true; + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function () { + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix, + 's3_delimiter' => $s3Delimiter, + ] + )->wait(); + + foreach ($objectKeys as $key => $validated) { + $this->assertTrue($validated, "Key {$key} should have been validated"); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['put_object_request_callback'] must be callable."); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'put_object_request_callback' => false, + ] + )->wait(); + } finally { + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryPutObjectRequestCallbackWorks(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) use (&$objectKeys) { + return new Command($commandName, $args); + }); + $client->method('executeAsync') + ->willReturnCallback(function ($command) { + $this->assertEquals("Test", $command['FooParameter']); + + return Create::promiseFor(new Result([])); + }); + $manager = new S3TransferManager( + $client, + ); + $called = 0; + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'put_object_request_callback' => function ( + &$requestArgs + ) use (&$called) { + $requestArgs["FooParameter"] = "Test"; + $called++; + }, + ] + )->wait(); + $this->assertEquals(count($files), $called); + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryUsesFailurePolicy(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = new S3Client([ + 'region' => 'us-east-2', + 'handler' => function ($command) { + if (str_contains($command['Key'], "dir-file-2.txt")) { + return Create::rejectionFor( + new Exception("Failed uploading second file") + ); + } + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + [ + 'concurrency' => 1, // To make uploads to be one after the other + ] + ); + $called = false; + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + UploadDirectoryResponse $uploadDirectoryResponse + ) use (&$called) { + $called = true; + $this->assertEquals( + "Failed uploading second file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsUploaded() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsFailed() + ); + }, + ] + )->wait(); + $this->assertTrue($called); + } finally { + foreach ($files as $file) { + unlink($file); + } + + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['failure_policy'] must be callable."); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [ + 'failure_policy' => false, + ] + )->wait(); + } finally { + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): void + { + $s3Delimiter = "*"; + $fileNameWithDelimiter = "dir-file-$s3Delimiter.txt"; + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage( + "The filename `$fileNameWithDelimiter` must not contain the provided delimiter `$s3Delimiter`" + ); + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + $directory . "/$fileNameWithDelimiter", + ]; + foreach ($files as $file) { + file_put_contents($file, "test"); + } + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + ['s3_delimiter' => $s3Delimiter] + )->wait(); + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return void + */ + public function testUploadDirectoryTracksMultipleFiles(): void + { + $directory = sys_get_temp_dir() . "/upload-directory-test"; + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + $files = [ + $directory . "/dir-file-1.txt", + $directory . "/dir-file-2.txt", + $directory . "/dir-file-3.txt", + $directory . "/dir-file-4.txt", + ]; + $objectKeys = []; + foreach ($files as $file) { + file_put_contents($file, "test"); + $objectKey = str_replace($directory . "/", "", $file); + $objectKeys[$objectKey] = false; + } + + try { + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $transferListener = $this->getMockBuilder(TransferListener::class) + ->disableOriginalConstructor() + ->getMock(); + $transferListener->expects($this->exactly(count($files))) + ->method('transferInitiated'); + $transferListener->expects($this->exactly(count($files))) + ->method('transferComplete'); + $transferListener->method('bytesTransferred') + ->willReturnCallback(function(array $context) use (&$objectKeys) { + /** @var TransferProgressSnapshot $snapshot */ + $snapshot = $context['progress_snapshot']; + $objectKeys[$snapshot->getIdentifier()] = true; + }); + $manager->uploadDirectory( + $directory, + "Bucket", + [], + [], + [ + $transferListener + ] + )->wait(); + foreach ($objectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The object key `$key` should have been validated." + ); + } + } finally { + foreach ($files as $file) { + unlink($file); + } + rmdir($directory); + } + } + + /** + * @return S3Client + */ + private function getS3ClientMock(): S3Client + { + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(['getCommand', 'executeAsync']) + ->getMock(); + $client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $client->method('executeAsync')->willReturnCallback( + function ($command) { + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'UploadPart', + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + } + ); + + return $client; + } +} From b27189790fc89640b07d4acd07bbcf2c35ebc534 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Mar 2025 08:33:35 -0700 Subject: [PATCH 17/62] chore: address naming feedback and test failures - Fix MultipartUpload tests by increasing the part size from 1024 to 10240000 so it gets between the allowed part size range 5MB-5GBs. - Rename tobe to to_be in the progress formatting. --- src/S3/S3Transfer/MultipartDownloader.php | 2 +- .../ColoredTransferProgressBarFormat.php | 4 ++-- .../Progress/SingleProgressTracker.php | 2 +- .../Progress/TransferProgressBarFormat.php | 4 ++-- .../S3/S3Transfer/MultipartDownloaderTest.php | 1 + tests/S3/S3Transfer/MultipartUploaderTest.php | 24 +++++++++---------- .../Progress/ConsoleProgressBarTest.php | 16 ++++++------- .../Progress/ProgressBarFormatTest.php | 14 +++++------ 8 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index ab42b59cd7..2e4de9c568 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -177,7 +177,7 @@ public function promise(): PromiseInterface yield Create::promiseFor(new DownloadResponse( $this->stream, - $result['@metadata'] + $result['@metadata'] ?? [] )); }); } diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php index e5a3eb95f0..cc7f80d6f3 100644 --- a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -16,7 +16,7 @@ public function getFormatTemplate(): string { return "|object_name|:\n" - ."\033|color_code|[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit| |message|\033[0m"; + ."\033|color_code|[|progress_bar|] |percent|% |transferred|/|to_be_transferred| |unit| |message|\033[0m"; } /** @@ -28,7 +28,7 @@ public function getFormatParameters(): array 'progress_bar', 'percent', 'transferred', - 'tobe_transferred', + 'to_be_transferred', 'unit', 'color_code', 'message', diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 1024068d4f..23867d7e22 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -195,7 +195,7 @@ private function updateProgressBar( $this->progressBar->getProgressBarFormat()->setArgs([ 'transferred' => $this->currentSnapshot->getTransferredBytes(), - 'tobe_transferred' => $this->currentSnapshot->getTotalBytes(), + 'to_be_transferred' => $this->currentSnapshot->getTotalBytes(), 'unit' => 'B', ]); // Display progress diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php index d6568e3b96..c7a40575a5 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -9,7 +9,7 @@ final class TransferProgressBarFormat extends ProgressBarFormat */ public function getFormatTemplate(): string { - return "|object_name|:\n[|progress_bar|] |percent|% |transferred|/|tobe_transferred| |unit|"; + return "|object_name|:\n[|progress_bar|] |percent|% |transferred|/|to_be_transferred| |unit|"; } /** @@ -22,7 +22,7 @@ public function getFormatParameters(): array 'progress_bar', 'percent', 'transferred', - 'tobe_transferred', + 'to_be_transferred', 'unit', ]; } diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 1ab1dd8fc8..6787a93231 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -68,6 +68,7 @@ public function testMultipartDownloader( -> willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); + $downloaderClassName = MultipartDownloader::chooseDownloaderClass( $multipartDownloadType ); diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 3a52b33b0c..27e4ab8793 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -85,58 +85,58 @@ public function multipartUploadProvider(): array { return [ '5_parts_upload' => [ 'stream' => Utils::streamFor( - str_repeat('*', 1024 * 5), + str_repeat('*', 10240000 * 5), ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 5, - 'bytesUploaded' => 1024 * 5, + 'bytesUploaded' => 10240000 * 5, ] ], '100_parts_upload' => [ 'stream' => Utils::streamFor( - str_repeat('*', 1024 * 100), + str_repeat('*', 10240000 * 100), ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 100, - 'bytesUploaded' => 1024 * 100, + 'bytesUploaded' => 10240000 * 100, ] ], '5_parts_no_seekable_stream' => [ 'stream' => new NoSeekStream( Utils::streamFor( - str_repeat('*', 1024 * 5) + str_repeat('*', 10240000 * 5) ) ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 5, - 'bytesUploaded' => 1024 * 5, + 'bytesUploaded' => 10240000 * 5, ] ], '100_parts_no_seekable_stream' => [ 'stream' => new NoSeekStream( Utils::streamFor( - str_repeat('*', 1024 * 100) + str_repeat('*', 10240000 * 100) ) ), 'config' => [ - 'part_size' => 1024 + 'part_size' => 10240000 ], 'expected' => [ 'succeed' => true, 'parts' => 100, - 'bytesUploaded' => 1024 * 100, + 'bytesUploaded' => 10240000 * 100, ] ] ]; diff --git a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php index 6de1dbdaaf..393bb13400 100644 --- a/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php +++ b/tests/S3/S3Transfer/Progress/ConsoleProgressBarTest.php @@ -175,7 +175,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 23, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[############ ] 23% 23/100 B" @@ -188,7 +188,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 75, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[################### ] 75% 75/100 B" @@ -201,7 +201,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[##############################] 100% 100/100 B" @@ -214,7 +214,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'FooObject', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "FooObject:\n[******************************] 100% 100/100 B" @@ -227,7 +227,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_1', 'transferred' => 10, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_output' => "ObjectName_1:\n\033[30m[## ] 10% 10/100 B \033[0m" @@ -240,7 +240,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_2', 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE ], @@ -254,7 +254,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_3', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE ], @@ -268,7 +268,7 @@ public function progressBarRenderingProvider(): array 'progress_bar_format_args' => [ 'object_name' => 'ObjectName_3', 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE ], diff --git a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php index 5e1c03a397..42fa59fc35 100644 --- a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php @@ -67,7 +67,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..........', 'percent' => 100, 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_format' => "foo:\n[..........] 100% 100/100 B", @@ -79,7 +79,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B' ], 'expected_format' => "foo:\n[..... ] 50% 50/100 B", @@ -90,7 +90,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject' ], @@ -102,7 +102,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE @@ -115,7 +115,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::GREEN_COLOR_CODE @@ -128,7 +128,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..... ', 'percent' => 50, 'transferred' => 50, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::RED_COLOR_CODE @@ -141,7 +141,7 @@ public function progressBarFormatProvider(): array 'progress_bar' => '..........', 'percent' => 100, 'transferred' => 100, - 'tobe_transferred' => 100, + 'to_be_transferred' => 100, 'unit' => 'B', 'object_name' => 'FooObject', 'color_code' => ColoredTransferProgressBarFormat::BLUE_COLOR_CODE From f4f1c88a4146e24facf1bc8d670b261cf5bdae74 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 13 Mar 2025 08:59:25 -0700 Subject: [PATCH 18/62] chore: address minor styling issues --- src/S3/S3Transfer/MultipartUploader.php | 4 ++-- src/S3/S3Transfer/S3TransferManager.php | 15 +++++++++++---- tests/S3/S3Transfer/MultipartUploaderTest.php | 1 - tests/S3/S3Transfer/S3TransferManagerTest.php | 1 - 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index a99162d21e..b40e1891ec 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -64,8 +64,8 @@ class MultipartUploader implements PromisorInterface * @param S3ClientInterface $s3Client * @param array $createMultipartArgs * @param array $config - * - part_size: (int, optional) - * - concurrency: (int, required) + * - part_size: (int, optional) + * - concurrency: (int, required) * @param string | StreamInterface $source * @param string|null $uploadId * @param array $parts diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 513908c264..bdb4f2eb80 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,7 +2,6 @@ namespace Aws\S3\S3Transfer; -use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; @@ -20,7 +19,6 @@ use GuzzleHttp\Promise\PromiseInterface; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; -use Throwable; use function Aws\filter; use function Aws\map; @@ -541,6 +539,14 @@ public function downloadDirectory( return call_user_func($filter, $key); }); } + $failurePolicyCallback = null; + if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { + throw new InvalidArgumentException( + "The parameter \$config['failure_policy'] must be callable." + ); + } elseif (isset($config['failure_policy'])) { + $failurePolicyCallback = $config['failure_policy']; + } $promises = []; $objectsDownloaded = 0; @@ -589,15 +595,16 @@ public function downloadDirectory( $result->getData()->close(); $objectsDownloaded++; })->otherwise(function ($reason) use ( + $failurePolicyCallback, &$objectsDownloaded, &$objectsFailed, $downloadArgs, $requestArgs ) { $objectsFailed++; - if (isset($config['failure_policy']) && is_callable($config['failure_policy'])) { + if ($failurePolicyCallback !== null) { call_user_func( - $config['failure_policy'], + $failurePolicyCallback, $requestArgs, $downloadArgs, $reason, diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 27e4ab8793..a5ca28bf69 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -18,7 +18,6 @@ class MultipartUploaderTest extends TestCase { - /** * @param StreamInterface $stream * @param array $config diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index cfea7c6c06..4ba9b5a0a5 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -13,7 +13,6 @@ use Aws\S3\S3Transfer\S3TransferManager; use Exception; use GuzzleHttp\Promise\Create; -use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; use PHPUnit\Framework\TestCase; From d987aff968802d7e329ab4481b8d7fa1be392f78 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 16 Mar 2025 20:54:06 -0700 Subject: [PATCH 19/62] chore: add download tests - Add download tests - Add download directory tests - Minor naming refactor --- .../RangeGetMultipartDownloader.php | 2 +- src/S3/S3Transfer/S3TransferManager.php | 175 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 1472 ++++++++++++++++- 3 files changed, 1557 insertions(+), 92 deletions(-) diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index 5fa10cde99..1022edc221 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -85,7 +85,7 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = array_slice($this->requestArgs, 0); + $nextRequestArgs = [...$this->requestArgs]; $from = ($this->currentPartNo - 1) * $this->partSize; $to = ($this->currentPartNo * $this->partSize) - 1; if ($this->objectSizeInBytes !== 0) { diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index bdb4f2eb80..fc761c2f1a 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,6 +2,7 @@ namespace Aws\S3\S3Transfer; +use Aws\Arn\ArnParser; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; @@ -68,7 +69,10 @@ public function __construct( ?S3ClientInterface $s3Client = null, array $config = [] ) { - $this->config = $config + self::$defaultConfig; + $this->config = [ + ...self::$defaultConfig, + ...$config, + ]; if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -132,7 +136,8 @@ public function upload( // Valid required parameters foreach (['Bucket', 'Key'] as $reqParam) { $this->requireNonEmpty( - $requestArgs[$reqParam] ?? null, + $requestArgs, + $reqParam, "The `$reqParam` parameter must be provided as part of the request arguments." ); } @@ -182,9 +187,9 @@ public function upload( } /** - * @param string $directory + * @param string $sourceDirectory * @param string $bucketTo - * @param array $requestArgs + * @param array $uploadDirectoryRequestArgs * @param array $config The config options for this request that are: * - follow_symbolic_links: (bool, optional, defaulted to false) * - recursive: (bool, optional, defaulted to false) @@ -219,21 +224,23 @@ public function upload( * @return PromiseInterface */ public function uploadDirectory( - string $directory, - string $bucketTo, - array $requestArgs = [], - array $config = [], - array $listeners = [], + string $sourceDirectory, + string $bucketTo, + array $uploadDirectoryRequestArgs = [], + array $config = [], + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { - if (!is_dir($directory)) { + if (!is_dir($sourceDirectory)) { throw new InvalidArgumentException( "Please provide a valid directory path. " - . "Provided = " . $directory + . "Provided = " . $sourceDirectory ); } + $bucketTo = $this->parseBucket($bucketTo); + if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = new MultiProgressTracker(); @@ -268,7 +275,7 @@ public function uploadDirectory( $failurePolicyCallback = $config['failure_policy']; } - $dirIterator = new \RecursiveDirectoryIterator($directory); + $dirIterator = new \RecursiveDirectoryIterator($sourceDirectory); $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); if (($config['follow_symbolic_links'] ?? false) === true) { $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); @@ -298,7 +305,7 @@ function ($file) use ($filter) { $objectsUploaded = 0; $objectsFailed = 0; foreach ($files as $file) { - $baseDir = rtrim($directory, '/') . '/'; + $baseDir = rtrim($sourceDirectory, '/') . '/'; $relativePath = substr($file, strlen($baseDir)); if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( @@ -312,7 +319,7 @@ function ($file) use ($filter) { $objectKey ); $uploadRequestArgs = [ - ...$requestArgs, + ...$uploadDirectoryRequestArgs, 'Bucket' => $bucketTo, 'Key' => $objectKey, ]; @@ -331,9 +338,11 @@ function ($file) use ($filter) { return $response; })->otherwise(function ($reason) use ( + $bucketTo, + $sourceDirectory, $failurePolicyCallback, $uploadRequestArgs, - $requestArgs, + $uploadDirectoryRequestArgs, &$objectsUploaded, &$objectsFailed ) { @@ -341,8 +350,12 @@ function ($file) use ($filter) { if($failurePolicyCallback !== null) { call_user_func( $failurePolicyCallback, - $requestArgs, $uploadRequestArgs, + [ + ...$uploadDirectoryRequestArgs, + "source_directory" => $sourceDirectory, + "bucket_to" => $bucketTo, + ], $reason, new UploadDirectoryResponse( $objectsUploaded, @@ -367,7 +380,7 @@ function ($file) use ($filter) { * @param string|array $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key * properties set. - * @param array $downloadArgs The getObject request arguments to be provided as part + * @param array $downloadRequestArgs The getObject request arguments to be provided as part * of each get object operation, except for the bucket and key, which * are already provided as the source. * @param array $config The configuration to be used for this operation: @@ -390,10 +403,10 @@ function ($file) use ($filter) { * @return PromiseInterface */ public function download( - string | array $source, - array $downloadArgs = [], - array $config = [], - array $listeners = [], + string | array $source, + array $downloadRequestArgs = [], + array $config = [], + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { @@ -401,19 +414,29 @@ public function download( $sourceArgs = $this->s3UriAsBucketAndKey($source); } elseif (is_array($source)) { $sourceArgs = [ - 'Bucket' => $this->requireNonEmpty($source['Bucket'], "A valid bucket must be provided."), - 'Key' => $this->requireNonEmpty($source['Key'], "A valid key must be provided."), + 'Bucket' => $this->requireNonEmpty( + $source, + 'Bucket', + "A valid bucket must be provided." + ), + 'Key' => $this->requireNonEmpty( + $source, + 'Key', + "A valid key must be provided." + ), ]; } else { - throw new InvalidArgumentException('Source must be a string or an array of strings'); + throw new S3TransferException( + "Unsupported source type `" . gettype($source) . "`" + ); } - if (!isset($requestArgs['ChecksumMode'])) { + if (!isset($downloadRequestArgs['ChecksumMode'])) { $checksumEnabled = $config['checksum_validation_enabled'] ?? $this->config['checksum_validation_enabled'] ?? false; if ($checksumEnabled) { - $requestArgs['ChecksumMode'] = 'enabled'; + $downloadRequestArgs['ChecksumMode'] = 'enabled'; } } @@ -427,8 +450,11 @@ public function download( } $listenerNotifier = new TransferListenerNotifier($listeners); - $requestArgs = $sourceArgs + $downloadArgs; - if (empty($downloadArgs['PartNumber']) && empty($downloadArgs['Range'])) { + $requestArgs = [ + ...$sourceArgs, + ...$downloadRequestArgs, + ]; + if (empty($downloadRequestArgs['PartNumber']) && empty($downloadRequestArgs['Range'])) { return $this->tryMultipartDownload( $requestArgs, [ @@ -449,7 +475,7 @@ public function download( * downloaded from. * @param string $destinationDirectory The destination path where the downloaded * files will be placed in. - * @param array $downloadArgs The getObject request arguments to be provided + * @param array $downloadDirectoryArgs The getObject request arguments to be provided * as part of each get object request sent to the service, except for the * bucket and key which will be resolved internally. * @param array $config The config options for this download directory operation. @@ -493,20 +519,22 @@ public function download( * @return PromiseInterface */ public function downloadDirectory( - string $bucket, - string $destinationDirectory, - array $downloadArgs, - array $config = [], - array $listeners = [], + string $bucket, + string $destinationDirectory, + array $downloadDirectoryArgs = [], + array $config = [], + array $listeners = [], ?TransferListener $progressTracker = null, ): PromiseInterface { if (!file_exists($destinationDirectory)) { throw new InvalidArgumentException( - "Destination directory '$destinationDirectory' MUST exists." + "Destination directory `$destinationDirectory` MUST exists." ); } + $bucket = $this->parseBucket($bucket); + if ($progressTracker === null && ($config['track_progress'] ?? $this->config['track_progress'])) { $progressTracker = new MultiProgressTracker(); @@ -526,9 +554,6 @@ public function downloadDirectory( $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) ->search('Contents[].Key'); - $objects = map($objects, function (string $key) use ($bucket) { - return "s3://$bucket/$key"; - }); if (isset($config['filter'])) { if (!is_callable($config['filter'])) { throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); @@ -539,6 +564,21 @@ public function downloadDirectory( return call_user_func($filter, $key); }); } + + $objects = map($objects, function (string $key) use ($bucket) { + return "s3://$bucket/$key"; + }); + $getObjectRequestCallback = null; + if (isset($config['get_object_request_callback'])) { + if (!is_callable($config['get_object_request_callback'])) { + throw new InvalidArgumentException( + "The parameter \$config['get_object_request_callback'] must be callable." + ); + } + + $getObjectRequestCallback = $config['get_object_request_callback']; + } + $failurePolicyCallback = null; if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { throw new InvalidArgumentException( @@ -562,15 +602,9 @@ public function downloadDirectory( ); } - $requestArgs = [...$downloadArgs]; - if (isset($config['get_object_request_callback'])) { - if (!is_callable($config['get_object_request_callback'])) { - throw new InvalidArgumentException( - "The parameter \$config['get_object_request_callback'] must be callable." - ); - } - - call_user_func($config['get_object_request_callback'], $requestArgs); + $requestArgs = [...$downloadDirectoryArgs]; + if ($getObjectRequestCallback !== null) { + call_user_func($getObjectRequestCallback, $requestArgs); } $promises[] = $this->download( @@ -595,10 +629,12 @@ public function downloadDirectory( $result->getData()->close(); $objectsDownloaded++; })->otherwise(function ($reason) use ( + $bucket, + $destinationDirectory, $failurePolicyCallback, &$objectsDownloaded, &$objectsFailed, - $downloadArgs, + $downloadDirectoryArgs, $requestArgs ) { $objectsFailed++; @@ -606,13 +642,19 @@ public function downloadDirectory( call_user_func( $failurePolicyCallback, $requestArgs, - $downloadArgs, + [ + ...$downloadDirectoryArgs, + "destination_directory" => $destinationDirectory, + "bucket" => $bucket, + ], $reason, new DownloadDirectoryResponse( $objectsDownloaded, $objectsFailed ) ); + + return; } throw $reason; @@ -888,18 +930,19 @@ private function defaultS3Client(): S3ClientInterface /** * Validates a provided value is not empty, and if so then * it throws an exception with the provided message. - * @param mixed $value + * @param array $array + * @param string $key * @param string $message * * @return mixed */ - private function requireNonEmpty(mixed $value, string $message): mixed + private function requireNonEmpty(array $array, string $key, string $message): mixed { - if (empty($value)) { + if (empty($array[$key])) { throw new InvalidArgumentException($message); } - return $value; + return $array[$key]; } /** @@ -927,7 +970,7 @@ private function isValidS3URI(string $uri): bool */ private function s3UriAsBucketAndKey(string $uri): array { - $errorMessage = "Invalid URI: $uri. A valid S3 URI must be s3://bucket/key"; + $errorMessage = "Invalid URI: `$uri` provided. \nA valid S3 URI looks as `s3://bucket/key`"; if (!$this->isValidS3URI($uri)) { throw new InvalidArgumentException($errorMessage); } @@ -945,6 +988,22 @@ private function s3UriAsBucketAndKey(string $uri): array ]; } + /** + * To parse the bucket name when the bucket is provided as an ARN. + * + * @param string $bucket + * + * @return string + */ + private function parseBucket(string $bucket): string + { + if (ArnParser::isArn($bucket)) { + return ArnParser::parse($bucket)->getResource(); + } + + return $bucket; + } + /** * @param string $sink * @param string $objectKey @@ -978,4 +1037,12 @@ private function resolvesOutsideTargetDirectory( return false; } + + /** + * @return array + */ + public static function getDefaultConfig(): array + { + return self::$defaultConfig; + } } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 4ba9b5a0a5..f66d84571b 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -2,15 +2,21 @@ namespace Aws\Test\S3\S3Transfer; +use Aws\Api\Service; use Aws\Command; +use Aws\CommandInterface; +use Aws\HandlerList; use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\S3TransferManager; +use Closure; use Exception; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; @@ -175,7 +181,7 @@ public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void */ public function testDoesMultipartUploadWhenApplicable(): void { - $client = $this->getS3ClientMock();; + $client = $this->getS3ClientMock(); $manager = new S3TransferManager( $client, ); @@ -473,15 +479,11 @@ private function testUploadResolvedChecksum( array $config, string $expectedChecksum ): void { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) - ->getMock(); - $manager = new S3TransferManager( - $client, - ); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) use ( + $client = $this->getS3ClientMock([ + 'getCommand' => function ( + string $commandName, + array $args + ) use ( $expectedChecksum ) { $this->assertEquals( @@ -490,11 +492,14 @@ private function testUploadResolvedChecksum( ); return new Command($commandName, $args); - }); - $client->method('executeAsync') - ->willReturnCallback(function ($command) { + }, + 'executeAsync' => function () { return Create::promiseFor(new Result([])); - }); + } + ]); + $manager = new S3TransferManager( + $client, + ); $manager->upload( Utils::streamFor(), [ @@ -1148,8 +1153,16 @@ public function testUploadDirectoryUsesFailurePolicy(): void array $uploadDirectoryRequestArgs, \Throwable $reason, UploadDirectoryResponse $uploadDirectoryResponse - ) use (&$called) { + ) use ($directory, &$called) { $called = true; + $this->assertEquals( + $directory, + $uploadDirectoryRequestArgs["source_directory"] + ); + $this->assertEquals( + "Bucket", + $uploadDirectoryRequestArgs["bucket_to"] + ); $this->assertEquals( "Failed uploading second file", $reason->getMessage() @@ -1312,32 +1325,1417 @@ public function testUploadDirectoryTracksMultipleFiles(): void } /** - * @return S3Client + * @return void */ - private function getS3ClientMock(): S3Client + public function testDownloadFailsOnInvalidS3UriSource(): void { - $client = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->onlyMethods(['getCommand', 'executeAsync']) - ->getMock(); - $client->method('getCommand') - ->willReturnCallback(function ($commandName, $args) { - return new Command($commandName, $args); - }); - $client->method('executeAsync')->willReturnCallback( - function ($command) { - return match ($command->getName()) { - 'CreateMultipartUpload' => Create::promiseFor(new Result([ - 'UploadId' => 'FooUploadId', - ])), - 'UploadPart', - 'CompleteMultipartUpload', - 'AbortMultipartUpload', - 'PutObject' => Create::promiseFor(new Result([])), - default => null, - }; + $invalidS3Uri = "invalid-s3-uri"; + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Invalid URI: `$invalidS3Uri` provided. " + . "\nA valid S3 URI looks as `s3://bucket/key`"); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $invalidS3Uri + ); + } + + /** + * @dataProvider downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider + * + * @param array $sourceAsArray + * @param string $expectedExceptionMessage + * + * @return void + */ + public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( + array $sourceAsArray, + string $expectedExceptionMessage, + ): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($expectedExceptionMessage); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $sourceAsArray + ); + } + + /** + * @return array + */ + public function downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider(): array + { + return [ + 'missing_key' => [ + 'source' => [ + 'Bucket' => 'bucket', + ], + 'expected_exception' => "A valid key must be provided." + ], + 'missing_bucket' => [ + 'source' => [ + 'Key' => 'key', + ], + 'expected_exception' => "A valid bucket must be provided." + ] + ]; + } + + /** + * @return void + */ + public function testDownloadWorksWithS3UriAsSource(): void + { + $sourceAsArray = [ + 'Bucket' => 'bucket', + 'Key' => 'key', + ]; + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function(CommandInterface $command) use ( + $sourceAsArray, + &$called + ) { + $called = true; + $this->assertEquals($sourceAsArray['Bucket'], $command['Bucket']); + $this->assertEquals($sourceAsArray['Key'], $command['Key']); + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + ]); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $sourceAsArray, + )->wait(); + $this->assertTrue($called); + } + + /** + * @return void + */ + public function testDownloadWorksWithBucketAndKeyAsSource(): void + { + $bucket = 'bucket'; + $key = 'key'; + $sourceAsS3Uri = "s3://$bucket/$key"; + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function(CommandInterface $command) use ( + $bucket, + $key, + &$called + ) { + $called = true; + $this->assertEquals($bucket, $command['Bucket']); + $this->assertEquals($key, $command['Key']); + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + ]); + $manager = new S3TransferManager( + $client + ); + $manager->download( + $sourceAsS3Uri, + )->wait(); + $this->assertTrue($called); + } + + /** + * + * @param array $transferManagerConfig + * @param array $downloadConfig + * @param array $downloadArgs + * @param bool $expectedChecksumMode + * + * @return void + * @dataProvider downloadAppliesChecksumProvider + * + */ + public function testDownloadAppliesChecksumMode( + array $transferManagerConfig, + array $downloadConfig, + array $downloadArgs, + bool $expectedChecksumMode, + ): void + { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedChecksumMode, + &$called + ) { + $called = true; + if ($expectedChecksumMode) { + $this->assertEquals( + 'enabled', + $command['ChecksumMode'], + ); + } else { + if (isset($command['ChecksumMode'])) { + $this->assertEquals( + 'disabled', + $command['ChecksumMode'], + ); + } + } + + if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + } + + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + $transferManagerConfig, + ); + $manager->download( + "s3://bucket/key", + $downloadArgs, + $downloadConfig + )->wait(); + $this->assertTrue($called); + } + + /** + * @return array + */ + public function downloadAppliesChecksumProvider(): array + { + return [ + 'checksum_mode_from_default_transfer_manager_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => S3TransferManager::getDefaultConfig()[ + 'checksum_validation_enabled' + ], + ], + 'checksum_mode_enabled_by_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'checksum_validation_enabled' => true + ], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_disabled_by_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'checksum_validation_enabled' => false + ], + 'download_config' => [], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => false, + ], + 'checksum_mode_enabled_by_download_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [ + 'checksum_validation_enabled' => true + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ], + 'checksum_mode_disabled_by_download_config' => [ + 'transfer_manager_config' => [], + 'download_config' => [ + 'checksum_validation_enabled' => false + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => false, + ], + 'checksum_mode_download_config_overrides_transfer_manager_config' => [ + 'transfer_manager_config' => [ + 'checksum_validation_enabled' => false + ], + 'download_config' => [ + 'checksum_validation_enabled' => true + ], + 'download_args' => [ + 'PartNumber' => 1 + ], + 'expected_checksum_mode' => true, + ] + ]; + } + + /** + * @param array $downloadArgs + * + * @dataProvider singleDownloadWhenPartNumberOrRangeArePresentProvider + * + * @return void + */ + public function testDoesSingleDownloadWhenPartNumberOrRangeArePresent( + array $downloadArgs, + ): void + { + $calledOnce = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use (&$calledOnce) { + if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { + if ($calledOnce) { + $this->fail(MultipartDownloader::GET_OBJECT_COMMAND . " should have been called once."); + } + + $calledOnce = true; + return Create::promiseFor(new Result([ + 'PartsCount' => 2, + 'ContentRange' => 10240000, + 'Body' => Utils::streamFor( + str_repeat("*", 1024 * 1024 * 20) + ), + '@metadata' => [] + ])); + } else { + $this->fail("Unexpected command execution `" . $command->getName() . "`."); + } + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + "s3://bucket/key", + $downloadArgs, + )->wait(); + $this->assertTrue($calledOnce); + } + + /** + * @return array + */ + public function singleDownloadWhenPartNumberOrRangeArePresentProvider(): array + { + return [ + 'part_number_present' => [ + 'download_args' => [ + 'PartNumber' => 1 + ] + ], + 'range_present' => [ + 'download_args' => [ + 'Range' => '100-1024' + ] + ] + ]; + } + + /** + * @param string $multipartDownloadType + * @param string $expectedParameter + * + * @dataProvider downloadChoosesMultipartDownloadTypeProvider + * + * @return void + */ + public function testDownloadChoosesMultipartDownloadType( + string $multipartDownloadType, + string $expectedParameter + ): void + { + $calledOnce = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$calledOnce, + $expectedParameter + ) { + $this->assertTrue( + isset($command[$expectedParameter]), + ); + $calledOnce = true; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + "s3://bucket/key", + [], + ['multipart_download_type' => $multipartDownloadType] + )->wait(); + $this->assertTrue($calledOnce); + } + + /** + * @return array + */ + public function downloadChoosesMultipartDownloadTypeProvider(): array + { + return [ + 'part_get_multipart_download' => [ + 'multipart_download_type' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, + 'expected_parameter' => 'PartNumber' + ], + 'range_get_multipart_download' => [ + 'multipart_download_type' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'expected_parameter' => 'Range' + ] + ]; + } + + /** + * @param int $minimumPartSize + * @param int $objectSize + * @param array $expectedPartsSize + * + * @dataProvider rangeGetMultipartDownloadMinimumPartSizeProvider + * + * @return void + */ + public function testRangeGetMultipartDownloadMinimumPartSize( + int $minimumPartSize, + int $objectSize, + array $expectedRangeSizes + ): void + { + $calledTimes = 0; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectSize, + $expectedRangeSizes, + &$calledTimes, + ) { + $this->assertTrue(isset($command['Range'])); + $range = str_replace("bytes=", "", $command['Range']); + $rangeParts = explode("-", $range); + $this->assertEquals( + (intval($rangeParts[1]) - intval($rangeParts[0])) + 1, + $expectedRangeSizes[$calledTimes] + ); + $calledTimes++; + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + 'ContentRange' => $objectSize, + '@metadata' => [] + ])); } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->download( + "s3://bucket/key", + [], + [ + 'multipart_download_type' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'minimum_part_size' => $minimumPartSize, + ] + )->wait(); + $this->assertEquals(count($expectedRangeSizes), $calledTimes); + } + + /** + * @return array + */ + public function rangeGetMultipartDownloadMinimumPartSizeProvider(): array + { + return [ + 'minimum_part_size_1' => [ + 'minimum_part_size' => 1024, + 'object_size' => 3072, + 'expected_range_sizes' => [ + 1024, + 1024, + 1024 + ] + ], + 'minimum_part_size_2' => [ + 'minimum_part_size' => 1024, + 'object_size' => 2000, + 'expected_range_sizes' => [ + 1024, + 977, + ] + ], + 'minimum_part_size_3' => [ + 'minimum_part_size' => 1024 * 1024 * 10, + 'object_size' => 1024 * 1024 * 25, + 'expected_range_sizes' => [ + 1024 * 1024 * 10, + 1024 * 1024 * 10, + (1024 * 1024 * 5) + 1 + ] + ], + 'minimum_part_size_4' => [ + 'minimum_part_size' => 1024 * 1024 * 25, + 'object_size' => 1024 * 1024 * 100, + 'expected_range_sizes' => [ + 1024 * 1024 * 25, + 1024 * 1024 * 25, + 1024 * 1024 * 25, + 1024 * 1024 * 25, + ] + ] + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryValidatesDestinationDirectory(): void + { + $destinationDirectory = "invalid-directory"; + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Destination directory `$destinationDirectory` MUST exists."); + $client = $this->getS3ClientMock(); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory ); + } + + /** + * @param array $config + * @param string $expectedS3Prefix + * + * @dataProvider downloadDirectoryAppliesS3PrefixProvider + * + * @return void + */ + public function testDownloadDirectoryAppliesS3Prefix( + array $config, + string $expectedS3Prefix + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Prefix, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Prefix, + $command['Prefix'] + ); + } + + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + $config + )->wait(); + + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return array + */ + public function downloadDirectoryAppliesS3PrefixProvider(): array + { + return [ + 's3_prefix_from_config' => [ + 'config' => [ + 's3_prefix' => 'TestPrefix', + ], + 'expected_s3_prefix' => 'TestPrefix' + ], + 's3_prefix_from_list_object_v2_args' => [ + 'config' => [ + 'list_object_v2_args' => [ + 'Prefix' => 'PrefixFromArgs' + ], + ], + 'expected_s3_prefix' => 'PrefixFromArgs' + ], + 's3_prefix_from_config_is_ignored_when_present_in_list_object_args' => [ + 'config' => [ + 's3_prefix' => 'TestPrefix', + 'list_object_v2_args' => [ + 'Prefix' => 'PrefixFromArgs' + ], + ], + 'expected_s3_prefix' => 'PrefixFromArgs' + ], + ]; + } + + /** + * @param array $config + * @param string $expectedS3Delimiter + * + * @dataProvider downloadDirectoryAppliesDelimiterProvider + * + * @return void + */ + public function testDownloadDirectoryAppliesDelimiter( + array $config, + string $expectedS3Delimiter + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsCalled = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $expectedS3Delimiter, + &$called, + &$listObjectsCalled, + ) { + $called = true; + if ($command->getName() === "ListObjectsV2") { + $listObjectsCalled = true; + $this->assertEquals( + $expectedS3Delimiter, + $command['Delimiter'] + ); + } + + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + $config + )->wait(); + + $this->assertTrue($called); + $this->assertTrue($listObjectsCalled); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return array + */ + public function downloadDirectoryAppliesDelimiterProvider(): array + { + return [ + 's3_delimiter_from_config' => [ + 'config' => [ + 's3_delimiter' => 'FooDelimiter', + ], + 'expected_s3_delimiter' => 'FooDelimiter' + ], + 's3_delimiter_from_list_object_v2_args' => [ + 'config' => [ + 'list_object_v2_args' => [ + 'Delimiter' => 'DelimiterFromArgs' + ], + ], + 'expected_s3_delimiter' => 'DelimiterFromArgs' + ], + 's3_delimiter_from_config_is_ignored_when_present_in_list_object_args' => [ + 'config' => [ + 's3_delimiter' => 'TestDelimiter', + 'list_object_v2_args' => [ + 'Delimiter' => 'DelimiterFromArgs' + ], + ], + 'expected_s3_delimiter' => 'DelimiterFromArgs' + ], + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidFilter(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['filter'] must be callable."); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['filter' => false] + )->wait(); + $this->assertTrue($called); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("The parameter \$config['failure_policy'] must be callable."); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + &$called, + ) { + $called = true; + return Create::promiseFor(new Result([])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => false] + )->wait(); + $this->assertTrue($called); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryUsesFailurePolicy(): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + + try { + $client = new S3Client([ + 'region' => 'us-west-2', + 'handler' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [ + [ + 'Key' => 'file1.txt', + ], + [ + 'Key' => 'file2.txt', + ] + ] + ])); + } elseif ($command->getName() === 'GetObject') { + if ($command['Key'] === 'file2.txt') { + return Create::rejectionFor( + new Exception("Failed downloading file") + ); + } + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + DownloadDirectoryResponse $downloadDirectoryResponse + ) use ($destinationDirectory, &$called) { + $called = true; + $this->assertEquals( + $destinationDirectory, + $uploadDirectoryRequestArgs['destination_directory'] + ); + $this->assertEquals( + "Failed downloading file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsDownloaded() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsFailed() + ); + }] + )->wait(); + $this->assertTrue($called); + } finally { + unlink($destinationDirectory . '/file1.txt'); + rmdir($destinationDirectory); + } + } + + /** + * @param Closure $filter + * @param array $objectList + * @param array $expectedObjectList + * + * @dataProvider downloadDirectoryAppliesFilter + * + * @return void + */ + public function testDownloadDirectoryAppliesFilter( + Closure $filter, + array $objectList, + array $expectedObjectList, + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $downloadObjectKeys = []; + foreach ($expectedObjectList as $objectKey) { + $downloadObjectKeys[$objectKey] = false; + } + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objectList, + &$called, + &$downloadObjectKeys + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objectList, + ])); + } elseif ($command->getName() === 'GetObject') { + $downloadObjectKeys[$command['Key']] = true; + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['filter' => $filter] + )->wait(); + + $this->assertTrue($called); + foreach ($downloadObjectKeys as $key => $validated) { + $this->assertTrue( + $validated, + "The key `$key` should have been validated" + ); + } + } finally { + $dirs = []; + foreach ($objectList as $object) { + if (file_exists($destinationDirectory . "/" . $object['Key'])) { + unlink($destinationDirectory . "/" . $object['Key']); + } + + $dirs [dirname($destinationDirectory . "/" . $object['Key'])] = true; + } + + foreach ($dirs as $dir => $_) { + if (is_dir($dir)) { + rmdir($dir); + } + } + + rmdir($destinationDirectory); + } + } + + /** + * @return array[] + */ + public function downloadDirectoryAppliesFilter(): array + { + return [ + 'filter_1' => [ + 'filter' => function (string $objectKey) { + return str_starts_with($objectKey, "folder_2/"); + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_1.txt", + "folder_2/key_2.txt", + ] + ], + 'filter_2' => [ + 'filter' => function (string $objectKey) { + return $objectKey === "folder_2/key_1.txt"; + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_1.txt", + ] + ], + 'filter_3' => [ + 'filter' => function (string $objectKey) { + return $objectKey !== "folder_2/key_1.txt"; + }, + 'object_list' => [ + [ + 'Key' => 'folder_1/key_1.txt', + ], + [ + 'Key' => 'folder_1/key_2.txt' + ], + [ + 'Key' => 'folder_2/key_1.txt' + ], + [ + 'Key' => 'folder_2/key_2.txt' + ] + ], + 'expected_object_list' => [ + "folder_2/key_2.txt", + "folder_1/key_1.txt", + "folder_1/key_1.txt", + ] + ] + ]; + } + + /** + * @return void + */ + public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "The parameter \$config['get_object_request_callback'] must be callable." + ); + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => [], + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [], + ['get_object_request_callback' => false] + )->wait(); + } finally { + rmdir($destinationDirectory); + } + } + + /** + * @return void + */ + public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $listObjectsContent = [ + [ + 'Key' => 'folder_1/key_1.txt', + ] + ]; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ($listObjectsContent) { + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor(), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $getObjectRequestCallback = function($requestArgs) use (&$called) { + $called = true; + $this->assertTrue(isset($requestArgs['CustomParameter'])); + $this->assertEquals( + 'CustomParameterValue', + $requestArgs['CustomParameter'] + ); + }; + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + [ + 'CustomParameter' => 'CustomParameterValue' + ], + ['get_object_request_callback' => $getObjectRequestCallback] + )->wait(); + $this->assertTrue($called); + } finally { + $dirs = []; + foreach ($listObjectsContent as $object) { + $file = $destinationDirectory . "/" . $object['Key']; + if (file_exists($file)) { + $dirs[dirname($file)] = true; + unlink($file); + } + } + + foreach (array_keys($dirs) as $dir) { + if (is_dir($dir)) { + rmdir($dir); + } + } + + rmdir($destinationDirectory); + } + } + + /** + * @param array $listObjectsContent + * @param array $expectedFileKeys + * + * @dataProvider downloadDirectoryCreateFilesProvider + * + * @return void + */ + public function testDownloadDirectoryCreateFiles( + array $listObjectsContent, + array $expectedFileKeys, + ): void + { + $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; + if (!is_dir($destinationDirectory)) { + mkdir($destinationDirectory, 0777, true); + } + try { + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $listObjectsContent, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $listObjectsContent, + ])); + } + + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor( + "Test file " . $command['Key'] + ), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + "Bucket", + $destinationDirectory, + )->wait(); + $this->assertTrue($called); + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . "/" . $key; + $this->assertFileExists($file); + $this->assertEquals( + "Test file " . $key, + file_get_contents($file) + ); + } + } finally { + $dirs = []; + foreach ($expectedFileKeys as $key) { + $file = $destinationDirectory . "/" . $key; + if (file_exists($file)) { + unlink($file); + } + + $dirs [dirname($file)] = true; + } + + foreach ($dirs as $dir => $_) { + if (is_dir($dir)) { + rmdir($dir); + } + } + + if (is_dir($destinationDirectory)) { + rmdir($destinationDirectory); + } + } + } + + /** + * @return array + */ + public function downloadDirectoryCreateFilesProvider(): array + { + return [ + 'files_1' => [ + 'list_objects_content' => [ + [ + 'Key' => 'file1.txt' + ], + [ + 'Key' => 'file2.txt' + ], + [ + 'Key' => 'file3.txt' + ], + [ + 'Key' => 'file4.txt' + ], + [ + 'Key' => 'file5.txt' + ] + ], + 'expected_file_keys' => [ + 'file1.txt', + 'file2.txt', + 'file3.txt', + 'file4.txt', + 'file5.txt' + ] + ] + ]; + } + + /** + * @param array $methodsCallback If any from the callbacks below + * is not provided then a default implementation will be provided. + * - getCommand: (Closure, optional) This callable will + * receive as parameters: + * - $commandName: (string, optional) + * - $args: (array, optional) + * - executeAsync: (Closure, optional) This callable will + * receive as parameter: + * - $command: (CommandInterface, optional) + * + * @return S3Client + */ + private function getS3ClientMock( + array $methodsCallback = [] + ): S3Client + { + if (isset($methodsCallback['getCommand']) && !is_callable($methodsCallback['getCommand'])) { + throw new InvalidArgumentException("getCommand should be callable"); + } elseif (!isset($methodsCallback['getCommand'])) { + $methodsCallback['getCommand'] = function ( + string $commandName, + array $args + ) { + return new Command($commandName, $args); + }; + } + + if (isset($methodsCallback['executeAsync']) && !is_callable($methodsCallback['executeAsync'])) { + throw new InvalidArgumentException("getObject should be callable"); + } elseif (!isset($methodsCallback['executeAsync'])) { + $methodsCallback['executeAsync'] = function ($command) { + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'UploadPart', + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + }; + } + + $client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->onlyMethods(array_keys($methodsCallback)) + ->getMock(); + foreach ($methodsCallback as $name => $callback) { + $client->method($name)->willReturnCallback($callback); + } return $client; } From 6d000f1cc9ccd4247056da9707c401b48d3c06c0 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 19 May 2025 06:30:02 -0700 Subject: [PATCH 20/62] chore: add integ tests - Add upload integ tests for: - Single uploads - Multipart uploads - Checksum in single uploads - Checksum in multipart uploads - Add download integ tests for: - Single downloads - Multipart downloads --- features/s3Transfer/s3TransferManager.feature | 85 ++++ tests/Integ/S3Context.php | 116 +----- tests/Integ/S3ContextTrait.php | 128 ++++++ tests/Integ/S3TransferManagerContext.php | 371 ++++++++++++++++++ tests/TestsUtility.php | 37 ++ tests/UserAgentMiddlewareTest.php | 32 +- 6 files changed, 626 insertions(+), 143 deletions(-) create mode 100644 features/s3Transfer/s3TransferManager.feature create mode 100644 tests/Integ/S3ContextTrait.php create mode 100644 tests/Integ/S3TransferManagerContext.php create mode 100644 tests/TestsUtility.php diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature new file mode 100644 index 0000000000..516adb6a7c --- /dev/null +++ b/features/s3Transfer/s3TransferManager.feature @@ -0,0 +1,85 @@ +@s3-transfer-manager @integ +Feature: S3 Transfer Manager + S3 Transfer Manager should successfully do: + - object uploads + - object multipart uploads + - object downloads + - object multipart downloads + - directory object uploads + - directory object downloads + + Scenario Outline: Successfully does a single file upload + Given I have a file with content + When I upload the file to a test bucket using the s3 transfer manager + Then The file should exist in the test bucket and its content should be + + Examples: + | filename | content | + | myfile.txt | Test content #1 | + | myfile-2.txt | Test content #2 | + | myfile-3.txt | Test content #3 | + + + Scenario Outline: Successfully does a single upload from a stream + Given I have a stream with content + When I do the upload to a test bucket with key + Then The object , once downloaded from the test bucket, should match the content + Examples: + | content | key | + | "This is a test text - 1" | file-1 | + | "This is a test text - 2" | file-2 | + | "This is a test text - 3" | file-3 | + + Scenario Outline: Successfully do multipart object upload from file + Given I have a file with name where its content's size is + When I do upload this file with name with the specified part size of + Then The object with name should have a total of parts and its size must be + Examples: + | filename | filesize | partsize | partnum | + | file-1 | 10485760 | 5242880 | 2 | + | file-2 | 24117248 | 5242880 | 5 | + | file-3 | 24117248 | 8388608 | 3 | + + Scenario Outline: Successfully do multipart object upload from streams + Given I have want to upload a stream of size + When I do upload this stream with name and the specified part size of + Then The object with name should have a total of parts and its size must be + Examples: + | filename | filesize | partsize | partnum | + | file-1 | 10485760 | 5242880 | 2 | + | file-2 | 24117248 | 5242880 | 5 | + | file-3 | 24117248 | 8388608 | 3 | + + Scenario Outline: Does single object upload with custom checksum + Given I have a file with name and its content is + When I upload this file with name by providing a custom checksum algorithm + Then The checksum from the object with name should be equals to the calculation of the object content with the checksum algorithm + Examples: + | filename | content | checksum_algorithm | + | file-1 | This is a test file content #1 | crc32 | + | file-2 | This is a test file content #2 | crc32c | + | file-3 | This is a test file content #3 | sha256 | + | file-4 | This is a test file content #4 | sha1 | + + Scenario Outline: Does single object download + Given I have an object in S3 with name and its content is + When I do a download of the object with name + Then The object with name should have been downloaded and its content should be + Examples: + | filename | content | + | file-1 | This is a test file content #1 | + | file-2 | This is a test file content #2 | + | file-3 | This is a test file content #3 | + + Scenario Outline: Successfully does multipart object download + Given I have an object in S3 with name and its size is + When I download the object with name by using the multipart download type + Then The content size for the object with name should be + Examples: + | filename | filesize | download_type | + | file-1 | 20971520 | partRange | + | file-2 | 28311552 | partRange | + | file-3 | 12582912 | partRange | + | file-1 | 20971520 | partGet | + | file-2 | 28311552 | partGet | + | file-3 | 12582912 | partGet | \ No newline at end of file diff --git a/tests/Integ/S3Context.php b/tests/Integ/S3Context.php index 2628417ccb..1a6ad99092 100644 --- a/tests/Integ/S3Context.php +++ b/tests/Integ/S3Context.php @@ -19,6 +19,7 @@ class S3Context implements Context, SnippetAcceptingContext { use IntegUtils; + use S3ContextTrait; const INTEG_LOG_BUCKET_PREFIX = 'aws-php-sdk-test-integ-logs'; @@ -38,18 +39,6 @@ class S3Context implements Context, SnippetAcceptingContext private $options; private $expires; - private static function getResourceName() - { - static $bucketName; - - if (empty($bucketName)) { - $bucketName = - self::getResourcePrefix() . 'aws-test-integ-s3-context'; - } - - return $bucketName; - } - /** * @BeforeSuite */ @@ -72,13 +61,7 @@ public static function deleteTempFile() */ public static function createTestBucket() { - $client = self::getSdk()->createS3(); - if (!$client->doesBucketExistV2(self::getResourceName())) { - $client->createBucket(['Bucket' => self::getResourceName()]); - $client->waitUntil('BucketExists', [ - 'Bucket' => self::getResourceName(), - ]); - } + self::doCreateTestBucket(); } /** @@ -86,64 +69,7 @@ public static function createTestBucket() */ public static function deleteTestBucket() { - $client = self::getSdk()->createS3(); - - $result = self::executeWithRetries( - $client, - 'listObjectsV2', - ['Bucket' => self::getResourceName()], - 10, - [404] - ); - - // Delete objects & wait until no longer available before deleting bucket - $client->deleteMatchingObjects(self::getResourceName(), '', '//'); - if (!empty($result['Contents']) && is_array($result['Contents'])) { - foreach ($result['Contents'] as $object) { - $client->waitUntil('ObjectNotExists', [ - 'Bucket' => self::getResourceName(), - 'Key' => $object['Key'], - '@waiter' => [ - 'maxAttempts' => 60, - 'delay' => 10, - ], - ]); - } - } - - // Delete bucket - $result = self::executeWithRetries( - $client, - 'deleteBucket', - ['Bucket' => self::getResourceName()], - 10, - [404] - ); - - // Use account number to generate a unique bucket name - $sts = new StsClient([ - 'version' => 'latest', - 'region' => 'us-east-1' - ]); - $identity = $sts->getCallerIdentity([]); - $logBucket = self::INTEG_LOG_BUCKET_PREFIX . "-{$identity['Account']}"; - - // Log bucket deletion result - if (!($client->doesBucketExistV2($logBucket))) { - $client->createBucket([ - 'Bucket' => $logBucket - ]); - } - $client->putObject([ - 'Bucket' => $logBucket, - 'Key' => self::getResourceName() . '-' . date('Y-M-d__H_i_s'), - 'Body' => print_r($result->toArray(), true) - ]); - - // Wait until bucket is no longer available - $client->waitUntil('BucketNotExists', [ - 'Bucket' => self::getResourceName(), - ]); + self::doDeleteTestBucket(); } /** @@ -353,40 +279,4 @@ private function preparePostData($postObject) 'filename' => 'file.ext', ]; } - - /** - * Executes S3 client method, adding retries for specified status codes. - * A practical work-around for the testing workflow, given eventual - * consistency constraints. - * - * @param S3Client $client - * @param string $command - * @param array $args - * @param int $retries - * @param array $statusCodes - * @return mixed - */ - private static function executeWithRetries( - $client, - $command, - $args, - $retries, - $statusCodes - ) { - $attempts = 0; - - while (true) { - try { - return call_user_func([$client, $command], $args); - } catch (S3Exception $e) { - if (!in_array($e->getStatusCode(), $statusCodes) - || $attempts >= $retries - ) { - throw $e; - } - $attempts++; - sleep((int) pow(1.2, $attempts)); - } - } - } } diff --git a/tests/Integ/S3ContextTrait.php b/tests/Integ/S3ContextTrait.php new file mode 100644 index 0000000000..e6defa7ba6 --- /dev/null +++ b/tests/Integ/S3ContextTrait.php @@ -0,0 +1,128 @@ +createS3(); + if (!$client->doesBucketExistV2(self::getResourceName())) { + $client->createBucket(['Bucket' => self::getResourceName()]); + $client->waitUntil('BucketExists', [ + 'Bucket' => self::getResourceName(), + ]); + } + } + + private static function doDeleteTestBucket(): void { + $client = self::getSdk()->createS3(); + $result = self::executeWithRetries( + $client, + 'listObjectsV2', + ['Bucket' => self::getResourceName()], + 10, + [404] + ); + + // Delete objects & wait until no longer available before deleting bucket + $client->deleteMatchingObjects(self::getResourceName(), '', '//'); + if (!empty($result['Contents']) && is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + $client->waitUntil('ObjectNotExists', [ + 'Bucket' => self::getResourceName(), + 'Key' => $object['Key'], + '@waiter' => [ + 'maxAttempts' => 60, + 'delay' => 10, + ], + ]); + } + } + + // Delete bucket + $result = self::executeWithRetries( + $client, + 'deleteBucket', + ['Bucket' => self::getResourceName()], + 10, + [404] + ); + + // Use account number to generate a unique bucket name + $sts = new StsClient([ + 'version' => 'latest', + 'region' => 'us-east-1' + ]); + $identity = $sts->getCallerIdentity([]); + $logBucket = self::INTEG_LOG_BUCKET_PREFIX . "-{$identity['Account']}"; + + // Log bucket deletion result + if (!($client->doesBucketExistV2($logBucket))) { + $client->createBucket([ + 'Bucket' => $logBucket + ]); + } + $client->putObject([ + 'Bucket' => $logBucket, + 'Key' => self::getResourceName() . '-' . date('Y-M-d__H_i_s'), + 'Body' => print_r($result->toArray(), true) + ]); + + // Wait until bucket is no longer available + $client->waitUntil('BucketNotExists', [ + 'Bucket' => self::getResourceName(), + ]); + } + + /** + * Executes S3 client method, adding retries for specified status codes. + * A practical work-around for the testing workflow, given eventual + * consistency constraints. + * + * @param S3Client $client + * @param string $command + * @param array $args + * @param int $retries + * @param array $statusCodes + * @return mixed + */ + private static function executeWithRetries( + $client, + $command, + $args, + $retries, + $statusCodes + ) { + $attempts = 0; + + while (true) { + try { + return call_user_func([$client, $command], $args); + } catch (S3Exception $e) { + if (!in_array($e->getStatusCode(), $statusCodes) + || $attempts >= $retries + ) { + throw $e; + } + $attempts++; + sleep((int) pow(1.2, $attempts)); + } + } + } +} \ No newline at end of file diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php new file mode 100644 index 0000000000..298571af6c --- /dev/null +++ b/tests/Integ/S3TransferManagerContext.php @@ -0,0 +1,371 @@ +stream = Utils::streamFor(''); + // Create temporary directory + self::$tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "s3-transfer-manager"; + if (is_dir(self::$tempDir)) { + TestsUtility::cleanUpDir(self::$tempDir); + } + + mkdir(self::$tempDir, 0777, true); + + // Create test bucket + // self::doCreateTestBucket(); + } + + /** + * @AfterScenario + */ + public function cleanUp(): void { + // Clean up temporary directory + TestsUtility::cleanUpDir(self::$tempDir); + + // Clean up test bucket + // self::doDeleteTestBucket(); + + // Clean up data holders + $this->stream?->close(); + } + + /** + * @Given /^I have a file (.*) with content (.*)$/ + */ + public function iHaveAFileWithContent($filename, $content): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, $content); + } + + /** + * @When /^I upload the file (.*) to a test bucket using the s3 transfer manager$/ + */ + public function iUploadTheFileToATestBucketUsingTheS3TransferManager($filename): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->upload( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ] + )->wait(); + } + + /** + * @Then /^The file (.*) should exist in the test bucket and its content should be (.*)$/ + */ + public function theFileShouldExistInTheTestBucketAndItsContentShouldBe($filename, $content): void + { + $client = self::getSdk()->createS3(); + $response = $client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + Assert::assertEquals($content, $response['Body']->getContents()); + } + + /** + * @Given /^I have a stream with content (.*)$/ + */ + public function iHaveAStreamWithContent($content): void + { + $this->stream = Utils::streamFor($content); + } + + /** + * @When /^I do the upload to a test bucket with key (.*)$/ + */ + public function iDoTheUploadToATestBucketWithKey($key): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $s3TransferManager->upload( + $this->stream, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ] + )->wait(); + } + + /** + * @Then /^The object (.*), once downloaded from the test bucket, should match the content (.*)$/ + */ + public function theObjectOnceDownloadedFromTheTestBucketShouldMatchTheContent($key, $content): void + { + $client = self::getSdk()->createS3(); + $response = $client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + Assert::assertEquals($content, $response['Body']->getContents()); + } + + /** + * @Given /^I have a file with name (.*) where its content's size is (.*)$/ + */ + public function iHaveAFileWithNameWhereItsContentSSizeIs($filename, $filesize): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, str_repeat('a', $filesize)); + } + + /** + * @When /^I do upload this file with name (.*) with the specified part size of (.*)$/ + */ + public function iDoUploadThisFileWithNameWithTheSpecifiedPartSizeOf($filename, $partsize): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + [ + 'multipart_upload_threshold_bytes' => $partsize, + ] + ); + $s3TransferManager->upload( + $fullFilePath, + [ + 'Bucket' => 'herrergy-sample-bucket', + 'Key' => $filename, + ], + [ + 'part_size' => intval($partsize), + ] + )->wait(); + } + + /** + * @Given /^I have want to upload a stream of size (.*)$/ + */ + public function iHaveWantToUploadAStreamOfSize($filesize): void + { + $this->stream = Utils::streamFor(str_repeat('a', $filesize)); + } + + /** + * @When /^I do upload this stream with name (.*) and the specified part size of (.*)$/ + */ + public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf($filename, $partsize): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + [ + 'multipart_upload_threshold_bytes' => $partsize, + ] + ); + $s3TransferManager->upload( + $this->stream, + [ + 'Bucket' => 'herrergy-sample-bucket', + 'Key' => $filename, + ], + [ + 'part_size' => intval($partsize), + ] + )->wait(); + } + + /** + * @Then /^The object with name (.*) should have a total of (.*) parts and its size must be (.*)$/ + */ + public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe($filename, $partnum, $filesize): void + { + $partNo = 1; + $s3Client = self::getSdk()->createS3(); + $response = $s3Client->headObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'PartNumber' => $partNo + ]); + Assert::assertEquals(206, $response['@metadata']['statusCode']); + Assert::assertEquals($partnum, $response['PartsCount']); + $contentLength = $response['@metadata']['headers']['content-length']; + $partNo++; + while ($partNo <= $partnum) { + $response = $s3Client->headObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'PartNumber' => $partNo + ]); + $contentLength += $response['@metadata']['headers']['content-length']; + $partNo++; + } + + Assert::assertEquals($filesize, $contentLength); + } + + /** + * @Given /^I have a file with name (.*) and its content is (.*)$/ + */ + public function iHaveAFileWithNameAndItsContentIs($filename, $content): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, $content); + } + + /** + * @When /^I upload this file with name (.*) by providing a custom checksum algorithm (.*)$/ + */ + public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm($filename, $checksum_algorithm): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->upload( + $fullFilePath, + [ + 'Bucket' => 'herrergy-sample-bucket', + 'Key' => $filename, + ], + [ + 'checksum_algorithm' => $checksum_algorithm, + ] + )->wait(); + } + + /** + * @Then /^The checksum from the object with name (.*) should be equals to the calculation of the object content with the checksum algorithm (.*)$/ + */ + public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOfTheObjectContentWithTheChecksumAlgorithm($filename, $checksum_algorithm): void + { + $s3Client = self::getSdk()->createS3(); + $response = $s3Client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'ChecksumMode' => 'ENABLED' + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + Assert::assertEquals( + ApplyChecksumMiddleware::getEncodedValue( + $checksum_algorithm, + $response['Body']->getContents() + ), + $response['Checksum' . strtoupper($checksum_algorithm)] + ); + } + + /** + * @Given /^I have an object in S3 with name (.*) and its content is (.*)$/ + */ + public function iHaveAnObjectInS3withNameAndItsContentIs($filename, $content): void + { + $client = self::getSdk()->createS3(); + $client->putObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'Body' => $content, + ]); + } + + /** + * @When /^I do a download of the object with name (.*)$/ + */ + public function iDoADownloadOfTheObjectWithName($filename): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->download([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ])->then(function (DownloadResponse $response) use ($filename) { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, $response->getData()); + })->wait(); + } + + /** + * @Then /^The object with name (.*) should have been downloaded and its content should be (.*)$/ + */ + public function theObjectWithNameShouldHaveBeenDownloadedAndItsContentShouldBe($filename, $content): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + Assert::assertFileExists($fullFilePath); + Assert::assertEquals($content, file_get_contents($fullFilePath)); + } + + /** + * @Given /^I have an object in S3 with name (.*) and its size is (.*)$/ + */ + public function iHaveAnObjectInS3withNameAndItsSizeIs($filename, $filesize): void + { + $client = self::getSdk()->createS3(); + $client->putObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + 'Body' => str_repeat('*', $filesize), + ]); + } + + /** + * @When /^I download the object with name (.*) by using the (.*) multipart download type$/ + */ + public function iDownloadTheObjectWithNameByUsingTheMultipartDownloadType($filename, $download_type): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->download([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ], + [], + [ + 'multipart_download_type' => $download_type, + ])->wait(); + } + + /** + * @Then /^The content size for the object with name (.*) should be (.*)$/ + */ + public function theContentSizeForTheObjectWithNameShouldBe($filename, $filesize): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + Assert::assertFileExists($fullFilePath); + Assert::assertEquals($filesize, filesize($fullFilePath)); + Assert::assertEquals(str_repeat('*', $filesize), file_get_contents($fullFilePath)); + } +} \ No newline at end of file diff --git a/tests/TestsUtility.php b/tests/TestsUtility.php new file mode 100644 index 0000000000..81a43b3f1a --- /dev/null +++ b/tests/TestsUtility.php @@ -0,0 +1,37 @@ +cleanUpDir($this->tempDir); + TestsUtility::cleanUpDir($this->tempDir); } /** @@ -1087,35 +1088,6 @@ public function testUserAgentCaptureCredentialsProfileStsWebIdTokenMetric() $s3Client->listBuckets(); } - /** - * Helper method to clean up temporary dirs. - * - * @param $dirPath - * - * @return void - */ - private function cleanUpDir($dirPath): void - { - if (!is_dir($dirPath)) { - return; - } - - $files = dir_iterator($dirPath); - foreach ($files as $file) { - if (in_array($file, ['.', '..'])) { - continue; - } - - $filePath = $dirPath . '/' . $file; - if (is_file($filePath) || !is_dir($filePath)) { - unlink($filePath); - } elseif (is_dir($filePath)) { - $this->cleanUpDir($filePath); - } - } - - rmdir($dirPath); - } /** * Test user agent captures metric for credentials resolved from From 27570d04acca87f41dee53be04a2ea89983ad845 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 20 May 2025 08:55:18 -0700 Subject: [PATCH 21/62] chore: add integ test - Add integ tests for directory uploads - Add integ tests for directory downloads --- behat.yml | 3 + features/s3Transfer/s3TransferManager.feature | 86 +++++---- src/S3/S3Transfer/MultipartDownloader.php | 1 + src/S3/S3Transfer/MultipartUploader.php | 1 - tests/Integ/S3TransferManagerContext.php | 167 +++++++++++++++--- tests/UserAgentMiddlewareTest.php | 2 - 6 files changed, 204 insertions(+), 56 deletions(-) diff --git a/behat.yml b/behat.yml index 3a19a2bc6f..d0b7119773 100644 --- a/behat.yml +++ b/behat.yml @@ -33,6 +33,9 @@ default: s3: paths: [ "%paths.base%/features/s3" ] contexts: [ Aws\Test\Integ\S3Context ] + s3TransferManager: + paths: [ "%paths.base%/features/s3Transfer" ] + contexts: [ Aws\Test\Integ\S3TransferManagerContext ] s3Encryption: paths: [ "%paths.base%/features/s3Encryption" ] contexts: [ Aws\Test\Integ\S3EncryptionContext ] diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature index 516adb6a7c..44a24e7cdf 100644 --- a/features/s3Transfer/s3TransferManager.feature +++ b/features/s3Transfer/s3TransferManager.feature @@ -11,75 +11,97 @@ Feature: S3 Transfer Manager Scenario Outline: Successfully does a single file upload Given I have a file with content When I upload the file to a test bucket using the s3 transfer manager - Then The file should exist in the test bucket and its content should be + Then the file should exist in the test bucket and its content should be Examples: | filename | content | - | myfile.txt | Test content #1 | - | myfile-2.txt | Test content #2 | - | myfile-3.txt | Test content #3 | + | myfile-test-1-1.txt | Test content #1 | + | myfile-test-1-2.txt | Test content #2 | + | myfile-test-1-3.txt | Test content #3 | Scenario Outline: Successfully does a single upload from a stream Given I have a stream with content When I do the upload to a test bucket with key - Then The object , once downloaded from the test bucket, should match the content + Then the object , once downloaded from the test bucket, should match the content Examples: | content | key | - | "This is a test text - 1" | file-1 | - | "This is a test text - 2" | file-2 | - | "This is a test text - 3" | file-3 | + | "This is a test text - 1" | myfile-test-2-1.txt | + | "This is a test text - 2" | myfile-test-2-2.txt | + | "This is a test text - 3" | myfile-test-2-3.txt | Scenario Outline: Successfully do multipart object upload from file Given I have a file with name where its content's size is When I do upload this file with name with the specified part size of - Then The object with name should have a total of parts and its size must be + Then the object with name should have a total of parts and its size must be Examples: | filename | filesize | partsize | partnum | - | file-1 | 10485760 | 5242880 | 2 | - | file-2 | 24117248 | 5242880 | 5 | - | file-3 | 24117248 | 8388608 | 3 | + | myfile-test-3-1.txt | 10485760 | 5242880 | 2 | + | myfile-test-3-2.txt | 24117248 | 5242880 | 5 | + | myfile-test-3-3.txt | 24117248 | 8388608 | 3 | Scenario Outline: Successfully do multipart object upload from streams Given I have want to upload a stream of size When I do upload this stream with name and the specified part size of - Then The object with name should have a total of parts and its size must be + Then the object with name should have a total of parts and its size must be Examples: | filename | filesize | partsize | partnum | - | file-1 | 10485760 | 5242880 | 2 | - | file-2 | 24117248 | 5242880 | 5 | - | file-3 | 24117248 | 8388608 | 3 | + | myfile-test-4-1.txt | 10485760 | 5242880 | 2 | + | myfile-test-4-2.txt | 24117248 | 5242880 | 5 | + | myfile-test-4-3.txt | 24117248 | 8388608 | 3 | Scenario Outline: Does single object upload with custom checksum Given I have a file with name and its content is When I upload this file with name by providing a custom checksum algorithm - Then The checksum from the object with name should be equals to the calculation of the object content with the checksum algorithm + Then the checksum from the object with name should be equals to the calculation of the object content with the checksum algorithm Examples: | filename | content | checksum_algorithm | - | file-1 | This is a test file content #1 | crc32 | - | file-2 | This is a test file content #2 | crc32c | - | file-3 | This is a test file content #3 | sha256 | - | file-4 | This is a test file content #4 | sha1 | + | myfile-test-5-1.txt | This is a test file content #1 | crc32 | + | myfile-test-5-2.txt | This is a test file content #2 | crc32c | + | myfile-test-5-3.txt | This is a test file content #3 | sha256 | + | myfile-test-5-4.txt | This is a test file content #4 | sha1 | Scenario Outline: Does single object download Given I have an object in S3 with name and its content is When I do a download of the object with name - Then The object with name should have been downloaded and its content should be + Then the object with name should have been downloaded and its content should be Examples: | filename | content | - | file-1 | This is a test file content #1 | - | file-2 | This is a test file content #2 | - | file-3 | This is a test file content #3 | + | myfile-test-6-1.txt | This is a test file content #1 | + | myfile-test-6-2.txt | This is a test file content #2 | + | myfile-test-6-3.txt | This is a test file content #3 | Scenario Outline: Successfully does multipart object download Given I have an object in S3 with name and its size is When I download the object with name by using the multipart download type - Then The content size for the object with name should be + Then the content size for the object with name should be Examples: | filename | filesize | download_type | - | file-1 | 20971520 | partRange | - | file-2 | 28311552 | partRange | - | file-3 | 12582912 | partRange | - | file-1 | 20971520 | partGet | - | file-2 | 28311552 | partGet | - | file-3 | 12582912 | partGet | \ No newline at end of file + | myfile-test-7-1.txt | 20971520 | rangeGet | + | myfile-test-7-2.txt | 28311552 | rangeGet | + | myfile-test-7-3.txt | 12582912 | rangeGet | + | myfile-test-7-4.txt | 20971520 | partGet | + | myfile-test-7-5.txt | 28311552 | partGet | + | myfile-test-7-6.txt | 12582912 | partGet | + + Scenario Outline: Successfully does directory upload + Given I have a directory with files that I want to upload + When I upload this directory + Then the files from this directory where its count should be should exist in the bucket + Examples: + | directory | numfile | + | directory-test-1-1 | 10 | + | directory-test-1-2 | 3 | + | directory-test-1-3 | 25 | + | directory-test-1-4 | 1 | + + Scenario Outline: Successfully does a directory download + Given I have a total of objects in a bucket prefixed with + When I download all of them into the directory + Then the objects should exist as files within the directory + Examples: + | numfile | directory | + | 15 | directory-test-2-1 | + | 12 | directory-test-2-2 | + | 1 | directory-test-2-3 | + | 30 | directory-test-2-4 | \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 2e4de9c568..d6bb8450d7 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -175,6 +175,7 @@ public function promise(): PromiseInterface // Transfer completed $this->downloadComplete(); + unset($result['Body']); yield Create::promiseFor(new DownloadResponse( $this->stream, $result['@metadata'] ?? [] diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index b40e1891ec..d9a86602f6 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -216,7 +216,6 @@ private function uploadParts(): PromiseInterface break; } - $uploadPartCommandArgs = [ ...$this->createMultipartArgs, 'UploadId' => $this->uploadId, diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index 298571af6c..eba874948a 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -29,10 +29,26 @@ class S3TransferManagerContext implements Context, SnippetAcceptingContext */ private StreamInterface | null $stream; + /** + * @BeforeSuite + */ + public static function beforeSuiteRuns(): void { + // Create test bucket + self::doCreateTestBucket(); + } + + /** + * @AfterSuite + */ + public static function afterSuiteRuns(): void { + // Clean up test bucket + self::doDeleteTestBucket(); + } + /** * @BeforeScenario */ - public function setup(): void { + public function beforeScenarioRuns(): void { $this->stream = Utils::streamFor(''); // Create temporary directory self::$tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "s3-transfer-manager"; @@ -41,21 +57,15 @@ public function setup(): void { } mkdir(self::$tempDir, 0777, true); - - // Create test bucket - // self::doCreateTestBucket(); } /** * @AfterScenario */ - public function cleanUp(): void { + public function afterScenarioRuns(): void { // Clean up temporary directory TestsUtility::cleanUpDir(self::$tempDir); - // Clean up test bucket - // self::doDeleteTestBucket(); - // Clean up data holders $this->stream?->close(); } @@ -88,7 +98,7 @@ public function iUploadTheFileToATestBucketUsingTheS3TransferManager($filename): } /** - * @Then /^The file (.*) should exist in the test bucket and its content should be (.*)$/ + * @Then /^the file (.*) should exist in the test bucket and its content should be (.*)$/ */ public function theFileShouldExistInTheTestBucketAndItsContentShouldBe($filename, $content): void { @@ -128,7 +138,7 @@ public function iDoTheUploadToATestBucketWithKey($key): void } /** - * @Then /^The object (.*), once downloaded from the test bucket, should match the content (.*)$/ + * @Then /^the object (.*), once downloaded from the test bucket, should match the content (.*)$/ */ public function theObjectOnceDownloadedFromTheTestBucketShouldMatchTheContent($key, $content): void { @@ -166,7 +176,7 @@ public function iDoUploadThisFileWithNameWithTheSpecifiedPartSizeOf($filename, $ $s3TransferManager->upload( $fullFilePath, [ - 'Bucket' => 'herrergy-sample-bucket', + 'Bucket' => self::getResourceName(), 'Key' => $filename, ], [ @@ -197,7 +207,7 @@ public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf($filename, $s3TransferManager->upload( $this->stream, [ - 'Bucket' => 'herrergy-sample-bucket', + 'Bucket' => self::getResourceName(), 'Key' => $filename, ], [ @@ -207,7 +217,7 @@ public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf($filename, } /** - * @Then /^The object with name (.*) should have a total of (.*) parts and its size must be (.*)$/ + * @Then /^the object with name (.*) should have a total of (.*) parts and its size must be (.*)$/ */ public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe($filename, $partnum, $filesize): void { @@ -256,7 +266,7 @@ public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm($file $s3TransferManager->upload( $fullFilePath, [ - 'Bucket' => 'herrergy-sample-bucket', + 'Bucket' => self::getResourceName(), 'Key' => $filename, ], [ @@ -266,7 +276,7 @@ public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm($file } /** - * @Then /^The checksum from the object with name (.*) should be equals to the calculation of the object content with the checksum algorithm (.*)$/ + * @Then /^the checksum from the object with name (.*) should be equals to the calculation of the object content with the checksum algorithm (.*)$/ */ public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOfTheObjectContentWithTheChecksumAlgorithm($filename, $checksum_algorithm): void { @@ -281,7 +291,7 @@ public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOf Assert::assertEquals( ApplyChecksumMiddleware::getEncodedValue( $checksum_algorithm, - $response['Body']->getContents() + $response['Body'] ), $response['Checksum' . strtoupper($checksum_algorithm)] ); @@ -313,12 +323,12 @@ public function iDoADownloadOfTheObjectWithName($filename): void 'Key' => $filename, ])->then(function (DownloadResponse $response) use ($filename) { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; - file_put_contents($fullFilePath, $response->getData()); + file_put_contents($fullFilePath, $response->getData()->getContents()); })->wait(); } /** - * @Then /^The object with name (.*) should have been downloaded and its content should be (.*)$/ + * @Then /^the object with name (.*) should have been downloaded and its content should be (.*)$/ */ public function theObjectWithNameShouldHaveBeenDownloadedAndItsContentShouldBe($filename, $content): void { @@ -355,17 +365,132 @@ public function iDownloadTheObjectWithNameByUsingTheMultipartDownloadType($filen [], [ 'multipart_download_type' => $download_type, - ])->wait(); + ])->then(function (DownloadResponse $response) use ($filename) { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; + file_put_contents($fullFilePath, $response->getData()->getContents()); + })->wait(); } /** - * @Then /^The content size for the object with name (.*) should be (.*)$/ + * @Then /^the content size for the object with name (.*) should be (.*)$/ */ public function theContentSizeForTheObjectWithNameShouldBe($filename, $filesize): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; Assert::assertFileExists($fullFilePath); Assert::assertEquals($filesize, filesize($fullFilePath)); - Assert::assertEquals(str_repeat('*', $filesize), file_get_contents($fullFilePath)); + } + + /** + * @Given /^I have a directory (.*) with (.*) files that I want to upload$/ + */ + public function iHaveADirectoryWithFilesThatIWantToUpload($directory, $numfile): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + if (!is_dir($fullDirectoryPath)) { + mkdir($fullDirectoryPath, 0777, true); + } + + for ($i = 0; $i < $numfile - 1; $i++) { + $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt"; + file_put_contents($fullFilePath, "This is a test file content #" . ($i + 1)); + } + + // 1 extra for multipart upload + $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt"; + file_put_contents($fullFilePath, str_repeat('*', 1024 * 1024 * 15)); + } + + /** + * @When /^I upload this directory (.*)$/ + */ + public function iUploadThisDirectory($directory): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->uploadDirectory( + $fullDirectoryPath, + self::getResourceName(), + )->wait(); + } + + /** + * @Then /^the files from this directory (.*) where its count should be (.*) should exist in the bucket$/ + */ + public function theFilesFromThisDirectoryWhereItsCountShouldBeShouldExistInTheBucket($directory, $numfile): void + { + $s3Client = self::getSdk()->createS3(); + $objects = $s3Client->getPaginator('ListObjectsV2', [ + 'Bucket' => self::getResourceName(), + 'Prefix' => $directory . DIRECTORY_SEPARATOR, + ]); + $count = 0; + foreach ($objects as $object) { + $fullObjectPath = self::$tempDir . DIRECTORY_SEPARATOR . $object; + Assert::assertFileExists($fullObjectPath); + $count++; + } + + Assert::assertEquals($numfile, $count); + } + + /** + * @Given /^I have a total of (.*) objects in a bucket prefixed with (.*)$/ + */ + public function iHaveATotalOfObjectsInABucketPrefixedWith($numfile, $directory): void + { + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + for ($i = 0; $i < $numfile - 1; $i++) { + $s3TransferManager->upload( + Utils::streamFor("This is a test file content #" . ($i + 1)), + [ + 'Bucket' => self::getResourceName(), + 'Key' => $directory . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt", + ] + )->wait(); + } + } + + /** + * @When /^I download all of them into the directory (.*)$/ + */ + public function iDownloadAllOfThemIntoTheDirectory($directory): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3(), + ); + $s3TransferManager->downloadDirectory( + self::getResourceName(), + $fullDirectoryPath, + [], + [ + 's3_prefix' => $directory . DIRECTORY_SEPARATOR, + ] + ); + } + + /** + * @Then /^the objects (.*) should exist as files within the directory (.*)$/ + */ + public function theObjectsShouldExistsAsFilesWithinTheDirectory($numfile, $directory): void + { + $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $s3Client = self::getSdk()->createS3(); + $objects = $s3Client->getPaginator('ListObjectsV2', [ + 'Bucket' => self::getResourceName(), + 'Prefix' => $directory . DIRECTORY_SEPARATOR, + ]); + $count = 0; + foreach ($objects as $object) { + Assert::assertFileExists($fullDirectoryPath . DIRECTORY_SEPARATOR . $object); + $count++; + } + + Assert::assertEquals($numfile, $count); } } \ No newline at end of file diff --git a/tests/UserAgentMiddlewareTest.php b/tests/UserAgentMiddlewareTest.php index 8468f09fc7..2b43a24925 100644 --- a/tests/UserAgentMiddlewareTest.php +++ b/tests/UserAgentMiddlewareTest.php @@ -29,9 +29,7 @@ use GuzzleHttp\Psr7\Utils; use Psr\Http\Message\RequestInterface; use GuzzleHttp\Psr7\Request; -use TestsUtility; use Yoast\PHPUnitPolyfills\TestCases\TestCase; -use function Aws\dir_iterator; /** * @covers \Aws\UserAgentMiddleware From 1dde7fcd5620b3229298e4bf62846dbe13697db8 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 26 May 2025 15:12:29 -0700 Subject: [PATCH 22/62] chore: address PR feedback - Move some fixed values out of the methods into consts. - Address a line exceeded 80 chars. - Declare keys used across different implementations as consts. --- src/S3/S3Transfer/MultipartDownloader.php | 22 +++---- src/S3/S3Transfer/MultipartUploader.php | 19 +++--- .../ColoredTransferProgressBarFormat.php | 28 ++++----- .../Progress/MultiProgressBarFormat.php | 20 ++++--- .../Progress/MultiProgressTracker.php | 12 ++-- .../Progress/PlainProgressBarFormat.php | 15 +++-- .../Progress/SingleProgressTracker.php | 10 ++-- .../S3Transfer/Progress/TransferListener.php | 4 ++ .../Progress/TransferProgressBarFormat.php | 22 ++++--- src/S3/S3Transfer/S3TransferManager.php | 28 ++++----- .../Progress/MultiProgressTrackerTest.php | 58 +++++++++---------- .../Progress/SingleProgressTrackerTest.php | 36 ++++++------ tests/S3/S3Transfer/S3TransferManagerTest.php | 6 +- 13 files changed, 151 insertions(+), 129 deletions(-) diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index d6bb8450d7..7e74a237fe 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -6,6 +6,7 @@ use Aws\ResultInterface; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; @@ -20,7 +21,8 @@ abstract class MultipartDownloader implements PromisorInterface public const GET_OBJECT_COMMAND = "GetObject"; public const PART_GET_MULTIPART_DOWNLOADER = "partGet"; public const RANGE_GET_MULTIPART_DOWNLOADER = "rangeGet"; - + private const OBJECT_SIZE_REGEX = "/\/(\d+)$/"; + /** @var array */ protected array $requestArgs; @@ -217,7 +219,7 @@ protected function computeObjectSize($sizeSource): int } // For extracting the object size from the ContentRange header value. - if (preg_match("/\/(\d+)$/", $sizeSource, $matches)) { + if (preg_match(self::OBJECT_SIZE_REGEX, $sizeSource, $matches)) { return $matches[1]; } @@ -252,8 +254,8 @@ private function downloadInitiated(array $commandArgs): void } $this->listenerNotifier?->transferInitiated([ - 'request_args' => $commandArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $commandArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -268,8 +270,8 @@ private function downloadFailed(\Throwable $reason): void { $this->stream->close(); $this->listenerNotifier?->transferFail([ - 'request_args' => $this->requestArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); } @@ -302,8 +304,8 @@ private function partDownloadCompleted( ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->bytesTransferred([ - 'request_args' => $this->requestArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -339,8 +341,8 @@ private function downloadComplete(): void ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->transferComplete([ - 'request_args' => $this->requestArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index d9a86602f6..e153f6e721 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -101,7 +101,8 @@ public function __construct( private function validateConfig(array &$config): void { if (isset($config['part_size'])) { - if ($config['part_size'] < self::PART_MIN_SIZE || $config['part_size'] > self::PART_MAX_SIZE) { + if ($config['part_size'] < self::PART_MIN_SIZE + || $config['part_size'] > self::PART_MAX_SIZE) { throw new \InvalidArgumentException( "The config `part_size` value must be between " . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE . "." @@ -398,8 +399,8 @@ private function uploadInitiated(array $requestArgs): void } $this->listenerNotifier?->transferInitiated([ - 'request_args' => $requestArgs, - 'progress_snapshot' => $this->currentSnapshot + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot ]); } @@ -413,8 +414,8 @@ private function uploadFailed(Throwable $reason): void { $this->abortMultipartUpload()->wait(); } $this->listenerNotifier?->transferFail([ - 'request_args' => $this->createMultipartArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); } @@ -433,8 +434,8 @@ private function uploadCompleted(ResultInterface $result): void { ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->transferComplete([ - 'request_args' => $this->createMultipartArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -456,8 +457,8 @@ private function partUploadCompleted( ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->bytesTransferred([ - 'request_args' => $requestArgs, - 'progress_snapshot' => $this->currentSnapshot, + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, $this->currentSnapshot ]); } diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php index cc7f80d6f3..3041f2a71e 100644 --- a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -8,15 +8,26 @@ final class ColoredTransferProgressBarFormat extends ProgressBarFormat public const BLUE_COLOR_CODE = '[34m'; public const GREEN_COLOR_CODE = '[32m'; public const RED_COLOR_CODE = '[31m'; + public const FORMAT_TEMPLATE = "|object_name|:\n" + ."\033|color_code|[|progress_bar|] |percent|% " + ."|transferred|/|to_be_transferred| |unit| |message|\033[0m"; + public const FORMAT_PARAMETERS = [ + 'progress_bar', + 'percent', + 'transferred', + 'to_be_transferred', + 'unit', + 'color_code', + 'message', + 'object_name' + ]; /** * @inheritDoc */ public function getFormatTemplate(): string { - return - "|object_name|:\n" - ."\033|color_code|[|progress_bar|] |percent|% |transferred|/|to_be_transferred| |unit| |message|\033[0m"; + return self::FORMAT_TEMPLATE; } /** @@ -24,16 +35,7 @@ public function getFormatTemplate(): string */ public function getFormatParameters(): array { - return [ - 'progress_bar', - 'percent', - 'transferred', - 'to_be_transferred', - 'unit', - 'color_code', - 'message', - 'object_name' - ]; + return self::FORMAT_PARAMETERS; } protected function getFormatDefaultParameterValues(): array diff --git a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php index 16f65a220c..a4185824ae 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php @@ -4,12 +4,22 @@ class MultiProgressBarFormat extends ProgressBarFormat { + public const FORMAT_TEMPLATE = "[|progress_bar|] |percent|%" + ."Completed: |completed|/|total|, Failed: |failed|/|total|"; + public const FORMAT_PARAMETERS = [ + 'completed', + 'failed', + 'total', + 'percent', + 'progress_bar' + ]; + /** * @return string */ public function getFormatTemplate(): string { - return "[|progress_bar|] |percent|% Completed: |completed|/|total|, Failed: |failed|/|total|"; + return self::FORMAT_TEMPLATE; } /** @@ -17,13 +27,7 @@ public function getFormatTemplate(): string */ public function getFormatParameters(): array { - return [ - 'completed', - 'failed', - 'total', - 'percent', - 'progress_bar' - ]; + return self::FORMAT_PARAMETERS; } /** diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php index c0d6ad196a..67895465bc 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressTracker.php +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -6,6 +6,8 @@ final class MultiProgressTracker extends TransferListener implements ProgressTrackerInterface { + private const CLEAR_ASCII_CODE = "\033[2J\033[H"; + /** @var array */ private array $singleProgressTrackers; @@ -103,7 +105,7 @@ public function getProgressBarFactory(): ProgressBarFactoryInterface | Closure | public function transferInitiated(array $context): void { $this->transferCount++; - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; if (isset($this->singleProgressTrackers[$snapshot->getIdentifier()])) { $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; } else { @@ -135,7 +137,7 @@ public function transferInitiated(array $context): void */ public function bytesTransferred(array $context): void { - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; $progressTracker->bytesTransferred($context); $this->showProgress(); @@ -147,7 +149,7 @@ public function bytesTransferred(array $context): void public function transferComplete(array $context): void { $this->completed++; - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; $progressTracker->transferComplete($context); $this->showProgress(); @@ -159,7 +161,7 @@ public function transferComplete(array $context): void public function transferFail(array $context): void { $this->failed++; - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressTracker = $this->singleProgressTrackers[$snapshot->getIdentifier()]; $progressTracker->transferFail($context); $this->showProgress(); @@ -170,7 +172,7 @@ public function transferFail(array $context): void */ public function showProgress(): void { - fwrite($this->output, "\033[2J\033[H"); + fwrite($this->output, self::CLEAR_ASCII_CODE); $percentsSum = 0; /** * @var $_ diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php index 0e89093fb5..0e077d6e5e 100644 --- a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -4,18 +4,21 @@ final class PlainProgressBarFormat extends ProgressBarFormat { + public const FORMAT_TEMPLATE = "|object_name|:\n[|progress_bar|] |percent|%"; + public const FORMAT_PARAMETERS = [ + 'object_name', + 'progress_bar', + 'percent', + ]; + public function getFormatTemplate(): string { - return "|object_name|:\n[|progress_bar|] |percent|%"; + return self::FORMAT_TEMPLATE; } public function getFormatParameters(): array { - return [ - 'object_name', - 'progress_bar', - 'percent', - ]; + return self::FORMAT_PARAMETERS; } protected function getFormatDefaultParameterValues(): array diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 23867d7e22..75c986f2a5 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -96,7 +96,7 @@ public function isShowProgressOnUpdate(): bool */ public function transferInitiated(array $context): void { - $this->currentSnapshot = $context['progress_snapshot']; + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressFormat = $this->progressBar->getProgressBarFormat(); // Probably a common argument $progressFormat->setArg( @@ -114,7 +114,7 @@ public function transferInitiated(array $context): void */ public function bytesTransferred(array $context): void { - $this->currentSnapshot = $context['progress_snapshot']; + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -133,7 +133,7 @@ public function bytesTransferred(array $context): void */ public function transferComplete(array $context): void { - $this->currentSnapshot = $context['progress_snapshot']; + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -154,7 +154,7 @@ public function transferComplete(array $context): void */ public function transferFail(array $context): void { - $this->currentSnapshot = $context['progress_snapshot']; + $this->currentSnapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $progressFormat = $this->progressBar->getProgressBarFormat(); if ($progressFormat instanceof ColoredTransferProgressBarFormat) { $progressFormat->setArg( @@ -163,7 +163,7 @@ public function transferFail(array $context): void ); $progressFormat->setArg( 'message', - $context['reason'] + $context[TransferListener::REASON_KEY] ); } diff --git a/src/S3/S3Transfer/Progress/TransferListener.php b/src/S3/S3Transfer/Progress/TransferListener.php index 811a3bd47d..3fea230bf3 100644 --- a/src/S3/S3Transfer/Progress/TransferListener.php +++ b/src/S3/S3Transfer/Progress/TransferListener.php @@ -4,6 +4,10 @@ abstract class TransferListener { + public const REQUEST_ARGS_KEY = TransferListener::REQUEST_ARGS_KEY; + public const PROGRESS_SNAPSHOT_KEY = TransferListener::PROGRESS_SNAPSHOT_KEY; + public const REASON_KEY = 'reason'; + /** * @param array $context * - request_args: (array) The request arguments that will be provided diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php index c7a40575a5..2737a1b513 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -4,12 +4,23 @@ final class TransferProgressBarFormat extends ProgressBarFormat { + public const FORMAT_TEMPLATE = "|object_name|:\n[|progress_bar|]" + ." |percent|% |transferred|/|to_be_transferred| |unit|"; + public const FORMAT_PARAMETERS = [ + 'object_name', + 'progress_bar', + 'percent', + 'transferred', + 'to_be_transferred', + 'unit' + ]; + /** * @inheritDoc */ public function getFormatTemplate(): string { - return "|object_name|:\n[|progress_bar|] |percent|% |transferred|/|to_be_transferred| |unit|"; + return self::FORMAT_TEMPLATE; } /** @@ -17,14 +28,7 @@ public function getFormatTemplate(): string */ public function getFormatParameters(): array { - return [ - 'object_name', - 'progress_bar', - 'percent', - 'transferred', - 'to_be_transferred', - 'unit', - ]; + return self::FORMAT_PARAMETERS; } protected function getFormatDefaultParameterValues(): array diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index fc761c2f1a..e2ce362624 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -715,8 +715,8 @@ private function trySingleDownload( { if ($listenerNotifier !== null) { $listenerNotifier->transferInitiated([ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], 0, 0 @@ -731,8 +731,8 @@ private function trySingleDownload( function ($result) use ($requestArgs, $listenerNotifier) { // Notify progress $progressContext = [ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], $result['Content-Length'] ?? 0, $result['Content-Length'] ?? 0, @@ -750,8 +750,8 @@ function ($result) use ($requestArgs, $listenerNotifier) { } )->otherwise(function ($reason) use ($requestArgs, $listenerNotifier) { $listenerNotifier->transferFail([ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], 0, 0, @@ -804,8 +804,8 @@ private function trySingleUpload( if (!empty($listenerNotifier)) { $listenerNotifier->transferInitiated( [ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], 0, $objectSize, @@ -818,8 +818,8 @@ private function trySingleUpload( function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { $listenerNotifier->bytesTransferred( [ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], $objectSize, $objectSize, @@ -829,8 +829,8 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { $listenerNotifier->transferComplete( [ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], $objectSize, $objectSize, @@ -844,8 +844,8 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { )->otherwise(function ($reason) use ($objectSize, $requestArgs, $listenerNotifier) { $listenerNotifier->transferFail( [ - 'request_args' => $requestArgs, - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( $requestArgs['Key'], 0, $objectSize, diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index 6c10b44df5..2802ed031d 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -159,16 +159,16 @@ public function multiProgressTrackerProvider(): array 'event_invoker' => function (MultiProgressTracker $tracker): void { $tracker->transferInitiated([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 0, 1024 ) ]); $tracker->bytesTransferred([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 512, 1024 @@ -202,26 +202,26 @@ public function multiProgressTrackerProvider(): array { $events = [ 'transfer_initiated' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024 ], 'transfer_progress_1' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 342, ], 'transfer_progress_2' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 684, ], 'transfer_progress_3' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 1024, ], 'transfer_complete' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 1024, ] @@ -230,8 +230,8 @@ public function multiProgressTrackerProvider(): array if ($eventName === 'transfer_initiated') { for ($i = 0; $i < 3; $i++) { $progressTracker->transferInitiated([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", 0, $event['total_bytes'], @@ -241,8 +241,8 @@ public function multiProgressTrackerProvider(): array } elseif (str_starts_with($eventName, 'transfer_progress')) { for ($i = 0; $i < 3; $i++) { $progressTracker->bytesTransferred([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", $event['bytes_transferred'], $event['total_bytes'], @@ -252,8 +252,8 @@ public function multiProgressTrackerProvider(): array } elseif ($eventName === 'transfer_complete') { for ($i = 0; $i < 3; $i++) { $progressTracker->transferComplete([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", $event['bytes_transferred'], $event['total_bytes'], @@ -412,31 +412,31 @@ public function multiProgressTrackerProvider(): array { $events = [ 'transfer_initiated' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024 ], 'transfer_progress_1' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 342, ], 'transfer_progress_2' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 684, ], 'transfer_progress_3' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 1024, ], 'transfer_complete' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 1024, ], 'transfer_fail' => [ - 'request_args' => [], + TransferListener::REQUEST_ARGS_KEY => [], 'total_bytes' => 1024, 'bytes_transferred' => 0, 'reason' => 'Transfer failed' @@ -446,8 +446,8 @@ public function multiProgressTrackerProvider(): array if ($eventName === 'transfer_initiated') { for ($i = 0; $i < 5; $i++) { $progressTracker->transferInitiated([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", 0, $event['total_bytes'], @@ -457,8 +457,8 @@ public function multiProgressTrackerProvider(): array } elseif (str_starts_with($eventName, 'transfer_progress')) { for ($i = 0; $i < 3; $i++) { $progressTracker->bytesTransferred([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", $event['bytes_transferred'], $event['total_bytes'], @@ -468,8 +468,8 @@ public function multiProgressTrackerProvider(): array } elseif ($eventName === 'transfer_complete') { for ($i = 0; $i < 3; $i++) { $progressTracker->transferComplete([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", $event['bytes_transferred'], $event['total_bytes'], @@ -480,8 +480,8 @@ public function multiProgressTrackerProvider(): array // Just two of them will fail for ($i = 3; $i < 5; $i++) { $progressTracker->transferFail([ - 'request_args' => $event['request_args'], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => $event[TransferListener::REQUEST_ARGS_KEY], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( "FooObject_$i", 0, $event['total_bytes'], diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index 04f8be80fe..a2c8fa8308 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -141,8 +141,8 @@ public function singleProgressTrackingProvider(): array 'event_invoker' => function (singleProgressTracker $progressTracker): void { $progressTracker->transferInitiated([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 0, 1024 @@ -167,16 +167,16 @@ public function singleProgressTrackingProvider(): array 'event_invoker' => function (singleProgressTracker $progressTracker): void { $progressTracker->transferInitiated([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 0, 1024 ) ]); $progressTracker->bytesTransferred([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 256, 1024 @@ -203,24 +203,24 @@ public function singleProgressTrackingProvider(): array 'event_invoker' => function (singleProgressTracker $progressTracker): void { $progressTracker->transferInitiated([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 0, 0 ) ]); $progressTracker->bytesTransferred([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 1024, 0 ) ]); $progressTracker->transferComplete([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 2048, 0 @@ -249,24 +249,24 @@ public function singleProgressTrackingProvider(): array 'event_invoker' => function (singleProgressTracker $progressTracker): void { $progressTracker->transferInitiated([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 0, 1024 ) ]); $progressTracker->bytesTransferred([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 512, 1024 ) ]); $progressTracker->transferFail([ - 'request_args' => [], - 'progress_snapshot' => new TransferProgressSnapshot( + TransferListener::REQUEST_ARGS_KEY => [], + TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( 'Foo', 512, 1024 diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index f66d84571b..8ab82918ea 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -297,7 +297,7 @@ public function testUploadUsesCustomMupThreshold( $transferListener->method('bytesTransferred') -> willReturnCallback(function ($context) use ($expectedPartSize, &$expectedIncrementalPartSize) { /** @var TransferProgressSnapshot $snapshot */ - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); $expectedIncrementalPartSize += $expectedPartSize; }); @@ -394,7 +394,7 @@ public function testUploadUsesCustomPartSize(): void &$expectedIncrementalPartSize ) { /** @var TransferProgressSnapshot $snapshot */ - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $this->assertEquals($expectedIncrementalPartSize, $snapshot->getTransferredBytes()); $expectedIncrementalPartSize += $expectedPartSize; }); @@ -1298,7 +1298,7 @@ public function testUploadDirectoryTracksMultipleFiles(): void $transferListener->method('bytesTransferred') ->willReturnCallback(function(array $context) use (&$objectKeys) { /** @var TransferProgressSnapshot $snapshot */ - $snapshot = $context['progress_snapshot']; + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; $objectKeys[$snapshot->getIdentifier()] = true; }); $manager->uploadDirectory( From 060e0e1fed9b9f38d6439a4395f0dd9119171504 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 27 May 2025 08:18:08 -0700 Subject: [PATCH 23/62] chore: fix and refactor - Fix keys declaration in TransferListener.php - Make use of DIRECTORY_SEPARATOR const instead of hardcoding `/` --- src/S3/S3Transfer/Progress/TransferListener.php | 4 ++-- src/S3/S3Transfer/S3TransferManager.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/S3/S3Transfer/Progress/TransferListener.php b/src/S3/S3Transfer/Progress/TransferListener.php index 3fea230bf3..1a75d1c9b7 100644 --- a/src/S3/S3Transfer/Progress/TransferListener.php +++ b/src/S3/S3Transfer/Progress/TransferListener.php @@ -4,8 +4,8 @@ abstract class TransferListener { - public const REQUEST_ARGS_KEY = TransferListener::REQUEST_ARGS_KEY; - public const PROGRESS_SNAPSHOT_KEY = TransferListener::PROGRESS_SNAPSHOT_KEY; + public const REQUEST_ARGS_KEY = 'request_args'; + public const PROGRESS_SNAPSHOT_KEY = 'progress_snapshot'; public const REASON_KEY = 'reason'; /** diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index e2ce362624..dce2c9fab8 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -305,7 +305,7 @@ function ($file) use ($filter) { $objectsUploaded = 0; $objectsFailed = 0; foreach ($files as $file) { - $baseDir = rtrim($sourceDirectory, '/') . '/'; + $baseDir = rtrim($sourceDirectory, '/') . DIRECTORY_SEPARATOR; $relativePath = substr($file, strlen($baseDir)); if (str_contains($relativePath, $delimiter) && $delimiter !== '/') { throw new S3TransferException( @@ -593,7 +593,7 @@ public function downloadDirectory( $objectsFailed = 0; foreach ($objects as $object) { $objectKey = $this->s3UriAsBucketAndKey($object)['Key']; - $destinationFile = $destinationDirectory . '/' . $objectKey; + $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { throw new S3TransferException( "Cannot download key ' . $objectKey From ee0fefbd4e200716b6c10a499666b9152e44ad41 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 27 May 2025 09:43:07 -0700 Subject: [PATCH 24/62] chore: fix TransferListener import - Some implementations using TransferListener were missing the import statement. --- src/S3/S3Transfer/MultipartUploader.php | 1 + src/S3/S3Transfer/Progress/MultiProgressBarFormat.php | 2 +- .../S3Transfer/Progress/MultiProgressTrackerTest.php | 1 + .../S3Transfer/Progress/SingleProgressTrackerTest.php | 1 + tests/S3/S3Transfer/S3TransferManagerTest.php | 11 +++++++++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index e153f6e721..fd8c343b69 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -8,6 +8,7 @@ use Aws\ResultInterface; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Models\UploadResponse; +use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Coroutine; diff --git a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php index a4185824ae..428a55b71d 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php @@ -4,7 +4,7 @@ class MultiProgressBarFormat extends ProgressBarFormat { - public const FORMAT_TEMPLATE = "[|progress_bar|] |percent|%" + public const FORMAT_TEMPLATE = "[|progress_bar|] |percent|% " ."Completed: |completed|/|total|, Failed: |failed|/|total|"; public const FORMAT_PARAMETERS = [ 'completed', diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index 2802ed031d..b3096e7e24 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -7,6 +7,7 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\ProgressBarFactoryInterface; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; +use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Closure; use PHPUnit\Framework\TestCase; diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index a2c8fa8308..586675c2f1 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -7,6 +7,7 @@ use Aws\S3\S3Transfer\Progress\PlainProgressBarFormat; use Aws\S3\S3Transfer\Progress\ProgressBarInterface; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; +use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use PHPUnit\Framework\TestCase; diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 8ab82918ea..0a776d28a8 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -2684,6 +2684,17 @@ public function downloadDirectoryCreateFilesProvider(): array ]; } + public function testFailsWhenKeyResolvesOutsideTargetDirectory() { + + } + + /** + * @return array + */ + public function failsWhenKeyResolvesOutsideTargetDirectoryProvider(): array { + return []; + } + /** * @param array $methodsCallback If any from the callbacks below * is not provided then a default implementation will be provided. From 9f639f1f44a6e04b18123ff9d3281019ef45a152 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 29 May 2025 14:56:08 -0700 Subject: [PATCH 25/62] chore: add test case - Add test cases for if a file being download `resolvesOutsideTargetDirectory`. --- tests/S3/S3Transfer/S3TransferManagerTest.php | 119 +++++++++++++++++- 1 file changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 0a776d28a8..5207ec1e7f 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -16,6 +16,7 @@ use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\S3TransferManager; +use Aws\Test\TestsUtility; use Closure; use Exception; use GuzzleHttp\Promise\Create; @@ -2684,15 +2685,129 @@ public function downloadDirectoryCreateFilesProvider(): array ]; } - public function testFailsWhenKeyResolvesOutsideTargetDirectory() { + /** + * @param array $objects + * + * @dataProvider failsWhenKeyResolvesOutsideTargetDirectoryProvider + * + * @return void + */ + public function testFailsWhenKeyResolvesOutsideTargetDirectory( + string $prefix, + array $objects, + ) { + $bucket = "test-bucket"; + $directory = "test-directory"; + try { + $fullDirectoryPath = sys_get_temp_dir() . DIRECTORY_SEPARATOR . $directory; + if (is_dir($fullDirectoryPath)) { + TestsUtility::cleanUpDir($fullDirectoryPath); + } + mkdir($fullDirectoryPath, 0777, true); + $this->expectException(S3TransferException::class); + $called = false; + $client = $this->getS3ClientMock([ + 'executeAsync' => function (CommandInterface $command) use ( + $objects, + &$called + ) { + $called = true; + if ($command->getName() === 'ListObjectsV2') { + return Create::promiseFor(new Result([ + 'Contents' => $objects, + ])); + } + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor( + "Test file " . $command['Key'] + ), + '@metadata' => [] + ])); + }, + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + $bucket, + $fullDirectoryPath, + [], + [ + 's3_prefix' => $prefix, + ] + )->wait(); + $this->assertTrue($called); + } finally { + TestsUtility::cleanUpDir($directory); + } } /** * @return array */ public function failsWhenKeyResolvesOutsideTargetDirectoryProvider(): array { - return []; + return [ + 'resolves_outside_target_directory_1' => [ + 'prefix' => 'foo-objects/', + 'objects' => [ + [ + 'Key' => '../outside/key1.txt' + ], + ], + ], + 'resolves_outside_target_directory_2' => [ + 'prefix' => 'foo-objects/', + 'objects' => [ + [ + 'Key' => '../../foo/key2.txt' + ] + ] + ], + 'resolves_outside_target_directory_3' => [ + 'prefix' => 'buzz/', + 'objects' => [ + [ + 'Key' => '..//inner//key3.txt' + ] + ] + ], + 'resolves_outside_target_directory_4' => [ + 'prefix' => 'test/', + 'objects' => [ + [ + 'Key' => './../../key4.txt' + ] + ] + ], + 'resolves_outside_target_directory_5' => [ + 'prefix' => 'test/', + 'objects' => [ + [ + 'Key' => './../another_dir/.././key1.txt', + ], + ] + ], + ]; } /** From bdce369d94598943869d21fe06d7002a037e4284 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 4 Jun 2025 09:50:55 -0700 Subject: [PATCH 26/62] chore: address PR feedback - Improve how the parts are created in multipart uploader so that it looks cleaner. - Create single class tests for PartGetMultipartDownloader and RangeGetMultipartDownloader. - Add tests for TransferListenerNotifier from MultipartUploader and MultipartDownloader implementations. --- src/S3/S3Transfer/MultipartDownloader.php | 7 + src/S3/S3Transfer/MultipartUploader.php | 31 +- .../S3/S3Transfer/MultipartDownloaderTest.php | 320 ++++++++++-------- tests/S3/S3Transfer/MultipartUploaderTest.php | 182 ++++++++++ .../PartGetMultipartDownloaderTest.php | 197 +++++++++++ .../RangeGetMultipartDownloaderTest.php | 306 +++++++++++++++++ 6 files changed, 893 insertions(+), 150 deletions(-) create mode 100644 tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php create mode 100644 tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 7e74a237fe..6d5cce35f9 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -123,6 +123,13 @@ public function getCurrentSnapshot(): TransferProgressSnapshot return $this->currentSnapshot; } + /** + * @return DownloadResponse + */ + public function download(): DownloadResponse { + return $this->promise()->wait(); + } + /** * Returns that resolves a multipart download operation, * or to a rejection in case of any failures. diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index fd8c343b69..b4e5c5f002 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -146,6 +146,13 @@ public function getCurrentSnapshot(): ?TransferProgressSnapshot return $this->currentSnapshot; } + /** + * @return UploadResponse + */ + public function upload(): UploadResponse { + return $this->promise()->wait(); + } + /** * @return PromiseInterface */ @@ -197,27 +204,19 @@ private function uploadParts(): PromiseInterface $isSeekable = $this->body->isSeekable(); $partSize = $this->config['part_size']; $commands = []; - for ($partNo = count($this->parts) + 1; - $isSeekable - ? $this->body->tell() < $this->body->getSize() - : !$this->body->eof(); - $partNo++ - ) { - if ($isSeekable) { - $readSize = min($partSize, $this->body->getSize() - $this->body->tell()); - } else { - $readSize = $partSize; - } - - $partBody = Utils::streamFor( - $this->body->read($readSize) - ); + $partNo = count($this->parts); + while (!$this->body->eof()) { + $partNo++; + $read = $this->body->read($partSize); // To make sure we do not create an empty part when // we already reached the end of file. - if (!$isSeekable && $this->body->eof() && $partBody->getSize() === 0) { + if (empty($read) && $this->body->eof()) { break; } + $partBody = Utils::streamFor( + $read + ); $uploadPartCommandArgs = [ ...$this->createMultipartArgs, 'UploadId' => $this->uploadId, diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 6787a93231..2675f2b2d4 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -8,174 +8,226 @@ use Aws\S3\S3Transfer\Models\DownloadResponse; use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\PartGetMultipartDownloader; +use Aws\S3\S3Transfer\Progress\TransferListener; +use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; /** - * Tests multipart download implementation. + * Tests MultipartDownloader abstract class implementation. */ class MultipartDownloaderTest extends TestCase { /** - * Tests part and range get multipart downloader. + * Tests chooseDownloaderClass factory method. * - * @param string $multipartDownloadType - * @param string $objectKey - * @param int $objectSizeInBytes - * @param int $targetPartSize + * @return void + */ + public function testChooseDownloaderClass(): void { + $multipartDownloadTypes = [ + MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + ]; + foreach ($multipartDownloadTypes as $multipartDownloadType => $class) { + $resolvedClass = MultipartDownloader::chooseDownloaderClass($multipartDownloadType); + $this->assertEquals($class, $resolvedClass); + } + } + + /** + * Tests chooseDownloaderClass throws exception for invalid type. * - * @dataProvider partGetMultipartDownloaderProvider + * @return void + */ + public function testChooseDownloaderClassThrowsExceptionForInvalidType(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('The config value for `multipart_download_type` must be one of:'); + + MultipartDownloader::chooseDownloaderClass('invalidType'); + } + + /** + * Tests constants are properly defined. * * @return void */ - public function testMultipartDownloader( - string $multipartDownloadType, - string $objectKey, - int $objectSizeInBytes, - int $targetPartSize - ): void { - $partsCount = (int) ceil($objectSizeInBytes / $targetPartSize); - $mockClient = $this->getMockBuilder(S3Client::class) + public function testConstants(): void + { + $this->assertEquals('GetObject', MultipartDownloader::GET_OBJECT_COMMAND); + $this->assertEquals('partGet', MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER); + $this->assertEquals('rangeGet', MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER); + } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener3 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener1->expects($this->once())->method('transferComplete'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener2->expects($this->once())->method('transferComplete'); + + $listener3->expects($this->once())->method('transferInitiated'); + $listener3->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener3->expects($this->once())->method('transferComplete'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2, $listener3]); + + $s3Client = $this->getMockBuilder(S3Client::class) ->disableOriginalConstructor() ->getMock(); - $remainingToTransfer = $objectSizeInBytes; - $mockClient->method('executeAsync') - -> willReturnCallback(function ($command) - use ( - $objectSizeInBytes, - $partsCount, - $targetPartSize, - &$remainingToTransfer - ) { - $currentPartLength = min( - $targetPartSize, - $remainingToTransfer - ); - $from = $objectSizeInBytes - $remainingToTransfer; - $to = $from + $currentPartLength; - $remainingToTransfer -= $currentPartLength; - return Create::promiseFor(new Result([ - 'Body' => Utils::streamFor('Foo'), - 'PartsCount' => $partsCount, - 'PartNumber' => $command['PartNumber'], - 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", - 'ContentLength' => $currentPartLength - ])); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('test data'), + 'ContentLength' => 9, + 'ContentRange' => 'bytes 0-8/9', + 'PartsCount' => 1, + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); }); - $mockClient->method('getCommand') - -> willReturnCallback(function ($commandName, $args) { + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { return new Command($commandName, $args); }); - $downloaderClassName = MultipartDownloader::chooseDownloaderClass( - $multipartDownloadType - ); - /** @var MultipartDownloader $downloader */ - $downloader = new $downloaderClassName( - $mockClient, - [ - 'Bucket' => 'FooBucket', - 'Key' => $objectKey, - ], - [ - 'minimum_part_size' => $targetPartSize, - ] + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [], + 0, + 0, + 0, + '', + null, + null, + $listenerNotifier ); - /** @var DownloadResponse $response */ - $response = $downloader->promise()->wait(); - $snapshot = $downloader->getCurrentSnapshot(); + $response = $multipartDownloader->promise()->wait(); $this->assertInstanceOf(DownloadResponse::class, $response); - $this->assertEquals($objectKey, $snapshot->getIdentifier()); - $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); - $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); - $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); - $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); } /** - * Part get multipart downloader data provider. - * - * @return array[] + * @return void */ - public function partGetMultipartDownloaderProvider(): array { - return [ - [ - 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_1', - 'objectSizeInBytes' => 1024 * 10, - 'targetPartSize' => 1024 * 2, - ], - [ - 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_2', - 'objectSizeInBytes' => 1024 * 100, - 'targetPartSize' => 1024 * 5, - ], - [ - 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_3', - 'objectSizeInBytes' => 512, - 'targetPartSize' => 512, - ], - [ - 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_4', - 'objectSizeInBytes' => 512, - 'targetPartSize' => 256, - ], - [ - 'multipartDownloadType' => MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_5', - 'objectSizeInBytes' => 512, - 'targetPartSize' => 458, - ], - [ - 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_1', - 'objectSizeInBytes' => 1024 * 10, - 'targetPartSize' => 1024 * 2, - ], - [ - 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_2', - 'objectSizeInBytes' => 1024 * 100, - 'targetPartSize' => 1024 * 5, - ], - [ - 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_3', - 'objectSizeInBytes' => 512, - 'targetPartSize' => 512, - ], - [ - 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_4', - 'objectSizeInBytes' => 512, - 'targetPartSize' => 256, - ], - [ - 'multipartDownloadType' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, - 'objectKey' => 'ObjectKey_5', - 'objectSizeInBytes' => 512, - 'targetPartSize' => 458, - ] + public function testTransferListenerNotifierNotifiesListenersOnFailure(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->once())->method('transferFail'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->once())->method('transferFail'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::rejectionFor(new \Exception('Download failed')); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', ]; + + $multipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [], + 0, + 0, + 0, + '', + null, + null, + $listenerNotifier + ); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Download failed'); + $multipartDownloader->promise()->wait(); } /** * @return void */ - public function testChooseDownloaderClass(): void { - $multipartDownloadTypes = [ - MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + public function testTransferListenerNotifierWithEmptyListeners(): void + { + $listenerNotifier = new TransferListenerNotifier([]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'GetObject') { + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('test'), + 'ContentLength' => 4, + 'ContentRange' => 'bytes 0-3/4', + 'PartsCount' => 1, + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', ]; - foreach ($multipartDownloadTypes as $multipartDownloadType => $class) { - $resolvedClass = MultipartDownloader::chooseDownloaderClass($multipartDownloadType); - $this->assertEquals($class, $resolvedClass); - } + + $multipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [], + 0, + 0, + 0, + '', + null, + null, + $listenerNotifier + ); + + $response = $multipartDownloader->promise()->wait(); + $this->assertInstanceOf(DownloadResponse::class, $response); } } \ No newline at end of file diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index a5ca28bf69..c62d9a5774 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -8,6 +8,8 @@ use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\MultipartUploader; +use Aws\S3\S3Transfer\Progress\TransferListener; +use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; @@ -208,4 +210,184 @@ public function testInvalidSourceTypeThrowsException(): void 12345 ); } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener3 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener1->expects($this->once())->method('transferComplete'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener2->expects($this->once())->method('transferComplete'); + + $listener3->expects($this->once())->method('transferInitiated'); + $listener3->expects($this->atLeastOnce())->method('bytesTransferred'); + $listener3->expects($this->once())->method('transferComplete'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2, $listener3]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 10240000)); // 10MB + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + [ + 'part_size' => 5242880, // 5MB + 'concurrency' => 1, + ], + $stream, + null, + [], + null, + $listenerNotifier + ); + + $response = $multipartUploader->promise()->wait(); + $this->assertInstanceOf(UploadResponse::class, $response); + } + + /** + * @return void + */ + public function testTransferListenerNotifierNotifiesListenersOnFailure(): void + { + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); + $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); + + $listener1->expects($this->once())->method('transferInitiated'); + $listener1->expects($this->once())->method('transferFail'); + + $listener2->expects($this->once())->method('transferInitiated'); + $listener2->expects($this->once())->method('transferFail'); + + $listenerNotifier = new TransferListenerNotifier([$listener1, $listener2]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::rejectionFor(new \Exception('Upload failed')); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 10240000)); + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + [ + 'part_size' => 5242880, // 5MB + 'concurrency' => 1, + ], + $stream, + null, + [], + null, + $listenerNotifier + ); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Upload failed'); + $multipartUploader->promise()->wait(); + } + + /** + * @return void + */ + public function testTransferListenerNotifierWithEmptyListeners(): void + { + $listenerNotifier = new TransferListenerNotifier([]); + + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 1024)); + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + [ + 'part_size' => 5242880, + 'concurrency' => 1, + ], + $stream, + null, + [], + null, + $listenerNotifier + ); + + $response = $multipartUploader->promise()->wait(); + $this->assertInstanceOf(UploadResponse::class, $response); + } } \ No newline at end of file diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php new file mode 100644 index 0000000000..b5ca7685f4 --- /dev/null +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -0,0 +1,197 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $remainingToTransfer = $objectSizeInBytes; + $mockClient->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'], + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength + ])); + }); + $mockClient->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $downloader = new PartGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'FooBucket', + 'Key' => $objectKey, + ], + [ + 'minimum_part_size' => $targetPartSize, + ] + ); + /** @var DownloadResponse $response */ + $response = $downloader->promise()->wait(); + $snapshot = $downloader->getCurrentSnapshot(); + + $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertEquals($objectKey, $snapshot->getIdentifier()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); + $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); + $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); + } + + /** + * Part get multipart downloader data provider. + * + * @return array[] + */ + public function partGetMultipartDownloaderProvider(): array { + return [ + [ + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ] + ]; + } + + /** + * Tests nextCommand method increments part number correctly. + * + * @return void + */ + public function testNextCommandIncrementsPartNumber(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $downloader = new PartGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ] + ); + + // Use reflection to test the protected nextCommand method + $reflection = new \ReflectionClass($downloader); + $nextCommandMethod = $reflection->getMethod('nextCommand'); + + // First call should set part number to 1 + $command1 = $nextCommandMethod->invoke($downloader); + $this->assertEquals(1, $command1['PartNumber']); + $this->assertEquals(1, $downloader->getCurrentPartNo()); + + // Second call should increment to 2 + $command2 = $nextCommandMethod->invoke($downloader); + $this->assertEquals(2, $command2['PartNumber']); + $this->assertEquals(2, $downloader->getCurrentPartNo()); + } + + /** + * Tests computeObjectDimensions method correctly calculates object size. + * + * @return void + */ + public function testComputeObjectDimensions(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $downloader = new PartGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ] + ); + + // Use reflection to test the protected computeObjectDimensions method + $reflection = new \ReflectionClass($downloader); + $computeObjectDimensionsMethod = $reflection->getMethod('computeObjectDimensions'); + + $result = new Result([ + 'PartsCount' => 5, + 'ContentRange' => 'bytes 0-1023/2048' + ]); + + $computeObjectDimensionsMethod->invoke($downloader, $result); + + $this->assertEquals(5, $downloader->getObjectPartsCount()); + $this->assertEquals(2048, $downloader->getObjectSizeInBytes()); + } +} \ No newline at end of file diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php new file mode 100644 index 0000000000..6a17928095 --- /dev/null +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -0,0 +1,306 @@ +getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $remainingToTransfer = $objectSizeInBytes; + $mockClient->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer + ) { + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'] ?? 1, + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength + ])); + }); + $mockClient->method('getCommand') + -> willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'FooBucket', + 'Key' => $objectKey, + ], + [ + 'minimum_part_size' => $targetPartSize, + ] + ); + /** @var DownloadResponse $response */ + $response = $downloader->promise()->wait(); + $snapshot = $downloader->getCurrentSnapshot(); + + $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertEquals($objectKey, $snapshot->getIdentifier()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); + $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); + $this->assertEquals($partsCount, $downloader->getObjectPartsCount()); + $this->assertEquals($partsCount, $downloader->getCurrentPartNo()); + } + + /** + * Range get multipart downloader data provider. + * + * @return array[] + */ + public function rangeGetMultipartDownloaderProvider(): array { + return [ + [ + 'objectKey' => 'ObjectKey_1', + 'objectSizeInBytes' => 1024 * 10, + 'targetPartSize' => 1024 * 2, + ], + [ + 'objectKey' => 'ObjectKey_2', + 'objectSizeInBytes' => 1024 * 100, + 'targetPartSize' => 1024 * 5, + ], + [ + 'objectKey' => 'ObjectKey_3', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 512, + ], + [ + 'objectKey' => 'ObjectKey_4', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 256, + ], + [ + 'objectKey' => 'ObjectKey_5', + 'objectSizeInBytes' => 512, + 'targetPartSize' => 458, + ] + ]; + } + + /** + * Tests constructor throws exception when minimum_part_size is not provided. + * + * @return void + */ + public function testConstructorThrowsExceptionWithoutMinimumPartSize(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('You must provide a valid minimum part size in bytes'); + + new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [] // Missing minimum_part_size + ); + } + + /** + * Tests nextCommand method generates correct range headers. + * + * @return void + */ + public function testNextCommandGeneratesCorrectRangeHeaders(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $partSize = 1024; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'minimum_part_size' => $partSize, + ] + ); + + // Use reflection to test the protected nextCommand method + $reflection = new \ReflectionClass($downloader); + $nextCommandMethod = $reflection->getMethod('nextCommand'); + $nextCommandMethod->setAccessible(true); + + // First call should create range 0-1023 + $command1 = $nextCommandMethod->invoke($downloader); + $this->assertEquals('bytes=0-1023', $command1['Range']); + $this->assertEquals(1, $downloader->getCurrentPartNo()); + + // Second call should create range 1024-2047 + $command2 = $nextCommandMethod->invoke($downloader); + $this->assertEquals('bytes=1024-2047', $command2['Range']); + $this->assertEquals(2, $downloader->getCurrentPartNo()); + } + + /** + * Tests computeObjectDimensions method with known object size. + * + * @return void + */ + public function testComputeObjectDimensionsWithKnownSize(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $objectSize = 5120; + $partSize = 1024; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'minimum_part_size' => $partSize, + ], + 0, // currentPartNo + 0, // objectPartsCount + $objectSize // objectSizeInBytes - known at construction + ); + + // With known object size, parts count should be calculated during construction + $this->assertEquals(5, $downloader->getObjectPartsCount()); + $this->assertEquals($objectSize, $downloader->getObjectSizeInBytes()); + } + + /** + * Tests computeObjectDimensions method for single part download. + * + * @return void + */ + public function testComputeObjectDimensionsForSinglePart(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $partSize = 2048; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'minimum_part_size' => $partSize, + ] + ); + + // Use reflection to test the protected computeObjectDimensions method + $reflection = new \ReflectionClass($downloader); + $computeObjectDimensionsMethod = $reflection->getMethod('computeObjectDimensions'); + $computeObjectDimensionsMethod->setAccessible(true); + + // Simulate object smaller than part size + $result = new Result([ + 'ContentRange' => 'bytes 0-511/512' + ]); + + $computeObjectDimensionsMethod->invoke($downloader, $result); + + // Should be single part download + $this->assertEquals(1, $downloader->getObjectPartsCount()); + $this->assertEquals(512, $downloader->getObjectSizeInBytes()); + } + + /** + * Tests nextCommand method includes IfMatch header when ETag is present. + * + * @return void + */ + public function testNextCommandIncludesIfMatchWhenETagPresent(): void + { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + + $mockClient->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $eTag = '"abc123"'; + $downloader = new RangeGetMultipartDownloader( + $mockClient, + [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ], + [ + 'minimum_part_size' => 1024, + ], + 0, // currentPartNo + 0, // objectPartsCount + 0, // objectSizeInBytes + $eTag // eTag + ); + + // Use reflection to test the protected nextCommand method + $reflection = new \ReflectionClass($downloader); + $nextCommandMethod = $reflection->getMethod('nextCommand'); + $nextCommandMethod->setAccessible(true); + + $command = $nextCommandMethod->invoke($downloader); + $this->assertEquals($eTag, $command['IfMatch']); + } +} \ No newline at end of file From cd2133a7b07ecc00c7d7444eda549067c9eddd57 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 4 Jun 2025 18:49:30 -0700 Subject: [PATCH 27/62] fix: prevent calling twice downloadFailed When a part download fails we trigger downloadFailed so it can be propagated to the listeners, and then we retrhow the exception, however, we also have a global exception catching for if something else fails during a multipart download also gets caught and propagated to the listeners as well, however, this causes the downloadFailed to be called twice. To prevent this we just check if the current snapshot has already a error message present there. --- src/S3/S3Transfer/MultipartDownloader.php | 12 ++++++++++ .../Progress/TransferProgressSnapshot.php | 23 ++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 6d5cce35f9..2c8f8d15fe 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -275,6 +275,18 @@ private function downloadInitiated(array $commandArgs): void */ private function downloadFailed(\Throwable $reason): void { + // Event already propagated. + if ($this->currentSnapshot->getReason() !== null) { + return; + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); $this->stream->close(); $this->listenerNotifier?->transferFail([ TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index 792929ac94..148357d43d 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -2,6 +2,8 @@ namespace Aws\S3\S3Transfer\Progress; +use Throwable; + class TransferProgressSnapshot { /** @var string */ @@ -16,6 +18,9 @@ class TransferProgressSnapshot /** @var array | null */ private array | null $response; + /** @var Throwable | string | null */ + private Throwable | string | null $reason; + /** * @param string $identifier * @param int $transferredBytes @@ -26,12 +31,14 @@ public function __construct( string $identifier, int $transferredBytes, int $totalBytes, - ?array $response = null + ?array $response = null, + Throwable | string | null $reason = null, ) { $this->identifier = $identifier; $this->transferredBytes = $transferredBytes; $this->totalBytes = $totalBytes; $this->response = $response; + $this->reason = $reason; } public function getIdentifier(): string @@ -56,9 +63,9 @@ public function getTotalBytes(): int } /** - * @return array + * @return array|null */ - public function getResponse(): array + public function getResponse(): array | null { return $this->response; } @@ -75,4 +82,14 @@ public function ratioTransferred(): float return $this->transferredBytes / $this->totalBytes; } + + /** + * @return Throwable|string|null + */ + public function getReason(): Throwable|string|null + { + return $this->reason; + } + + } \ No newline at end of file From 0426e11a1400b201bdcff2720a9da03f9c034944 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 5 Jun 2025 07:36:09 -0700 Subject: [PATCH 28/62] chore: fix exception throwing - In MultipartUploader, when a part upload fails, the exception should be thrown, and it was not being to. --- src/S3/S3Transfer/MultipartUploader.php | 15 +++++++++++++++ tests/S3/S3Transfer/MultipartUploaderTest.php | 5 +++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index b4e5c5f002..275d070f84 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -261,6 +261,8 @@ private function uploadParts(): PromiseInterface }, 'rejected' => function (Throwable $e) { $this->partUploadFailed($e); + + throw $e; } ] ))->promise(); @@ -410,6 +412,19 @@ private function uploadInitiated(array $requestArgs): void * @return void */ private function uploadFailed(Throwable $reason): void { + // Event has been already propagated + if ($this->currentSnapshot->getReason() !== null) { + return; + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); + if (!empty($this->uploadId)) { $this->abortMultipartUpload()->wait(); } diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index c62d9a5774..216541446b 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -284,6 +284,9 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void */ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void { + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Upload failed'); + $listener1 = $this->getMockBuilder(TransferListener::class)->getMock(); $listener2 = $this->getMockBuilder(TransferListener::class)->getMock(); @@ -334,8 +337,6 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $listenerNotifier ); - $this->expectException(\Exception::class); - $this->expectExceptionMessage('Upload failed'); $multipartUploader->promise()->wait(); } From a548ce2ff3b639133cc1b0be607fa33bf401b07b Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 17 Jun 2025 08:25:57 -0700 Subject: [PATCH 29/62] feat: consider checksum mode from command Read `request_checksum_calculation` from command arguments, being this config value considered over the one from client config. This is useful for when we need to disable checksums per operation basis. --- src/S3/ApplyChecksumMiddleware.php | 3 +- tests/S3/ApplyChecksumMiddlewareTest.php | 167 ++++++++++++++--------- 2 files changed, 101 insertions(+), 69 deletions(-) diff --git a/src/S3/ApplyChecksumMiddleware.php b/src/S3/ApplyChecksumMiddleware.php index ed3b9bf73a..5e4f3ecb17 100644 --- a/src/S3/ApplyChecksumMiddleware.php +++ b/src/S3/ApplyChecksumMiddleware.php @@ -76,7 +76,8 @@ public function __invoke( $name = $command->getName(); $body = $request->getBody(); $operation = $this->api->getOperation($name); - $mode = $this->config['request_checksum_calculation'] + $mode = $command['@context']['request_checksum_calculation'] + ?? $this->config['request_checksum_calculation'] ?? self::DEFAULT_CALCULATION_MODE; // Trigger warning if AddContentMD5 is specified for PutObject or UploadPart diff --git a/tests/S3/ApplyChecksumMiddlewareTest.php b/tests/S3/ApplyChecksumMiddlewareTest.php index 257a2fa3c6..5abae7fb63 100644 --- a/tests/S3/ApplyChecksumMiddlewareTest.php +++ b/tests/S3/ApplyChecksumMiddlewareTest.php @@ -57,112 +57,143 @@ public function testFlexibleChecksums( public function getFlexibleChecksumUseCases() { return [ - // httpChecksum not modeled - [ - 'GetObject', - [], - [ + 'http_checksum_not_modeled' => [ + 'operation' => 'GetObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumMode' => 'ENABLED' ], - null, - false, - '' + 'body' => null, + 'headers_added' => false, + 'header_value' => '' ], - // default: when_supported. defaults to crc32 - [ - 'PutObject', - [], - [ + 'default_when_supported_defaults_to_crc32' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'Body' => 'abc' ], - 'abc', - true, - 'NSRBwg==' + 'body' => 'abc', + 'headers_added' => true, + 'header_value' => 'NSRBwg==' ], - // when_required when not required and no requested algorithm - [ - 'PutObject', - ['request_checksum_calculation' => 'when_required'], - [ - 'Bucket' => 'foo', - 'Key' => 'bar', - 'Body' => 'abc' - ], - 'abc', - false, - '' + 'when_required_when_not_required_and_no_requested_algorithm' => [ + 'operation' => 'PutObject', + 'config' => ['request_checksum_calculation' => 'when_required'], + 'command_args' => [], + 'body' => 'abc', + 'headers_added' => false, + 'header_value' => '' ], - // when_required when required and no requested algorithm - [ - 'PutObjectLockConfiguration', - ['request_checksum_calculation' => 'when_required'], - [ + 'when_required_when_required_and_no_requested_algorithm' => [ + 'operation' => 'PutObjectLockConfiguration', + 'config' => ['request_checksum_calculation' => 'when_required'], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ObjectLockConfiguration' => 'blah' ], - 'blah', - true, - 'zilhXA==' + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' ], - // when_required when not required and requested algorithm - [ - 'PutObject', - ['request_checksum_calculation' => 'when_required'], - [ + 'when_required_when_not_required_and_requested_algorithm' => [ + 'operation' => 'PutObject', + 'config' => ['request_checksum_calculation' => 'when_required'], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'Body' => 'blah', 'ChecksumAlgorithm' => 'crc32', ], - 'blah', - true, - 'zilhXA==' + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' ], - // when_supported and requested algorithm - [ - 'PutObject', - [], - [ + 'when_supported_and_requested_algorithm_1' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'crc32c', 'Body' => 'abc' ], - 'abc', - true, - 'Nks/tw==' + 'body' => 'abc', + 'headers_added' => true, + 'header_value' => 'Nks/tw==' ], - // when_supported and requested algorithm - [ - 'PutObject', - [], - [ + 'when_supported_and_requested_algorithm_2' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'sha256' ], - '', - true, - '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' + 'body' => '', + 'headers_added' => true, + 'header_value' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' ], - // when_supported and requested algorithm - [ - 'PutObject', - [], - [ + 'when_supported_and_requested_algorithm_3' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ 'Bucket' => 'foo', 'Key' => 'bar', 'ChecksumAlgorithm' => 'SHA1' ], - '', - true, - '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' + 'body' => '', + 'headers_added' => true, + 'header_value' => '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' ], + 'when_required_when_not_required_and_no_requested_algorithm_from_command_args' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ + '@context' => [ + 'request_checksum_calculation' => 'when_required' + ] + ], + 'body' => 'abc', + 'headers_added' => false, + 'header_value' => '' + ], + 'when_required_when_required_and_no_requested_algorithm_from_command_args' => [ + 'operation' => 'PutObjectLockConfiguration', + 'config' => [], + 'command_args' => [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'ObjectLockConfiguration' => 'blah', + '@context' => [ + 'request_checksum_calculation' => 'when_required' + ] + ], + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' + ], + 'when_required_when_not_required_and_requested_algorithm_from_command_args' => [ + 'operation' => 'PutObject', + 'config' => [], + 'command_args' => [ + 'Bucket' => 'foo', + 'Key' => 'bar', + 'Body' => 'blah', + 'ChecksumAlgorithm' => 'crc32', + '@context' => [ + 'request_checksum_calculation' => 'when_required' + ] + ], + 'body' => 'blah', + 'headers_added' => true, + 'header_value' => 'zilhXA==' + ] ]; } From 5a25ad835732fe64d0954e7ada4b2d50487dd3d2 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 18 Jun 2025 06:54:47 -0700 Subject: [PATCH 30/62] chore: tests and minor fixes - Fix abort multipart upload called more than once. - Add test coverage for multipart uploads with custom checksums. - Add test coverage for multipart upload abortion, to make sure it is called just once when a upload process fails. - Add test coverage to make sure the different multipart operations are called. --- src/S3/S3Transfer/MultipartUploader.php | 66 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 568 ++++++++++++++++-- 2 files changed, 557 insertions(+), 77 deletions(-) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 275d070f84..adf02c4196 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -18,7 +18,6 @@ use GuzzleHttp\Psr7\LazyOpenStream; use GuzzleHttp\Psr7\Utils; use Psr\Http\Message\StreamInterface; -use Psr\Http\Message\StreamInterface as Stream; use Throwable; /** @@ -106,7 +105,8 @@ private function validateConfig(array &$config): void || $config['part_size'] > self::PART_MAX_SIZE) { throw new \InvalidArgumentException( "The config `part_size` value must be between " - . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE . "." + . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE + . " but ${config['part_size']} given." ); } } else { @@ -181,6 +181,15 @@ public function promise(): PromiseInterface private function createMultipartUpload(): PromiseInterface { $requestArgs = [...$this->createMultipartArgs]; + $checksum = $this->filterChecksum($requestArgs); + // Customer provided checksum + if ($checksum !== null) { + $requestArgs['ChecksumType'] = 'FULL_OBJECT'; + $requestArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); + $requestArgs['@context']['request_checksum_calculation'] = 'when_required'; + unset($requestArgs[$checksum]); + } + $this->uploadInitiated($requestArgs); $command = $this->s3Client->getCommand( 'CreateMultipartUpload', @@ -201,10 +210,21 @@ private function createMultipartUpload(): PromiseInterface private function uploadParts(): PromiseInterface { $this->calculatedObjectSize = 0; - $isSeekable = $this->body->isSeekable(); $partSize = $this->config['part_size']; $commands = []; $partNo = count($this->parts); + $baseUploadPartCommandArgs = [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + ]; + // Customer provided checksum + $checksum = $this->filterChecksum($this->createMultipartArgs); + if ($checksum !== null) { + unset($baseUploadPartCommandArgs['ChecksumAlgorithm']); + unset($baseUploadPartCommandArgs[$checksum]); + $baseUploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + } + while (!$this->body->eof()) { $partNo++; $read = $this->body->read($partSize); @@ -218,11 +238,11 @@ private function uploadParts(): PromiseInterface $read ); $uploadPartCommandArgs = [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, + ...$baseUploadPartCommandArgs, 'PartNumber' => $partNo, 'ContentLength' => $partBody->getSize(), ]; + // To get `requestArgs` when notifying the bytesTransfer listeners. $uploadPartCommandArgs['requestArgs'] = [...$uploadPartCommandArgs]; // Attach body @@ -282,8 +302,12 @@ private function completeMultipartUpload(): PromiseInterface 'Parts' => $this->parts, ] ]; - if ($this->containsChecksum($this->createMultipartArgs)) { + $checksum = $this->filterChecksum($completeMultipartUploadArgs); + // Customer provided checksum + if ($checksum !== null) { + $completeMultipartUploadArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $completeMultipartUploadArgs['@context']['request_checksum_calculation'] = 'when_required'; } $command = $this->s3Client->getCommand( @@ -359,7 +383,7 @@ private function parseBody(string | StreamInterface $source): StreamInterface // Make sure the files exists if (!is_readable($source)) { throw new \InvalidArgumentException( - "The source for this upload must be either a readable file or a valid stream." + "The source for this upload must be either a readable file path or a valid stream." ); } $body = new LazyOpenStream($source, 'r'); @@ -371,7 +395,7 @@ private function parseBody(string | StreamInterface $source): StreamInterface $body = $source; } else { throw new \InvalidArgumentException( - "The source must be a string or a StreamInterface." + "The source must be a valid string file path or a StreamInterface." ); } @@ -396,7 +420,8 @@ private function uploadInitiated(array $requestArgs): void $this->currentSnapshot->getIdentifier(), $this->currentSnapshot->getTransferredBytes(), $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse() + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), ); } @@ -445,7 +470,8 @@ private function uploadCompleted(ResultInterface $result): void { $this->currentSnapshot->getIdentifier(), $this->currentSnapshot->getTransferredBytes(), $this->currentSnapshot->getTotalBytes(), - $result->toArray() + $result->toArray(), + $this->currentSnapshot->getReason(), ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->transferComplete([ @@ -468,7 +494,9 @@ private function partUploadCompleted( $newSnapshot = new TransferProgressSnapshot( $this->currentSnapshot->getIdentifier(), $this->currentSnapshot->getTransferredBytes() + $partCompletedBytes, - $this->currentSnapshot->getTotalBytes() + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->bytesTransferred([ @@ -501,13 +529,13 @@ private function callDeferredFns(): void } /** - * Verifies if a checksum was provided. + * Filters a provided checksum if one was provided. * * @param array $requestArgs * - * @return bool + * @return string | null */ - private function containsChecksum(array $requestArgs): bool + private function filterChecksum(array $requestArgs):? string { static $algorithms = [ 'ChecksumCRC32', @@ -518,20 +546,20 @@ private function containsChecksum(array $requestArgs): bool ]; foreach ($algorithms as $algorithm) { if (isset($requestArgs[$algorithm])) { - return true; + return $algorithm; } } - return false; + return null; } /** - * @param Stream $stream + * @param StreamInterface $stream * @param array $data * - * @return Stream + * @return StreamInterface */ - private function decorateWithHashes(Stream $stream, array &$data): StreamInterface + private function decorateWithHashes(StreamInterface $stream, array &$data): StreamInterface { // Decorate source with a hashing stream $hash = new PhpHash('sha256'); diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 216541446b..364bbaceba 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -3,13 +3,16 @@ namespace Aws\Test\S3\S3Transfer; use Aws\Command; +use Aws\CommandInterface; use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; +use Aws\Test\TestsUtility; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; @@ -21,7 +24,7 @@ class MultipartUploaderTest extends TestCase { /** - * @param StreamInterface $stream + * @param array $sourceConfig * @param array $config * @param array $expected * @return void @@ -30,7 +33,8 @@ class MultipartUploaderTest extends TestCase * */ public function testMultipartUpload( - StreamInterface $stream, + array $sourceConfig, + array $commandArgs, array $config, array $expected ): void @@ -39,7 +43,7 @@ public function testMultipartUpload( ->disableOriginalConstructor() ->getMock(); $s3Client->method('executeAsync') - -> willReturnCallback(function ($command) + -> willReturnCallback(function ($command) use ($expected) { if ($command->getName() === 'CreateMultipartUpload') { return Create::promiseFor(new Result([ @@ -51,6 +55,14 @@ public function testMultipartUpload( ])); } + if (isset($expected[$command->getName()])) { + $expectedOperationLevel = $expected['operations'][$command->getName()] ?? []; + foreach ($expectedOperationLevel as $key => $value) { + $this->assertArrayHasKey($key, $command); + $this->assertEquals($value, $command[$key]); + } + } + return Create::promiseFor(new Result([])); }); $s3Client->method('getCommand') @@ -60,23 +72,59 @@ public function testMultipartUpload( $requestArgs = [ 'Key' => 'FooKey', 'Bucket' => 'FooBucket', + ...$commandArgs ]; - $multipartUploader = new MultipartUploader( - $s3Client, - $requestArgs, - $config + [ - 'concurrency' => 3, - ], - $stream - ); - /** @var UploadResponse $response */ - $response = $multipartUploader->promise()->wait(); - $snapshot = $multipartUploader->getCurrentSnapshot(); - - $this->assertInstanceOf(UploadResponse::class, $response); - $this->assertCount($expected['parts'], $multipartUploader->getParts()); - $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); - $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); + $cleanUpFns = []; + if ($sourceConfig['type'] === 'stream') { + $source = Utils::streamFor( + str_repeat('*', $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'no_seekable_stream') { + $source = Utils::streamFor( + str_repeat('*', $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'file') { + $tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-uploader-test/'; + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + $source = $tempDir . DIRECTORY_SEPARATOR . 'temp-file.txt'; + file_put_contents($source, str_repeat('*', $sourceConfig['size'])); + $cleanUpFns[] = function () use ($tempDir, $source) { + TestsUtility::cleanUpDir($tempDir); + }; + } else { + $this->fail("Unsupported Source type"); + } + + if ($sourceConfig['type'] !== 'file') { + $cleanUpFns[] = function () use ($source) { + $source->close(); + }; + } + + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + $config + [ + 'concurrency' => 3, + ], + $source, + ); + /** @var UploadResponse $response */ + $response = $multipartUploader->promise()->wait(); + $snapshot = $multipartUploader->getCurrentSnapshot(); + + $this->assertInstanceOf(UploadResponse::class, $response); + $this->assertCount($expected['parts'], $multipartUploader->getParts()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); + $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); + } finally { + foreach ($cleanUpFns as $fn) { + $fn(); + } + } } /** @@ -85,9 +133,11 @@ public function testMultipartUpload( public function multipartUploadProvider(): array { return [ '5_parts_upload' => [ - 'stream' => Utils::streamFor( - str_repeat('*', 10240000 * 5), - ), + 'source_config' => [ + 'type' => 'stream', + 'size' => 10240000 * 5 + ], + 'command_args' => [], 'config' => [ 'part_size' => 10240000 ], @@ -98,9 +148,11 @@ public function multipartUploadProvider(): array { ] ], '100_parts_upload' => [ - 'stream' => Utils::streamFor( - str_repeat('*', 10240000 * 100), - ), + 'source_config' => [ + 'type' => 'stream', + 'size' => 10240000 * 100 + ], + 'command_args' => [], 'config' => [ 'part_size' => 10240000 ], @@ -111,11 +163,11 @@ public function multipartUploadProvider(): array { ] ], '5_parts_no_seekable_stream' => [ - 'stream' => new NoSeekStream( - Utils::streamFor( - str_repeat('*', 10240000 * 5) - ) - ), + 'source_config' => [ + 'type' => 'no_seekable_stream', + 'size' => 10240000 * 5 + ], + 'command_args' => [], 'config' => [ 'part_size' => 10240000 ], @@ -126,11 +178,11 @@ public function multipartUploadProvider(): array { ] ], '100_parts_no_seekable_stream' => [ - 'stream' => new NoSeekStream( - Utils::streamFor( - str_repeat('*', 10240000 * 100) - ) - ), + 'source_config' => [ + 'type' => 'no_seekable_stream', + 'size' => 10240000 * 100 + ], + 'command_args' => [], 'config' => [ 'part_size' => 10240000 ], @@ -139,14 +191,40 @@ public function multipartUploadProvider(): array { 'parts' => 100, 'bytesUploaded' => 10240000 * 100, ] - ] + ], + '100_parts_with_custom_checksum' => [ + 'source_config' => [ + 'type' => 'file', + 'size' => 10240000 * 100 + ], + 'command_args' => [ + 'ChecksumCRC32' => 'FooChecksum', + ], + 'config' => [ + 'part_size' => 10240000 + ], + 'expected' => [ + 'succeed' => true, + 'parts' => 100, + 'bytesUploaded' => 10240000 * 100, + 'CreateMultipartUpload' => [ + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'CRC32' + ], + 'CompleteMultipartUpload' => [ + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'CRC32', + 'ChecksumCRC32' => 'FooChecksum', + ] + ] + ], ]; } /** * @return S3ClientInterface */ - private function multipartUploadS3Client(): S3ClientInterface + private function getMultipartUploadS3Client(): S3ClientInterface { return new S3Client([ 'region' => 'us-east-2', @@ -176,39 +254,133 @@ private function multipartUploadS3Client(): S3ClientInterface ]); } + /** + * @param int $partSize + * @param bool $expectError + * + * @dataProvider validatePartSizeProvider + * * @return void */ - public function testInvalidSourceStringThrowsException(): void - { - $nonExistentFile = '/path/to/nonexistent/file.txt'; - $this->expectException(\InvalidArgumentException::class); - $this->expectExceptionMessage( - "The source for this upload must be either a readable file or a valid stream." - ); + public function testValidatePartSize( + int $partSize, + bool $expectError + ): void { + if ($expectError) { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The config `part_size` value must be between " + . MultipartUploader::PART_MIN_SIZE . " and " . MultipartUploader::PART_MAX_SIZE + . " but ${partSize} given." + ); + } else { + $this->assertTrue(true); + } + new MultipartUploader( - $this->multipartUploadS3Client(), + $this->getMultipartUploadS3Client(), ['Bucket' => 'test-bucket', 'Key' => 'test-key'], - [], - $nonExistentFile + [ + 'part_size' => $partSize, + ], + Utils::streamFor('') ); } /** + * @return array + */ + public function validatePartSizeProvider(): array { + return [ + 'part_size_over_max' => [ + 'part_size' => MultipartUploader::PART_MAX_SIZE + 1, + 'expectError' => true, + ], + 'part_size_under_min' => [ + 'part_size' => MultipartUploader::PART_MIN_SIZE - 1, + 'expectError' => true, + ], + 'part_size_between_valid_range_1' => [ + 'part_size' => MultipartUploader::PART_MAX_SIZE - 1, + 'expectError' => false, + ], + 'part_size_between_valid_range_2' => [ + 'part_size' => MultipartUploader::PART_MIN_SIZE + 1, + 'expectError' => false, + ] + ]; + } + + /** + * @param string|int $source + * @param bool $expectError + * + * @dataProvider invalidSourceStringProvider + * * @return void */ - public function testInvalidSourceTypeThrowsException(): void + public function testInvalidSourceStringThrowsException( + string|int $source, + bool $expectError + ): void { - $this->expectException(\InvalidArgumentException::class); - $this->expectExceptionMessage( - "The source for this upload must be either a readable file or a valid stream." - ); - new MultipartUploader( - $this->multipartUploadS3Client(), - ['Bucket' => 'test-bucket', 'Key' => 'test-key'], - [], - 12345 - ); + $cleanUpFns = []; + if ($expectError) { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage( + "The source for this upload must be either a readable file path or a valid stream." + ); + } else { + $this->assertTrue(true); + $tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-upload-test'; + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + + $source = $tempDir . DIRECTORY_SEPARATOR . $source; + file_put_contents($source, 'foo'); + $cleanUpFns[] = function () use ($tempDir) { + TestsUtility::cleanUpDir($tempDir); + }; + } + + try { + new MultipartUploader( + $this->getMultipartUploadS3Client(), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [], + $source + ); + } finally { + foreach ($cleanUpFns as $cleanUpFn) { + $cleanUpFn(); + } + } + } + + /** + * @return array[] + */ + public function invalidSourceStringProvider(): array { + return [ + 'invalid_source_file_path_1' => [ + 'source' => 'invalid', + 'expectError' => true, + ], + 'invalid_source_file_path_2' => [ + 'source' => 'invalid_2', + 'expectError' => true, + ], + 'invalid_source_3' => [ + 'source' => 12345, + 'expectError' => true, + ], + 'valid_source' => [ + 'source' => 'myfile.txt', + 'expectError' => false, + ], + ]; } /** @@ -279,6 +451,286 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $this->assertInstanceOf(UploadResponse::class, $response); } + /** + * Test to make sure createMultipart, uploadPart, and completeMultipart + * operations are called. + * + * @return void + */ + public function testMultipartOperationsAreCalled(): void { + $operationsCalled = [ + 'CreateMultipartUpload' => false, + 'UploadPart' => false, + 'CompleteMultipartUpload' => false, + ]; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) use (&$operationsCalled) { + $operationsCalled[$command->getName()] = true; + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + return Create::promiseFor(new Result([ + 'ETag' => 'TestETag' + ])); + } + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + + $stream = Utils::streamFor(str_repeat('*', 1024 * 1024 * 5)); + $requestArgs = [ + 'Key' => 'test-key', + 'Bucket' => 'test-bucket', + ]; + + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + [ + 'concurrency' => 1, + ], + $stream + ); + + $multipartUploader->promise()->wait(); + foreach ($operationsCalled as $key => $value) { + $this->assertTrue($value, 'Operation {' . $key . '} was not called'); + } + } + + /** + * @param array $sourceConfig + * @param array $checksumConfig + * @param array $expectedOperationHeaders + * + * @dataProvider multipartUploadWithCustomChecksumProvider + * + * @return void + */ + public function testMultipartUploadWithCustomChecksum( + array $sourceConfig, + array $checksumConfig, + array $expectedOperationHeaders, + ): void { + // $operationsCalled: To make sure each expected operation is invoked. + $operationsCalled = []; + foreach (array_keys($expectedOperationHeaders) as $key) { + $operationsCalled[$key] = false; + } + + $s3Client = $this->getMultipartUploadS3Client(); + $s3Client->getHandlerList()->appendSign( + function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) { + return function ( + CommandInterface $command, + RequestInterface $request + ) use ($handler, &$operationsCalled, $expectedOperationHeaders) { + $operationsCalled[$command->getName()] = true; + $expectedHeaders = $expectedOperationHeaders[$command->getName()] ?? []; + $has = $expectedHeaders['has'] ?? []; + $hasNot = $expectedHeaders['has_not'] ?? []; + foreach ($has as $key => $value) { + $this->assertArrayHasKey($key, $request->getHeaders()); + $this->assertEquals($value, $request->getHeader($key)[0]); + } + + foreach ($hasNot as $value) { + $this->assertArrayNotHasKey($value, $request->getHeaders()); + } + + return $handler($command, $request); + }; + } + ); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ...$checksumConfig, + ]; + $cleanUpFns = []; + if ($sourceConfig['type'] === 'stream') { + $source = Utils::streamFor( + str_repeat($sourceConfig['char'], $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'no_seekable_stream') { + $source = Utils::streamFor( + str_repeat($sourceConfig['char'], $sourceConfig['size']) + ); + } elseif ($sourceConfig['type'] === 'file') { + $tempDir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'multipart-uploader-test/'; + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + $source = $tempDir . DIRECTORY_SEPARATOR . 'temp-file.txt'; + file_put_contents($source, str_repeat($sourceConfig['char'], $sourceConfig['size'])); + $cleanUpFns[] = function () use ($tempDir, $source) { + TestsUtility::cleanUpDir($tempDir); + }; + } else { + $this->fail("Unsupported Source type"); + } + + if ($sourceConfig['type'] !== 'file') { + $cleanUpFns[] = function () use ($source) { + $source->close(); + }; + } + + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + [ + 'concurrency' => 3, + ], + $source, + ); + /** @var UploadResponse $response */ + $response = $multipartUploader->promise()->wait(); + foreach ($operationsCalled as $key => $value) { + $this->assertTrue($value, 'Operation {' . $key . '} was not called'); + } + $this->assertInstanceOf(UploadResponse::class, $response); + } finally { + foreach ($cleanUpFns as $fn) { + $fn(); + } + } + } + + /** + * @return array + */ + public function multipartUploadWithCustomChecksumProvider(): array { + return [ + 'custom_checksum_sha256_1' => [ + 'source_config' => [ + 'type' => 'stream', + 'size' => 1024 * 1024 * 20, + 'char' => '*' + ], + 'checksum_config' => [ + 'ChecksumSHA256' => '0c58gNl31EVxhClRWw5+WHiAUp2B3/3g1zQDCvY4bmQ=', + ], + 'expected_operation_headers' => [ + 'CreateMultipartUpload' => [ + 'has' => [ + 'x-amz-checksum-algorithm' => 'SHA256', + 'x-amz-checksum-type' => 'FULL_OBJECT' + ] + ], + 'UploadPart' => [ + 'has_not' => [ + 'x-amz-checksum-algorithm', + 'x-amz-checksum-type', + 'x-amz-checksum-sha256' + ] + ], + 'CompleteMultipartUpload' => [ + 'has' => [ + 'x-amz-checksum-sha256' => '0c58gNl31EVxhClRWw5+WHiAUp2B3/3g1zQDCvY4bmQ=', + 'x-amz-checksum-type' => 'FULL_OBJECT', + ], + ] + ] + ], + 'custom_checksum_crc32_1' => [ + 'source_config' => [ + 'type' => 'stream', + 'size' => 1024 * 1024 * 20, + 'char' => '*' + ], + 'checksum_config' => [ + 'ChecksumCRC32' => '+IIKcQ==', + ], + 'expected_operation_headers' => [ + 'CreateMultipartUpload' => [ + 'has' => [ + 'x-amz-checksum-algorithm' => 'CRC32', + 'x-amz-checksum-type' => 'FULL_OBJECT' + ] + ], + 'UploadPart' => [ + 'has_not' => [ + 'x-amz-checksum-algorithm', + 'x-amz-checksum-type', + 'x-amz-checksum-crc32' + ] + ], + 'CompleteMultipartUpload' => [ + 'has' => [ + 'x-amz-checksum-crc32' => '+IIKcQ==', + 'x-amz-checksum-type' => 'FULL_OBJECT', + ], + ] + ] + ] + ]; + } + + /** + * @return void + */ + public function testMultipartUploadAbort() { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessage('Upload failed'); + $abortMultipartCalled = false; + $abortMultipartCalledTimes = 0; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) + use (&$abortMultipartCalled, &$abortMultipartCalledTimes) { + if ($command->getName() === 'CreateMultipartUpload') { + return Create::promiseFor(new Result([ + 'UploadId' => 'TestUploadId' + ])); + } elseif ($command->getName() === 'UploadPart') { + if ($command['PartNumber'] == 3) { + return Create::rejectionFor(new S3TransferException('Upload failed')); + } + } elseif ($command->getName() === 'AbortMultipartUpload') { + $abortMultipartCalled = true; + $abortMultipartCalledTimes++; + } + + return Create::promiseFor(new Result([])); + }); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) { + return new Command($commandName, $args); + }); + $requestArgs = [ + 'Bucket' => 'test-bucket', + 'Key' => 'test-key', + ]; + $source = Utils::streamFor(str_repeat('*', 1024 * 1024 * 20)); + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + [ + 'concurrency' => 3, + ], + $source, + ); + $multipartUploader->upload(); + } finally { + $this->assertTrue($abortMultipartCalled); + $this->assertEquals(1, $abortMultipartCalledTimes); + $source->close(); + } + } + /** * @return void */ From 64fc3f63345b2570dc13e5df500b35a63bf321be Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 18 Jun 2025 08:45:39 -0700 Subject: [PATCH 31/62] tests: Add integ test for abort Abort multipart upload integ test, that makes sure a multipart upload is aborted properly and cleaned from the bucket after a failure. --- tests/Integ/S3TransferManagerContext.php | 87 +++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index eba874948a..4cfbf7593a 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -4,10 +4,13 @@ use Aws\S3\ApplyChecksumMiddleware; use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Progress\TransferListener; +use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\S3TransferManager; use Aws\Test\TestsUtility; use Behat\Behat\Context\Context; use Behat\Behat\Context\SnippetAcceptingContext; +use Behat\Behat\Tester\Exception\PendingException; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\Assert; use Psr\Http\Message\StreamInterface; @@ -477,7 +480,10 @@ public function iDownloadAllOfThemIntoTheDirectory($directory): void /** * @Then /^the objects (.*) should exist as files within the directory (.*)$/ */ - public function theObjectsShouldExistsAsFilesWithinTheDirectory($numfile, $directory): void + public function theObjectsShouldExistsAsFilesWithinTheDirectory( + $numfile, + $directory + ): void { $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; $s3Client = self::getSdk()->createS3(); @@ -493,4 +499,83 @@ public function theObjectsShouldExistsAsFilesWithinTheDirectory($numfile, $direc Assert::assertEquals($numfile, $count); } + + /** + * @Given /^I am uploading the file (.*) with size (.*)$/ + */ + public function iAmUploadingTheFileWithSize($file, $size): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + file_put_contents($fullFilePath, str_repeat('*', $size)); + } + + /** + * @When /^I upload the file (.*) using multipart upload and fails at part number (.*)$/ + */ + public function iUploadTheFileUsingMultipartUploadAndFailsAtPartNumber( + $file, + $partNumberFail + ): void + { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; + $s3TransferManager = new S3TransferManager( + self::getSdk()->createS3() + ); + $transferListener = new class($partNumberFail) extends TransferListener { + private int $partNumber; + private int $partNumberFail; + + public function __construct(int $partNumberFail) { + $this->partNumberFail = $partNumberFail; + $this->partNumber = 0; + } + + public function bytesTransferred(array $context): void + { + $this->partNumber++; + if ($this->partNumber === $this->partNumberFail) { + throw new \RuntimeException( + "Transfer failed at part number {$this->partNumber} failed" + ); + } + } + }; + $s3TransferManager->upload( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + [], + [ + $transferListener, + ] + )->wait(); + } + + /** + * @Then /^The multipart upload should have been aborted for file (.*)$/ + */ + public function theMultipartUploadShouldHaveBeenAbortedForFile($file): void + { + $client = self::getSdk()->createS3(); + $inProgressMultipartUploads = $client->listMultipartUploads([ + 'Bucket' => self::getResourceName(), + ]); + // Make sure that, if there are in progress multipart upload + // it is not for the file being uploaded in this test. + $multipartUploadCount = count($inProgressMultipartUploads); + if ($multipartUploadCount > 0) { + $multipartUploadCount = 0; + foreach ($inProgressMultipartUploads as $inProgressMultipartUpload) { + if ($inProgressMultipartUpload['Key'] === $file) { + $multipartUploadCount++; + } + } + + Assert::assertEquals(0, $multipartUploadCount); + } + + Assert::assertEquals(0, $multipartUploadCount); + } } \ No newline at end of file From 908b4a01ea48389e6c9d95eb648db2ccf63eb83c Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 18 Jun 2025 08:56:49 -0700 Subject: [PATCH 32/62] chore: update integ test Update multipart upload integ test to make sure it validates transfer fails is called. --- features/s3Transfer/s3TransferManager.feature | 12 +++++++++++- tests/Integ/S3TransferManagerContext.php | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature index 44a24e7cdf..e86a3cdd65 100644 --- a/features/s3Transfer/s3TransferManager.feature +++ b/features/s3Transfer/s3TransferManager.feature @@ -104,4 +104,14 @@ Feature: S3 Transfer Manager | 15 | directory-test-2-1 | | 12 | directory-test-2-2 | | 1 | directory-test-2-3 | - | 30 | directory-test-2-4 | \ No newline at end of file + | 30 | directory-test-2-4 | + + Scenario Outline: Abort a multipart upload + Given I am uploading the file with size + When I upload the file using multipart upload and fails at part number + Then The multipart upload should have been aborted for file + Examples: + | file | size | partNumberFail | + | abort-file-1.txt | 1024 * 1024 * 20 | 3 | + | abort-file-2.txt | 1024 * 1024 * 40 | 5 | + | abort-file-3.txt | 1024 * 1024 * 10 | 1 | \ No newline at end of file diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index 4cfbf7593a..dad95f9f65 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -13,6 +13,7 @@ use Behat\Behat\Tester\Exception\PendingException; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\Assert; +use PHPUnit\Framework\TestCase; use Psr\Http\Message\StreamInterface; class S3TransferManagerContext implements Context, SnippetAcceptingContext @@ -540,6 +541,14 @@ public function bytesTransferred(array $context): void } } }; + // To make sure transferFail is called + $testCase = new class extends TestCase {}; + $transferListener2 = $testCase->getMockBuilder( + TransferListener::class + )->getMock(); + $transferListener2->expects($testCase->once())->method('transferInitiated'); + $transferListener2->expects($testCase->once())->method('transferFail'); + $s3TransferManager->upload( $fullFilePath, [ @@ -549,6 +558,7 @@ public function bytesTransferred(array $context): void [], [ $transferListener, + $transferListener2 ] )->wait(); } From 1e643e5c504d3839477102e6ed505dba809c30ec Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 10 Jul 2025 09:25:01 -0700 Subject: [PATCH 33/62] feat: update to use modeled inputs - S3TransferManager: Uses modeled input for config. - MultipartUpload: - Use abstract base class for multipart uploader in order to share functionalities with multipart copier. - Use modeled input for config and request args. - Update unit testing - Update Integ Tests --- .../S3Transfer/AbstractMultipartUploader.php | 499 +++++++++ .../Models/MultipartUploaderConfig.php | 86 ++ src/S3/S3Transfer/Models/PutObjectRequest.php | 964 ++++++++++++++++++ .../S3Transfer/Models/PutObjectResponse.php | 310 ++++++ .../Models/S3TransferManagerConfig.php | 187 ++++ src/S3/S3Transfer/Models/TransferRequest.php | 47 + .../Models/TransferRequestConfig.php | 31 + src/S3/S3Transfer/Models/UploadRequest.php | 140 +++ .../S3Transfer/Models/UploadRequestConfig.php | 120 +++ src/S3/S3Transfer/MultipartUploader.php | 543 ++-------- src/S3/S3Transfer/S3TransferManager.php | 167 +-- tests/Integ/S3TransferManagerContext.php | 122 ++- tests/S3/S3Transfer/MultipartUploaderTest.php | 103 +- 13 files changed, 2665 insertions(+), 654 deletions(-) create mode 100644 src/S3/S3Transfer/AbstractMultipartUploader.php create mode 100644 src/S3/S3Transfer/Models/MultipartUploaderConfig.php create mode 100644 src/S3/S3Transfer/Models/PutObjectRequest.php create mode 100644 src/S3/S3Transfer/Models/PutObjectResponse.php create mode 100644 src/S3/S3Transfer/Models/S3TransferManagerConfig.php create mode 100644 src/S3/S3Transfer/Models/TransferRequest.php create mode 100644 src/S3/S3Transfer/Models/TransferRequestConfig.php create mode 100644 src/S3/S3Transfer/Models/UploadRequest.php create mode 100644 src/S3/S3Transfer/Models/UploadRequestConfig.php diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php new file mode 100644 index 0000000000..d860382a74 --- /dev/null +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -0,0 +1,499 @@ +s3Client = $s3Client; + $this->putObjectRequest = $putObjectRequest; + $this->config = $config; + $this->validateConfig(); + $this->uploadId = $uploadId; + $this->parts = $parts; + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; + // Evaluation for custom provided checksums + $objectRequestArgs = $putObjectRequest->toArray(); + $checksumName = self::filterChecksum($objectRequestArgs); + if ($checksumName !== null) { + $this->requestChecksum = $objectRequestArgs[$checksumName]; + $this->requestChecksumAlgorithm = str_replace( + 'Checksum', + '', + $checksumName + ); + } else { + $this->requestChecksum = null; + $this->requestChecksumAlgorithm = null; + } + } + + /** + * @return void + */ + protected function validateConfig(): void + { + $partSize = $this->config->getTargetPartSizeBytes(); + if ($partSize < self::PART_MIN_SIZE || $partSize > self::PART_MAX_SIZE) { + throw new \InvalidArgumentException( + "Part size config must be between " . self::PART_MIN_SIZE + ." and " . self::PART_MAX_SIZE . " bytes " + ."but it is configured to $partSize" + ); + } + } + + /** + * @return string|null + */ + public function getUploadId(): ?string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getParts(): array + { + return $this->parts; + } + + /** + * Get the current progress snapshot. + * @return TransferProgressSnapshot|null + */ + public function getCurrentSnapshot(): ?TransferProgressSnapshot + { + return $this->currentSnapshot; + } + + /** + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + try { + yield $this->createMultipartUpload(); + yield $this->processMultipartOperation(); + $result = yield $this->completeMultipartUpload(); + yield Create::promiseFor($this->createResponse($result)); + } catch (Throwable $e) { + $this->operationFailed($e); + yield Create::rejectionFor($e); + } finally { + $this->callDeferredFns(); + } + }); + } + + /** + * @return PromiseInterface + */ + protected function createMultipartUpload(): PromiseInterface + { + $createMultipartUploadArgs = $this->putObjectRequest->toCreateMultipartRequest(); + if ($this->requestChecksum !== null) { + $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + } elseif ($this->config->getRequestChecksumCalculation() === 'when_supported') { + $this->requestChecksumAlgorithm = self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM; + $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + } + + $this->operationInitiated($createMultipartUploadArgs); + $command = $this->s3Client->getCommand( + 'CreateMultipartUpload', + $createMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadId = $result['UploadId']; + return $result; + }); + } + + /** + * @return PromiseInterface + */ + protected function completeMultipartUpload(): PromiseInterface + { + $this->sortParts(); + $completeMultipartUploadArgs = $this->putObjectRequest->toCompleteMultipartUploadRequest(); + $completeMultipartUploadArgs['UploadId'] = $this->uploadId; + $completeMultipartUploadArgs['MultipartUpload'] = [ + 'Parts' => $this->parts + ]; + $completeMultipartUploadArgs['MpuObjectSize'] = $this->getTotalSize(); + + if ($this->requestChecksum !== null) { + $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $completeMultipartUploadArgs[ + 'Checksum' . $this->requestChecksumAlgorithm + ] = $this->requestChecksum; + } + + $command = $this->s3Client->getCommand( + 'CompleteMultipartUpload', + $completeMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->operationCompleted($result); + return $result; + }); + } + + /** + * @return PromiseInterface + */ + protected function abortMultipartUpload(): PromiseInterface + { + $abortMultipartUploadArgs = $this->putObjectRequest->toAbortMultipartRequest(); + $abortMultipartUploadArgs['UploadId'] = $this->uploadId; + $command = $this->s3Client->getCommand( + 'AbortMultipartUpload', + $abortMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command); + } + + /** + * @return void + */ + protected function sortParts(): void + { + usort($this->parts, function ($partOne, $partTwo) { + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; + }); + } + + /** + * @param ResultInterface $result + * @param CommandInterface $command + * @return void + */ + protected function collectPart + ( + ResultInterface $result, + CommandInterface $command + ): void + { + $checksumResult = match($command->getName()) { + 'UploadPart' => $result, + 'UploadPartCopy' => $result['CopyPartResult'], + default => $result[$command->getName() . 'Result'] + }; + + $partData = [ + 'PartNumber' => $command['PartNumber'], + 'ETag' => $checksumResult['ETag'], + ]; + + if (isset($command['ChecksumAlgorithm'])) { + $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); + $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; + } + + $this->parts[] = $partData; + } + + /** + * @param array $commands + * @param callable $fulfilledCallback + * @param callable $rejectedCallback + * @return PromiseInterface + */ + protected function createCommandPool + ( + array $commands, + callable $fulfilledCallback, + callable $rejectedCallback + ): PromiseInterface + { + return (new CommandPool( + $this->s3Client, + $commands, + [ + 'concurrency' => $this->config['concurrency'], + 'fulfilled' => $fulfilledCallback, + 'rejected' => $rejectedCallback + ] + ))->promise(); + } + + /** + * @param array $requestArgs + * @return void + */ + protected function operationInitiated(array $requestArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $this->getTotalSize() + ); + } + + $this->listenerNotifier?->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @param ResultInterface $result + * @return void + */ + protected function operationCompleted(ResultInterface $result): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $result->toArray(), + $this->currentSnapshot->getReason(), + ); + + $this->currentSnapshot = $newSnapshot; + + $this->listenerNotifier?->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => + $this->putObjectRequest->toCompleteMultipartUploadRequest(), + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @param Throwable $reason + * @return void + * + */ + protected function operationFailed(Throwable $reason): void + { + // Event already propagated + if ($this->currentSnapshot?->getReason() !== null) { + return; + } + + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + 'Unknown', + 0, + 0, + ); + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); + + if (!empty($this->uploadId)) { + error_log( + "Multipart Upload with id: " . $this->uploadId . " failed", + E_USER_WARNING + ); + $this->abortMultipartUpload()->wait(); + } + + $this->listenerNotifier?->transferFail([ + TransferListener::REQUEST_ARGS_KEY => + $this->putObjectRequest->toAbortMultipartRequest(), + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + 'reason' => $reason, + ]); + } + + /** + * @param int $partSize + * @param array $requestArgs + * @return void + */ + protected function partCompleted( + int $partSize, + array $requestArgs + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partSize, + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), + ); + + $this->currentSnapshot = $newSnapshot; + + $this->listenerNotifier?->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @return void + */ + protected function callDeferredFns(): void + { + foreach ($this->deferFns as $fn) { + $fn(); + } + + $this->deferFns = []; + } + + /** + * @param Throwable $reason + * @return void + */ + protected function partFailed(Throwable $reason): void + { + $this->operationFailed($reason); + } + + /** + * @return int + */ + protected function calculatePartSize(): int + { + return max( + $this->getTotalSize() / self::PART_MAX_NUM, + $this->config->getTargetPartSizeBytes() + ); + } + + /** + * @return PromiseInterface + */ + abstract protected function processMultipartOperation(): PromiseInterface; + + /** + * @return int + */ + abstract protected function getTotalSize(): int; + + /** + * @param ResultInterface $result + * + * @return UploadResponse + */ + abstract protected function createResponse(ResultInterface $result): UploadResponse; + + /** + * Filters a provided checksum if one was provided. + * + * @param array $requestArgs + * + * @return string | null + */ + private static function filterChecksum(array $requestArgs):? string + { + static $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return $algorithm; + } + } + + return null; + } +} diff --git a/src/S3/S3Transfer/Models/MultipartUploaderConfig.php b/src/S3/S3Transfer/Models/MultipartUploaderConfig.php new file mode 100644 index 0000000000..3dd4d63f8d --- /dev/null +++ b/src/S3/S3Transfer/Models/MultipartUploaderConfig.php @@ -0,0 +1,86 @@ +targetPartSizeBytes = $targetPartSizeBytes; + $this->concurrency = $concurrency; + $this->requestChecksumCalculation = $requestChecksumCalculation; + } + + /** + * @return int + */ + public function getTargetPartSizeBytes(): int { + return $this->targetPartSizeBytes; + } + + /** + * @return int + */ + public function getConcurrency(): int { + return $this->concurrency; + } + + /** + * @return string + */ + public function getRequestChecksumCalculation(): string { + return $this->requestChecksumCalculation; + } + + /** + * Convert the configuration to an array + * + * @return array + */ + public function toArray(): array + { + return [ + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'concurrency' => $this->concurrency, + 'request_checksum_calculation' => $this->requestChecksumCalculation, + ]; + } + + /** + * Create an MultipartUploaderConfig instance from an array + * + * @param array $data Array containing configuration data + * + * @return static + */ + public static function fromArray(array $data): self + { + return new self( + $data['target_part_size_bytes'] + ?? S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, + $data['concurrency'] + ?? S3TransferManagerConfig::DEFAULT_CONCURRENCY, + $data['request_checksum_calculation'] + ?? S3TransferManagerConfig::DEFAULT_REQUEST_CHECKSUM_CALCULATION + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/PutObjectRequest.php b/src/S3/S3Transfer/Models/PutObjectRequest.php new file mode 100644 index 0000000000..cfa3ea1d19 --- /dev/null +++ b/src/S3/S3Transfer/Models/PutObjectRequest.php @@ -0,0 +1,964 @@ +acl = $acl; + $this->bucket = $bucket; + $this->bucketKeyEnabled = $bucketKeyEnabled; + $this->cacheControl = $cacheControl; + $this->checksumAlgorithm = $checksumAlgorithm; + $this->contentDisposition = $contentDisposition; + $this->contentEncoding = $contentEncoding; + $this->contentLanguage = $contentLanguage; + $this->contentType = $contentType; + $this->expectedBucketOwner = $expectedBucketOwner; + $this->expires = $expires; + $this->grantFullControl = $grantFullControl; + $this->grantRead = $grantRead; + $this->grantReadACP = $grantReadACP; + $this->grantWriteACP = $grantWriteACP; + $this->key = $key; + $this->metadata = $metadata; + $this->objectLockLegalHoldStatus = $objectLockLegalHoldStatus; + $this->objectLockMode = $objectLockMode; + $this->objectLockRetainUntilDate = $objectLockRetainUntilDate; + $this->requestPayer = $requestPayer; + $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; + $this->sseCustomerKey = $sseCustomerKey; + $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; + $this->ssekmsEncryptionContext = $ssekmsEncryptionContext; + $this->ssekmsKeyId = $ssekmsKeyId; + $this->serverSideEncryption = $serverSideEncryption; + $this->storageClass = $storageClass; + $this->tagging = $tagging; + $this->websiteRedirectLocation = $websiteRedirectLocation; + $this->checksumCRC32 = $checksumCRC32; + $this->checksumCRC32C = $checksumCRC32C; + $this->checksumCRC64NVME = $checksumCRC64NVME; + $this->checksumSHA1 = $checksumSHA1; + $this->checksumSHA256 = $checksumSHA256; + $this->ifMatch = $ifMatch; + $this->ifNoneMatch = $ifNoneMatch; + } + + /** + * @return string|null + */ + public function getAcl(): ?string + { + return $this->acl; + } + + /** + * @return string|null + */ + public function getBucket(): ?string + { + return $this->bucket; + } + + /** + * @return bool|null + */ + public function getBucketKeyEnabled(): ?bool + { + return $this->bucketKeyEnabled; + } + + /** + * @return string|null + */ + public function getCacheControl(): ?string + { + return $this->cacheControl; + } + + /** + * @return string|null + */ + public function getChecksumAlgorithm(): ?string + { + return $this->checksumAlgorithm; + } + + /** + * @return string|null + */ + public function getContentDisposition(): ?string + { + return $this->contentDisposition; + } + + /** + * @return string|null + */ + public function getContentEncoding(): ?string + { + return $this->contentEncoding; + } + + /** + * @return string|null + */ + public function getContentLanguage(): ?string + { + return $this->contentLanguage; + } + + /** + * @return string|null + */ + public function getContentType(): ?string + { + return $this->contentType; + } + + /** + * @return string|null + */ + public function getExpectedBucketOwner(): ?string + { + return $this->expectedBucketOwner; + } + + /** + * @return string|null + */ + public function getExpires(): ?string + { + return $this->expires; + } + + /** + * @return string|null + */ + public function getGrantFullControl(): ?string + { + return $this->grantFullControl; + } + + /** + * @return string|null + */ + public function getGrantRead(): ?string + { + return $this->grantRead; + } + + /** + * @return string|null + */ + public function getGrantReadACP(): ?string + { + return $this->grantReadACP; + } + + /** + * @return string|null + */ + public function getGrantWriteACP(): ?string + { + return $this->grantWriteACP; + } + + /** + * @return string|null + */ + public function getKey(): ?string + { + return $this->key; + } + + /** + * @return string|null + */ + public function getMetadata(): ?string + { + return $this->metadata; + } + + /** + * @return string|null + */ + public function getObjectLockLegalHoldStatus(): ?string + { + return $this->objectLockLegalHoldStatus; + } + + /** + * @return string|null + */ + public function getObjectLockMode(): ?string + { + return $this->objectLockMode; + } + + /** + * @return string|null + */ + public function getObjectLockRetainUntilDate(): ?string + { + return $this->objectLockRetainUntilDate; + } + + /** + * @return string|null + */ + public function getRequestPayer(): ?string + { + return $this->requestPayer; + } + + /** + * @return string|null + */ + public function getSseCustomerAlgorithm(): ?string + { + return $this->sseCustomerAlgorithm; + } + + /** + * @return string|null + */ + public function getSseCustomerKey(): ?string + { + return $this->sseCustomerKey; + } + + /** + * @return string|null + */ + public function getSseCustomerKeyMD5(): ?string + { + return $this->sseCustomerKeyMD5; + } + + /** + * @return string|null + */ + public function getSsekmsEncryptionContext(): ?string + { + return $this->ssekmsEncryptionContext; + } + + /** + * @return string|null + */ + public function getSsekmsKeyId(): ?string + { + return $this->ssekmsKeyId; + } + + /** + * @return string|null + */ + public function getServerSideEncryption(): ?string + { + return $this->serverSideEncryption; + } + + /** + * @return string|null + */ + public function getStorageClass(): ?string + { + return $this->storageClass; + } + + /** + * @return string|null + */ + public function getTagging(): ?string + { + return $this->tagging; + } + + /** + * @return string|null + */ + public function getWebsiteRedirectLocation(): ?string + { + return $this->websiteRedirectLocation; + } + + /** + * @return string|null + */ + public function getChecksumCRC32(): ?string + { + return $this->checksumCRC32; + } + + /** + * @return string|null + */ + public function getChecksumCRC32C(): ?string + { + return $this->checksumCRC32C; + } + + /** + * @return string|null + */ + public function getChecksumCRC64NVME(): ?string + { + return $this->checksumCRC64NVME; + } + + /** + * @return string|null + */ + public function getChecksumSHA1(): ?string + { + return $this->checksumSHA1; + } + + /** + * @return string|null + */ + public function getChecksumSHA256(): ?string + { + return $this->checksumSHA256; + } + + /** + * @return string|null + */ + public function getIfMatch(): ?string + { + return $this->ifMatch; + } + + /** + * @return string|null + */ + public function getIfNoneMatch(): ?string + { + return $this->ifNoneMatch; + } + + /** + * Convert to single object request array + * + * @return array + */ + public function toSingleObjectRequest(): array + { + $requestArgs = []; + + if ($this->bucket !== null) { + $requestArgs['Bucket'] = $this->bucket; + } + + if ($this->checksumAlgorithm !== null) { + $requestArgs['ChecksumAlgorithm'] = $this->checksumAlgorithm; + } + + if ($this->checksumCRC32 !== null) { + $requestArgs['ChecksumCRC32'] = $this->checksumCRC32; + } + + if ($this->checksumCRC32C !== null) { + $requestArgs['ChecksumCRC32C'] = $this->checksumCRC32C; + } + + if ($this->checksumCRC64NVME !== null) { + $requestArgs['ChecksumCRC64NVME'] = $this->checksumCRC64NVME; + } + + if ($this->checksumSHA1 !== null) { + $requestArgs['ChecksumSHA1'] = $this->checksumSHA1; + } + + if ($this->checksumSHA256 !== null) { + $requestArgs['ChecksumSHA256'] = $this->checksumSHA256; + } + + if ($this->expectedBucketOwner !== null) { + $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; + } + + if ($this->key !== null) { + $requestArgs['Key'] = $this->key; + } + + if ($this->requestPayer !== null) { + $requestArgs['RequestPayer'] = $this->requestPayer; + } + + if ($this->sseCustomerAlgorithm !== null) { + $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; + } + + if ($this->sseCustomerKey !== null) { + $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; + } + + if ($this->sseCustomerKeyMD5 !== null) { + $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; + } + + return $requestArgs; + } + + /** + * Convert to create multipart request array + * + * @return array + */ + public function toCreateMultipartRequest(): array + { + $requestArgs = []; + + if ($this->acl !== null) { + $requestArgs['ACL'] = $this->acl; + } + + if ($this->bucket !== null) { + $requestArgs['Bucket'] = $this->bucket; + } + + if ($this->bucketKeyEnabled !== null) { + $requestArgs['BucketKeyEnabled'] = $this->bucketKeyEnabled; + } + + if ($this->cacheControl !== null) { + $requestArgs['CacheControl'] = $this->cacheControl; + } + + if ($this->checksumAlgorithm !== null) { + $requestArgs['ChecksumAlgorithm'] = $this->checksumAlgorithm; + } + + if ($this->contentDisposition !== null) { + $requestArgs['ContentDisposition'] = $this->contentDisposition; + } + + if ($this->contentEncoding !== null) { + $requestArgs['ContentEncoding'] = $this->contentEncoding; + } + + if ($this->contentLanguage !== null) { + $requestArgs['ContentLanguage'] = $this->contentLanguage; + } + + if ($this->contentType !== null) { + $requestArgs['ContentType'] = $this->contentType; + } + + if ($this->expectedBucketOwner !== null) { + $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; + } + + if ($this->expires !== null) { + $requestArgs['Expires'] = $this->expires; + } + + if ($this->grantFullControl !== null) { + $requestArgs['GrantFullControl'] = $this->grantFullControl; + } + + if ($this->grantRead !== null) { + $requestArgs['GrantRead'] = $this->grantRead; + } + + if ($this->grantReadACP !== null) { + $requestArgs['GrantReadACP'] = $this->grantReadACP; + } + + if ($this->grantWriteACP !== null) { + $requestArgs['GrantWriteACP'] = $this->grantWriteACP; + } + + if ($this->key !== null) { + $requestArgs['Key'] = $this->key; + } + + if ($this->metadata !== null) { + $requestArgs['Metadata'] = $this->metadata; + } + + if ($this->objectLockLegalHoldStatus !== null) { + $requestArgs['ObjectLockLegalHoldStatus'] = $this->objectLockLegalHoldStatus; + } + + if ($this->objectLockMode !== null) { + $requestArgs['ObjectLockMode'] = $this->objectLockMode; + } + + if ($this->objectLockRetainUntilDate !== null) { + $requestArgs['ObjectLockRetainUntilDate'] = $this->objectLockRetainUntilDate; + } + + if ($this->requestPayer !== null) { + $requestArgs['RequestPayer'] = $this->requestPayer; + } + + if ($this->sseCustomerAlgorithm !== null) { + $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; + } + + if ($this->sseCustomerKey !== null) { + $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; + } + + if ($this->sseCustomerKeyMD5 !== null) { + $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; + } + + if ($this->ssekmsEncryptionContext !== null) { + $requestArgs['SSEKMSEncryptionContext'] = $this->ssekmsEncryptionContext; + } + + if ($this->ssekmsKeyId !== null) { + $requestArgs['SSEKMSKeyId'] = $this->ssekmsKeyId; + } + + if ($this->serverSideEncryption !== null) { + $requestArgs['ServerSideEncryption'] = $this->serverSideEncryption; + } + + if ($this->storageClass !== null) { + $requestArgs['StorageClass'] = $this->storageClass; + } + + if ($this->tagging !== null) { + $requestArgs['Tagging'] = $this->tagging; + } + + if ($this->websiteRedirectLocation !== null) { + $requestArgs['WebsiteRedirectLocation'] = $this->websiteRedirectLocation; + } + + return $requestArgs; + } + + /** + * Convert to upload part request array + * + * @return array + */ + public function toUploadPartRequest(): array + { + $requestArgs = []; + + if ($this->bucket !== null) { + $requestArgs['Bucket'] = $this->bucket; + } + + if ($this->checksumAlgorithm !== null) { + $requestArgs['ChecksumAlgorithm'] = $this->checksumAlgorithm; + } + + if ($this->expectedBucketOwner !== null) { + $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; + } + + if ($this->key !== null) { + $requestArgs['Key'] = $this->key; + } + + if ($this->requestPayer !== null) { + $requestArgs['RequestPayer'] = $this->requestPayer; + } + + if ($this->sseCustomerAlgorithm !== null) { + $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; + } + + if ($this->sseCustomerKey !== null) { + $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; + } + + if ($this->sseCustomerKeyMD5 !== null) { + $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; + } + + return $requestArgs; + } + + /** + * Convert to complete multipart upload request array + * + * @return array + */ + public function toCompleteMultipartUploadRequest(): array + { + $requestArgs = []; + + if ($this->bucket !== null) { + $requestArgs['Bucket'] = $this->bucket; + } + + if ($this->checksumCRC32 !== null) { + $requestArgs['ChecksumCRC32'] = $this->checksumCRC32; + } + + if ($this->checksumCRC32C !== null) { + $requestArgs['ChecksumCRC32C'] = $this->checksumCRC32C; + } + + if ($this->checksumCRC64NVME !== null) { + $requestArgs['ChecksumCRC64NVME'] = $this->checksumCRC64NVME; + } + + if ($this->checksumSHA1 !== null) { + $requestArgs['ChecksumSHA1'] = $this->checksumSHA1; + } + + if ($this->checksumSHA256 !== null) { + $requestArgs['ChecksumSHA256'] = $this->checksumSHA256; + } + + if ($this->expectedBucketOwner !== null) { + $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; + } + + if ($this->ifMatch !== null) { + $requestArgs['IfMatch'] = $this->ifMatch; + } + + if ($this->ifNoneMatch !== null) { + $requestArgs['IfNoneMatch'] = $this->ifNoneMatch; + } + + if ($this->key !== null) { + $requestArgs['Key'] = $this->key; + } + + if ($this->requestPayer !== null) { + $requestArgs['RequestPayer'] = $this->requestPayer; + } + + if ($this->sseCustomerAlgorithm !== null) { + $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; + } + + if ($this->sseCustomerKey !== null) { + $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; + } + + if ($this->sseCustomerKeyMD5 !== null) { + $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; + } + + return $requestArgs; + } + + /** + * Convert to abort multipart upload request array + * + * @return array + */ + public function toAbortMultipartRequest(): array + { + $requestArgs = []; + + if ($this->bucket !== null) { + $requestArgs['Bucket'] = $this->bucket; + } + + if ($this->expectedBucketOwner !== null) { + $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; + } + + if ($this->key !== null) { + $requestArgs['Key'] = $this->key; + } + + if ($this->requestPayer !== null) { + $requestArgs['RequestPayer'] = $this->requestPayer; + } + + return $requestArgs; + } + + /** + * Convert the object to an array + * + * @return array + */ + public function toArray(): array + { + return [ + 'ACL' => $this->acl, + 'Bucket' => $this->bucket, + 'BucketKeyEnabled' => $this->bucketKeyEnabled, + 'CacheControl' => $this->cacheControl, + 'ChecksumAlgorithm' => $this->checksumAlgorithm, + 'ContentDisposition' => $this->contentDisposition, + 'ContentEncoding' => $this->contentEncoding, + 'ContentLanguage' => $this->contentLanguage, + 'ContentType' => $this->contentType, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'Expires' => $this->expires, + 'GrantFullControl' => $this->grantFullControl, + 'GrantRead' => $this->grantRead, + 'GrantReadACP' => $this->grantReadACP, + 'GrantWriteACP' => $this->grantWriteACP, + 'Key' => $this->key, + 'Metadata' => $this->metadata, + 'ObjectLockLegalHoldStatus' => $this->objectLockLegalHoldStatus, + 'ObjectLockMode' => $this->objectLockMode, + 'ObjectLockRetainUntilDate' => $this->objectLockRetainUntilDate, + 'RequestPayer' => $this->requestPayer, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKey' => $this->sseCustomerKey, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + 'SSEKMSEncryptionContext' => $this->ssekmsEncryptionContext, + 'SSEKMSKeyId' => $this->ssekmsKeyId, + 'ServerSideEncryption' => $this->serverSideEncryption, + 'StorageClass' => $this->storageClass, + 'Tagging' => $this->tagging, + 'WebsiteRedirectLocation' => $this->websiteRedirectLocation, + 'ChecksumCRC32' => $this->checksumCRC32, + 'ChecksumCRC32C' => $this->checksumCRC32C, + 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, + 'ChecksumSHA1' => $this->checksumSHA1, + 'ChecksumSHA256' => $this->checksumSHA256, + 'IfMatch' => $this->ifMatch, + 'IfNoneMatch' => $this->ifNoneMatch, + ]; + } + + /** + * Create instance from array + * + * @param array $data + * @return static + */ + public static function fromArray(array $data): static + { + return new static( + $data['ACL'] ?? null, + $data['Bucket'] ?? null, + $data['BucketKeyEnabled'] ?? null, + $data['CacheControl'] ?? null, + $data['ChecksumAlgorithm'] ?? null, + $data['ContentDisposition'] ?? null, + $data['ContentEncoding'] ?? null, + $data['ContentLanguage'] ?? null, + $data['ContentType'] ?? null, + $data['ExpectedBucketOwner'] ?? null, + $data['Expires'] ?? null, + $data['GrantFullControl'] ?? null, + $data['GrantRead'] ?? null, + $data['GrantReadACP'] ?? null, + $data['GrantWriteACP'] ?? null, + $data['Key'] ?? null, + $data['Metadata'] ?? null, + $data['ObjectLockLegalHoldStatus'] ?? null, + $data['ObjectLockMode'] ?? null, + $data['ObjectLockRetainUntilDate'] ?? null, + $data['RequestPayer'] ?? null, + $data['SSECustomerAlgorithm'] ?? null, + $data['SSECustomerKey'] ?? null, + $data['SSECustomerKeyMD5'] ?? null, + $data['SSEKMSEncryptionContext'] ?? null, + $data['SSEKMSKeyId'] ?? null, + $data['ServerSideEncryption'] ?? null, + $data['StorageClass'] ?? null, + $data['Tagging'] ?? null, + $data['WebsiteRedirectLocation'] ?? null, + $data['ChecksumCRC32'] ?? null, + $data['ChecksumCRC32C'] ?? null, + $data['ChecksumCRC64NVME'] ?? null, + $data['ChecksumSHA1'] ?? null, + $data['ChecksumSHA256'] ?? null, + $data['IfMatch'] ?? null, + $data['IfNoneMatch'] ?? null + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/PutObjectResponse.php b/src/S3/S3Transfer/Models/PutObjectResponse.php new file mode 100644 index 0000000000..f920418d9d --- /dev/null +++ b/src/S3/S3Transfer/Models/PutObjectResponse.php @@ -0,0 +1,310 @@ +bucketKeyEnabled = $bucketKeyEnabled; + $this->checksumCRC32 = $checksumCRC32; + $this->checksumCRC32C = $checksumCRC32C; + $this->checksumCRC64NVME = $checksumCRC64NVME; + $this->checksumSHA1 = $checksumSHA1; + $this->checksumSHA256 = $checksumSHA256; + $this->checksumType = $checksumType; + $this->eTag = $eTag; + $this->expiration = $expiration; + $this->requestCharged = $requestCharged; + $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; + $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; + $this->ssekmsEncryptionContext = $ssekmsEncryptionContext; + $this->ssekmsKeyId = $ssekmsKeyId; + $this->serverSideEncryption = $serverSideEncryption; + $this->size = $size; + $this->versionId = $versionId; + } + + /** + * @return bool|null + */ + public function getBucketKeyEnabled(): ?bool + { + return $this->bucketKeyEnabled; + } + + /** + * @return string|null + */ + public function getChecksumCRC32(): ?string + { + return $this->checksumCRC32; + } + + /** + * @return string|null + */ + public function getChecksumCRC32C(): ?string + { + return $this->checksumCRC32C; + } + + /** + * @return string|null + */ + public function getChecksumCRC64NVME(): ?string + { + return $this->checksumCRC64NVME; + } + + /** + * @return string|null + */ + public function getChecksumSHA1(): ?string + { + return $this->checksumSHA1; + } + + /** + * @return string|null + */ + public function getChecksumSHA256(): ?string + { + return $this->checksumSHA256; + } + + /** + * @return string|null + */ + public function getChecksumType(): ?string + { + return $this->checksumType; + } + + /** + * @return string|null + */ + public function getETag(): ?string + { + return $this->eTag; + } + + /** + * @return string|null + */ + public function getExpiration(): ?string + { + return $this->expiration; + } + + /** + * @return string|null + */ + public function getRequestCharged(): ?string + { + return $this->requestCharged; + } + + /** + * @return string|null + */ + public function getSseCustomerAlgorithm(): ?string + { + return $this->sseCustomerAlgorithm; + } + + /** + * @return string|null + */ + public function getSseCustomerKeyMD5(): ?string + { + return $this->sseCustomerKeyMD5; + } + + /** + * @return string|null + */ + public function getSsekmsEncryptionContext(): ?string + { + return $this->ssekmsEncryptionContext; + } + + /** + * @return string|null + */ + public function getSsekmsKeyId(): ?string + { + return $this->ssekmsKeyId; + } + + /** + * @return string|null + */ + public function getServerSideEncryption(): ?string + { + return $this->serverSideEncryption; + } + + /** + * @return int|null + */ + public function getSize(): ?int + { + return $this->size; + } + + /** + * @return string|null + */ + public function getVersionId(): ?string + { + return $this->versionId; + } + + /** + * Convert the object to an array format suitable for multipart upload response + * + * @return array Array containing AWS S3 response fields with their corresponding values + */ + public function toMultipartUploadResponse(): array { + return [ + 'BucketKeyEnabled' => $this->bucketKeyEnabled, + 'ChecksumCRC32' => $this->checksumCRC32, + 'ChecksumCRC32C' => $this->checksumCRC32C, + 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, + 'ChecksumSHA1' => $this->checksumSHA1, + 'ChecksumSHA256' => $this->checksumSHA256, + 'ChecksumType' => $this->checksumType, + 'ETag' => $this->eTag, + 'Expiration' => $this->expiration, + 'RequestCharged' => $this->requestCharged, + 'SSEKMSKeyId' => $this->ssekmsKeyId, + 'ServerSideEncryption' => $this->serverSideEncryption, + 'VersionId' => $this->versionId + ]; + } + + /** + * Convert the object to an array format suitable for single upload response + * + * @return array Array containing AWS S3 response fields with their corresponding values + */ + public function toSingleUploadResponse(): array { + return [ + 'BucketKeyEnabled' => $this->bucketKeyEnabled, + 'ChecksumCRC32' => $this->checksumCRC32, + 'ChecksumCRC32C' => $this->checksumCRC32C, + 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, + 'ChecksumSHA1' => $this->checksumSHA1, + 'ChecksumSHA256' => $this->checksumSHA256, + 'ChecksumType' => $this->checksumType, + 'ETag' => $this->eTag, + 'Expiration' => $this->expiration, + 'RequestCharged' => $this->requestCharged, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + 'SSEKMSEncryptionContext' => $this->ssekmsEncryptionContext, + 'SSEKMSKeyId' => $this->ssekmsKeyId, + 'ServerSideEncryption' => $this->serverSideEncryption, + 'Size' => $this->size, + 'VersionId' => $this->versionId + ]; + } + + /** + * Create an instance from an array of data + * + * @param array $data + * @return self + */ + public static function fromArray(array $data): self + { + return new self( + $data['BucketKeyEnabled'] ?? null, + $data['ChecksumCRC32'] ?? null, + $data['ChecksumCRC32C'] ?? null, + $data['ChecksumCRC64NVME'] ?? null, + $data['ChecksumSHA1'] ?? null, + $data['ChecksumSHA256'] ?? null, + $data['ChecksumType'] ?? null, + $data['ETag'] ?? null, + $data['Expiration'] ?? null, + $data['RequestCharged'] ?? null, + $data['SSECustomerAlgorithm'] ?? null, + $data['SSECustomerKeyMD5'] ?? null, + $data['SSEKMSEncryptionContext'] ?? null, + $data['SSEKMSKeyId'] ?? null, + $data['ServerSideEncryption'] ?? null, + $data['Size'] ?? null, + $data['VersionId'] ?? null + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php new file mode 100644 index 0000000000..dfd5bd3e79 --- /dev/null +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -0,0 +1,187 @@ +targetPartSizeBytes = $targetPartSizeBytes; + $this->multipartUploadThresholdBytes = $multipartUploadThresholdBytes; + $this->requestChecksumCalculation = $requestChecksumCalculation; + $this->responseChecksumValidation = $responseChecksumValidation; + $this->multipartDownloadType = $multipartDownloadType; + $this->concurrency = $concurrency; + $this->trackProgress = $trackProgress; + $this->defaultRegion = $defaultRegion; + } + + /** + * @return int + */ + public function getTargetPartSizeBytes(): int + { + return $this->targetPartSizeBytes; + } + + /** + * @return int + */ + public function getMultipartUploadThresholdBytes(): int + { + return $this->multipartUploadThresholdBytes; + } + + /** + * @return string + */ + public function getRequestChecksumCalculation(): string + { + return $this->requestChecksumCalculation; + } + + /** + * @return string + */ + public function getResponseChecksumValidation(): string + { + return $this->responseChecksumValidation; + } + + /** + * @return string + */ + public function getMultipartDownloadType(): string + { + return $this->multipartDownloadType; + } + + /** + * @return int + */ + public function getConcurrency(): int + { + return $this->concurrency; + } + + /** + * @return bool + */ + public function isTrackProgress(): bool + { + return $this->trackProgress; + } + + /** + * @return string + */ + public function getDefaultRegion(): string + { + return $this->defaultRegion; + } + + /** + * @return array + */ + public function toArray(): array { + return [ + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'multipart_upload_threshold_bytes' => $this->multipartUploadThresholdBytes, + 'request_checksum_calculation' => $this->requestChecksumCalculation, + 'response_checksum_validation' => $this->responseChecksumValidation, + 'multipart_download_type' => $this->multipartDownloadType, + 'concurrency' => $this->concurrency, + 'track_progress' => $this->trackProgress, + 'default_region' => $this->defaultRegion, + ]; + } + + + /** $config: + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) + * The minimum part size to be used in a multipart upload/download. + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) + * The threshold to decided whether a multipart upload is needed. + * - request_checksum_calculation: (string, default=`when_supported`) + * To decide whether a checksum validation will be applied to the response. + * - request_checksum_validation: (string, default=`when_supported`) + * - multipart_download_type: (string, default='part') + * The download type to be used in a multipart download. + * - concurrency: (int, default=5) + * Maximum number of concurrent operations allowed during a multipart + * upload/download. + * - track_progress: (bool, default=false) + * To enable progress tracker in a multipart upload/download, and or + * a directory upload/download operation. + * - default_region: (string, default="us-east-2") + */ + public static function fromArray(array $config): self { + return new self( + $config['target_part_size_bytes'] + ?? self::DEFAULT_TARGET_PART_SIZE_BYTES, + $config['multipart_upload_threshold_bytes'] + ?? self::DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES, + 'request_checksum_calculation' + ?? self::DEFAULT_REQUEST_CHECKSUM_CALCULATION, + $config['response_checksum_validation'] + ?? self::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, + $config['multipart_download_type'] + ?? self::DEFAULT_MULTIPART_DOWNLOAD_TYPE, + $config['concurrency'] + ?? self::DEFAULT_CONCURRENCY, + $config['track_progress'] ?? self::DEFAULT_TRACK_PROGRESS, + $config['default_region'] ?? self::DEFAULT_REGION + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/TransferRequest.php b/src/S3/S3Transfer/Models/TransferRequest.php new file mode 100644 index 0000000000..3a470c4d74 --- /dev/null +++ b/src/S3/S3Transfer/Models/TransferRequest.php @@ -0,0 +1,47 @@ +listeners = $listeners; + $this->progressTracker = $progressTracker; + } + + /** + * Get current listeners. + * + * @return array + */ + public function getListeners(): array + { + return $this->listeners; + } + + /** + * Get the progress tracker. + * + * @return TransferListener|null + */ + public function getProgressTracker(): ?TransferListener + { + return $this->progressTracker; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/TransferRequestConfig.php b/src/S3/S3Transfer/Models/TransferRequestConfig.php new file mode 100644 index 0000000000..40e950a1f2 --- /dev/null +++ b/src/S3/S3Transfer/Models/TransferRequestConfig.php @@ -0,0 +1,31 @@ +trackProgress = $trackProgress; + } + + /** + * @return bool|null + */ + public function getTrackProgress(): ?bool + { + return $this->trackProgress; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php new file mode 100644 index 0000000000..6749728b6b --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -0,0 +1,140 @@ +source = $source; + $this->putObjectRequest = $putObjectRequest; + $this->config = $config; + } + + /** + * Get the source. + * + * @return StreamInterface|string + */ + public function getSource(): StreamInterface|string + { + return $this->source; + } + + /** + * Get the put object request. + * + * @return PutObjectRequest + */ + public function getPutObjectRequest(): PutObjectRequest + { + return $this->putObjectRequest; + } + + public function getConfig(): UploadRequestConfig { + return $this->config; + } + + /** + * Helper method for validating the given source. + * + * @return void + */ + public function validateSource(): void + { + if (is_string($this->getSource()) && !is_readable($this->getSource())) { + throw new InvalidArgumentException( + "Please provide a valid readable file path or a valid stream as source." + ); + } + } + + /** + * Helper method for validating required parameters. + * + * @param string|null $customMessage + * @return void + */ + public function validateRequiredParameters( + ?string $customMessage = null + ): void + { + $requiredParametersWithArgs = [ + 'Bucket' => $this->getPutObjectRequest()->getBucket(), + 'Key' => $this->getPutObjectRequest()->getKey() + ]; + foreach ($requiredParametersWithArgs as $key => $value) { + if (empty($value)) { + if ($customMessage !== null) { + throw new InvalidArgumentException($customMessage); + } + + // Fallback to default error message + throw new InvalidArgumentException( + "The `$key` parameter must be provided as part of the request arguments." + ); + } + } + } + + /** + * @param string|StreamInterface $source + * @param array $requestArgs The putObject request arguments. + * Required parameters would be: + * - Bucket: (string, required) + * - Key: (string, required) + * @param array $config The config options for this upload operation. + * - multipart_upload_threshold_bytes: (int, optional) + * To override the default threshold for when to use multipart upload. + * - target_part_size_bytes: (int, optional) To override the default + * target part size in bytes. + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. This option is intended to make the operation to use + * a default progress tracker implementation when $progressTracker is null. + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker + * + * @return UploadRequest + */ + public static function fromLegacyArgs(string | StreamInterface $source, + array $requestArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null): UploadRequest { + return new UploadRequest( + $source, + PutObjectRequest::fromArray($requestArgs), + UploadRequestConfig::fromArray($config), + $listeners, + $progressTracker + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadRequestConfig.php b/src/S3/S3Transfer/Models/UploadRequestConfig.php new file mode 100644 index 0000000000..6a57bf97c2 --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadRequestConfig.php @@ -0,0 +1,120 @@ +multipartUploadThresholdBytes = $multipartUploadThresholdBytes; + $this->targetPartSizeBytes = $targetPartSizeBytes; + $this->concurrency = $concurrency; + $this->requestChecksumCalculation = $requestChecksumCalculation; + } + + /** + * Get the multipart upload threshold in bytes + * + * @return int|null + */ + public function getMultipartUploadThresholdBytes(): ?int + { + return $this->multipartUploadThresholdBytes; + } + + /** + * Get the target part size in bytes + * + * @return int|null + */ + public function getTargetPartSizeBytes(): ?int + { + return $this->targetPartSizeBytes; + } + + /** + * @return int|null + */ + public function getConcurrency(): ?int + { + return $this->concurrency; + } + + /** + * @return string|null + */ + public function getRequestChecksumCalculation(): ?string + { + return $this->requestChecksumCalculation; + } + + /** + * Convert the configuration to an array + * + * @return array + */ + public function toArray(): array + { + return [ + 'multipart_upload_threshold_bytes' => $this->multipartUploadThresholdBytes, + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'track_progress' => $this->trackProgress, + 'concurrency' => $this->concurrency, + 'request_checksum_calculation' => $this->requestChecksumCalculation, + ]; + } + + /** + * Create an UploadConfig instance from an array + * + * @param array $data Array containing configuration data + * + * @return static + */ + public static function fromArray(array $data): self + { + return new self( + $data['multipart_upload_threshold_bytes'] ?? null, + $data['target_part_size_bytes'] ?? null, + $data['track_progress'] ?? null, + $data['concurrency'] ?? null, + $data['request_checksum_calculation'] ?? null, + ); + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index adf02c4196..30912ef368 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -1,228 +1,115 @@ s3Client = $s3Client; - $this->createMultipartArgs = $createMultipartArgs; - $this->validateConfig($config); - $this->config = $config; + ) + { + parent::__construct( + $s3Client, + $putObjectRequest, + $config, + $uploadId, + $parts, + $currentSnapshot, + $listenerNotifier + ); $this->body = $this->parseBody($source); - $this->uploadId = $uploadId; - $this->parts = $parts; - $this->currentSnapshot = $currentSnapshot; - $this->listenerNotifier = $listenerNotifier; + $this->calculatedObjectSize = 0; } /** - * @param array $config + * @param string|StreamInterface $source * - * @return void + * @return StreamInterface */ - private function validateConfig(array &$config): void + private function parseBody( + string | StreamInterface $source + ): StreamInterface { - if (isset($config['part_size'])) { - if ($config['part_size'] < self::PART_MIN_SIZE - || $config['part_size'] > self::PART_MAX_SIZE) { + if (is_string($source)) { + // Make sure the files exists + if (!is_readable($source)) { throw new \InvalidArgumentException( - "The config `part_size` value must be between " - . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE - . " but ${config['part_size']} given." + "The source for this upload must be either a readable file path or a valid stream." ); } + $body = new LazyOpenStream($source, 'r'); + // To make sure the resource is closed. + $this->deferFns[] = function () use ($body) { + $body->close(); + }; + } elseif ($source instanceof StreamInterface) { + $body = $source; } else { - $config['part_size'] = self::PART_MIN_SIZE; - } - } - - /** - * @return string|null - */ - public function getUploadId(): ?string - { - return $this->uploadId; - } - - /** - * @return array - */ - public function getParts(): array - { - return $this->parts; - } - - /** - * @return int - */ - public function getCalculatedObjectSize(): int - { - return $this->calculatedObjectSize; - } - - /** - * @return TransferProgressSnapshot|null - */ - public function getCurrentSnapshot(): ?TransferProgressSnapshot - { - return $this->currentSnapshot; - } - - /** - * @return UploadResponse - */ - public function upload(): UploadResponse { - return $this->promise()->wait(); - } - - /** - * @return PromiseInterface - */ - public function promise(): PromiseInterface - { - return Coroutine::of(function () { - try { - yield $this->createMultipartUpload(); - yield $this->uploadParts(); - $result = yield $this->completeMultipartUpload(); - yield Create::promiseFor( - new UploadResponse($result->toArray()) - ); - } catch (Throwable $e) { - $this->uploadFailed($e); - yield Create::rejectionFor($e); - } finally { - $this->callDeferredFns(); - } - }); - } - - /** - * @return PromiseInterface - */ - private function createMultipartUpload(): PromiseInterface - { - $requestArgs = [...$this->createMultipartArgs]; - $checksum = $this->filterChecksum($requestArgs); - // Customer provided checksum - if ($checksum !== null) { - $requestArgs['ChecksumType'] = 'FULL_OBJECT'; - $requestArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); - $requestArgs['@context']['request_checksum_calculation'] = 'when_required'; - unset($requestArgs[$checksum]); + throw new \InvalidArgumentException( + "The source must be a valid string file path or a StreamInterface." + ); } - $this->uploadInitiated($requestArgs); - $command = $this->s3Client->getCommand( - 'CreateMultipartUpload', - $requestArgs - ); - - return $this->s3Client->executeAsync($command) - ->then(function (ResultInterface $result) { - $this->uploadId = $result['UploadId']; - - return $result; - }); + return $body; } - /** - * @return PromiseInterface - */ - private function uploadParts(): PromiseInterface + protected function processMultipartOperation(): PromiseInterface { $this->calculatedObjectSize = 0; - $partSize = $this->config['part_size']; + $partSize = $this->calculatePartSize(); + $partsCount = ceil($this->getTotalSize() / $partSize); $commands = []; $partNo = count($this->parts); - $baseUploadPartCommandArgs = [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, - ]; + $uploadPartCommandArgs = $this->putObjectRequest->toUploadPartRequest(); + $uploadPartCommandArgs['UploadId'] = $this->uploadId; // Customer provided checksum - $checksum = $this->filterChecksum($this->createMultipartArgs); - if ($checksum !== null) { - unset($baseUploadPartCommandArgs['ChecksumAlgorithm']); - unset($baseUploadPartCommandArgs[$checksum]); - $baseUploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + $hashBody = false; + if ($this->requestChecksum !== null) { + // To avoid default calculation + $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + } elseif ($this->requestChecksumAlgorithm === self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM) { + $hashBody = true; + $this->hashContext = hash_init('crc32b'); + // To avoid default calculation + $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; } while (!$this->body->eof()) { @@ -234,23 +121,24 @@ private function uploadParts(): PromiseInterface break; } + if ($hashBody) { + hash_update($this->hashContext, $read); + } + $partBody = Utils::streamFor( $read ); - $uploadPartCommandArgs = [ - ...$baseUploadPartCommandArgs, - 'PartNumber' => $partNo, - 'ContentLength' => $partBody->getSize(), - ]; - - // To get `requestArgs` when notifying the bytesTransfer listeners. - $uploadPartCommandArgs['requestArgs'] = [...$uploadPartCommandArgs]; + $uploadPartCommandArgs['PartNumber'] = $partNo; + $uploadPartCommandArgs['ContentLength'] = $partBody->getSize(); // Attach body $uploadPartCommandArgs['Body'] = $this->decorateWithHashes( $partBody, $uploadPartCommandArgs ); - $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); + $command = $this->s3Client->getCommand( + 'UploadPart', + $uploadPartCommandArgs + ); $commands[] = $command; $this->calculatedObjectSize += $partBody->getSize(); if ($partNo > self::PART_MAX_NUM) { @@ -259,28 +147,38 @@ private function uploadParts(): PromiseInterface "Max = " . self::PART_MAX_NUM ); } + + if ($partNo > $partsCount) { + return Create::rejectionFor( + "The current part `$partNo` is over the expected number of parts `$partsCount`" + ); + } + } + + if ($hashBody) { + $this->requestChecksum = hash_final($this->hashContext); } return (new CommandPool( $this->s3Client, $commands, [ - 'concurrency' => $this->config['concurrency'], + 'concurrency' => $this->config->getConcurrency(), 'fulfilled' => function (ResultInterface $result, $index) - use ($commands) { - $command = $commands[$index]; - $this->collectPart( - $result, - $command - ); - // Part Upload Completed Event - $this->partUploadCompleted( - $command['ContentLength'], - $command['requestArgs'] - ); + use ($commands) { + $command = $commands[$index]; + $this->collectPart( + $result, + $command + ); + // Part Upload Completed Event + $this->partCompleted( + $command['ContentLength'], + $command->toArray() + ); }, 'rejected' => function (Throwable $e) { - $this->partUploadFailed($e); + $this->partFailed($e); throw $e; } @@ -289,268 +187,29 @@ private function uploadParts(): PromiseInterface } /** - * @return PromiseInterface - */ - private function completeMultipartUpload(): PromiseInterface - { - $this->sortParts(); - $completeMultipartUploadArgs = [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, - 'MpuObjectSize' => $this->calculatedObjectSize, - 'MultipartUpload' => [ - 'Parts' => $this->parts, - ] - ]; - $checksum = $this->filterChecksum($completeMultipartUploadArgs); - // Customer provided checksum - if ($checksum !== null) { - $completeMultipartUploadArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); - $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; - $completeMultipartUploadArgs['@context']['request_checksum_calculation'] = 'when_required'; - } - - $command = $this->s3Client->getCommand( - 'CompleteMultipartUpload', - $completeMultipartUploadArgs - ); - - return $this->s3Client->executeAsync($command) - ->then(function (ResultInterface $result) { - $this->uploadCompleted($result); - - return $result; - }); - } - - /** - * @return PromiseInterface - */ - private function abortMultipartUpload(): PromiseInterface - { - $command = $this->s3Client->getCommand('AbortMultipartUpload', [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, - ]); - - return $this->s3Client->executeAsync($command); - } - - /** - * @param ResultInterface $result - * @param CommandInterface $command - * - * @return void - */ - private function collectPart( - ResultInterface $result, - CommandInterface $command, - ): void - { - $checksumResult = $command->getName() === 'UploadPart' - ? $result - : $result[$command->getName() . 'Result']; - $partData = [ - 'PartNumber' => $command['PartNumber'], - 'ETag' => $result['ETag'], - ]; - if (isset($command['ChecksumAlgorithm'])) { - $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); - $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; - } - - $this->parts[] = $partData; - } - - /** - * @return void - */ - private function sortParts(): void - { - usort($this->parts, function($partOne, $partTwo) { - return $partOne['PartNumber'] <=> $partTwo['PartNumber']; - }); - } - - /** - * @param string|StreamInterface $source - * - * @return StreamInterface - */ - private function parseBody(string | StreamInterface $source): StreamInterface - { - if (is_string($source)) { - // Make sure the files exists - if (!is_readable($source)) { - throw new \InvalidArgumentException( - "The source for this upload must be either a readable file path or a valid stream." - ); - } - $body = new LazyOpenStream($source, 'r'); - // To make sure the resource is closed. - $this->deferFns[] = function () use ($body) { - $body->close(); - }; - } elseif ($source instanceof StreamInterface) { - $body = $source; - } else { - throw new \InvalidArgumentException( - "The source must be a valid string file path or a StreamInterface." - ); - } - - return $body; - } - - /** - * @param array $requestArgs - * - * @return void + * @return int */ - private function uploadInitiated(array $requestArgs): void + protected function getTotalSize(): int { - if ($this->currentSnapshot === null) { - $this->currentSnapshot = new TransferProgressSnapshot( - $requestArgs['Key'], - 0, - $this->body->getSize(), - ); - } else { - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $this->currentSnapshot->getReason(), - ); - } - - $this->listenerNotifier?->transferInitiated([ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot - ]); - } - - /** - * @param Throwable $reason - * - * @return void - */ - private function uploadFailed(Throwable $reason): void { - // Event has been already propagated - if ($this->currentSnapshot->getReason() !== null) { - return; + if ($this->calculatedObjectSize > 0) { + return $this->calculatedObjectSize; } - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $reason - ); - - if (!empty($this->uploadId)) { - $this->abortMultipartUpload()->wait(); - } - $this->listenerNotifier?->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - 'reason' => $reason, - ]); + return $this->body->getSize(); } /** * @param ResultInterface $result * - * @return void - */ - private function uploadCompleted(ResultInterface $result): void { - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $result->toArray(), - $this->currentSnapshot->getReason(), - ); - $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->transferComplete([ - TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); - } - - /** - * @param int $partCompletedBytes - * @param array $requestArgs - * - * @return void + * @return UploadResponse */ - private function partUploadCompleted( - int $partCompletedBytes, - array $requestArgs - ): void + protected function createResponse(ResultInterface $result): UploadResponse { - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partCompletedBytes, - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $this->currentSnapshot->getReason(), + return new UploadResponse( + PutObjectResponse::fromArray( + $result->toArray() + )->toMultipartUploadResponse() ); - $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->bytesTransferred([ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - $this->currentSnapshot - ]); - } - - /** - * @param Throwable $reason - * - * @return void - */ - private function partUploadFailed(Throwable $reason): void - { - $this->uploadFailed($reason); - } - - /** - * @return void - */ - private function callDeferredFns(): void - { - foreach ($this->deferFns as $fn) { - $fn(); - } - - $this->deferFns = []; - } - - /** - * Filters a provided checksum if one was provided. - * - * @param array $requestArgs - * - * @return string | null - */ - private function filterChecksum(array $requestArgs):? string - { - static $algorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - foreach ($algorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return $algorithm; - } - } - - return null; } /** diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index dce2c9fab8..bd533b4c90 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -8,7 +8,11 @@ use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; +use Aws\S3\S3Transfer\Models\PutObjectResponse; +use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\Progress\MultiProgressTracker; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; @@ -20,59 +24,30 @@ use GuzzleHttp\Promise\PromiseInterface; use InvalidArgumentException; use Psr\Http\Message\StreamInterface; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; use function Aws\filter; use function Aws\map; class S3TransferManager { - private static array $defaultConfig = [ - 'target_part_size_bytes' => 8 * 1024 * 1024, - 'multipart_upload_threshold_bytes' => 16 * 1024 * 1024, - 'checksum_validation_enabled' => true, - 'checksum_algorithm' => 'crc32', - 'multipart_download_type' => 'partGet', - 'concurrency' => 5, - 'track_progress' => false, - 'region' => 'us-east-1', - ]; - /** @var S3Client */ private S3ClientInterface $s3Client; - /** @var array */ - private array $config; + /** @var S3TransferManagerConfig */ + private S3TransferManagerConfig $config; /** * @param S3ClientInterface | null $s3Client If provided as null then, * a default client will be created where its region will be the one * resolved from either the default from the config or the provided. - * @param array $config - * - target_part_size_bytes: (int, default=(8388608 `8MB`)) - * The minimum part size to be used in a multipart upload/download. - * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) - * The threshold to decided whether a multipart upload is needed. - * - checksum_validation_enabled: (bool, default=true) - * To decide whether a checksum validation will be applied to the response. - * - checksum_algorithm: (string, default='crc32') - * The checksum algorithm to be used in an upload request. - * - multipart_download_type: (string, default='partGet') - * The download type to be used in a multipart download. - * - concurrency: (int, default=5) - * Maximum number of concurrent operations allowed during a multipart - * upload/download. - * - track_progress: (bool, default=false) - * To enable progress tracker in a multipart upload/download, and or - * a directory upload/download operation. - * - region: (string, default="us-east-2") + * @param S3TransferManagerConfig|null $config */ public function __construct( ?S3ClientInterface $s3Client = null, - array $config = [] + ?S3TransferManagerConfig $config = null ) { - $this->config = [ - ...self::$defaultConfig, - ...$config, - ]; + $this->config = $config ?? S3TransferManagerConfig::fromArray([]); if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -89,99 +64,64 @@ public function getS3Client(): S3ClientInterface } /** - * @return array + * @return S3TransferManagerConfig */ - public function getConfig(): array + public function getConfig(): S3TransferManagerConfig { return $this->config; } /** - * @param string|StreamInterface $source - * @param array $requestArgs The putObject request arguments. - * Required parameters would be: - * - Bucket: (string, required) - * - Key: (string, required) - * @param array $config The config options for this upload operation. - * - multipart_upload_threshold_bytes: (int, optional) - * To override the default threshold for when to use multipart upload. - * - part_size: (int, optional) To override the default - * target part size in bytes. - * - track_progress: (bool, optional) To override the default option for - * enabling progress tracking. If this option is resolved as true and - * a progressTracker parameter is not provided then, a default implementation - * will be resolved. This option is intended to make the operation to use - * a default progress tracker implementation when $progressTracker is null. - * - checksum_algorithm: (bool, optional) To override the default - * checksum algorithm. - * @param TransferListener[]|null $listeners - * @param TransferListener|null $progressTracker + * @param UploadRequest $uploadRequest * * @return PromiseInterface */ - public function upload( - string | StreamInterface $source, - array $requestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null, - ): PromiseInterface + public function upload(UploadRequest $uploadRequest): PromiseInterface { // Make sure it is a valid in path in case of a string - if (is_string($source) && !is_readable($source)) { - throw new InvalidArgumentException( - "Please provide a valid readable file path or a valid stream as source." - ); - } - // Valid required parameters - foreach (['Bucket', 'Key'] as $reqParam) { - $this->requireNonEmpty( - $requestArgs, - $reqParam, - "The `$reqParam` parameter must be provided as part of the request arguments." - ); - } + $uploadRequest->validateSource(); - $mupThreshold = $config['multipart_upload_threshold_bytes'] - ?? $this->config['multipart_upload_threshold_bytes']; - if ($mupThreshold < MultipartUploader::PART_MIN_SIZE) { - throw new InvalidArgumentException( - "The provided config `multipart_upload_threshold_bytes`" - ."must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE - ); - } + // Valid required parameters + $uploadRequest->validateRequiredParameters(); - if (!isset($requestArgs['ChecksumAlgorithm'])) { - $algorithm = $config['checksum_algorithm'] - ?? $this->config['checksum_algorithm']; - $requestArgs['ChecksumAlgorithm'] = strtoupper($algorithm); - } + $config = $uploadRequest->getConfig(); + // Validate progress tracker + $progressTracker = $uploadRequest->getProgressTracker(); if ($progressTracker === null - && ($config['track_progress'] ?? $this->config['track_progress'])) { + && ($config->getTrackProgress() + ?? $this->config->isTrackProgress())) { $progressTracker = new SingleProgressTracker(); } + // Append progress tracker to listeners if not null + $listeners = $uploadRequest->getListeners(); if ($progressTracker !== null) { $listeners[] = $progressTracker; } $listenerNotifier = new TransferListenerNotifier($listeners); - if ($this->requiresMultipartUpload($source, $mupThreshold)) { + + // Validate multipart upload threshold + $mupThreshold = $config->getMultipartUploadThresholdBytes() + ?? $this->config->getMultipartUploadThresholdBytes(); + if ($mupThreshold < AbstractMultipartUploader::PART_MIN_SIZE) { + throw new InvalidArgumentException( + "The provided config `multipart_upload_threshold_bytes`" + ."must be greater than or equal to " . AbstractMultipartUploader::PART_MIN_SIZE + ); + } + + if ($this->requiresMultipartUpload($uploadRequest->getSource(), $mupThreshold)) { return $this->tryMultipartUpload( - $source, - $requestArgs, - [ - 'part_size' => $config['part_size'] ?? $this->config['target_part_size_bytes'], - 'concurrency' => $this->config['concurrency'], - ], + $uploadRequest, $listenerNotifier ); } return $this->trySingleUpload( - $source, - $requestArgs, + $uploadRequest->getSource(), + $uploadRequest->getPutObjectRequest()->toSingleObjectRequest(), $listenerNotifier ); } @@ -275,14 +215,14 @@ public function uploadDirectory( $failurePolicyCallback = $config['failure_policy']; } - $dirIterator = new \RecursiveDirectoryIterator($sourceDirectory); + $dirIterator = new RecursiveDirectoryIterator($sourceDirectory); $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); if (($config['follow_symbolic_links'] ?? false) === true) { $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); } if (($config['recursive'] ?? false) === true) { - $dirIterator = new \RecursiveIteratorIterator($dirIterator); + $dirIterator = new RecursiveIteratorIterator($dirIterator); } $files = filter( @@ -839,7 +779,11 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { ] ); - return new UploadResponse($result->toArray()); + return new UploadResponse( + PutObjectResponse::fromArray( + $result->toArray() + )->toSingleUploadResponse() + ); } )->otherwise(function ($reason) use ($objectSize, $requestArgs, $listenerNotifier) { $listenerNotifier->transferFail( @@ -867,25 +811,22 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { } /** - * @param string|StreamInterface $source - * @param array $requestArgs - * @param array $config + * @param UploadRequest $uploadRequest * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartUpload( - string | StreamInterface $source, - array $requestArgs, - array $config = [], + UploadRequest $uploadRequest, ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { - $createMultipartArgs = [...$requestArgs]; return (new MultipartUploader( $this->s3Client, - $createMultipartArgs, - $config, - $source, + $uploadRequest->getPutObjectRequest(), + MultipartUploaderConfig::fromArray( + $uploadRequest->getConfig()->toArray() + ), + $uploadRequest->getSource(), listenerNotifier: $listenerNotifier, ))->promise(); } diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index dad95f9f65..0efb9b747c 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -4,6 +4,8 @@ use Aws\S3\ApplyChecksumMiddleware; use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; +use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\S3TransferManager; @@ -93,11 +95,13 @@ public function iUploadTheFileToATestBucketUsingTheS3TransferManager($filename): self::getSdk()->createS3() ); $s3TransferManager->upload( - $fullFilePath, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $filename, - ] + UploadRequest::fromLegacyArgs( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ] + ) )->wait(); } @@ -133,11 +137,13 @@ public function iDoTheUploadToATestBucketWithKey($key): void self::getSdk()->createS3() ); $s3TransferManager->upload( - $this->stream, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $key, - ] + UploadRequest::fromLegacyArgs( + $this->stream, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ] + ) )->wait(); } @@ -173,19 +179,21 @@ public function iDoUploadThisFileWithNameWithTheSpecifiedPartSizeOf($filename, $ $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), - [ + S3TransferManagerConfig::fromArray([ 'multipart_upload_threshold_bytes' => $partsize, - ] + ]) ); $s3TransferManager->upload( - $fullFilePath, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $filename, - ], - [ - 'part_size' => intval($partsize), - ] + UploadRequest::fromLegacyArgs( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ], + [ + 'part_size' => intval($partsize), + ] + ) )->wait(); } @@ -204,19 +212,21 @@ public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf($filename, { $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), - [ + S3TransferManagerConfig::fromArray([ 'multipart_upload_threshold_bytes' => $partsize, - ] + ]) ); $s3TransferManager->upload( - $this->stream, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $filename, - ], - [ - 'part_size' => intval($partsize), - ] + UploadRequest::fromLegacyArgs( + $this->stream, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ], + [ + 'part_size' => intval($partsize), + ] + ) )->wait(); } @@ -268,14 +278,16 @@ public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm($file self::getSdk()->createS3(), ); $s3TransferManager->upload( - $fullFilePath, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $filename, - ], - [ - 'checksum_algorithm' => $checksum_algorithm, - ] + UploadRequest::fromLegacyArgs( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ], + [ + 'checksum_algorithm' => $checksum_algorithm, + ] + ) )->wait(); } @@ -450,11 +462,13 @@ public function iHaveATotalOfObjectsInABucketPrefixedWith($numfile, $directory): ); for ($i = 0; $i < $numfile - 1; $i++) { $s3TransferManager->upload( - Utils::streamFor("This is a test file content #" . ($i + 1)), - [ - 'Bucket' => self::getResourceName(), - 'Key' => $directory . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt", - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor("This is a test file content #" . ($i + 1)), + [ + 'Bucket' => self::getResourceName(), + 'Key' => $directory . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt", + ] + ) )->wait(); } } @@ -550,16 +564,18 @@ public function bytesTransferred(array $context): void $transferListener2->expects($testCase->once())->method('transferFail'); $s3TransferManager->upload( - $fullFilePath, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $file, - ], - [], - [ - $transferListener, - $transferListener2 - ] + UploadRequest::fromLegacyArgs( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + [], + [ + $transferListener, + $transferListener2 + ] + ) )->wait(); } diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 364bbaceba..e77ee6a230 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -7,7 +7,11 @@ use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\AbstractMultipartUploader; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; +use Aws\S3\S3Transfer\Models\PutObjectRequest; +use Aws\S3\S3Transfer\Models\UploadRequestConfig; use Aws\S3\S3Transfer\Models\UploadResponse; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; @@ -25,12 +29,12 @@ class MultipartUploaderTest extends TestCase { /** * @param array $sourceConfig + * @param array $commandArgs * @param array $config * @param array $expected * @return void * * @dataProvider multipartUploadProvider - * */ public function testMultipartUpload( array $sourceConfig, @@ -106,11 +110,13 @@ public function testMultipartUpload( try { $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - $config + [ - 'concurrency' => 3, - ], - $source, + PutObjectRequest::fromArray( + $requestArgs, + ), + MultipartUploaderConfig::fromArray( + $config + ), + $source ); /** @var UploadResponse $response */ $response = $multipartUploader->promise()->wait(); @@ -139,7 +145,7 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'part_size' => 10240000 + 'target_part_size_bytes' => 10240000 ], 'expected' => [ 'succeed' => true, @@ -154,7 +160,7 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'part_size' => 10240000 + 'target_part_size_bytes' => 10240000 ], 'expected' => [ 'succeed' => true, @@ -169,7 +175,7 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'part_size' => 10240000 + 'target_part_size_bytes' => 10240000 ], 'expected' => [ 'succeed' => true, @@ -184,7 +190,7 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'part_size' => 10240000 + 'target_part_size_bytes' => 10240000 ], 'expected' => [ 'succeed' => true, @@ -201,7 +207,7 @@ public function multipartUploadProvider(): array { 'ChecksumCRC32' => 'FooChecksum', ], 'config' => [ - 'part_size' => 10240000 + 'target_part_size_bytes' => 10240000 ], 'expected' => [ 'succeed' => true, @@ -270,9 +276,9 @@ public function testValidatePartSize( if ($expectError) { $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessage( - "The config `part_size` value must be between " - . MultipartUploader::PART_MIN_SIZE . " and " . MultipartUploader::PART_MAX_SIZE - . " but ${partSize} given." + "Part size config must be between " . AbstractMultipartUploader::PART_MIN_SIZE + ." and " . AbstractMultipartUploader::PART_MAX_SIZE . " bytes " + ."but it is configured to $partSize" ); } else { $this->assertTrue(true); @@ -280,10 +286,12 @@ public function testValidatePartSize( new MultipartUploader( $this->getMultipartUploadS3Client(), - ['Bucket' => 'test-bucket', 'Key' => 'test-key'], - [ - 'part_size' => $partSize, - ], + PutObjectRequest::fromArray( + ['Bucket' => 'test-bucket', 'Key' => 'test-key'] + ), + MultipartUploaderConfig::fromArray([ + 'target_part_size_bytes' => $partSize + ]), Utils::streamFor('') ); } @@ -294,19 +302,19 @@ public function testValidatePartSize( public function validatePartSizeProvider(): array { return [ 'part_size_over_max' => [ - 'part_size' => MultipartUploader::PART_MAX_SIZE + 1, + 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE + 1, 'expectError' => true, ], 'part_size_under_min' => [ - 'part_size' => MultipartUploader::PART_MIN_SIZE - 1, + 'part_size' => AbstractMultipartUploader::PART_MIN_SIZE - 1, 'expectError' => true, ], 'part_size_between_valid_range_1' => [ - 'part_size' => MultipartUploader::PART_MAX_SIZE - 1, + 'part_size' => AbstractMultipartUploader::PART_MAX_SIZE - 1, 'expectError' => false, ], 'part_size_between_valid_range_2' => [ - 'part_size' => MultipartUploader::PART_MIN_SIZE + 1, + 'part_size' => AbstractMultipartUploader::PART_MIN_SIZE + 1, 'expectError' => false, ] ]; @@ -348,8 +356,11 @@ public function testInvalidSourceStringThrowsException( try { new MultipartUploader( $this->getMultipartUploadS3Client(), - ['Bucket' => 'test-bucket', 'Key' => 'test-key'], - [], + PutObjectRequest::fromArray([ + 'Bucket' => 'test-bucket', + 'Key' => 'test-key' + ]), + MultipartUploaderConfig::fromArray([]), $source ); } finally { @@ -435,11 +446,11 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - [ - 'part_size' => 5242880, // 5MB + PutObjectRequest::fromArray($requestArgs), + MultipartUploaderConfig::fromArray([ + 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ], + ]), $stream, null, [], @@ -493,10 +504,10 @@ public function testMultipartOperationsAreCalled(): void { $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - [ + PutObjectRequest::fromArray($requestArgs), + MultipartUploaderConfig::fromArray([ 'concurrency' => 1, - ], + ]), $stream ); @@ -587,10 +598,10 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) try { $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - [ + PutObjectRequest::fromArray($requestArgs), + MultipartUploaderConfig::fromArray([ 'concurrency' => 3, - ], + ]), $source, ); /** @var UploadResponse $response */ @@ -717,13 +728,13 @@ public function testMultipartUploadAbort() { try { $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - [ + PutObjectRequest::fromArray($requestArgs), + MultipartUploaderConfig::fromArray([ 'concurrency' => 3, - ], + ]), $source, ); - $multipartUploader->upload(); + $multipartUploader->promise()->wait(); } finally { $this->assertTrue($abortMultipartCalled); $this->assertEquals(1, $abortMultipartCalledTimes); @@ -777,11 +788,11 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - [ - 'part_size' => 5242880, // 5MB + PutObjectRequest::fromArray($requestArgs), + MultipartUploaderConfig::fromArray([ + 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ], + ]), $stream, null, [], @@ -828,11 +839,11 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $multipartUploader = new MultipartUploader( $s3Client, - $requestArgs, - [ - 'part_size' => 5242880, + PUtObjectRequest::fromArray($requestArgs), + MultipartUploaderConfig::fromArray([ + 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ], + ]), $stream, null, [], From 48a2822234315f173bf2c1fcbc154d1c934a9c7d Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 15 Jul 2025 07:01:52 -0700 Subject: [PATCH 34/62] chore: multipart download updates Work In Progress... --- .../Exceptions/FileDownloadException.php | 8 + .../Models/DownloadDirectoryRequest.php | 163 +++++ .../Models/DownloadDirectoryRequestConfig.php | 184 +++++ .../S3Transfer/Models/DownloadFileRequest.php | 63 ++ src/S3/S3Transfer/Models/DownloadHandler.php | 18 + src/S3/S3Transfer/Models/DownloadRequest.php | 183 +++++ .../Models/DownloadRequestConfig.php | 92 +++ src/S3/S3Transfer/Models/DownloadResponse.php | 20 +- src/S3/S3Transfer/Models/GetObjectRequest.php | 345 ++++++++++ .../S3Transfer/Models/GetObjectResponse.php | 629 ++++++++++++++++++ .../Models/MultipartDownloaderConfig.php | 81 +++ .../Models/MultipartUploaderConfig.php | 40 +- src/S3/S3Transfer/Models/PutObjectRequest.php | 463 ++++--------- .../S3Transfer/Models/PutObjectResponse.php | 70 +- .../Models/S3TransferManagerConfig.php | 77 ++- .../Models/UploadDirectoryRequest.php | 123 ++++ .../Models/UploadDirectoryRequestConfig.php | 167 +++++ src/S3/S3Transfer/Models/UploadRequest.php | 70 +- .../S3Transfer/Models/UploadRequestConfig.php | 38 +- src/S3/S3Transfer/MultipartDownloader.php | 191 +++--- .../S3Transfer/MultipartDownloaderInitial.php | 388 +++++++++++ .../S3Transfer/MultipartUploaderInitial.php | 570 ++++++++++++++++ .../S3Transfer/PartGetMultipartDownloader.php | 14 +- .../Progress/TransferListenerNotifier.php | 17 +- .../RangeGetMultipartDownloader.php | 85 +-- src/S3/S3Transfer/S3TransferManager.php | 572 +++++----------- .../S3Transfer/Utils/FileDownloadHandler.php | 166 +++++ .../Utils/StreamDownloadHandler.php | 84 +++ 28 files changed, 3877 insertions(+), 1044 deletions(-) create mode 100644 src/S3/S3Transfer/Exceptions/FileDownloadException.php create mode 100644 src/S3/S3Transfer/Models/DownloadDirectoryRequest.php create mode 100644 src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php create mode 100644 src/S3/S3Transfer/Models/DownloadFileRequest.php create mode 100644 src/S3/S3Transfer/Models/DownloadHandler.php create mode 100644 src/S3/S3Transfer/Models/DownloadRequest.php create mode 100644 src/S3/S3Transfer/Models/DownloadRequestConfig.php create mode 100644 src/S3/S3Transfer/Models/GetObjectRequest.php create mode 100644 src/S3/S3Transfer/Models/GetObjectResponse.php create mode 100644 src/S3/S3Transfer/Models/MultipartDownloaderConfig.php create mode 100644 src/S3/S3Transfer/Models/UploadDirectoryRequest.php create mode 100644 src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php create mode 100644 src/S3/S3Transfer/MultipartDownloaderInitial.php create mode 100644 src/S3/S3Transfer/MultipartUploaderInitial.php create mode 100644 src/S3/S3Transfer/Utils/FileDownloadHandler.php create mode 100644 src/S3/S3Transfer/Utils/StreamDownloadHandler.php diff --git a/src/S3/S3Transfer/Exceptions/FileDownloadException.php b/src/S3/S3Transfer/Exceptions/FileDownloadException.php new file mode 100644 index 0000000000..9cad8fa555 --- /dev/null +++ b/src/S3/S3Transfer/Exceptions/FileDownloadException.php @@ -0,0 +1,8 @@ +getResource(); + } + + $this->sourceBucket = $sourceBucket; + $this->destinationDirectory = $destinationDirectory; + $this->getObjectRequest = $getObjectRequest; + $this->config = $config; + } + + /** + * @param string $sourceBucket The bucket from where the files are going to be + * downloaded from. + * @param string $destinationDirectory The destination path where the downloaded + * files will be placed in. + * @param array $downloadDirectoryArgs The getObject request arguments to be provided + * as part of each get object request sent to the service, except for the + * bucket and key which will be resolved internally. + * @param array $config The config options for this download directory operation. + * - s3_prefix: (string, optional) This parameter will be considered just if + * not provided as part of the list_object_v2_args config option. + * - s3_delimiter: (string, optional, defaulted to '/') This parameter will be + * considered just if not provided as part of the list_object_v2_args config + * option. + * - filter: (Closure, optional) A callable which will receive an object key as + * parameter and should return true or false in order to determine + * whether the object should be downloaded. + * - get_object_request_callback: (Closure, optional) A function that will + * be invoked right before the download request is performed and that will + * receive as parameter the request arguments for each request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on a download failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $downloadDirectoryRequestArgs: (array) The arguments for the download + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $downloadDirectoryResponse: (DownloadDirectoryResponse) The download response + * to that point in the upload process. + * - track_progress: (bool, optional) Overrides the config option set + * in the transfer manager instantiation to decide whether transfer + * progress should be tracked. + * - minimum_part_size: (int, optional) The minimum part size in bytes + * to be used in a range multipart download. + * - list_object_v2_args: (array, optional) The arguments to be included + * as part of the listObjectV2 request in order to fetch the objects to + * be downloaded. The most common arguments would be: + * - MaxKeys: (int) Sets the maximum number of keys returned in the response. + * - Prefix: (string) To limit the response to keys that begin with the + * specified prefix. + * @param TransferListener[] $listeners The listeners for watching + * transfer events. Each listener will be cloned per file upload. + * @param TransferListener|null $progressTracker Ideally the progress + * tracker implementation provided here should be able to track multiple + * transfers at once. Please see MultiProgressTracker implementation. + * + * @return DownloadDirectoryRequest + */ + public static function fromLegacyArgs( + string $sourceBucket, + string $destinationDirectory, + array $downloadDirectoryArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null, + ): DownloadDirectoryRequest { + return new self( + $sourceBucket, + $destinationDirectory, + GetObjectRequest::fromArray($downloadDirectoryArgs), + DownloadDirectoryRequestConfig::fromArray($config), + $listeners, + $progressTracker + ); + } + + /** + * @return string + */ + public function getSourceBucket(): string + { + return $this->sourceBucket; + } + + /** + * @return string + */ + public function getDestinationDirectory(): string + { + return $this->destinationDirectory; + } + + /** + * @return GetObjectRequest + */ + public function getGetObjectRequest(): GetObjectRequest + { + return $this->getObjectRequest; + } + + /** + * @return DownloadDirectoryRequestConfig + */ + public function getConfig(): DownloadDirectoryRequestConfig + { + return $this->config; + } + + /** + * Helper method to validate the destination directory exists. + * + * @return void + */ + public function validateDestinationDirectory(): void + { + if (!file_exists($this->destinationDirectory)) { + mkdir($this->destinationDirectory, 0755, true); + } + + if (!is_dir($this->destinationDirectory)) { + throw new InvalidArgumentException( + "Destination directory `$this->destinationDirectory` is not a directory." + ); + } + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php new file mode 100644 index 0000000000..14d0490a98 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php @@ -0,0 +1,184 @@ +s3Prefix = $s3Prefix; + $this->s3Delimiter = $s3Delimiter; + $this->filter = $filter; + $this->getObjectRequestCallback = $getObjectRequestCallback; + $this->failurePolicy = $failurePolicy; + $this->trackProgress = $trackProgress; + $this->targetPartSizeBytes = $targetPartSizeBytes; + $this->listObjectV2Args = $listObjectV2Args; + $this->failsWhenDestinationExists = $failsWhenDestinationExists; + } + + /** + * @param array $config + * @return DownloadDirectoryRequestConfig + */ + public static function fromArray(array $config): DownloadDirectoryRequestConfig + { + return new self( + s3Prefix: $config['s3_prefix'] ?? null, + s3Delimiter: $config['s3_delimiter'] ?? '/', + filter: $config['filter'] ?? null, + getObjectRequestCallback: $config['get_object_request_callback'] ?? null, + failurePolicy: $config['failure_policy'] ?? null, + targetPartSizeBytes: $config['target_part_size_bytes'] ?? null, + listObjectV2Args: $config['list_object_v2_args'] ?? [], + failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, + trackProgress: $config['track_progress'] ?? null + ); + } + + /** + * @return string|null + */ + public function getS3Prefix(): ?string + { + return $this->s3Prefix; + } + + /** + * @return string + */ + public function getS3Delimiter(): string + { + return $this->s3Delimiter; + } + + /** + * @return Closure|null + */ + public function getFilter(): ?Closure + { + return $this->filter; + } + + /** + * @return Closure|null + */ + public function getGetObjectRequestCallback(): ?Closure + { + return $this->getObjectRequestCallback; + } + + /** + * @return Closure|null + */ + public function getFailurePolicy(): ?Closure + { + return $this->failurePolicy; + } + + /** + * @return int|null + */ + public function getTargetPartSizeBytes(): ?int + { + return $this->targetPartSizeBytes; + } + + /** + * @return array + */ + public function getListObjectV2Args(): array + { + return $this->listObjectV2Args; + } + + + /** + * @return string|null + */ + public function getEffectivePrefix(): ?string + { + return $this->listObjectV2Args['Prefix'] ?? $this->s3Prefix; + } + + /** + * @return string + */ + public function getEffectiveDelimiter(): string + { + return $this->listObjectV2Args['Delimiter'] ?? $this->s3Delimiter; + } + + /** + * @return bool + */ + public function isFailsWhenDestinationExists(): bool + { + return $this->failsWhenDestinationExists; + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 's3_prefix' => $this->s3Prefix, + 's3_delimiter' => $this->s3Delimiter, + 'filter' => $this->filter, + 'get_object_request_callback' => $this->getObjectRequestCallback, + 'failure_policy' => $this->failurePolicy, + 'track_progress' => $this->trackProgress, + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'list_object_v2_args' => $this->listObjectV2Args, + 'fails_when_destination_exists' => $this->failsWhenDestinationExists, + ]; + } +} diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php new file mode 100644 index 0000000000..9ad8568776 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -0,0 +1,63 @@ +destination = $destination; + $this->failsWhenDestinationExists = $failsWhenDestinationExists; + $this->downloadRequest = DownloadRequest::fromDownloadRequestAndDownloadHandler( + $downloadRequest, + new FileDownloadHandler( + $destination, + $failsWhenDestinationExists + ) + ); + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->destination; + } + + /** + * @return bool + */ + public function isFailsWhenDestinationExists(): bool + { + return $this->failsWhenDestinationExists; + } + + /** + * @return DownloadRequest + */ + public function getDownloadRequest(): DownloadRequest + { + return $this->downloadRequest; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadHandler.php b/src/S3/S3Transfer/Models/DownloadHandler.php new file mode 100644 index 0000000000..f5337abfab --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadHandler.php @@ -0,0 +1,18 @@ +source = $source; + $this->getObjectRequest = $getObjectRequest; + $this->config = $config; + if ($downloadHandler === null) { + $downloadHandler = new StreamDownloadHandler(); + } + $this->downloadHandler = $downloadHandler; + } + + /** + * @param string|array|null $source The object to be downloaded from S3. + * It can be either a string with a S3 URI or an array with a Bucket and Key + * properties set. + * @param array $downloadRequestArgs The getObject request arguments to be provided as part + * of each get object operation, except for the bucket and key, which + * are already provided as the source. + * @param array $config The configuration to be used for this operation: + * - multipart_download_type: (string, optional) + * Overrides the resolved value from the transfer manager config. + * - checksum_validation_enabled: (bool, optional) Overrides the resolved + * value from transfer manager config for whether checksum validation + * should be done. This option will be considered just if ChecksumMode + * is not present in the request args. + * - track_progress: (bool) Overrides the config option set in the transfer + * manager instantiation to decide whether transfer progress should be + * tracked. + * - minimum_part_size: (int) The minimum part size in bytes to be used + * in a range multipart download. If this parameter is not provided + * then it fallbacks to the transfer manager `target_part_size_bytes` + * config value. + * @param DownloadHandler|null $downloadHandler + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker + * + * @return static + */ + public static function fromLegacyArgs( + string | array | null $source, + array $downloadRequestArgs = [], + array $config = [], + ?DownloadHandler $downloadHandler = null, + array $listeners = [], + ?TransferListener $progressTracker = null, + ): DownloadRequest + { + return new DownloadRequest( + $source, + GetObjectRequest::fromArray($downloadRequestArgs), + DownloadRequestConfig::fromArray($config), + $downloadHandler, + $listeners, + $progressTracker + ); + } + + /** + * @param DownloadRequest $downloadRequest + * @param FileDownloadHandler $downloadHandler + * + * @return DownloadRequest + */ + public static function fromDownloadRequestAndDownloadHandler( + DownloadRequest $downloadRequest, + FileDownloadHandler $downloadHandler + ): DownloadRequest + { + return new DownloadRequest( + $downloadRequest->getSource(), + $downloadRequest->getGetObjectRequest(), + $downloadRequest->getConfig(), + $downloadHandler, + $downloadRequest->getListeners(), + $downloadRequest->getProgressTracker() + ); + } + + /** + * @return array|string|null + */ + public function getSource(): array|string|null + { + return $this->source; + } + + /** + * @return GetObjectRequest + */ + public function getGetObjectRequest(): GetObjectRequest + { + return $this->getObjectRequest; + } + + /** + * @return DownloadRequestConfig + */ + public function getConfig(): DownloadRequestConfig + { + return $this->config; + } + + /** + * @return DownloadHandler + */ + public function getDownloadHandler(): DownloadHandler { + return $this->downloadHandler; + } + + /** + * Helper method to normalize the source as an array with: + * - Bucket + * - Key + * + * @return array + */ + public function normalizeSourceAsArray(): array { + // If source is null then fall back to getObjectRequest. + $source = $this->getSource() ?? [ + 'Bucket' => $this->getObjectRequest->getBucket(), + 'Key' => $this->getObjectRequest->getKey(), + ]; + if (is_string($source)) { + $sourceAsArray = S3TransferManager::s3UriAsBucketAndKey($source); + } elseif (is_array($source)) { + $sourceAsArray = $source; + } else { + throw new S3TransferException( + "Unsupported source type `" . gettype($source) . "`" + ); + } + + foreach (['Bucket', 'Key'] as $reqKey) { + if (empty($sourceAsArray[$reqKey])) { + throw new \InvalidArgumentException( + "`$reqKey` is required but not provided in " + . implode(', ', array_keys($sourceAsArray)) . "." + ); + } + } + + return $sourceAsArray; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadRequestConfig.php b/src/S3/S3Transfer/Models/DownloadRequestConfig.php new file mode 100644 index 0000000000..84581c812b --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadRequestConfig.php @@ -0,0 +1,92 @@ +multipartDownloadType = $multipartDownloadType; + $this->requestChecksumValidation = $requestChecksumValidation; + $this->targetPartSizeBytes = $targetPartSizeBytes; + } + + /** + * Convert the DownloadRequestConfig instance to an array + * + * @return array + */ + public function toArray(): array + { + return [ + 'multipart_download_type' => $this->multipartDownloadType, + 'request_checksum_validation' => $this->requestChecksumValidation, + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'track_progress' => $this->getTrackProgress(), // Assuming this getter exists in parent class + ]; + } + + /** + * Create a DownloadRequestConfig instance from an array + * + * @param array $data + * @return static + */ + public static function fromArray(array $data): static + { + return new self( + $data['multipart_download_type'] ?? null, + $data['request_checksum_validation'] ?? null, + $data['target_part_size_bytes'] ?? null, + $data['track_progress'] ?? null + ); + } + + /** + * @return string|null + */ + public function getMultipartDownloadType(): ?string + { + return $this->multipartDownloadType; + } + + /** + * @return string|null + */ + public function getRequestChecksumValidation(): ?string + { + return $this->requestChecksumValidation; + } + + /** + * @return int|null + */ + public function getTargetPartSizeBytes(): ?int + { + return $this->targetPartSizeBytes; + } +} diff --git a/src/S3/S3Transfer/Models/DownloadResponse.php b/src/S3/S3Transfer/Models/DownloadResponse.php index 836ba428bc..6c601118ee 100644 --- a/src/S3/S3Transfer/Models/DownloadResponse.php +++ b/src/S3/S3Transfer/Models/DownloadResponse.php @@ -2,32 +2,30 @@ namespace Aws\S3\S3Transfer\Models; -use Psr\Http\Message\StreamInterface; - class DownloadResponse { /** - * @param StreamInterface $data - * @param array $metadata + * @param mixed $downloadDataResult + * @param array $downloadResponse */ public function __construct( - private readonly StreamInterface $data, - private readonly array $metadata = [] + private readonly mixed $downloadDataResult, + private readonly array $downloadResponse = [] ) {} /** - * @return StreamInterface + * @return mixed */ - public function getData(): StreamInterface + public function getDownloadDataResult(): mixed { - return $this->data; + return $this->downloadDataResult; } /** * @return array */ - public function getMetadata(): array + public function getDownloadResponse(): array { - return $this->metadata; + return $this->downloadResponse; } } \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/GetObjectRequest.php b/src/S3/S3Transfer/Models/GetObjectRequest.php new file mode 100644 index 0000000000..0844616026 --- /dev/null +++ b/src/S3/S3Transfer/Models/GetObjectRequest.php @@ -0,0 +1,345 @@ +bucket = $bucket; + $this->checksumMode = $checksumMode; + $this->expectedBucketOwner = $expectedBucketOwner; + $this->ifMatch = $ifMatch; + $this->ifModifiedSince = $ifModifiedSince; + $this->ifNoneMatch = $ifNoneMatch; + $this->ifUnmodifiedSince = $ifUnmodifiedSince; + $this->key = $key; + $this->requestPayer = $requestPayer; + $this->responseCacheControl = $responseCacheControl; + $this->responseContentDisposition = $responseContentDisposition; + $this->responseContentEncoding = $responseContentEncoding; + $this->responseContentLanguage = $responseContentLanguage; + $this->responseContentType = $responseContentType; + $this->responseExpires = $responseExpires; + $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; + $this->sseCustomerKey = $sseCustomerKey; + $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; + $this->versionId = $versionId; + } + + /** + * Create an instance from an array of data + * + * @param array $data + * @return self + */ + public static function fromArray(array $data): self + { + return new self( + $data['Bucket'] ?? null, + $data['ChecksumMode'] ?? null, + $data['ExpectedBucketOwner'] ?? null, + $data['IfMatch'] ?? null, + $data['IfModifiedSince'] ?? null, + $data['IfNoneMatch'] ?? null, + $data['IfUnmodifiedSince'] ?? null, + $data['Key'] ?? null, + $data['RequestPayer'] ?? null, + $data['ResponseCacheControl'] ?? null, + $data['ResponseContentDisposition'] ?? null, + $data['ResponseContentEncoding'] ?? null, + $data['ResponseContentLanguage'] ?? null, + $data['ResponseContentType'] ?? null, + $data['ResponseExpires'] ?? null, + $data['SSECustomerAlgorithm'] ?? null, + $data['SSECustomerKey'] ?? null, + $data['SSECustomerKeyMD5'] ?? null, + $data['VersionId'] ?? null + ); + } + + /** + * @return string|null + */ + public function getBucket(): ?string + { + return $this->bucket; + } + + /** + * @return string|null + */ + public function getChecksumMode(): ?string + { + return $this->checksumMode; + } + + /** + * @return string|null + */ + public function getExpectedBucketOwner(): ?string + { + return $this->expectedBucketOwner; + } + + /** + * @return string|null + */ + public function getIfMatch(): ?string + { + return $this->ifMatch; + } + + /** + * @return string|null + */ + public function getIfModifiedSince(): ?string + { + return $this->ifModifiedSince; + } + + /** + * @return string|null + */ + public function getIfNoneMatch(): ?string + { + return $this->ifNoneMatch; + } + + /** + * @return string|null + */ + public function getIfUnmodifiedSince(): ?string + { + return $this->ifUnmodifiedSince; + } + + /** + * @return string|null + */ + public function getKey(): ?string + { + return $this->key; + } + + /** + * @return string|null + */ + public function getRequestPayer(): ?string + { + return $this->requestPayer; + } + + /** + * @return string|null + */ + public function getResponseCacheControl(): ?string + { + return $this->responseCacheControl; + } + + /** + * @return string|null + */ + public function getResponseContentDisposition(): ?string + { + return $this->responseContentDisposition; + } + + /** + * @return string|null + */ + public function getResponseContentEncoding(): ?string + { + return $this->responseContentEncoding; + } + + /** + * @return string|null + */ + public function getResponseContentLanguage(): ?string + { + return $this->responseContentLanguage; + } + + /** + * @return string|null + */ + public function getResponseContentType(): ?string + { + return $this->responseContentType; + } + + /** + * @return string|null + */ + public function getResponseExpires(): ?string + { + return $this->responseExpires; + } + + /** + * @return string|null + */ + public function getSseCustomerAlgorithm(): ?string + { + return $this->sseCustomerAlgorithm; + } + + /** + * @return string|null + */ + public function getSseCustomerKey(): ?string + { + return $this->sseCustomerKey; + } + + /** + * @return string|null + */ + public function getSseCustomerKeyMD5(): ?string + { + return $this->sseCustomerKeyMD5; + } + + /** + * @return string|null + */ + public function getVersionId(): ?string + { + return $this->versionId; + } + + /** + * Convert the object to an array format suitable for AWS S3 API request + * + * @return array Array containing AWS S3 request fields with their corresponding values + */ + public function toArray(): array + { + $array = [ + 'Bucket' => $this->bucket, + 'ChecksumMode' => $this->checksumMode, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'IfMatch' => $this->ifMatch, + 'IfModifiedSince' => $this->ifModifiedSince, + 'IfNoneMatch' => $this->ifNoneMatch, + 'IfUnmodifiedSince' => $this->ifUnmodifiedSince, + 'Key' => $this->key, + 'RequestPayer' => $this->requestPayer, + 'ResponseCacheControl' => $this->responseCacheControl, + 'ResponseContentDisposition' => $this->responseContentDisposition, + 'ResponseContentEncoding' => $this->responseContentEncoding, + 'ResponseContentLanguage' => $this->responseContentLanguage, + 'ResponseContentType' => $this->responseContentType, + 'ResponseExpires' => $this->responseExpires, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKey' => $this->sseCustomerKey, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + 'VersionId' => $this->versionId + ]; + + remove_nulls_from_array($array); + + return $array; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/GetObjectResponse.php b/src/S3/S3Transfer/Models/GetObjectResponse.php new file mode 100644 index 0000000000..ae59bb830c --- /dev/null +++ b/src/S3/S3Transfer/Models/GetObjectResponse.php @@ -0,0 +1,629 @@ +acceptRanges = $acceptRanges; + $this->bucketKeyEnabled = $bucketKeyEnabled; + $this->cacheControl = $cacheControl; + $this->checksumCRC32 = $checksumCRC32; + $this->checksumCRC32C = $checksumCRC32C; + $this->checksumCRC64NVME = $checksumCRC64NVME; + $this->checksumSHA1 = $checksumSHA1; + $this->checksumSHA256 = $checksumSHA256; + $this->checksumType = $checksumType; + $this->contentDisposition = $contentDisposition; + $this->contentEncoding = $contentEncoding; + $this->contentLanguage = $contentLanguage; + $this->contentLength = $contentLength; + $this->contentRange = $contentRange; + $this->contentType = $contentType; + $this->deleteMarker = $deleteMarker; + $this->eTag = $eTag; + $this->expiration = $expiration; + $this->expires = $expires; + $this->lastModified = $lastModified; + $this->metadata = $metadata; + $this->missingMeta = $missingMeta; + $this->objectLockLegalHoldStatus = $objectLockLegalHoldStatus; + $this->objectLockMode = $objectLockMode; + $this->objectLockRetainUntilDate = $objectLockRetainUntilDate; + $this->partsCount = $partsCount; + $this->replicationStatus = $replicationStatus; + $this->requestCharged = $requestCharged; + $this->restore = $restore; + $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; + $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; + $this->sseKMSKeyId = $sseKMSKeyId; + $this->serverSideEncryption = $serverSideEncryption; + $this->storageClass = $storageClass; + $this->tagCount = $tagCount; + $this->versionId = $versionId; + $this->websiteRedirectLocation = $websiteRedirectLocation; + } + + /** + * @param array $array + * @return GetObjectResponse + */ + public static function fromArray(array $array): GetObjectResponse + { + return new GetObjectResponse( + acceptRanges: $array['AcceptRanges'] ?? null, + bucketKeyEnabled: $array['BucketKeyEnabled'] ?? null, + cacheControl: $array['CacheControl'] ?? null, + checksumCRC32: $array['ChecksumCRC32'] ?? null, + checksumCRC32C: $array['ChecksumCRC32C'] ?? null, + checksumCRC64NVME: $array['ChecksumCRC64NVME'] ?? null, + checksumSHA1: $array['ChecksumSHA1'] ?? null, + checksumSHA256: $array['ChecksumSHA256'] ?? null, + checksumType: $array['ChecksumType'] ?? null, + contentDisposition: $array['ContentDisposition'] ?? null, + contentEncoding: $array['ContentEncoding'] ?? null, + contentLanguage: $array['ContentLanguage'] ?? null, + contentLength: $array['ContentLength'] ?? null, + contentRange: $array['ContentRange'] ?? null, + contentType: $array['ContentType'] ?? null, + deleteMarker: $array['DeleteMarker'] ?? null, + eTag: $array['ETag'] ?? null, + expiration: $array['Expiration'] ?? null, + expires: $array['Expires'] ?? null, + lastModified: $array['LastModified'] ?? null, + metadata: $array['@metadata'] ?? null, + missingMeta: $array['MissingMeta'] ?? null, + objectLockLegalHoldStatus: $array['ObjectLockLegalHoldStatus'] ?? null, + objectLockMode: $array['ObjectLockMode'] ?? null, + objectLockRetainUntilDate: $array['ObjectLockRetainUntilDate'] ?? null, + partsCount: $array['PartsCount'] ?? null, + replicationStatus: $array['ReplicationStatus'] ?? null, + requestCharged: $array['RequestCharged'] ?? null, + restore: $array['Restore'] ?? null, + sseCustomerAlgorithm: $array['SSECustomerAlgorithm'] ?? null, + sseCustomerKeyMD5: $array['SSECustomerKeyMD5'] ?? null, + sseKMSKeyId: $array['SSEKMSKeyId'] ?? null, + serverSideEncryption: $array['ServerSideEncryption'] ?? null, + storageClass: $array['StorageClass'] ?? null, + tagCount: $array['TagCount'] ?? null, + versionId: $array['VersionId'] ?? null, + websiteRedirectLocation: $array['WebsiteRedirectLocation'] ?? null + ); + } + + /** + * @return string|null + */ + public function getAcceptRanges(): ?string + { + return $this->acceptRanges; + } + + /** + * @return string|null + */ + public function getBucketKeyEnabled(): ?string + { + return $this->bucketKeyEnabled; + } + + /** + * @return string|null + */ + public function getCacheControl(): ?string + { + return $this->cacheControl; + } + + /** + * @return string|null + */ + public function getChecksumCRC32(): ?string + { + return $this->checksumCRC32; + } + + /** + * @return string|null + */ + public function getChecksumCRC32C(): ?string + { + return $this->checksumCRC32C; + } + + /** + * @return string|null + */ + public function getChecksumCRC64NVME(): ?string + { + return $this->checksumCRC64NVME; + } + + /** + * @return string|null + */ + public function getChecksumSHA1(): ?string + { + return $this->checksumSHA1; + } + + /** + * @return string|null + */ + public function getChecksumSHA256(): ?string + { + return $this->checksumSHA256; + } + + /** + * @return string|null + */ + public function getChecksumType(): ?string + { + return $this->checksumType; + } + + /** + * @return string|null + */ + public function getContentDisposition(): ?string + { + return $this->contentDisposition; + } + + /** + * @return string|null + */ + public function getContentEncoding(): ?string + { + return $this->contentEncoding; + } + + /** + * @return string|null + */ + public function getContentLanguage(): ?string + { + return $this->contentLanguage; + } + + /** + * @return string|null + */ + public function getContentLength(): ?string + { + return $this->contentLength; + } + + /** + * @return string|null + */ + public function getContentRange(): ?string + { + return $this->contentRange; + } + + /** + * @return string|null + */ + public function getContentType(): ?string + { + return $this->contentType; + } + + /** + * @return string|null + */ + public function getDeleteMarker(): ?string + { + return $this->deleteMarker; + } + + /** + * @return string|null + */ + public function getETag(): ?string + { + return $this->eTag; + } + + /** + * @return string|null + */ + public function getExpiration(): ?string + { + return $this->expiration; + } + + /** + * @return string|null + */ + public function getExpires(): ?string + { + return $this->expires; + } + + /** + * @return string|null + */ + public function getLastModified(): ?string + { + return $this->lastModified; + } + + /** + * @return array|null + */ + public function getMetadata(): ?array + { + return $this->metadata; + } + + /** + * @return string|null + */ + public function getMissingMeta(): ?string + { + return $this->missingMeta; + } + + /** + * @return string|null + */ + public function getObjectLockLegalHoldStatus(): ?string + { + return $this->objectLockLegalHoldStatus; + } + + /** + * @return string|null + */ + public function getObjectLockMode(): ?string + { + return $this->objectLockMode; + } + + /** + * @return string|null + */ + public function getObjectLockRetainUntilDate(): ?string + { + return $this->objectLockRetainUntilDate; + } + + /** + * @return string|null + */ + public function getPartsCount(): ?string + { + return $this->partsCount; + } + + /** + * @return string|null + */ + public function getReplicationStatus(): ?string + { + return $this->replicationStatus; + } + + /** + * @return string|null + */ + public function getRequestCharged(): ?string + { + return $this->requestCharged; + } + + /** + * @return string|null + */ + public function getRestore(): ?string + { + return $this->restore; + } + + /** + * @return string|null + */ + public function getSSECustomerAlgorithm(): ?string + { + return $this->sseCustomerAlgorithm; + } + + /** + * @return string|null + */ + public function getSSECustomerKeyMD5(): ?string + { + return $this->sseCustomerKeyMD5; + } + + /** + * @return string|null + */ + public function getSSEKMSKeyId(): ?string + { + return $this->sseKMSKeyId; + } + + /** + * @return string|null + */ + public function getServerSideEncryption(): ?string + { + return $this->serverSideEncryption; + } + + /** + * @return string|null + */ + public function getStorageClass(): ?string + { + return $this->storageClass; + } + + /** + * @return string|null + */ + public function getTagCount(): ?string + { + return $this->tagCount; + } + + /** + * @return string|null + */ + public function getVersionId(): ?string + { + return $this->versionId; + } + + /** + * @return string|null + */ + public function getWebsiteRedirectLocation(): ?string + { + return $this->websiteRedirectLocation; + } + + /** + * @return array + */ + public function toArray(): array + { + $array = [ + 'AcceptRanges' => $this->acceptRanges, + 'BucketKeyEnabled' => $this->bucketKeyEnabled, + 'CacheControl' => $this->cacheControl, + 'ChecksumCRC32' => $this->checksumCRC32, + 'ChecksumCRC32C' => $this->checksumCRC32C, + 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, + 'ChecksumSHA1' => $this->checksumSHA1, + 'ChecksumSHA256' => $this->checksumSHA256, + 'ChecksumType' => $this->checksumType, + 'ContentDisposition' => $this->contentDisposition, + 'ContentEncoding' => $this->contentEncoding, + 'ContentLanguage' => $this->contentLanguage, + 'ContentLength' => $this->contentLength, + 'ContentRange' => $this->contentRange, + 'ContentType' => $this->contentType, + 'DeleteMarker' => $this->deleteMarker, + 'ETag' => $this->eTag, + 'Expiration' => $this->expiration, + 'Expires' => $this->expires, + 'LastModified' => $this->lastModified, + 'Metadata' => $this->metadata, + 'MissingMeta' => $this->missingMeta, + 'ObjectLockLegalHoldStatus' => $this->objectLockLegalHoldStatus, + 'ObjectLockMode' => $this->objectLockMode, + 'ObjectLockRetainUntilDate' => $this->objectLockRetainUntilDate, + 'PartsCount' => $this->partsCount, + 'ReplicationStatus' => $this->replicationStatus, + 'RequestCharged' => $this->requestCharged, + 'Restore' => $this->restore, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + 'SSEKMSKeyId' => $this->sseKMSKeyId, + 'ServerSideEncryption' => $this->serverSideEncryption, + 'StorageClass' => $this->storageClass, + 'TagCount' => $this->tagCount, + 'VersionId' => $this->versionId, + 'WebsiteRedirectLocation' => $this->websiteRedirectLocation, + ]; + + remove_nulls_from_array($array); + + return $array; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/MultipartDownloaderConfig.php b/src/S3/S3Transfer/Models/MultipartDownloaderConfig.php new file mode 100644 index 0000000000..cfd3689148 --- /dev/null +++ b/src/S3/S3Transfer/Models/MultipartDownloaderConfig.php @@ -0,0 +1,81 @@ +targetPartSizeBytes = $targetPartSizeBytes; + $this->responseChecksumValidationEnabled = $responseChecksumValidationEnabled; + $this->multipartDownloadType = $multipartDownloadType; + } + + /** + * @param array $array + * + * @return MultipartDownloaderConfig + */ + public static function fromArray(array $array): MultipartDownloaderConfig { + return new self( + $array['target_part_size_bytes'] + ?? S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, + $array['response_checksum_validation_enabled'] + ?? true, + $array['multipart_download_type'] + ?? S3TransferManagerConfig::DEFAULT_MULTIPART_DOWNLOAD_TYPE + ); + } + + /** + * @return int + */ + public function getTargetPartSizeBytes(): int + { + return $this->targetPartSizeBytes; + } + + /** + * @return bool + */ + public function getResponseChecksumValidationEnabled(): bool + { + return $this->responseChecksumValidationEnabled; + } + + /** + * @return string + */ + public function getMultipartDownloadType(): string + { + return $this->multipartDownloadType; + } + + /** + * @return array + */ + public function toArray(): array { + return [ + 'target_part_size_bytes' => $this->targetPartSizeBytes, + 'response_checksum_validation_enabled' => $this->responseChecksumValidationEnabled, + 'multipart_download_type' => $this->multipartDownloadType, + ]; + } +} diff --git a/src/S3/S3Transfer/Models/MultipartUploaderConfig.php b/src/S3/S3Transfer/Models/MultipartUploaderConfig.php index 3dd4d63f8d..43a249b3a4 100644 --- a/src/S3/S3Transfer/Models/MultipartUploaderConfig.php +++ b/src/S3/S3Transfer/Models/MultipartUploaderConfig.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Models; -class MultipartUploaderConfig +final class MultipartUploaderConfig { /** @var int */ @@ -30,6 +30,25 @@ public function __construct( $this->requestChecksumCalculation = $requestChecksumCalculation; } + /** + * Create an MultipartUploaderConfig instance from an array + * + * @param array $data Array containing configuration data + * + * @return MultipartUploaderConfig + */ + public static function fromArray(array $data): MultipartUploaderConfig + { + return new self( + $data['target_part_size_bytes'] + ?? S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, + $data['concurrency'] + ?? S3TransferManagerConfig::DEFAULT_CONCURRENCY, + $data['request_checksum_calculation'] + ?? S3TransferManagerConfig::DEFAULT_REQUEST_CHECKSUM_CALCULATION + ); + } + /** * @return int */ @@ -64,23 +83,4 @@ public function toArray(): array 'request_checksum_calculation' => $this->requestChecksumCalculation, ]; } - - /** - * Create an MultipartUploaderConfig instance from an array - * - * @param array $data Array containing configuration data - * - * @return static - */ - public static function fromArray(array $data): self - { - return new self( - $data['target_part_size_bytes'] - ?? S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, - $data['concurrency'] - ?? S3TransferManagerConfig::DEFAULT_CONCURRENCY, - $data['request_checksum_calculation'] - ?? S3TransferManagerConfig::DEFAULT_REQUEST_CHECKSUM_CALCULATION - ); - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/PutObjectRequest.php b/src/S3/S3Transfer/Models/PutObjectRequest.php index cfa3ea1d19..c1e5cc526c 100644 --- a/src/S3/S3Transfer/Models/PutObjectRequest.php +++ b/src/S3/S3Transfer/Models/PutObjectRequest.php @@ -2,6 +2,8 @@ namespace Aws\S3\S3Transfer\Models; +use function Aws\remove_nulls_from_array; + class PutObjectRequest { /** @var string|null */ @@ -233,6 +235,54 @@ public function __construct( $this->ifNoneMatch = $ifNoneMatch; } + /** + * @param array $array + * + * @return PutObjectRequest + */ + public static function fromArray(array $array): PutObjectRequest + { + return new self( + $array['ACL'] ?? null, + $array['Bucket'] ?? null, + $array['BucketKeyEnabled'] ?? null, + $array['CacheControl'] ?? null, + $array['ChecksumAlgorithm'] ?? null, + $array['ContentDisposition'] ?? null, + $array['ContentEncoding'] ?? null, + $array['ContentLanguage'] ?? null, + $array['ContentType'] ?? null, + $array['ExpectedBucketOwner'] ?? null, + $array['Expires'] ?? null, + $array['GrantFullControl'] ?? null, + $array['GrantRead'] ?? null, + $array['GrantReadACP'] ?? null, + $array['GrantWriteACP'] ?? null, + $array['Key'] ?? null, + $array['Metadata'] ?? null, + $array['ObjectLockLegalHoldStatus'] ?? null, + $array['ObjectLockMode'] ?? null, + $array['ObjectLockRetainUntilDate'] ?? null, + $array['RequestPayer'] ?? null, + $array['SSECustomerAlgorithm'] ?? null, + $array['SSECustomerKey'] ?? null, + $array['SSECustomerKeyMD5'] ?? null, + $array['SSEKMSEncryptionContext'] ?? null, + $array['SSEKMSKeyId'] ?? null, + $array['ServerSideEncryption'] ?? null, + $array['StorageClass'] ?? null, + $array['Tagging'] ?? null, + $array['WebsiteRedirectLocation'] ?? null, + $array['ChecksumCRC32'] ?? null, + $array['ChecksumCRC32C'] ?? null, + $array['ChecksumCRC64NVME'] ?? null, + $array['ChecksumSHA1'] ?? null, + $array['ChecksumSHA256'] ?? null, + $array['IfMatch'] ?? null, + $array['IfNoneMatch'] ?? null + ); + } + /** * @return string|null */ @@ -536,59 +586,23 @@ public function getIfNoneMatch(): ?string */ public function toSingleObjectRequest(): array { - $requestArgs = []; - - if ($this->bucket !== null) { - $requestArgs['Bucket'] = $this->bucket; - } - - if ($this->checksumAlgorithm !== null) { - $requestArgs['ChecksumAlgorithm'] = $this->checksumAlgorithm; - } - - if ($this->checksumCRC32 !== null) { - $requestArgs['ChecksumCRC32'] = $this->checksumCRC32; - } - - if ($this->checksumCRC32C !== null) { - $requestArgs['ChecksumCRC32C'] = $this->checksumCRC32C; - } - - if ($this->checksumCRC64NVME !== null) { - $requestArgs['ChecksumCRC64NVME'] = $this->checksumCRC64NVME; - } - - if ($this->checksumSHA1 !== null) { - $requestArgs['ChecksumSHA1'] = $this->checksumSHA1; - } - - if ($this->checksumSHA256 !== null) { - $requestArgs['ChecksumSHA256'] = $this->checksumSHA256; - } - - if ($this->expectedBucketOwner !== null) { - $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; - } - - if ($this->key !== null) { - $requestArgs['Key'] = $this->key; - } - - if ($this->requestPayer !== null) { - $requestArgs['RequestPayer'] = $this->requestPayer; - } - - if ($this->sseCustomerAlgorithm !== null) { - $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; - } - - if ($this->sseCustomerKey !== null) { - $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; - } + $requestArgs = [ + 'Bucket' => $this->bucket, + 'ChecksumAlgorithm' => $this->checksumAlgorithm, + 'ChecksumCRC32' => $this->checksumCRC32, + 'ChecksumCRC32C' => $this->checksumCRC32C, + 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, + 'ChecksumSHA1' => $this->checksumSHA1, + 'ChecksumSHA256' => $this->checksumSHA256, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'Key' => $this->key, + 'RequestPayer' => $this->requestPayer, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKey' => $this->sseCustomerKey, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + ]; - if ($this->sseCustomerKeyMD5 !== null) { - $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; - } + remove_nulls_from_array($requestArgs); return $requestArgs; } @@ -600,127 +614,40 @@ public function toSingleObjectRequest(): array */ public function toCreateMultipartRequest(): array { - $requestArgs = []; - - if ($this->acl !== null) { - $requestArgs['ACL'] = $this->acl; - } - - if ($this->bucket !== null) { - $requestArgs['Bucket'] = $this->bucket; - } - - if ($this->bucketKeyEnabled !== null) { - $requestArgs['BucketKeyEnabled'] = $this->bucketKeyEnabled; - } - - if ($this->cacheControl !== null) { - $requestArgs['CacheControl'] = $this->cacheControl; - } - - if ($this->checksumAlgorithm !== null) { - $requestArgs['ChecksumAlgorithm'] = $this->checksumAlgorithm; - } - - if ($this->contentDisposition !== null) { - $requestArgs['ContentDisposition'] = $this->contentDisposition; - } - - if ($this->contentEncoding !== null) { - $requestArgs['ContentEncoding'] = $this->contentEncoding; - } - - if ($this->contentLanguage !== null) { - $requestArgs['ContentLanguage'] = $this->contentLanguage; - } - - if ($this->contentType !== null) { - $requestArgs['ContentType'] = $this->contentType; - } - - if ($this->expectedBucketOwner !== null) { - $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; - } - - if ($this->expires !== null) { - $requestArgs['Expires'] = $this->expires; - } - - if ($this->grantFullControl !== null) { - $requestArgs['GrantFullControl'] = $this->grantFullControl; - } - - if ($this->grantRead !== null) { - $requestArgs['GrantRead'] = $this->grantRead; - } - - if ($this->grantReadACP !== null) { - $requestArgs['GrantReadACP'] = $this->grantReadACP; - } - - if ($this->grantWriteACP !== null) { - $requestArgs['GrantWriteACP'] = $this->grantWriteACP; - } - - if ($this->key !== null) { - $requestArgs['Key'] = $this->key; - } - - if ($this->metadata !== null) { - $requestArgs['Metadata'] = $this->metadata; - } - - if ($this->objectLockLegalHoldStatus !== null) { - $requestArgs['ObjectLockLegalHoldStatus'] = $this->objectLockLegalHoldStatus; - } - - if ($this->objectLockMode !== null) { - $requestArgs['ObjectLockMode'] = $this->objectLockMode; - } - - if ($this->objectLockRetainUntilDate !== null) { - $requestArgs['ObjectLockRetainUntilDate'] = $this->objectLockRetainUntilDate; - } - - if ($this->requestPayer !== null) { - $requestArgs['RequestPayer'] = $this->requestPayer; - } - - if ($this->sseCustomerAlgorithm !== null) { - $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; - } - - if ($this->sseCustomerKey !== null) { - $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; - } - - if ($this->sseCustomerKeyMD5 !== null) { - $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; - } - - if ($this->ssekmsEncryptionContext !== null) { - $requestArgs['SSEKMSEncryptionContext'] = $this->ssekmsEncryptionContext; - } - - if ($this->ssekmsKeyId !== null) { - $requestArgs['SSEKMSKeyId'] = $this->ssekmsKeyId; - } - - if ($this->serverSideEncryption !== null) { - $requestArgs['ServerSideEncryption'] = $this->serverSideEncryption; - } - - if ($this->storageClass !== null) { - $requestArgs['StorageClass'] = $this->storageClass; - } - - if ($this->tagging !== null) { - $requestArgs['Tagging'] = $this->tagging; - } + $requestArgs = [ + 'ACL' => $this->acl, + 'Bucket' => $this->bucket, + 'BucketKeyEnabled' => $this->bucketKeyEnabled, + 'CacheControl' => $this->cacheControl, + 'ChecksumAlgorithm' => $this->checksumAlgorithm, + 'ContentDisposition' => $this->contentDisposition, + 'ContentEncoding' => $this->contentEncoding, + 'ContentLanguage' => $this->contentLanguage, + 'ContentType' => $this->contentType, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'Expires' => $this->expires, + 'GrantFullControl' => $this->grantFullControl, + 'GrantRead' => $this->grantRead, + 'GrantReadACP' => $this->grantReadACP, + 'GrantWriteACP' => $this->grantWriteACP, + 'Key' => $this->key, + 'Metadata' => $this->metadata, + 'ObjectLockLegalHoldStatus' => $this->objectLockLegalHoldStatus, + 'ObjectLockMode' => $this->objectLockMode, + 'ObjectLockRetainUntilDate' => $this->objectLockRetainUntilDate, + 'RequestPayer' => $this->requestPayer, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKey' => $this->sseCustomerKey, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + 'SSEKMSEncryptionContext' => $this->ssekmsEncryptionContext, + 'SSEKMSKeyId' => $this->ssekmsKeyId, + 'ServerSideEncryption' => $this->serverSideEncryption, + 'StorageClass' => $this->storageClass, + 'Tagging' => $this->tagging, + 'WebsiteRedirectLocation' => $this->websiteRedirectLocation, + ]; - if ($this->websiteRedirectLocation !== null) { - $requestArgs['WebsiteRedirectLocation'] = $this->websiteRedirectLocation; - } + remove_nulls_from_array($requestArgs); return $requestArgs; } @@ -732,39 +659,18 @@ public function toCreateMultipartRequest(): array */ public function toUploadPartRequest(): array { - $requestArgs = []; - - if ($this->bucket !== null) { - $requestArgs['Bucket'] = $this->bucket; - } - - if ($this->checksumAlgorithm !== null) { - $requestArgs['ChecksumAlgorithm'] = $this->checksumAlgorithm; - } - - if ($this->expectedBucketOwner !== null) { - $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; - } - - if ($this->key !== null) { - $requestArgs['Key'] = $this->key; - } - - if ($this->requestPayer !== null) { - $requestArgs['RequestPayer'] = $this->requestPayer; - } - - if ($this->sseCustomerAlgorithm !== null) { - $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; - } - - if ($this->sseCustomerKey !== null) { - $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; - } + $requestArgs = [ + 'Bucket' => $this->bucket, + 'ChecksumAlgorithm' => $this->checksumAlgorithm, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'Key' => $this->key, + 'RequestPayer' => $this->requestPayer, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKey' => $this->sseCustomerKey, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + ]; - if ($this->sseCustomerKeyMD5 !== null) { - $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; - } + remove_nulls_from_array($requestArgs); return $requestArgs; } @@ -776,63 +682,24 @@ public function toUploadPartRequest(): array */ public function toCompleteMultipartUploadRequest(): array { - $requestArgs = []; - - if ($this->bucket !== null) { - $requestArgs['Bucket'] = $this->bucket; - } - - if ($this->checksumCRC32 !== null) { - $requestArgs['ChecksumCRC32'] = $this->checksumCRC32; - } - - if ($this->checksumCRC32C !== null) { - $requestArgs['ChecksumCRC32C'] = $this->checksumCRC32C; - } - - if ($this->checksumCRC64NVME !== null) { - $requestArgs['ChecksumCRC64NVME'] = $this->checksumCRC64NVME; - } - - if ($this->checksumSHA1 !== null) { - $requestArgs['ChecksumSHA1'] = $this->checksumSHA1; - } - - if ($this->checksumSHA256 !== null) { - $requestArgs['ChecksumSHA256'] = $this->checksumSHA256; - } - - if ($this->expectedBucketOwner !== null) { - $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; - } - - if ($this->ifMatch !== null) { - $requestArgs['IfMatch'] = $this->ifMatch; - } - - if ($this->ifNoneMatch !== null) { - $requestArgs['IfNoneMatch'] = $this->ifNoneMatch; - } - - if ($this->key !== null) { - $requestArgs['Key'] = $this->key; - } - - if ($this->requestPayer !== null) { - $requestArgs['RequestPayer'] = $this->requestPayer; - } - - if ($this->sseCustomerAlgorithm !== null) { - $requestArgs['SSECustomerAlgorithm'] = $this->sseCustomerAlgorithm; - } - - if ($this->sseCustomerKey !== null) { - $requestArgs['SSECustomerKey'] = $this->sseCustomerKey; - } + $requestArgs = [ + 'Bucket' => $this->bucket, + 'ChecksumCRC32' => $this->checksumCRC32, + 'ChecksumCRC32C' => $this->checksumCRC32C, + 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, + 'ChecksumSHA1' => $this->checksumSHA1, + 'ChecksumSHA256' => $this->checksumSHA256, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'IfMatch' => $this->ifMatch, + 'IfNoneMatch' => $this->ifNoneMatch, + 'Key' => $this->key, + 'RequestPayer' => $this->requestPayer, + 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, + 'SSECustomerKey' => $this->sseCustomerKey, + 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, + ]; - if ($this->sseCustomerKeyMD5 !== null) { - $requestArgs['SSECustomerKeyMD5'] = $this->sseCustomerKeyMD5; - } + remove_nulls_from_array($requestArgs); return $requestArgs; } @@ -844,23 +711,14 @@ public function toCompleteMultipartUploadRequest(): array */ public function toAbortMultipartRequest(): array { - $requestArgs = []; - - if ($this->bucket !== null) { - $requestArgs['Bucket'] = $this->bucket; - } - - if ($this->expectedBucketOwner !== null) { - $requestArgs['ExpectedBucketOwner'] = $this->expectedBucketOwner; - } - - if ($this->key !== null) { - $requestArgs['Key'] = $this->key; - } + $requestArgs = [ + 'Bucket' => $this->bucket, + 'ExpectedBucketOwner' => $this->expectedBucketOwner, + 'Key' => $this->key, + 'RequestPayer' => $this->requestPayer, + ]; - if ($this->requestPayer !== null) { - $requestArgs['RequestPayer'] = $this->requestPayer; - } + remove_nulls_from_array($requestArgs); return $requestArgs; } @@ -872,7 +730,7 @@ public function toAbortMultipartRequest(): array */ public function toArray(): array { - return [ + $array = [ 'ACL' => $this->acl, 'Bucket' => $this->bucket, 'BucketKeyEnabled' => $this->bucketKeyEnabled, @@ -911,54 +769,9 @@ public function toArray(): array 'IfMatch' => $this->ifMatch, 'IfNoneMatch' => $this->ifNoneMatch, ]; - } - /** - * Create instance from array - * - * @param array $data - * @return static - */ - public static function fromArray(array $data): static - { - return new static( - $data['ACL'] ?? null, - $data['Bucket'] ?? null, - $data['BucketKeyEnabled'] ?? null, - $data['CacheControl'] ?? null, - $data['ChecksumAlgorithm'] ?? null, - $data['ContentDisposition'] ?? null, - $data['ContentEncoding'] ?? null, - $data['ContentLanguage'] ?? null, - $data['ContentType'] ?? null, - $data['ExpectedBucketOwner'] ?? null, - $data['Expires'] ?? null, - $data['GrantFullControl'] ?? null, - $data['GrantRead'] ?? null, - $data['GrantReadACP'] ?? null, - $data['GrantWriteACP'] ?? null, - $data['Key'] ?? null, - $data['Metadata'] ?? null, - $data['ObjectLockLegalHoldStatus'] ?? null, - $data['ObjectLockMode'] ?? null, - $data['ObjectLockRetainUntilDate'] ?? null, - $data['RequestPayer'] ?? null, - $data['SSECustomerAlgorithm'] ?? null, - $data['SSECustomerKey'] ?? null, - $data['SSECustomerKeyMD5'] ?? null, - $data['SSEKMSEncryptionContext'] ?? null, - $data['SSEKMSKeyId'] ?? null, - $data['ServerSideEncryption'] ?? null, - $data['StorageClass'] ?? null, - $data['Tagging'] ?? null, - $data['WebsiteRedirectLocation'] ?? null, - $data['ChecksumCRC32'] ?? null, - $data['ChecksumCRC32C'] ?? null, - $data['ChecksumCRC64NVME'] ?? null, - $data['ChecksumSHA1'] ?? null, - $data['ChecksumSHA256'] ?? null, - $data['IfMatch'] ?? null, - $data['IfNoneMatch'] ?? null - ); + remove_nulls_from_array($array); + + return $array; } } \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/PutObjectResponse.php b/src/S3/S3Transfer/Models/PutObjectResponse.php index f920418d9d..0c33ee8cd4 100644 --- a/src/S3/S3Transfer/Models/PutObjectResponse.php +++ b/src/S3/S3Transfer/Models/PutObjectResponse.php @@ -2,6 +2,8 @@ namespace Aws\S3\S3Transfer\Models; +use function Aws\remove_nulls_from_array; + class PutObjectResponse { /** @var bool|null */ @@ -93,6 +95,35 @@ public function __construct( $this->versionId = $versionId; } + /** + * Create an instance from an array of data + * + * @param array $data + * @return self + */ + public static function fromArray(array $data): self + { + return new self( + $data['BucketKeyEnabled'] ?? null, + $data['ChecksumCRC32'] ?? null, + $data['ChecksumCRC32C'] ?? null, + $data['ChecksumCRC64NVME'] ?? null, + $data['ChecksumSHA1'] ?? null, + $data['ChecksumSHA256'] ?? null, + $data['ChecksumType'] ?? null, + $data['ETag'] ?? null, + $data['Expiration'] ?? null, + $data['RequestCharged'] ?? null, + $data['SSECustomerAlgorithm'] ?? null, + $data['SSECustomerKeyMD5'] ?? null, + $data['SSEKMSEncryptionContext'] ?? null, + $data['SSEKMSKeyId'] ?? null, + $data['ServerSideEncryption'] ?? null, + $data['Size'] ?? null, + $data['VersionId'] ?? null + ); + } + /** * @return bool|null */ @@ -235,7 +266,7 @@ public function getVersionId(): ?string * @return array Array containing AWS S3 response fields with their corresponding values */ public function toMultipartUploadResponse(): array { - return [ + $array = [ 'BucketKeyEnabled' => $this->bucketKeyEnabled, 'ChecksumCRC32' => $this->checksumCRC32, 'ChecksumCRC32C' => $this->checksumCRC32C, @@ -250,6 +281,10 @@ public function toMultipartUploadResponse(): array { 'ServerSideEncryption' => $this->serverSideEncryption, 'VersionId' => $this->versionId ]; + + remove_nulls_from_array($array); + + return $array; } /** @@ -258,7 +293,7 @@ public function toMultipartUploadResponse(): array { * @return array Array containing AWS S3 response fields with their corresponding values */ public function toSingleUploadResponse(): array { - return [ + $array = [ 'BucketKeyEnabled' => $this->bucketKeyEnabled, 'ChecksumCRC32' => $this->checksumCRC32, 'ChecksumCRC32C' => $this->checksumCRC32C, @@ -277,34 +312,9 @@ public function toSingleUploadResponse(): array { 'Size' => $this->size, 'VersionId' => $this->versionId ]; - } - /** - * Create an instance from an array of data - * - * @param array $data - * @return self - */ - public static function fromArray(array $data): self - { - return new self( - $data['BucketKeyEnabled'] ?? null, - $data['ChecksumCRC32'] ?? null, - $data['ChecksumCRC32C'] ?? null, - $data['ChecksumCRC64NVME'] ?? null, - $data['ChecksumSHA1'] ?? null, - $data['ChecksumSHA256'] ?? null, - $data['ChecksumType'] ?? null, - $data['ETag'] ?? null, - $data['Expiration'] ?? null, - $data['RequestCharged'] ?? null, - $data['SSECustomerAlgorithm'] ?? null, - $data['SSECustomerKeyMD5'] ?? null, - $data['SSEKMSEncryptionContext'] ?? null, - $data['SSEKMSKeyId'] ?? null, - $data['ServerSideEncryption'] ?? null, - $data['Size'] ?? null, - $data['VersionId'] ?? null - ); + remove_nulls_from_array($array); + + return $array; } } \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php index dfd5bd3e79..53bc161a00 100644 --- a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -8,7 +8,7 @@ class S3TransferManagerConfig private const DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES = 16777216; // 16MB public const DEFAULT_REQUEST_CHECKSUM_CALCULATION = 'when_supported'; private const DEFAULT_RESPONSE_CHECKSUM_VALIDATION = 'when_supported'; - private const DEFAULT_MULTIPART_DOWNLOAD_TYPE = 'part'; + public const DEFAULT_MULTIPART_DOWNLOAD_TYPE = 'part'; public const DEFAULT_CONCURRENCY = 5; private const DEFAULT_TRACK_PROGRESS = false; private const DEFAULT_REGION = 'us-east-1'; @@ -67,6 +67,43 @@ public function __construct( $this->defaultRegion = $defaultRegion; } + /** $config: + * - target_part_size_bytes: (int, default=(8388608 `8MB`)) + * The minimum part size to be used in a multipart upload/download. + * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) + * The threshold to decided whether a multipart upload is needed. + * - request_checksum_calculation: (string, default=`when_supported`) + * To decide whether a checksum validation will be applied to the response. + * - request_checksum_validation: (string, default=`when_supported`) + * - multipart_download_type: (string, default='part') + * The download type to be used in a multipart download. + * - concurrency: (int, default=5) + * Maximum number of concurrent operations allowed during a multipart + * upload/download. + * - track_progress: (bool, default=false) + * To enable progress tracker in a multipart upload/download, and or + * a directory upload/download operation. + * - default_region: (string, default="us-east-2") + */ + public static function fromArray(array $config): self { + return new self( + $config['target_part_size_bytes'] + ?? self::DEFAULT_TARGET_PART_SIZE_BYTES, + $config['multipart_upload_threshold_bytes'] + ?? self::DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES, + 'request_checksum_calculation' + ?? self::DEFAULT_REQUEST_CHECKSUM_CALCULATION, + $config['response_checksum_validation'] + ?? self::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, + $config['multipart_download_type'] + ?? self::DEFAULT_MULTIPART_DOWNLOAD_TYPE, + $config['concurrency'] + ?? self::DEFAULT_CONCURRENCY, + $config['track_progress'] ?? self::DEFAULT_TRACK_PROGRESS, + $config['default_region'] ?? self::DEFAULT_REGION + ); + } + /** * @return int */ @@ -146,42 +183,4 @@ public function toArray(): array { 'default_region' => $this->defaultRegion, ]; } - - - /** $config: - * - target_part_size_bytes: (int, default=(8388608 `8MB`)) - * The minimum part size to be used in a multipart upload/download. - * - multipart_upload_threshold_bytes: (int, default=(16777216 `16 MB`)) - * The threshold to decided whether a multipart upload is needed. - * - request_checksum_calculation: (string, default=`when_supported`) - * To decide whether a checksum validation will be applied to the response. - * - request_checksum_validation: (string, default=`when_supported`) - * - multipart_download_type: (string, default='part') - * The download type to be used in a multipart download. - * - concurrency: (int, default=5) - * Maximum number of concurrent operations allowed during a multipart - * upload/download. - * - track_progress: (bool, default=false) - * To enable progress tracker in a multipart upload/download, and or - * a directory upload/download operation. - * - default_region: (string, default="us-east-2") - */ - public static function fromArray(array $config): self { - return new self( - $config['target_part_size_bytes'] - ?? self::DEFAULT_TARGET_PART_SIZE_BYTES, - $config['multipart_upload_threshold_bytes'] - ?? self::DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES, - 'request_checksum_calculation' - ?? self::DEFAULT_REQUEST_CHECKSUM_CALCULATION, - $config['response_checksum_validation'] - ?? self::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, - $config['multipart_download_type'] - ?? self::DEFAULT_MULTIPART_DOWNLOAD_TYPE, - $config['concurrency'] - ?? self::DEFAULT_CONCURRENCY, - $config['track_progress'] ?? self::DEFAULT_TRACK_PROGRESS, - $config['default_region'] ?? self::DEFAULT_REGION - ); - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php new file mode 100644 index 0000000000..1f91247c3f --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -0,0 +1,123 @@ +sourceDirectory = $sourceDirectory; + if (ArnParser::isArn($targetBucket)) { + $targetBucket = ArnParser::parse($targetBucket)->getResource(); + } + $this->targetBucket = $targetBucket; + $this->putObjectRequest = $putObjectRequest; + $this->config = $config; + } + + /** + * @param string $sourceDirectory + * @param string $targetBucket + * @param array $uploadDirectoryRequestArgs + * @param array $config + * @param array $listeners + * @param TransferListener|null $progressTracker + * + * @return UploadDirectoryRequest + */ + public static function fromLegacyArgs( + string $sourceDirectory, + string $targetBucket, + array $uploadDirectoryRequestArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null, + ): UploadDirectoryRequest { + return new self( + $sourceDirectory, + $targetBucket, + PutObjectRequest::fromArray($uploadDirectoryRequestArgs), + UploadDirectoryRequestConfig::fromArray($config), + $listeners, + $progressTracker + ); + } + + /** + * @return string + */ + public function getSourceDirectory(): string + { + return $this->sourceDirectory; + } + + /** + * @return string + */ + public function getTargetBucket(): string + { + return $this->targetBucket; + } + + /** + * @return PutObjectRequest + */ + public function getPutObjectRequest(): PutObjectRequest + { + return $this->putObjectRequest; + } + + /** + * @return UploadDirectoryRequestConfig + */ + public function getConfig(): UploadDirectoryRequestConfig + { + return $this->config; + } + + /** + * Helper method to validate source directory + * @return void + */ + public function validateSourceDirectory(): void + { + if (!is_dir($this->sourceDirectory)) { + throw new InvalidArgumentException( + "Please provide a valid directory path. " + . "Provided = " . $this->sourceDirectory + ); + } + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php b/src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php new file mode 100644 index 0000000000..7d52d83903 --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php @@ -0,0 +1,167 @@ +followSymbolicLinks = $followSymbolicLinks; + $this->recursive = $recursive; + $this->s3Prefix = $s3Prefix; + $this->filter = $filter; + $this->s3Delimiter = $s3Delimiter; + $this->putObjectRequestCallback = $putObjectRequestCallback; + $this->failurePolicy = $failurePolicy; + } + + /* + * @param array $config The config options for this request that are: + * - follow_symbolic_links: (bool, optional, defaulted to false) + * - recursive: (bool, optional, defaulted to false) + * - s3_prefix: (string, optional, defaulted to null) + * - filter: (Closure(SplFileInfo|string), optional) + * By default an instance of SplFileInfo will be provided, however + * you can annotate the parameter with a string type and by doing + * so you will get the full path of the file. + * - s3_delimiter: (string, optional, defaulted to `/`) + * - put_object_request_callback: (Closure, optional) A callback function + * to be invoked right before the request initiates and that will receive + * as parameter the request arguments for each upload request. + * - failure_policy: (Closure, optional) A function that will be invoked + * on an upload failure and that will receive as parameters: + * - $requestArgs: (array) The arguments for the request that originated + * the failure. + * - $uploadDirectoryRequestArgs: (array) The arguments for the upload + * directory request. + * - $reason: (Throwable) The exception that originated the request failure. + * - $uploadDirectoryResponse: (UploadDirectoryResponse) The upload response + * to that point in the upload process. + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. + */ + public static function fromArray(array $array): UploadDirectoryRequestConfig { + return new self( + $array['follow_symbolic_links'] ?? false, + $array['recursive'] ?? false, + $array['s3_prefix'] ?? null, + $array['filter'] ?? null, + $array['s3_delimiter'] ?? '/', + $array['failure_policy'] ?? null, + $array['track_progress'] ?? false + ); + } + + /** + * @return bool + */ + public function isFollowSymbolicLinks(): bool + { + return $this->followSymbolicLinks; + } + + /** + * @return bool + */ + public function isRecursive(): bool + { + return $this->recursive; + } + + /** + * @return string|null + */ + public function getS3Prefix(): ?string + { + return $this->s3Prefix; + } + + /** + * @return Closure|null + */ + public function getFilter(): ?Closure + { + return $this->filter; + } + + /** + * @return string + */ + public function getS3Delimiter(): string + { + return $this->s3Delimiter; + } + + /** + * @return Closure|null + */ + public function getPutObjectRequestCallback(): ?Closure { + return $this->putObjectRequestCallback; + } + + /** + * @return Closure|null + */ + public function getFailurePolicy(): ?Closure + { + return $this->failurePolicy; + } + + /** + * @return array + */ + public function toArray(): array { + return [ + 'follow_symbolic_links' => $this->followSymbolicLinks, + 'recursive' => $this->recursive, + 's3_prefix' => $this->s3Prefix, + 'filter' => $this->filter, + 's3_delimiter' => $this->s3Delimiter, + 'failure_policy' => $this->failurePolicy, + 'track_progress' => $this->trackProgress, + ]; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 6749728b6b..809a3a6521 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -37,6 +37,41 @@ public function __construct( $this->config = $config; } + /** + * @param string|StreamInterface $source + * @param array $requestArgs The putObject request arguments. + * Required parameters would be: + * - Bucket: (string, required) + * - Key: (string, required) + * @param array $config The config options for this upload operation. + * - multipart_upload_threshold_bytes: (int, optional) + * To override the default threshold for when to use multipart upload. + * - target_part_size_bytes: (int, optional) To override the default + * target part size in bytes. + * - track_progress: (bool, optional) To override the default option for + * enabling progress tracking. If this option is resolved as true and + * a progressTracker parameter is not provided then, a default implementation + * will be resolved. This option is intended to make the operation to use + * a default progress tracker implementation when $progressTracker is null. + * @param TransferListener[]|null $listeners + * @param TransferListener|null $progressTracker + * + * @return UploadRequest + */ + public static function fromLegacyArgs(string | StreamInterface $source, + array $requestArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null): UploadRequest { + return new UploadRequest( + $source, + PutObjectRequest::fromArray($requestArgs), + UploadRequestConfig::fromArray($config), + $listeners, + $progressTracker + ); + } + /** * Get the source. * @@ -102,39 +137,4 @@ public function validateRequiredParameters( } } } - - /** - * @param string|StreamInterface $source - * @param array $requestArgs The putObject request arguments. - * Required parameters would be: - * - Bucket: (string, required) - * - Key: (string, required) - * @param array $config The config options for this upload operation. - * - multipart_upload_threshold_bytes: (int, optional) - * To override the default threshold for when to use multipart upload. - * - target_part_size_bytes: (int, optional) To override the default - * target part size in bytes. - * - track_progress: (bool, optional) To override the default option for - * enabling progress tracking. If this option is resolved as true and - * a progressTracker parameter is not provided then, a default implementation - * will be resolved. This option is intended to make the operation to use - * a default progress tracker implementation when $progressTracker is null. - * @param TransferListener[]|null $listeners - * @param TransferListener|null $progressTracker - * - * @return UploadRequest - */ - public static function fromLegacyArgs(string | StreamInterface $source, - array $requestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null): UploadRequest { - return new UploadRequest( - $source, - PutObjectRequest::fromArray($requestArgs), - UploadRequestConfig::fromArray($config), - $listeners, - $progressTracker - ); - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadRequestConfig.php b/src/S3/S3Transfer/Models/UploadRequestConfig.php index 6a57bf97c2..cedb0e6c5a 100644 --- a/src/S3/S3Transfer/Models/UploadRequestConfig.php +++ b/src/S3/S3Transfer/Models/UploadRequestConfig.php @@ -21,7 +21,7 @@ class UploadRequestConfig extends TransferRequestConfig */ private ?int $targetPartSizeBytes; - /** @var int */ + /** @var int|null */ private ?int $concurrency; /** @var string|null */ @@ -48,6 +48,24 @@ public function __construct( $this->requestChecksumCalculation = $requestChecksumCalculation; } + /** + * Create an UploadConfig instance from an array + * + * @param array $data Array containing configuration data + * + * @return self + */ + public static function fromArray(array $data): self + { + return new self( + $data['multipart_upload_threshold_bytes'] ?? null, + $data['target_part_size_bytes'] ?? null, + $data['track_progress'] ?? null, + $data['concurrency'] ?? null, + $data['request_checksum_calculation'] ?? null, + ); + } + /** * Get the multipart upload threshold in bytes * @@ -99,22 +117,4 @@ public function toArray(): array 'request_checksum_calculation' => $this->requestChecksumCalculation, ]; } - - /** - * Create an UploadConfig instance from an array - * - * @param array $data Array containing configuration data - * - * @return static - */ - public static function fromArray(array $data): self - { - return new self( - $data['multipart_upload_threshold_bytes'] ?? null, - $data['target_part_size_bytes'] ?? null, - $data['track_progress'] ?? null, - $data['concurrency'] ?? null, - $data['request_checksum_calculation'] ?? null, - ); - } } \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 2c8f8d15fe..9ba454e3a1 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -5,7 +5,12 @@ use Aws\CommandInterface; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadHandler; use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\GetObjectRequest; +use Aws\S3\S3Transfer\Models\GetObjectResponse; +use Aws\S3\S3Transfer\Models\MultipartDownloaderConfig; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; @@ -13,18 +18,22 @@ use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Promise\PromisorInterface; -use GuzzleHttp\Psr7\Utils; -use Psr\Http\Message\StreamInterface; abstract class MultipartDownloader implements PromisorInterface { public const GET_OBJECT_COMMAND = "GetObject"; public const PART_GET_MULTIPART_DOWNLOADER = "partGet"; - public const RANGE_GET_MULTIPART_DOWNLOADER = "rangeGet"; + public const RANGED_GET_MULTIPART_DOWNLOADER = "rangedGet"; private const OBJECT_SIZE_REGEX = "/\/(\d+)$/"; - /** @var array */ - protected array $requestArgs; + /** @var GetObjectRequest */ + protected GetObjectRequest $getObjectRequest; + + /** @var MultipartDownloaderConfig */ + protected readonly MultipartDownloaderConfig $config; + + /** @var DownloadHandler */ + private DownloadHandler $downloadHandler; /** @var int */ protected int $currentPartNo; @@ -38,9 +47,6 @@ abstract class MultipartDownloader implements PromisorInterface /** @var string */ protected string $eTag; - /** @var StreamInterface */ - private StreamInterface $stream; - /** @var TransferListenerNotifier | null */ private readonly ?TransferListenerNotifier $listenerNotifier; @@ -49,45 +55,41 @@ abstract class MultipartDownloader implements PromisorInterface /** * @param S3ClientInterface $s3Client - * @param array $requestArgs - * @param array $config - * - minimum_part_size: The minimum part size for a multipart download - * using range get. This option MUST be set when using range get. + * @param GetObjectRequest $getObjectRequest + * @param MultipartDownloaderConfig $config + * @param DownloadHandler $downloadHandler * @param int $currentPartNo * @param int $objectPartsCount * @param int $objectSizeInBytes * @param string $eTag - * @param StreamInterface|null $stream * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier */ public function __construct( protected readonly S3ClientInterface $s3Client, - array $requestArgs, - protected readonly array $config = [], + GetObjectRequest $getObjectRequest, + MultipartDownloaderConfig $config, + DownloadHandler $downloadHandler, int $currentPartNo = 0, int $objectPartsCount = 0, int $objectSizeInBytes = 0, string $eTag = "", - ?StreamInterface $stream = null, ?TransferProgressSnapshot $currentSnapshot = null, - ?TransferListenerNotifier $listenerNotifier = null, + ?TransferListenerNotifier $listenerNotifier = null ) { - $this->requestArgs = $requestArgs; + $this->getObjectRequest = $getObjectRequest; + $this->config = $config; + $this->downloadHandler = $downloadHandler; $this->currentPartNo = $currentPartNo; $this->objectPartsCount = $objectPartsCount; $this->objectSizeInBytes = $objectSizeInBytes; $this->eTag = $eTag; - if ($stream === null) { - $this->stream = Utils::streamFor( - fopen('php://temp', 'w+') - ); - } else { - $this->stream = $stream; - // Position at the end of the stream - $this->stream->seek($stream->getSize()); - } $this->currentSnapshot = $currentSnapshot; + if ($listenerNotifier === null) { + $listenerNotifier = new TransferListenerNotifier(); + } + // Add download handler to the listener notifier + $listenerNotifier->addListener($downloadHandler); $this->listenerNotifier = $listenerNotifier; } @@ -139,30 +141,19 @@ public function download(): DownloadResponse { public function promise(): PromiseInterface { return Coroutine::of(function () { - $this->downloadInitiated($this->requestArgs); - $result = ['@metadata'=>[]]; try { - $result = yield $this->s3Client->executeAsync($this->nextCommand()) - ->then(function (ResultInterface $result) { - // Calculate object size and parts count. - $this->computeObjectDimensions($result); - // Trigger first part completed - $this->partDownloadCompleted($result); - - return $result; - })->otherwise(function ($reason) { - $this->partDownloadFailed($reason); - - throw $reason; - }); - } catch (\Throwable $e) { - $this->downloadFailed($e); - // TODO: yield transfer exception modeled with a transfer failed response. - yield Create::rejectionFor($e); - } + $initialRequestResult = yield $this->initialRequest(); + $prevPartNo = $this->currentPartNo - 1; + while ($this->currentPartNo < $this->objectPartsCount) { + // To prevent infinite loops + if ($prevPartNo === $this->currentPartNo) { + throw new S3TransferException( + "Current part `$this->currentPartNo` MUST increment." + ); + } + + $prevPartNo = $this->currentPartNo; - while ($this->currentPartNo < $this->objectPartsCount) { - try { yield $this->s3Client->executeAsync($this->nextCommand()) ->then(function ($result) { $this->partDownloadCompleted($result); @@ -173,23 +164,57 @@ public function promise(): PromiseInterface throw $reason; }); - } catch (\Throwable $reason) { - $this->downloadFailed($reason); - // TODO: yield transfer exception modeled with a transfer failed response. - yield Create::rejectionFor($reason); } + // Transfer completed + $this->downloadComplete(); + + // Return response + yield Create::promiseFor(new DownloadResponse( + $this->downloadHandler->getHandlerResult(), + GetObjectResponse::fromArray( + $initialRequestResult->toArray() + )->toArray(), + )); + } catch (\Throwable $e) { + $this->downloadFailed($e); + yield Create::rejectionFor($e); } + }); + } - // Transfer completed - $this->downloadComplete(); + /** + * Perform the initial download request. + * + * @return PromiseInterface + */ + protected function initialRequest(): PromiseInterface { + $command = $this->nextCommand(); + // Notify download initiated + $this->downloadInitiated($command->toArray()); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + // Compute object dimensions such as parts count and object size + $this->computeObjectDimensions($result); + + // If a multipart is likely to happen then save the ETag + if ($this->objectPartsCount > 1) { + $this->eTag = $result['ETag']; + } - unset($result['Body']); - yield Create::promiseFor(new DownloadResponse( - $this->stream, - $result['@metadata'] ?? [] - )); - }); + // Notify listeners + $this->partDownloadCompleted($result); + + // Assign custom fields in the result + $result['ContentLength'] = $this->objectSizeInBytes; + + return $result; + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); + + throw $reason; + }); } /** @@ -197,7 +222,7 @@ public function promise(): PromiseInterface * * @return CommandInterface */ - abstract protected function nextCommand() : CommandInterface; + abstract protected function nextCommand(): CommandInterface; /** * Compute the object dimensions, such as size and parts count. @@ -209,28 +234,28 @@ abstract protected function nextCommand() : CommandInterface; abstract protected function computeObjectDimensions(ResultInterface $result): void; /** - * Calculates the object size dynamically. + * Calculates the object size from content range. * - * @param $sizeSource + * @param string $sizeSource * * @return int */ - protected function computeObjectSize($sizeSource): int + protected function computeObjectSizeFromContentRange( + string $contentRange + ): int { - if (is_int($sizeSource)) { - return (int) $sizeSource; - } - - if (empty($sizeSource)) { + if (empty($contentRange)) { return 0; } // For extracting the object size from the ContentRange header value. - if (preg_match(self::OBJECT_SIZE_REGEX, $sizeSource, $matches)) { + if (preg_match(self::OBJECT_SIZE_REGEX, $contentRange, $matches)) { return $matches[1]; } - throw new \RuntimeException('Invalid source size format'); + throw new S3TransferException( + "Invalid content range \"$contentRange\"" + ); } /** @@ -287,9 +312,9 @@ private function downloadFailed(\Throwable $reason): void $this->currentSnapshot->getResponse(), $reason ); - $this->stream->close(); + $this->listenerNotifier?->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequest->toArray(), TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); @@ -298,9 +323,6 @@ private function downloadFailed(\Throwable $reason): void /** * Propagates part-download-completed to listeners. * It also does some computation in order to maintain internal states. - * In this specific method we move each part content into an accumulative - * stream, which is meant to hold the full object content once the download - * is completed. * * @param ResultInterface $result * @@ -314,7 +336,7 @@ private function partDownloadCompleted( if (isset($result['ETag'])) { $this->eTag = $result['ETag']; } - Utils::copyToStream($result['Body'], $this->stream); + $newSnapshot = new TransferProgressSnapshot( $this->currentSnapshot->getIdentifier(), $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, @@ -323,7 +345,7 @@ private function partDownloadCompleted( ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->bytesTransferred([ - TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequest->toArray(), TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -344,14 +366,11 @@ private function partDownloadFailed( /** * Propagates object-download-completed event to listeners. - * It also resets the pointer of the stream to the first position, - * so that the stream is ready to be consumed once returned. * * @return void */ private function downloadComplete(): void { - $this->stream->rewind(); $newSnapshot = new TransferProgressSnapshot( $this->currentSnapshot->getIdentifier(), $this->currentSnapshot->getTransferredBytes(), @@ -360,7 +379,7 @@ private function downloadComplete(): void ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->transferComplete([ - TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequest->toArray(), TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -376,13 +395,13 @@ public static function chooseDownloaderClass( { return match ($multipartDownloadType) { MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + MultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, default => throw new \InvalidArgumentException( "The config value for `multipart_download_type` must be one of:\n" . "\t* " . MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER ."\n" - . "\t* " . MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER + . "\t* " . MultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER ) }; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/MultipartDownloaderInitial.php b/src/S3/S3Transfer/MultipartDownloaderInitial.php new file mode 100644 index 0000000000..8da9fc8176 --- /dev/null +++ b/src/S3/S3Transfer/MultipartDownloaderInitial.php @@ -0,0 +1,388 @@ +requestArgs = $requestArgs; + $this->currentPartNo = $currentPartNo; + $this->objectPartsCount = $objectPartsCount; + $this->objectSizeInBytes = $objectSizeInBytes; + $this->eTag = $eTag; + if ($stream === null) { + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } else { + $this->stream = $stream; + // Position at the end of the stream + $this->stream->seek($stream->getSize()); + } + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; + } + + /** + * @return int + */ + public function getCurrentPartNo(): int + { + return $this->currentPartNo; + } + + /** + * @return int + */ + public function getObjectPartsCount(): int + { + return $this->objectPartsCount; + } + + /** + * @return int + */ + public function getObjectSizeInBytes(): int + { + return $this->objectSizeInBytes; + } + + /** + * @return TransferProgressSnapshot + */ + public function getCurrentSnapshot(): TransferProgressSnapshot + { + return $this->currentSnapshot; + } + + /** + * @return DownloadResponse + */ + public function download(): DownloadResponse { + return $this->promise()->wait(); + } + + /** + * Returns that resolves a multipart download operation, + * or to a rejection in case of any failures. + * + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + $this->downloadInitiated($this->requestArgs); + $result = ['@metadata'=>[]]; + try { + $result = yield $this->s3Client->executeAsync($this->nextCommand()) + ->then(function (ResultInterface $result) { + // Calculate object size and parts count. + $this->computeObjectDimensions($result); + // Trigger first part completed + $this->partDownloadCompleted($result); + + return $result; + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); + + throw $reason; + }); + } catch (\Throwable $e) { + $this->downloadFailed($e); + // TODO: yield transfer exception modeled with a transfer failed response. + yield Create::rejectionFor($e); + } + + while ($this->currentPartNo < $this->objectPartsCount) { + try { + yield $this->s3Client->executeAsync($this->nextCommand()) + ->then(function ($result) { + $this->partDownloadCompleted($result); + + return $result; + })->otherwise(function ($reason) { + $this->partDownloadFailed($reason); + + throw $reason; + }); + } catch (\Throwable $reason) { + $this->downloadFailed($reason); + // TODO: yield transfer exception modeled with a transfer failed response. + yield Create::rejectionFor($reason); + } + + } + + // Transfer completed + $this->downloadComplete(); + + unset($result['Body']); + yield Create::promiseFor(new DownloadResponse( + $this->stream, + $result['@metadata'] ?? [] + )); + }); + } + + /** + * Returns the next command for fetching the next object part. + * + * @return CommandInterface + */ + abstract protected function nextCommand() : CommandInterface; + + /** + * Compute the object dimensions, such as size and parts count. + * + * @param ResultInterface $result + * + * @return void + */ + abstract protected function computeObjectDimensions(ResultInterface $result): void; + + /** + * Calculates the object size dynamically. + * + * @param $sizeSource + * + * @return int + */ + protected function computeObjectSize($sizeSource): int + { + if (is_int($sizeSource)) { + return (int) $sizeSource; + } + + if (empty($sizeSource)) { + return 0; + } + + // For extracting the object size from the ContentRange header value. + if (preg_match(self::OBJECT_SIZE_REGEX, $sizeSource, $matches)) { + return $matches[1]; + } + + throw new \RuntimeException('Invalid source size format'); + } + + /** + * Main purpose of this method is to propagate + * the download-initiated event to listeners, but + * also it does some computation regarding internal states + * that need to be maintained. + * + * @param array $commandArgs + * + * @return void + */ + private function downloadInitiated(array $commandArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $commandArgs['Key'], + 0, + $this->objectSizeInBytes + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse() + ); + } + + $this->listenerNotifier?->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $commandArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * Propagates download-failed event to listeners. + * + * @param \Throwable $reason + * + * @return void + */ + private function downloadFailed(\Throwable $reason): void + { + // Event already propagated. + if ($this->currentSnapshot->getReason() !== null) { + return; + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); + $this->stream->close(); + $this->listenerNotifier?->transferFail([ + TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + 'reason' => $reason, + ]); + } + + /** + * Propagates part-download-completed to listeners. + * It also does some computation in order to maintain internal states. + * In this specific method we move each part content into an accumulative + * stream, which is meant to hold the full object content once the download + * is completed. + * + * @param ResultInterface $result + * + * @return void + */ + private function partDownloadCompleted( + ResultInterface $result + ): void + { + $partDownloadBytes = $result['ContentLength']; + if (isset($result['ETag'])) { + $this->eTag = $result['ETag']; + } + Utils::copyToStream($result['Body'], $this->stream); + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, + $this->objectSizeInBytes, + $result->toArray() + ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * Propagates part-download-failed event to listeners. + * + * @param \Throwable $reason + * + * @return void + */ + private function partDownloadFailed( + \Throwable $reason, + ): void + { + $this->downloadFailed($reason); + } + + /** + * Propagates object-download-completed event to listeners. + * It also resets the pointer of the stream to the first position, + * so that the stream is ready to be consumed once returned. + * + * @return void + */ + private function downloadComplete(): void + { + $this->stream->rewind(); + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->objectSizeInBytes, + $this->currentSnapshot->getResponse() + ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * @param mixed $multipartDownloadType + * + * @return string + */ + public static function chooseDownloaderClass( + string $multipartDownloadType + ): string + { + return match ($multipartDownloadType) { + MultipartDownloaderInitial::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, + MultipartDownloaderInitial::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + default => throw new \InvalidArgumentException( + "The config value for `multipart_download_type` must be one of:\n" + . "\t* " . MultipartDownloaderInitial::PART_GET_MULTIPART_DOWNLOADER + ."\n" + . "\t* " . MultipartDownloaderInitial::RANGE_GET_MULTIPART_DOWNLOADER + ) + }; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/MultipartUploaderInitial.php b/src/S3/S3Transfer/MultipartUploaderInitial.php new file mode 100644 index 0000000000..612e64797a --- /dev/null +++ b/src/S3/S3Transfer/MultipartUploaderInitial.php @@ -0,0 +1,570 @@ +s3Client = $s3Client; + $this->createMultipartArgs = $createMultipartArgs; + $this->validateConfig($config); + $this->config = $config; + $this->body = $this->parseBody($source); + $this->uploadId = $uploadId; + $this->parts = $parts; + $this->currentSnapshot = $currentSnapshot; + $this->listenerNotifier = $listenerNotifier; + } + + /** + * @param array $config + * + * @return void + */ + private function validateConfig(array &$config): void + { + if (isset($config['part_size'])) { + if ($config['part_size'] < self::PART_MIN_SIZE + || $config['part_size'] > self::PART_MAX_SIZE) { + throw new \InvalidArgumentException( + "The config `part_size` value must be between " + . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE + . " but ${config['part_size']} given." + ); + } + } else { + $config['part_size'] = self::PART_MIN_SIZE; + } + } + + /** + * @return string|null + */ + public function getUploadId(): ?string + { + return $this->uploadId; + } + + /** + * @return array + */ + public function getParts(): array + { + return $this->parts; + } + + /** + * @return int + */ + public function getCalculatedObjectSize(): int + { + return $this->calculatedObjectSize; + } + + /** + * @return TransferProgressSnapshot|null + */ + public function getCurrentSnapshot(): ?TransferProgressSnapshot + { + return $this->currentSnapshot; + } + + /** + * @return UploadResponse + */ + public function upload(): UploadResponse { + return $this->promise()->wait(); + } + + /** + * @return PromiseInterface + */ + public function promise(): PromiseInterface + { + return Coroutine::of(function () { + try { + yield $this->createMultipartUpload(); + yield $this->uploadParts(); + $result = yield $this->completeMultipartUpload(); + yield Create::promiseFor( + new UploadResponse($result->toArray()) + ); + } catch (Throwable $e) { + $this->uploadFailed($e); + yield Create::rejectionFor($e); + } finally { + $this->callDeferredFns(); + } + }); + } + + /** + * @return PromiseInterface + */ + private function createMultipartUpload(): PromiseInterface + { + $requestArgs = [...$this->createMultipartArgs]; + $checksum = $this->filterChecksum($requestArgs); + // Customer provided checksum + if ($checksum !== null) { + $requestArgs['ChecksumType'] = 'FULL_OBJECT'; + $requestArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); + $requestArgs['@context']['request_checksum_calculation'] = 'when_required'; + unset($requestArgs[$checksum]); + } + + $this->uploadInitiated($requestArgs); + $command = $this->s3Client->getCommand( + 'CreateMultipartUpload', + $requestArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadId = $result['UploadId']; + + return $result; + }); + } + + /** + * @return PromiseInterface + */ + private function uploadParts(): PromiseInterface + { + $this->calculatedObjectSize = 0; + $partSize = $this->config['part_size']; + $commands = []; + $partNo = count($this->parts); + $baseUploadPartCommandArgs = [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + ]; + // Customer provided checksum + $checksum = $this->filterChecksum($this->createMultipartArgs); + if ($checksum !== null) { + unset($baseUploadPartCommandArgs['ChecksumAlgorithm']); + unset($baseUploadPartCommandArgs[$checksum]); + $baseUploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + } + + while (!$this->body->eof()) { + $partNo++; + $read = $this->body->read($partSize); + // To make sure we do not create an empty part when + // we already reached the end of file. + if (empty($read) && $this->body->eof()) { + break; + } + + $partBody = Utils::streamFor( + $read + ); + $uploadPartCommandArgs = [ + ...$baseUploadPartCommandArgs, + 'PartNumber' => $partNo, + 'ContentLength' => $partBody->getSize(), + ]; + + // To get `requestArgs` when notifying the bytesTransfer listeners. + $uploadPartCommandArgs['requestArgs'] = [...$uploadPartCommandArgs]; + // Attach body + $uploadPartCommandArgs['Body'] = $this->decorateWithHashes( + $partBody, + $uploadPartCommandArgs + ); + $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); + $commands[] = $command; + $this->calculatedObjectSize += $partBody->getSize(); + if ($partNo > self::PART_MAX_NUM) { + return Create::rejectionFor( + "The max number of parts has been exceeded. " . + "Max = " . self::PART_MAX_NUM + ); + } + } + + return (new CommandPool( + $this->s3Client, + $commands, + [ + 'concurrency' => $this->config['concurrency'], + 'fulfilled' => function (ResultInterface $result, $index) + use ($commands) { + $command = $commands[$index]; + $this->collectPart( + $result, + $command + ); + // Part Upload Completed Event + $this->partUploadCompleted( + $command['ContentLength'], + $command['requestArgs'] + ); + }, + 'rejected' => function (Throwable $e) { + $this->partUploadFailed($e); + + throw $e; + } + ] + ))->promise(); + } + + /** + * @return PromiseInterface + */ + private function completeMultipartUpload(): PromiseInterface + { + $this->sortParts(); + $completeMultipartUploadArgs = [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + 'MpuObjectSize' => $this->calculatedObjectSize, + 'MultipartUpload' => [ + 'Parts' => $this->parts, + ] + ]; + $checksum = $this->filterChecksum($completeMultipartUploadArgs); + // Customer provided checksum + if ($checksum !== null) { + $completeMultipartUploadArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); + $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $completeMultipartUploadArgs['@context']['request_checksum_calculation'] = 'when_required'; + } + + $command = $this->s3Client->getCommand( + 'CompleteMultipartUpload', + $completeMultipartUploadArgs + ); + + return $this->s3Client->executeAsync($command) + ->then(function (ResultInterface $result) { + $this->uploadCompleted($result); + + return $result; + }); + } + + /** + * @return PromiseInterface + */ + private function abortMultipartUpload(): PromiseInterface + { + $command = $this->s3Client->getCommand('AbortMultipartUpload', [ + ...$this->createMultipartArgs, + 'UploadId' => $this->uploadId, + ]); + + return $this->s3Client->executeAsync($command); + } + + /** + * @param ResultInterface $result + * @param CommandInterface $command + * + * @return void + */ + private function collectPart( + ResultInterface $result, + CommandInterface $command, + ): void + { + $checksumResult = $command->getName() === 'UploadPart' + ? $result + : $result[$command->getName() . 'Result']; + $partData = [ + 'PartNumber' => $command['PartNumber'], + 'ETag' => $result['ETag'], + ]; + if (isset($command['ChecksumAlgorithm'])) { + $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); + $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; + } + + $this->parts[] = $partData; + } + + /** + * @return void + */ + private function sortParts(): void + { + usort($this->parts, function($partOne, $partTwo) { + return $partOne['PartNumber'] <=> $partTwo['PartNumber']; + }); + } + + /** + * @param string|StreamInterface $source + * + * @return StreamInterface + */ + private function parseBody(string | StreamInterface $source): StreamInterface + { + if (is_string($source)) { + // Make sure the files exists + if (!is_readable($source)) { + throw new \InvalidArgumentException( + "The source for this upload must be either a readable file path or a valid stream." + ); + } + $body = new LazyOpenStream($source, 'r'); + // To make sure the resource is closed. + $this->deferFns[] = function () use ($body) { + $body->close(); + }; + } elseif ($source instanceof StreamInterface) { + $body = $source; + } else { + throw new \InvalidArgumentException( + "The source must be a valid string file path or a StreamInterface." + ); + } + + return $body; + } + + /** + * @param array $requestArgs + * + * @return void + */ + private function uploadInitiated(array $requestArgs): void + { + if ($this->currentSnapshot === null) { + $this->currentSnapshot = new TransferProgressSnapshot( + $requestArgs['Key'], + 0, + $this->body->getSize(), + ); + } else { + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), + ); + } + + $this->listenerNotifier?->transferInitiated([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot + ]); + } + + /** + * @param Throwable $reason + * + * @return void + */ + private function uploadFailed(Throwable $reason): void { + // Event has been already propagated + if ($this->currentSnapshot->getReason() !== null) { + return; + } + + $this->currentSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $reason + ); + + if (!empty($this->uploadId)) { + $this->abortMultipartUpload()->wait(); + } + $this->listenerNotifier?->transferFail([ + TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + 'reason' => $reason, + ]); + } + + /** + * @param ResultInterface $result + * + * @return void + */ + private function uploadCompleted(ResultInterface $result): void { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes(), + $this->currentSnapshot->getTotalBytes(), + $result->toArray(), + $this->currentSnapshot->getReason(), + ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->transferComplete([ + TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + ]); + } + + /** + * @param int $partCompletedBytes + * @param array $requestArgs + * + * @return void + */ + private function partUploadCompleted( + int $partCompletedBytes, + array $requestArgs + ): void + { + $newSnapshot = new TransferProgressSnapshot( + $this->currentSnapshot->getIdentifier(), + $this->currentSnapshot->getTransferredBytes() + $partCompletedBytes, + $this->currentSnapshot->getTotalBytes(), + $this->currentSnapshot->getResponse(), + $this->currentSnapshot->getReason(), + ); + $this->currentSnapshot = $newSnapshot; + $this->listenerNotifier?->bytesTransferred([ + TransferListener::REQUEST_ARGS_KEY => $requestArgs, + TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, + $this->currentSnapshot + ]); + } + + /** + * @param Throwable $reason + * + * @return void + */ + private function partUploadFailed(Throwable $reason): void + { + $this->uploadFailed($reason); + } + + /** + * @return void + */ + private function callDeferredFns(): void + { + foreach ($this->deferFns as $fn) { + $fn(); + } + + $this->deferFns = []; + } + + /** + * Filters a provided checksum if one was provided. + * + * @param array $requestArgs + * + * @return string | null + */ + private function filterChecksum(array $requestArgs):? string + { + static $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return $algorithm; + } + } + + return null; + } + + /** + * @param StreamInterface $stream + * @param array $data + * + * @return StreamInterface + */ + private function decorateWithHashes(StreamInterface $stream, array &$data): StreamInterface + { + // Decorate source with a hashing stream + $hash = new PhpHash('sha256'); + return new HashingStream($stream, $hash, function ($result) use (&$data) { + $data['ContentSHA256'] = bin2hex($result); + }); + } +} diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index bb30b3e952..95b717ce5e 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -11,6 +11,7 @@ */ class PartGetMultipartDownloader extends MultipartDownloader { + /** * @inheritDoc * @@ -24,8 +25,15 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = array_slice($this->requestArgs, 0); + $nextRequestArgs = $this->getObjectRequest->toArray(); $nextRequestArgs['PartNumber'] = $this->currentPartNo; + if ($this->config->getResponseChecksumValidationEnabled()) { + $nextRequestArgs['ChecksumMode'] = 'ENABLED'; + } + + if (!empty($this->eTag)) { + $nextRequestArgs['IfMatch'] = $this->eTag; + } return $this->s3Client->getCommand( self::GET_OBJECT_COMMAND, @@ -46,8 +54,8 @@ protected function computeObjectDimensions(ResultInterface $result): void $this->objectPartsCount = $result['PartsCount']; } - $this->objectSizeInBytes = $this->computeObjectSize( + $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( $result['ContentRange'] ?? "" ); } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php index a4d956872a..d4f455171c 100644 --- a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php +++ b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php @@ -22,6 +22,15 @@ public function __construct(array $listeners = []) $this->listeners = $listeners; } + /** + * @param TransferListener $listener + * + * @return void + */ + public function addListener(TransferListener $listener): void { + $this->listeners[] = $listener; + } + /** * @inheritDoc * @@ -29,7 +38,7 @@ public function __construct(array $listeners = []) */ public function transferInitiated(array $context): void { - foreach ($this->listeners as $name => $listener) { + foreach ($this->listeners as $listener) { $listener->transferInitiated($context); } } @@ -41,7 +50,7 @@ public function transferInitiated(array $context): void */ public function bytesTransferred(array $context): void { - foreach ($this->listeners as $name => $listener) { + foreach ($this->listeners as $listener) { $listener->bytesTransferred($context); } } @@ -53,7 +62,7 @@ public function bytesTransferred(array $context): void */ public function transferComplete(array $context): void { - foreach ($this->listeners as $name => $listener) { + foreach ($this->listeners as $listener) { $listener->transferComplete($context); } } @@ -65,7 +74,7 @@ public function transferComplete(array $context): void */ public function transferFail(array $context): void { - foreach ($this->listeners as $name => $listener) { + foreach ($this->listeners as $listener) { $listener->transferFail($context); } } diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index 1022edc221..d4e96b6a36 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -5,71 +5,9 @@ use Aws\CommandInterface; use Aws\Result; use Aws\ResultInterface; -use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Exceptions\S3TransferException; -use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; -use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; -use Psr\Http\Message\StreamInterface; class RangeGetMultipartDownloader extends MultipartDownloader { - - /** @var int */ - private int $partSize; - - /** - * @param S3ClientInterface $s3Client - * @param array $requestArgs - * @param array $config - * - minimum_part_size: The minimum part size for a multipart download - * using range get. This option MUST be set when using range get. - * @param int $currentPartNo - * @param int $objectPartsCount - * @param int $objectSizeInBytes - * @param string $eTag - * @param StreamInterface|null $stream - * @param TransferProgressSnapshot|null $currentSnapshot - * @param TransferListenerNotifier|null $listenerNotifier - */ - public function __construct( - S3ClientInterface $s3Client, - array $requestArgs, - array $config = [], - int $currentPartNo = 0, - int $objectPartsCount = 0, - int $objectSizeInBytes = 0, - string $eTag = "", - ?StreamInterface $stream = null, - ?TransferProgressSnapshot $currentSnapshot = null, - ?TransferListenerNotifier $listenerNotifier = null, - ) { - parent::__construct( - $s3Client, - $requestArgs, - $config, - $currentPartNo, - $objectPartsCount, - $objectSizeInBytes, - $eTag, - $stream, - $currentSnapshot, - $listenerNotifier, - ); - if (empty($config['minimum_part_size'])) { - throw new S3TransferException( - 'You must provide a valid minimum part size in bytes' - ); - } - $this->partSize = $config['minimum_part_size']; - // If object size is known at instantiation time then, we can compute - // the object dimensions. - if ($this->objectSizeInBytes !== 0) { - $this->computeObjectDimensions( - new Result(['ContentRange' => $this->objectSizeInBytes]) - ); - } - } - /** * @inheritDoc * @@ -77,22 +15,27 @@ public function __construct( */ protected function nextCommand(): CommandInterface { - // If currentPartNo is not know then lets initialize it to 1 - // otherwise just increment it. if ($this->currentPartNo === 0) { $this->currentPartNo = 1; } else { $this->currentPartNo++; } - $nextRequestArgs = [...$this->requestArgs]; - $from = ($this->currentPartNo - 1) * $this->partSize; - $to = ($this->currentPartNo * $this->partSize) - 1; + $nextRequestArgs = $this->getObjectRequest->toArray(); + $partSize = $this->config->getTargetPartSizeBytes(); + $from = ($this->currentPartNo - 1) * $partSize; + $to = ($this->currentPartNo * $partSize) - 1; + if ($this->objectSizeInBytes !== 0) { $to = min($this->objectSizeInBytes, $to); } $nextRequestArgs['Range'] = "bytes=$from-$to"; + + if ($this->config->getResponseChecksumValidationEnabled()) { + $nextRequestArgs['ChecksumMode'] = 'ENABLED'; + } + if (!empty($this->eTag)) { $nextRequestArgs['IfMatch'] = $this->eTag; } @@ -114,18 +57,18 @@ protected function computeObjectDimensions(ResultInterface $result): void { // Assign object size just if needed. if ($this->objectSizeInBytes === 0) { - $this->objectSizeInBytes = $this->computeObjectSize( + $this->objectSizeInBytes = $this->computeObjectSizeFromContentRange( $result['ContentRange'] ?? "" ); } - if ($this->objectSizeInBytes > $this->partSize) { + $partSize = $this->config->getTargetPartSizeBytes(); + if ($this->objectSizeInBytes > $partSize) { $this->objectPartsCount = intval( - ceil($this->objectSizeInBytes / $this->partSize) + ceil($this->objectSizeInBytes / $partSize) ); } else { // Single download since partSize will be set to full object size. - $this->partSize = $this->objectSizeInBytes; $this->objectPartsCount = 1; $this->currentPartNo = 1; } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index bd533b4c90..55e32cb69b 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -6,11 +6,19 @@ use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; +use Aws\S3\S3Transfer\Models\DownloadFileRequest; +use Aws\S3\S3Transfer\Models\DownloadHandler; +use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\GetObjectRequest; +use Aws\S3\S3Transfer\Models\MultipartDownloaderConfig; use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; +use Aws\S3\S3Transfer\Models\PutObjectRequest; use Aws\S3\S3Transfer\Models\PutObjectResponse; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; +use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Models\UploadResponse; @@ -127,101 +135,48 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface } /** - * @param string $sourceDirectory - * @param string $bucketTo - * @param array $uploadDirectoryRequestArgs - * @param array $config The config options for this request that are: - * - follow_symbolic_links: (bool, optional, defaulted to false) - * - recursive: (bool, optional, defaulted to false) - * - s3_prefix: (string, optional, defaulted to null) - * - filter: (Closure(SplFileInfo|string), optional) - * By default an instance of SplFileInfo will be provided, however - * you can annotate the parameter with a string type and by doing - * so you will get the full path of the file. - * - s3_delimiter: (string, optional, defaulted to `/`) - * - put_object_request_callback: (Closure, optional) A callback function - * to be invoked right before the request initiates and that will receive - * as parameter the request arguments for each upload request. - * - failure_policy: (Closure, optional) A function that will be invoked - * on an upload failure and that will receive as parameters: - * - $requestArgs: (array) The arguments for the request that originated - * the failure. - * - $uploadDirectoryRequestArgs: (array) The arguments for the upload - * directory request. - * - $reason: (Throwable) The exception that originated the request failure. - * - $uploadDirectoryResponse: (UploadDirectoryResponse) The upload response - * to that point in the upload process. - * - track_progress: (bool, optional) To override the default option for - * enabling progress tracking. If this option is resolved as true and - * a progressTracker parameter is not provided then, a default implementation - * will be resolved. - * @param TransferListener[]|null $listeners The listeners for watching - * transfer events. Each listener will be cloned per file upload. - * @param TransferListener|null $progressTracker Ideally the progress - * tracker implementation provided here should be able to track multiple - * transfers at once. Please see MultiProgressTracker implementation. - * + * @param UploadDirectoryRequest $uploadDirectoryRequest + * * @return PromiseInterface */ public function uploadDirectory( - string $sourceDirectory, - string $bucketTo, - array $uploadDirectoryRequestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null, + UploadDirectoryRequest $uploadDirectoryRequest, ): PromiseInterface { - if (!is_dir($sourceDirectory)) { - throw new InvalidArgumentException( - "Please provide a valid directory path. " - . "Provided = " . $sourceDirectory - ); - } - - $bucketTo = $this->parseBucket($bucketTo); - + $uploadDirectoryRequest->validateSourceDirectory(); + $targetBucket = $uploadDirectoryRequest->getTargetBucket(); + $config = $uploadDirectoryRequest->getConfig(); + $progressTracker = $uploadDirectoryRequest->getProgressTracker(); if ($progressTracker === null - && ($config['track_progress'] ?? $this->config['track_progress'])) { + && ($config->getTrackProgress() ?? $this->config->isTrackProgress())) { $progressTracker = new MultiProgressTracker(); } $filter = null; - if (isset($config['filter'])) { - if (!is_callable($config['filter'])) { - throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); - } - - $filter = $config['filter']; + if ($config->getFilter() !== null) { + $filter = $config->getFilter(); } $putObjectRequestCallback = null; - if (isset($config['put_object_request_callback'])) { - if (!is_callable($config['put_object_request_callback'])) { - throw new InvalidArgumentException( - "The parameter \$config['put_object_request_callback'] must be callable." - ); - } - - $putObjectRequestCallback = $config['put_object_request_callback']; + if ($config->getPutObjectRequestCallback() !== null) { + $putObjectRequestCallback = $config->getPutObjectRequestCallback(); } $failurePolicyCallback = null; - if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { - throw new InvalidArgumentException( - "The parameter \$config['failure_policy'] must be callable." - ); - } elseif (isset($config['failure_policy'])) { - $failurePolicyCallback = $config['failure_policy']; + if ($config->getFailurePolicy() !== null) { + $failurePolicyCallback = $config->getFailurePolicy(); } - $dirIterator = new RecursiveDirectoryIterator($sourceDirectory); + $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); + $dirIterator = new RecursiveDirectoryIterator( + $sourceDirectory + ); $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); - if (($config['follow_symbolic_links'] ?? false) === true) { + if ($config->isFollowSymbolicLinks()) { $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); } - if (($config['recursive'] ?? false) === true) { + if ($config->isRecursive()) { $dirIterator = new RecursiveIteratorIterator($dirIterator); } @@ -236,11 +191,12 @@ function ($file) use ($filter) { } ); - $prefix = $config['s3_prefix'] ?? ''; + $prefix = $config->getS3Prefix() ?? ''; if ($prefix !== '' && !str_ends_with($prefix, '/')) { $prefix .= '/'; } - $delimiter = $config['s3_delimiter'] ?? '/'; + + $delimiter = $config->getS3Delimiter(); $promises = []; $objectsUploaded = 0; $objectsFailed = 0; @@ -258,31 +214,35 @@ function ($file) use ($filter) { $delimiter, $objectKey ); - $uploadRequestArgs = [ - ...$uploadDirectoryRequestArgs, - 'Bucket' => $bucketTo, - 'Key' => $objectKey, + $putObjectRequestArgs = [ + ...$uploadDirectoryRequest->getPutObjectRequest()->toArray(), + 'Bucket' => $targetBucket, + 'Key' => $objectKey ]; if ($putObjectRequestCallback !== null) { - $putObjectRequestCallback($uploadRequestArgs); + $putObjectRequestCallback($putObjectRequestArgs); } $promises[] = $this->upload( - $file, - $uploadRequestArgs, - $config, - array_map(function ($listener) { return clone $listener; }, $listeners), - $progressTracker, + UploadRequest::fromLegacyArgs( + $file, + $putObjectRequestArgs, + $config->toArray(), + array_map( + function ($listener) { return clone $listener; }, + $uploadDirectoryRequest->getListeners() + ), + $progressTracker + ) )->then(function (UploadResponse $response) use (&$objectsUploaded) { $objectsUploaded++; return $response; })->otherwise(function ($reason) use ( - $bucketTo, + $targetBucket, $sourceDirectory, $failurePolicyCallback, - $uploadRequestArgs, - $uploadDirectoryRequestArgs, + $putObjectRequestArgs, &$objectsUploaded, &$objectsFailed ) { @@ -290,11 +250,10 @@ function ($file) use ($filter) { if($failurePolicyCallback !== null) { call_user_func( $failurePolicyCallback, - $uploadRequestArgs, + $putObjectRequestArgs, [ - ...$uploadDirectoryRequestArgs, "source_directory" => $sourceDirectory, - "bucket_to" => $bucketTo, + "bucket_to" => $targetBucket, ], $reason, new UploadDirectoryResponse( @@ -317,222 +276,136 @@ function ($file) use ($filter) { } /** - * @param string|array $source The object to be downloaded from S3. - * It can be either a string with a S3 URI or an array with a Bucket and Key - * properties set. - * @param array $downloadRequestArgs The getObject request arguments to be provided as part - * of each get object operation, except for the bucket and key, which - * are already provided as the source. - * @param array $config The configuration to be used for this operation: - * - multipart_download_type: (string, optional) - * Overrides the resolved value from the transfer manager config. - * - checksum_validation_enabled: (bool, optional) Overrides the resolved - * value from transfer manager config for whether checksum validation - * should be done. This option will be considered just if ChecksumMode - * is not present in the request args. - * - track_progress: (bool) Overrides the config option set in the transfer - * manager instantiation to decide whether transfer progress should be - * tracked. - * - minimum_part_size: (int) The minimum part size in bytes to be used - * in a range multipart download. If this parameter is not provided - * then it fallbacks to the transfer manager `target_part_size_bytes` - * config value. - * @param TransferListener[]|null $listeners - * @param TransferListener|null $progressTracker + * @param DownloadRequest $downloadRequest * * @return PromiseInterface */ - public function download( - string | array $source, - array $downloadRequestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null, - ): PromiseInterface + public function download(DownloadRequest $downloadRequest): PromiseInterface { - if (is_string($source)) { - $sourceArgs = $this->s3UriAsBucketAndKey($source); - } elseif (is_array($source)) { - $sourceArgs = [ - 'Bucket' => $this->requireNonEmpty( - $source, - 'Bucket', - "A valid bucket must be provided." - ), - 'Key' => $this->requireNonEmpty( - $source, - 'Key', - "A valid key must be provided." - ), - ]; - } else { - throw new S3TransferException( - "Unsupported source type `" . gettype($source) . "`" - ); - } + $sourceArgs = $downloadRequest->normalizeSourceAsArray(); + $getObjectRequest = $downloadRequest->getGetObjectRequest(); + $config = [ + 'response_checksum_validation_enabled' => false + ]; + if (empty($getObjectRequest->getChecksumMode())) { + $requestChecksumValidation = + $downloadRequest->getConfig()->getRequestChecksumValidation() + ?? $this->config->getRequestChecksumCalculation(); - if (!isset($downloadRequestArgs['ChecksumMode'])) { - $checksumEnabled = $config['checksum_validation_enabled'] - ?? $this->config['checksum_validation_enabled'] - ?? false; - if ($checksumEnabled) { - $downloadRequestArgs['ChecksumMode'] = 'enabled'; + if ($requestChecksumValidation === 'when_supported') { + $config['response_checksum_validation_enabled'] = true; } + } else { + $config['response_checksum_validation_enabled'] = true; } + $config['multipart_download_type'] = $downloadRequest->getConfig() + ->getMultipartDownloadType() ?? $this->config->getMultipartDownloadType(); + + $progressTracker = $downloadRequest->getProgressTracker(); if ($progressTracker === null - && ($config['track_progress'] ?? $this->config['track_progress'])) { + && ($downloadRequest->getConfig()->getTrackProgress() + ?? $this->getConfig()->isTrackProgress())) { $progressTracker = new SingleProgressTracker(); } + $listeners = $downloadRequest->getListeners(); if ($progressTracker !== null) { $listeners[] = $progressTracker; } + // Build listener notifier for notifying listeners $listenerNotifier = new TransferListenerNotifier($listeners); - $requestArgs = [ - ...$sourceArgs, - ...$downloadRequestArgs, - ]; - if (empty($downloadRequestArgs['PartNumber']) && empty($downloadRequestArgs['Range'])) { - return $this->tryMultipartDownload( - $requestArgs, - [ - 'minimum_part_size' => $config['minimum_part_size'] - ?? $this->config['target_part_size_bytes'], - 'multipart_download_type' => $config['multipart_download_type'] - ?? $this->config['multipart_download_type'], - ], - $listenerNotifier, - ); + + // Assign source + $getObjectRequestArray = $getObjectRequest->toArray(); + foreach ($sourceArgs as $key => $value) { + $getObjectRequestArray[$key] = $value; } - return $this->trySingleDownload($requestArgs, $progressTracker); + return $this->tryMultipartDownload( + GetObjectRequest::fromArray($getObjectRequestArray), + MultipartDownloaderConfig::fromArray($config), + $downloadRequest->getDownloadHandler(), + $listenerNotifier, + ); } /** - * @param string $bucket The bucket from where the files are going to be - * downloaded from. - * @param string $destinationDirectory The destination path where the downloaded - * files will be placed in. - * @param array $downloadDirectoryArgs The getObject request arguments to be provided - * as part of each get object request sent to the service, except for the - * bucket and key which will be resolved internally. - * @param array $config The config options for this download directory operation. - * - s3_prefix: (string, optional) This parameter will be considered just if - * not provided as part of the list_object_v2_args config option. - * - s3_delimiter: (string, optional, defaulted to '/') This parameter will be - * considered just if not provided as part of the list_object_v2_args config - * option. - * - filter: (Closure, optional) A callable which will receive an object key as - * parameter and should return true or false in order to determine - * whether the object should be downloaded. - * - get_object_request_callback: (Closure, optional) A function that will - * be invoked right before the download request is performed and that will - * receive as parameter the request arguments for each request. - * - failure_policy: (Closure, optional) A function that will be invoked - * on a download failure and that will receive as parameters: - * - $requestArgs: (array) The arguments for the request that originated - * the failure. - * - $downloadDirectoryRequestArgs: (array) The arguments for the download - * directory request. - * - $reason: (Throwable) The exception that originated the request failure. - * - $downloadDirectoryResponse: (DownloadDirectoryResponse) The download response - * to that point in the upload process. - * - track_progress: (bool, optional) Overrides the config option set - * in the transfer manager instantiation to decide whether transfer - * progress should be tracked. - * - minimum_part_size: (int, optional) The minimum part size in bytes - * to be used in a range multipart download. - * - list_object_v2_args: (array, optional) The arguments to be included - * as part of the listObjectV2 request in order to fetch the objects to - * be downloaded. The most common arguments would be: - * - MaxKeys: (int) Sets the maximum number of keys returned in the response. - * - Prefix: (string) To limit the response to keys that begin with the - * specified prefix. - * @param TransferListener[] $listeners The listeners for watching - * transfer events. Each listener will be cloned per file upload. - * @param TransferListener|null $progressTracker Ideally the progress - * tracker implementation provided here should be able to track multiple - * transfers at once. Please see MultiProgressTracker implementation. + * @param DownloadFileRequest $downloadFileRequest * * @return PromiseInterface */ - public function downloadDirectory( - string $bucket, - string $destinationDirectory, - array $downloadDirectoryArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null, + public function downloadFile( + DownloadFileRequest $downloadFileRequest ): PromiseInterface { - if (!file_exists($destinationDirectory)) { - throw new InvalidArgumentException( - "Destination directory `$destinationDirectory` MUST exists." - ); - } - - $bucket = $this->parseBucket($bucket); + return $this->download($downloadFileRequest->getDownloadRequest()); + } + /** + * @param DownloadDirectoryRequest $downloadDirectoryRequest + * + * @return PromiseInterface + */ + public function downloadDirectory( + DownloadDirectoryRequest $downloadDirectoryRequest + ): PromiseInterface + { + $downloadDirectoryRequest->validateDestinationDirectory(); + $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); + $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); + $progressTracker = $downloadDirectoryRequest->getProgressTracker(); + $config = $downloadDirectoryRequest->getConfig(); if ($progressTracker === null - && ($config['track_progress'] ?? $this->config['track_progress'])) { + && ($config->getTrackProgress() ?? $this->config->isTrackProgress())) { $progressTracker = new MultiProgressTracker(); } $listArgs = [ - 'Bucket' => $bucket - ] + ($config['list_object_v2_args'] ?? []); - if (isset($config['s3_prefix']) && !isset($listArgs['Prefix'])) { - $listArgs['Prefix'] = $config['s3_prefix']; + 'Bucket' => $sourceBucket, + ] + ($config->getListObjectV2Args()); + $s3Prefix = $config->getEffectivePrefix(); + if ($s3Prefix !== null) { + $listArgs['Prefix'] = $s3Prefix; } - if (isset($config['s3_delimiter']) && !isset($listArgs['Delimiter'])) { - $listArgs['Delimiter'] = $config['s3_delimiter']; - } + $listArgs['Delimiter'] = $listArgs['Delimiter'] ?? null; $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) ->search('Contents[].Key'); - if (isset($config['filter'])) { - if (!is_callable($config['filter'])) { - throw new InvalidArgumentException("The parameter \$config['filter'] must be callable."); - } - $filter = $config['filter']; + $filter = $config->getFilter(); + if ($filter !== null) { $objects = filter($objects, function (string $key) use ($filter) { - return call_user_func($filter, $key); + return call_user_func($filter, $key) && !str_ends_with($key, "/"); + }); + } else { + $objects = filter($objects, function (string $key) use ($filter) { + return !str_ends_with($key, "/"); }); } - $objects = map($objects, function (string $key) use ($bucket) { - return "s3://$bucket/$key"; + $objects = map($objects, function (string $key) use ($sourceBucket) { + return self::formatAsS3URI($sourceBucket, $key); }); $getObjectRequestCallback = null; - if (isset($config['get_object_request_callback'])) { - if (!is_callable($config['get_object_request_callback'])) { - throw new InvalidArgumentException( - "The parameter \$config['get_object_request_callback'] must be callable." - ); - } - - $getObjectRequestCallback = $config['get_object_request_callback']; + if ($config->getGetObjectRequestCallback() !== null) { + $getObjectRequestCallback = $config->getGetObjectRequestCallback(); } $failurePolicyCallback = null; - if (isset($config['failure_policy']) && !is_callable($config['failure_policy'])) { - throw new InvalidArgumentException( - "The parameter \$config['failure_policy'] must be callable." - ); - } elseif (isset($config['failure_policy'])) { - $failurePolicyCallback = $config['failure_policy']; + if ($config->getFailurePolicy() !== null) { + $failurePolicyCallback = $config->getFailurePolicy(); } $promises = []; $objectsDownloaded = 0; $objectsFailed = 0; foreach ($objects as $object) { - $objectKey = $this->s3UriAsBucketAndKey($object)['Key']; + $bucketAndKeyArray = self::s3UriAsBucketAndKey($object); + $objectKey = $bucketAndKeyArray['Key']; $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { throw new S3TransferException( @@ -542,39 +415,42 @@ public function downloadDirectory( ); } - $requestArgs = [...$downloadDirectoryArgs]; + $requestArgs = $downloadDirectoryRequest->getGetObjectRequest()->toArray(); + foreach ($bucketAndKeyArray as $key => $value) { + $requestArgs[$key] = $value; + } if ($getObjectRequestCallback !== null) { call_user_func($getObjectRequestCallback, $requestArgs); } - $promises[] = $this->download( - $object, - $requestArgs, - [ - 'minimum_part_size' => $config['minimum_part_size'] ?? 0, - ], - array_map(function ($listener) { return clone $listener; }, $listeners), - $progressTracker, - )->then(function (DownloadResponse $result) use ( - &$objectsDownloaded, - $destinationFile + $promises[] = $this->downloadFile( + new DownloadFileRequest( + $destinationFile, + $config->isFailsWhenDestinationExists(), + DownloadRequest::fromLegacyArgs( + null, + $requestArgs, + [ + 'target_part_size_bytes' => $config->getTargetPartSizeBytes() ?? 0, + ], + null, + array_map( + function ($listener) { return clone $listener; }, + $downloadDirectoryRequest->getListeners() + ), + $progressTracker, + ) + ), + )->then(function () use ( + &$objectsDownloaded ) { - $directory = dirname($destinationFile); - if (!is_dir($directory)) { - mkdir($directory, 0777, true); - } - - file_put_contents($destinationFile, $result->getData()); - // Close the stream - $result->getData()->close(); $objectsDownloaded++; })->otherwise(function ($reason) use ( - $bucket, + $sourceBucket, $destinationDirectory, $failurePolicyCallback, &$objectsDownloaded, &$objectsFailed, - $downloadDirectoryArgs, $requestArgs ) { $objectsFailed++; @@ -583,9 +459,8 @@ public function downloadDirectory( $failurePolicyCallback, $requestArgs, [ - ...$downloadDirectoryArgs, "destination_directory" => $destinationDirectory, - "bucket" => $bucket, + "bucket" => $sourceBucket, ], $reason, new DownloadDirectoryResponse( @@ -601,7 +476,7 @@ public function downloadDirectory( }); } - return Each::ofLimitAll($promises, $this->config['concurrency']) + return Each::ofLimitAll($promises, $this->config->getConcurrency()) ->then(function ($_) use (&$objectsFailed, &$objectsDownloaded) { return new DownloadDirectoryResponse( $objectsDownloaded, @@ -613,110 +488,34 @@ public function downloadDirectory( /** * Tries an object multipart download. * - * @param array $requestArgs - * @param array $config - * - minimum_part_size: (int) The minimum part size in bytes for a - * range multipart download. + * @param GetObjectRequest $getObjectRequest + * @param MultipartDownloaderConfig $config + * @param DownloadHandler $downloadHandler * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartDownload( - array $requestArgs, - array $config = [], + GetObjectRequest $getObjectRequest, + MultipartDownloaderConfig $config, + DownloadHandler $downloadHandler, ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { $downloaderClassName = MultipartDownloader::chooseDownloaderClass( - $config['multipart_download_type'] + $config->getMultipartDownloadType() ); $multipartDownloader = new $downloaderClassName( $this->s3Client, - $requestArgs, + $getObjectRequest, $config, + $downloadHandler, listenerNotifier: $listenerNotifier, ); return $multipartDownloader->promise(); } - /** - * Does a single object download. - * - * @param array $requestArgs - * @param TransferListenerNotifier|null $listenerNotifier - * - * @return PromiseInterface - */ - private function trySingleDownload( - array $requestArgs, - ?TransferListenerNotifier $listenerNotifier = null, - ): PromiseInterface - { - if ($listenerNotifier !== null) { - $listenerNotifier->transferInitiated([ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( - $requestArgs['Key'], - 0, - 0 - ) - ]); - $command = $this->s3Client->getCommand( - MultipartDownloader::GET_OBJECT_COMMAND, - $requestArgs - ); - - return $this->s3Client->executeAsync($command)->then( - function ($result) use ($requestArgs, $listenerNotifier) { - // Notify progress - $progressContext = [ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( - $requestArgs['Key'], - $result['Content-Length'] ?? 0, - $result['Content-Length'] ?? 0, - $result->toArray() - ) - ]; - $listenerNotifier->bytesTransferred($progressContext); - // Notify Completion - $listenerNotifier->transferComplete($progressContext); - - return new DownloadResponse( - data: $result['Body'], - metadata: $result['@metadata'], - ); - } - )->otherwise(function ($reason) use ($requestArgs, $listenerNotifier) { - $listenerNotifier->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => new TransferProgressSnapshot( - $requestArgs['Key'], - 0, - 0, - ), - 'reason' => $reason - ]); - - throw $reason; - }); - } - - $command = $this->s3Client->getCommand( - MultipartDownloader::GET_OBJECT_COMMAND, - $requestArgs - ); - - return $this->s3Client->executeAsync($command) - ->then(function ($result) { - return new DownloadResponse( - data: $result['Body'], - metadata: $result['@metadata'], - ); - }); - } - /** * @param string|StreamInterface $source * @param array $requestArgs @@ -853,7 +652,9 @@ private function requiresMultipartUpload( return $source->getSize() >= $mupThreshold; } - throw new S3TransferException("Unable to determine if a multipart is required"); + throw new S3TransferException( + "Unable to determine if a multipart is required" + ); } /** @@ -868,24 +669,6 @@ private function defaultS3Client(): S3ClientInterface ]); } - /** - * Validates a provided value is not empty, and if so then - * it throws an exception with the provided message. - * @param array $array - * @param string $key - * @param string $message - * - * @return mixed - */ - private function requireNonEmpty(array $array, string $key, string $message): mixed - { - if (empty($array[$key])) { - throw new InvalidArgumentException($message); - } - - return $array[$key]; - } - /** * Validates a string value is a valid S3 URI. * Valid S3 URI Example: S3://mybucket.dev/myobject.txt @@ -894,7 +677,7 @@ private function requireNonEmpty(array $array, string $key, string $message): mi * * @return bool */ - private function isValidS3URI(string $uri): bool + public static function isValidS3URI(string $uri): bool { // in the expression `substr($uri, 5)))` the 5 belongs to the size of `s3://`. return str_starts_with(strtolower($uri), 's3://') @@ -909,10 +692,10 @@ private function isValidS3URI(string $uri): bool * * @return array */ - private function s3UriAsBucketAndKey(string $uri): array + public static function s3UriAsBucketAndKey(string $uri): array { $errorMessage = "Invalid URI: `$uri` provided. \nA valid S3 URI looks as `s3://bucket/key`"; - if (!$this->isValidS3URI($uri)) { + if (!self::isValidS3URI($uri)) { throw new InvalidArgumentException($errorMessage); } @@ -930,19 +713,14 @@ private function s3UriAsBucketAndKey(string $uri): array } /** - * To parse the bucket name when the bucket is provided as an ARN. - * - * @param string $bucket + * @param $bucket + * @param $key * * @return string */ - private function parseBucket(string $bucket): string + private static function formatAsS3URI($bucket, $key): string { - if (ArnParser::isArn($bucket)) { - return ArnParser::parse($bucket)->getResource(); - } - - return $bucket; + return "s3://$bucket/$key"; } /** @@ -978,12 +756,4 @@ private function resolvesOutsideTargetDirectory( return false; } - - /** - * @return array - */ - public static function getDefaultConfig(): array - { - return self::$defaultConfig; - } } diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php new file mode 100644 index 0000000000..6fdda5f04e --- /dev/null +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -0,0 +1,166 @@ +destination = $destination; + $this->failsWhenDestinationExists = $failsWhenDestinationExists; + $this->temporaryDestination = ""; + } + + /** + * @return string + */ + public function getDestination(): string + { + return $this->destination; + } + + /** + * @return bool + */ + public function isFailsWhenDestinationExists(): bool + { + return $this->failsWhenDestinationExists; + } + + /** + * @param array $context + * + * @return void + */ + public function transferInitiated(array $context): void + { + if ($this->failsWhenDestinationExists && file_exists($this->destination)) { + throw new FileDownloadException( + "The destination '$this->destination' already exists." + ); + } elseif (is_dir($this->destination)) { + throw new FileDownloadException( + "The destination '$this->destination' can't be a directory." + ); + } + + // Create directory if necessary + $directory = dirname($this->destination); + if (!is_dir($directory)) { + mkdir($directory, 0777, true); + } + + $uniqueId = self::getUniqueIdentifier(); + $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; + while (file_exists($temporaryName)) { + $uniqueId = self::getUniqueIdentifier(); + $temporaryName = $this->destination . self::TEMP_INFIX . $uniqueId; + } + + // Create the file + file_put_contents($temporaryName, ""); + $this->temporaryDestination = $temporaryName; + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $response = $snapshot->getResponse(); + file_put_contents( + $this->temporaryDestination, + $response['Body'], + FILE_APPEND + ); + } + + /** + * @param array $context + * + * @return void + */ + public function transferComplete(array $context): void + { + // Make sure the file is deleted if exists + if (file_exists($this->destination) && is_file($this->destination)) { + if ($this->failsWhenDestinationExists) { + throw new FileDownloadException( + "The destination '$this->destination' already exists." + ); + } + } + + if (!rename($this->temporaryDestination, $this->destination)) { + throw new FileDownloadException( + "Unable to rename the file `$this->temporaryDestination` to `$this->destination`." + ); + } + } + + /** + * @param array $context + * + * @return void + */ + public function transferFail(array $context): void + { + if (file_exists($this->temporaryDestination)) { + unlink($this->temporaryDestination); + } elseif (file_exists($this->destination) + && !str_contains( + $context[self::REASON_KEY], + "The destination '$this->destination' already exists.") + ) { + unlink($this->destination); + } + } + + /** + * @return string + */ + private static function getUniqueIdentifier(): string + { + $uniqueId = uniqid(); + if (strlen($uniqueId) > self::IDENTIFIER_LENGTH) { + $uniqueId = substr($uniqueId, 0, self::IDENTIFIER_LENGTH); + } else { + $uniqueId = str_pad($uniqueId, self::IDENTIFIER_LENGTH, "0"); + } + + return $uniqueId; + } + + /** + * @return string + */ + public function getHandlerResult(): string + { + return $this->destination; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Utils/StreamDownloadHandler.php b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php new file mode 100644 index 0000000000..ce8194c750 --- /dev/null +++ b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php @@ -0,0 +1,84 @@ +stream = $stream; + } + + /** + * @param array $context + * + * @return void + */ + public function transferInitiated(array $context): void + { + if ($this->stream === null) { + $this->stream = Utils::streamFor( + fopen('php://temp', 'w+') + ); + } else { + $this->stream->seek($this->stream->getSize()); + } + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void + { + $snapshot = $context[TransferListener::PROGRESS_SNAPSHOT_KEY]; + $response = $snapshot->getResponse(); + Utils::copyToStream( + $response['Body'], + $this->stream + ); + } + + /** + * @param array $context + * + * @return void + */ + public function transferComplete(array $context): void + { + $this->stream->rewind(); + } + + /** + * @param array $context + * + * @return void + */ + public function transferFail(array $context): void + { + $this->stream->close(); + $this->stream = null; + } + + /** + * @inheritDoc + * + * @return StreamInterface + */ + public function getHandlerResult(): StreamInterface + { + return $this->stream; + } +} \ No newline at end of file From 02b43f0448dc41b73d4844fad6a6a67ddadb6c8e Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 17 Jul 2025 13:47:24 -0700 Subject: [PATCH 35/62] chore: s3 transfer manager updates - This change adds download handlers when dealing with downloads - Add a new API for downlaoding files "downloadFile" - Create directories by default when downloading files - Overrides destination files if the flag to fail is not enabled - Wrap API parameters in a separated dataclass. For example, UploadRequest - Make responses in download and upload to extends Result, which allows array access in the response. - Others code refactoring, such as test fixing based on new updates, etc. --- .../S3Transfer/AbstractMultipartUploader.php | 121 +-- src/S3/S3Transfer/Data/tm-model-schema.json | 304 ++++++ src/S3/S3Transfer/Data/tm-model.json | 287 ++++++ .../Models/DownloadDirectoryRequest.php | 80 +- .../Models/DownloadDirectoryRequestConfig.php | 184 ---- src/S3/S3Transfer/Models/DownloadRequest.php | 96 +- .../Models/DownloadRequestConfig.php | 92 -- src/S3/S3Transfer/Models/DownloadResponse.php | 31 - src/S3/S3Transfer/Models/DownloadResult.php | 30 + src/S3/S3Transfer/Models/GetObjectRequest.php | 345 ------- .../S3Transfer/Models/GetObjectResponse.php | 629 ------------ .../Models/MultipartDownloaderConfig.php | 81 -- .../Models/MultipartUploaderConfig.php | 86 -- src/S3/S3Transfer/Models/PutObjectRequest.php | 777 -------------- .../S3Transfer/Models/PutObjectResponse.php | 320 ------ .../Models/S3TransferManagerConfig.php | 6 +- src/S3/S3Transfer/Models/TransferRequest.php | 33 +- .../Models/TransferRequestConfig.php | 31 - .../Models/UploadDirectoryRequest.php | 41 +- .../Models/UploadDirectoryRequestConfig.php | 167 --- src/S3/S3Transfer/Models/UploadRequest.php | 90 +- .../S3Transfer/Models/UploadRequestConfig.php | 120 --- src/S3/S3Transfer/Models/UploadResponse.php | 21 - src/S3/S3Transfer/Models/UploadResult.php | 16 + src/S3/S3Transfer/MultipartDownloader.php | 76 +- .../S3Transfer/MultipartDownloaderInitial.php | 8 +- src/S3/S3Transfer/MultipartUploader.php | 84 +- .../S3Transfer/MultipartUploaderInitial.php | 8 +- .../S3Transfer/PartGetMultipartDownloader.php | 4 +- .../RangeGetMultipartDownloader.php | 8 +- src/S3/S3Transfer/S3TransferManager.php | 211 ++-- .../{Models => Utils}/DownloadHandler.php | 2 +- .../S3Transfer/Utils/FileDownloadHandler.php | 1 - .../Utils/StreamDownloadHandler.php | 1 - tests/Integ/S3TransferManagerContext.php | 6 +- .../S3/S3Transfer/MultipartDownloaderTest.php | 74 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 113 ++- .../PartGetMultipartDownloaderTest.php | 18 +- .../RangeGetMultipartDownloaderTest.php | 77 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 948 ++++++++++-------- 40 files changed, 1772 insertions(+), 3855 deletions(-) create mode 100644 src/S3/S3Transfer/Data/tm-model-schema.json create mode 100644 src/S3/S3Transfer/Data/tm-model.json delete mode 100644 src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php delete mode 100644 src/S3/S3Transfer/Models/DownloadRequestConfig.php delete mode 100644 src/S3/S3Transfer/Models/DownloadResponse.php create mode 100644 src/S3/S3Transfer/Models/DownloadResult.php delete mode 100644 src/S3/S3Transfer/Models/GetObjectRequest.php delete mode 100644 src/S3/S3Transfer/Models/GetObjectResponse.php delete mode 100644 src/S3/S3Transfer/Models/MultipartDownloaderConfig.php delete mode 100644 src/S3/S3Transfer/Models/MultipartUploaderConfig.php delete mode 100644 src/S3/S3Transfer/Models/PutObjectRequest.php delete mode 100644 src/S3/S3Transfer/Models/PutObjectResponse.php delete mode 100644 src/S3/S3Transfer/Models/TransferRequestConfig.php delete mode 100644 src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php delete mode 100644 src/S3/S3Transfer/Models/UploadRequestConfig.php delete mode 100644 src/S3/S3Transfer/Models/UploadResponse.php create mode 100644 src/S3/S3Transfer/Models/UploadResult.php rename src/S3/S3Transfer/{Models => Utils}/DownloadHandler.php (92%) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index d860382a74..4e3a89e51e 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -6,9 +6,7 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; -use Aws\S3\S3Transfer\Models\PutObjectRequest; -use Aws\S3\S3Transfer\Models\UploadResponse; +use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; @@ -26,16 +24,16 @@ abstract class AbstractMultipartUploader implements PromisorInterface public const PART_MIN_SIZE = 5 * 1024 * 1024; // 5 MiB public const PART_MAX_SIZE = 5 * 1024 * 1024 * 1024; // 5 GiB public const PART_MAX_NUM = 10000; - protected const DEFAULT_CHECKSUM_CALCULATION_ALGORITHM = 'crc32'; + public const DEFAULT_CHECKSUM_CALCULATION_ALGORITHM = 'crc32'; /** @var S3ClientInterface */ protected readonly S3ClientInterface $s3Client; - /** @var PutObjectRequest @ */ - protected readonly PutObjectRequest $putObjectRequest; + /** @var array @ */ + protected readonly array $putObjectRequestArgs; - /** @var MultipartUploaderConfig @ */ - protected readonly MultipartUploaderConfig $config; + /** @var array @ */ + protected readonly array $config; /** @var string|null */ protected string|null $uploadId; @@ -44,7 +42,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface protected array $parts; /** @var array */ - private array $deferFns = []; + protected array $onCompletionCallbacks = []; /** @var TransferListenerNotifier|null */ protected ?TransferListenerNotifier $listenerNotifier; @@ -69,8 +67,11 @@ abstract class AbstractMultipartUploader implements PromisorInterface /** * @param S3ClientInterface $s3Client - * @param PutObjectRequest $putObjectRequest - * @param MultipartUploaderConfig $config + * @param array $putObjectRequestArgs + * @param array $config + * - target_part_size_bytes: (int, optional) + * - request_checksum_calculation: (string, optional) + * - concurrency: (int, optional) * @param string|null $uploadId * @param array $parts * @param TransferProgressSnapshot|null $currentSnapshot @@ -79,8 +80,8 @@ abstract class AbstractMultipartUploader implements PromisorInterface public function __construct ( S3ClientInterface $s3Client, - PutObjectRequest $putObjectRequest, - MultipartUploaderConfig $config, + array $putObjectRequestArgs, + array $config, ?string $uploadId = null, array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, @@ -88,35 +89,35 @@ public function __construct ) { $this->s3Client = $s3Client; - $this->putObjectRequest = $putObjectRequest; + $this->putObjectRequestArgs = $putObjectRequestArgs; + $this->validateConfig($config); $this->config = $config; - $this->validateConfig(); $this->uploadId = $uploadId; $this->parts = $parts; $this->currentSnapshot = $currentSnapshot; $this->listenerNotifier = $listenerNotifier; - // Evaluation for custom provided checksums - $objectRequestArgs = $putObjectRequest->toArray(); - $checksumName = self::filterChecksum($objectRequestArgs); - if ($checksumName !== null) { - $this->requestChecksum = $objectRequestArgs[$checksumName]; - $this->requestChecksumAlgorithm = str_replace( - 'Checksum', - '', - $checksumName - ); - } else { - $this->requestChecksum = null; - $this->requestChecksumAlgorithm = null; - } } /** + * @param array $config + * * @return void */ - protected function validateConfig(): void + protected function validateConfig(array &$config): void { - $partSize = $this->config->getTargetPartSizeBytes(); + if (!isset($config['target_part_size_bytes'])) { + $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; + } + + if (!isset($config['concurrency'])) { + $config['concurrency'] = S3TransferManagerConfig::DEFAULT_CONCURRENCY; + } + + if (!isset($config['request_checksum_calculation'])) { + $config['request_checksum_calculation'] = 'when_supported'; + } + + $partSize = $config['target_part_size_bytes']; if ($partSize < self::PART_MIN_SIZE || $partSize > self::PART_MAX_SIZE) { throw new \InvalidArgumentException( "Part size config must be between " . self::PART_MIN_SIZE @@ -166,7 +167,7 @@ public function promise(): PromiseInterface $this->operationFailed($e); yield Create::rejectionFor($e); } finally { - $this->callDeferredFns(); + $this->callOnCompletionCallbacks(); } }); } @@ -176,12 +177,13 @@ public function promise(): PromiseInterface */ protected function createMultipartUpload(): PromiseInterface { - $createMultipartUploadArgs = $this->putObjectRequest->toCreateMultipartRequest(); + $createMultipartUploadArgs = $this->putObjectRequestArgs; if ($this->requestChecksum !== null) { - $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; - } elseif ($this->config->getRequestChecksumCalculation() === 'when_supported') { - $this->requestChecksumAlgorithm = self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM; + } elseif ($this->config['request_checksum_calculation'] === 'when_supported') { + $this->requestChecksumAlgorithm = $createMultipartUploadArgs['ChecksumAlgorithm'] + ?? self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM; $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; } @@ -205,7 +207,7 @@ protected function createMultipartUpload(): PromiseInterface protected function completeMultipartUpload(): PromiseInterface { $this->sortParts(); - $completeMultipartUploadArgs = $this->putObjectRequest->toCompleteMultipartUploadRequest(); + $completeMultipartUploadArgs = $this->putObjectRequestArgs; $completeMultipartUploadArgs['UploadId'] = $this->uploadId; $completeMultipartUploadArgs['MultipartUpload'] = [ 'Parts' => $this->parts @@ -215,7 +217,7 @@ protected function completeMultipartUpload(): PromiseInterface if ($this->requestChecksum !== null) { $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; $completeMultipartUploadArgs[ - 'Checksum' . $this->requestChecksumAlgorithm + 'Checksum' . ucfirst($this->requestChecksumAlgorithm) ] = $this->requestChecksum; } @@ -236,7 +238,7 @@ protected function completeMultipartUpload(): PromiseInterface */ protected function abortMultipartUpload(): PromiseInterface { - $abortMultipartUploadArgs = $this->putObjectRequest->toAbortMultipartRequest(); + $abortMultipartUploadArgs = $this->putObjectRequestArgs; $abortMultipartUploadArgs['UploadId'] = $this->uploadId; $command = $this->s3Client->getCommand( 'AbortMultipartUpload', @@ -348,7 +350,7 @@ protected function operationCompleted(ResultInterface $result): void $this->listenerNotifier?->transferComplete([ TransferListener::REQUEST_ARGS_KEY => - $this->putObjectRequest->toCompleteMultipartUploadRequest(), + $this->putObjectRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot ]); } @@ -391,7 +393,7 @@ protected function operationFailed(Throwable $reason): void $this->listenerNotifier?->transferFail([ TransferListener::REQUEST_ARGS_KEY => - $this->putObjectRequest->toAbortMultipartRequest(), + $this->putObjectRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); @@ -426,13 +428,13 @@ protected function partCompleted( /** * @return void */ - protected function callDeferredFns(): void + protected function callOnCompletionCallbacks(): void { - foreach ($this->deferFns as $fn) { + foreach ($this->onCompletionCallbacks as $fn) { $fn(); } - $this->deferFns = []; + $this->onCompletionCallbacks = []; } /** @@ -451,7 +453,7 @@ protected function calculatePartSize(): int { return max( $this->getTotalSize() / self::PART_MAX_NUM, - $this->config->getTargetPartSizeBytes() + $this->config['target_part_size_bytes'] ); } @@ -468,32 +470,7 @@ abstract protected function getTotalSize(): int; /** * @param ResultInterface $result * - * @return UploadResponse + * @return mixed */ - abstract protected function createResponse(ResultInterface $result): UploadResponse; - - /** - * Filters a provided checksum if one was provided. - * - * @param array $requestArgs - * - * @return string | null - */ - private static function filterChecksum(array $requestArgs):? string - { - static $algorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - foreach ($algorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return $algorithm; - } - } - - return null; - } + abstract protected function createResponse(ResultInterface $result): mixed; } diff --git a/src/S3/S3Transfer/Data/tm-model-schema.json b/src/S3/S3Transfer/Data/tm-model-schema.json new file mode 100644 index 0000000000..11c90ef03e --- /dev/null +++ b/src/S3/S3Transfer/Data/tm-model-schema.json @@ -0,0 +1,304 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://aws.amazon.com/schemas/s3-transfer-manager/tm-model.json", + "title": "S3 Transfer Manager Model Configuration", + "description": "Configuration schema for S3 Transfer Manager field mappings and conversions between request/response types", + "type": "object", + "required": ["Definition", "Conversion"], + "properties": { + "Definition": { + "type": "object", + "description": "Defines the base field mappings for S3 Transfer Manager operations", + "required": ["UploadRequest", "UploadResponse", "DownloadRequest", "DownloadResponse"], + "properties": { + "UploadRequest": { + "type": "object", + "description": "Field mappings for upload request operations", + "required": ["PutObjectRequest"], + "properties": { + "PutObjectRequest": { + "type": "array", + "description": "List of fields available in PutObjectRequest that should be added to UploadRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "UploadResponse": { + "type": "object", + "description": "Field mappings for upload response operations", + "required": ["PutObjectResponse"], + "properties": { + "PutObjectResponse": { + "type": "array", + "description": "List of fields available in PutObjectResponse that should be added to UploadResponse", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "DownloadRequest": { + "type": "object", + "description": "Field mappings for download request operations", + "required": ["GetObjectRequest"], + "properties": { + "GetObjectRequest": { + "type": "array", + "description": "List of fields available in GetObjectRequest that should be added to DownloadRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "DownloadResponse": { + "type": "object", + "description": "Field mappings for download response operations", + "required": ["GetObjectResponse"], + "properties": { + "GetObjectResponse": { + "type": "array", + "description": "List of fields available in GetObjectResponse that should be added to DownloadResponse", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "Conversion": { + "type": "object", + "description": "Defines field conversion mappings between different S3 operation types", + "required": ["UploadRequest", "CompleteMultipartResponse", "PutObjectResponse", "GetObjectResponse"], + "properties": { + "UploadRequest": { + "type": "object", + "description": "Field conversion mappings for upload request operations", + "required": ["PutObjectRequest", "CreateMultipartRequest", "UploadPartReuqest", "CompleteMultipartRequest", "AbortMultipartRequest"], + "properties": { + "PutObjectRequest": { + "type": "array", + "description": "Fields to copy when converting UploadRequest to PutObjectRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + }, + "CreateMultipartRequest": { + "type": "array", + "description": "Fields to copy when converting UploadRequest to CreateMultipartRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + }, + "UploadPartReuqest": { + "type": "array", + "description": "Fields to copy when converting UploadRequest to UploadPartRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + }, + "CompleteMultipartRequest": { + "type": "array", + "description": "Fields to convert from UploadRequest to CompleteMultipartRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + }, + "AbortMultipartRequest": { + "type": "array", + "description": "Fields to copy when converting UploadRequest to AbortMultipartRequest", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "CompleteMultipartResponse": { + "type": "object", + "description": "Field conversion mappings for complete multipart response operations", + "required": ["UploadResponse"], + "properties": { + "UploadResponse": { + "type": "array", + "description": "Fields to copy when converting CompleteMultipartResponse to UploadResponse", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "PutObjectResponse": { + "type": "object", + "description": "Field conversion mappings for put object response operations", + "required": ["UploadResponse"], + "properties": { + "UploadResponse": { + "type": "array", + "description": "Fields to copy when converting PutObjectResponse to UploadResponse", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + }, + "GetObjectResponse": { + "type": "object", + "description": "Field conversion mappings for get object response operations", + "required": ["DowloadResponse"], + "properties": { + "DowloadResponse": { + "type": "array", + "description": "Fields to copy when converting GetObjectResponse to DownloadResponse", + "items": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$" + }, + "uniqueItems": true, + "minItems": 1 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "definitions": { + "S3FieldName": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9]*$", + "description": "Valid S3 API field name following PascalCase convention" + }, + "S3FieldList": { + "type": "array", + "items": { + "$ref": "#/definitions/S3FieldName" + }, + "uniqueItems": true, + "minItems": 1, + "description": "List of unique S3 API field names" + } + }, + "examples": [ + { + "Definition": { + "UploadRequest": { + "PutObjectRequest": [ + "Bucket", + "Key", + "ContentType", + "Metadata" + ] + }, + "UploadResponse": { + "PutObjectResponse": [ + "ETag", + "VersionId" + ] + }, + "DownloadRequest": { + "GetObjectRequest": [ + "Bucket", + "Key", + "Range" + ] + }, + "DownloadResponse": { + "GetObjectResponse": [ + "ContentLength", + "ContentType", + "ETag" + ] + } + }, + "Conversion": { + "UploadRequest": { + "PutObjectRequest": [ + "Bucket", + "Key" + ], + "CreateMultipartRequest": [ + "Bucket", + "Key", + "ContentType" + ], + "UploadPartReuqest": [ + "Bucket", + "Key" + ], + "CompleteMultipartRequest": [ + "Bucket", + "Key" + ], + "AbortMultipartRequest": [ + "Bucket", + "Key" + ] + }, + "CompleteMultipartResponse": { + "UploadResponse": [ + "ETag", + "VersionId" + ] + }, + "PutObjectResponse": { + "UploadResponse": [ + "ETag", + "VersionId" + ] + }, + "GetObjectResponse": { + "DowloadResponse": [ + "ContentLength", + "ContentType" + ] + } + } + } + ] +} diff --git a/src/S3/S3Transfer/Data/tm-model.json b/src/S3/S3Transfer/Data/tm-model.json new file mode 100644 index 0000000000..02aa71cc6d --- /dev/null +++ b/src/S3/S3Transfer/Data/tm-model.json @@ -0,0 +1,287 @@ +{ + "Definition": { + "UploadRequest": { + "PutObjectRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME" + ] + }, + "UploadResponse": { + "PutObjectResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "DownloadRequest": { + "GetObjectRequest": [ + "Bucket", + "ChecksumMode", + "ExpectedBucketOwner", + "IfMatch", + "IfModifiedSince", + "IfNoneMatch", + "IfUnmodifiedSince", + "Key", + "PartNumber", + "Range", + "RequestPayer", + "ResponseCacheControl", + "ResponseContentDisposition", + "ResponseContentEncoding", + "ResponseContentLanguage", + "ResponseContentType", + "ResponseExpires", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "VersionId" + ] + }, + "DownloadResponse": { + "GetObjectResponse": [ + "AcceptRanges", + "BucketKeyEnabled", + "CacheControl", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentLength", + "ContentRange", + "ContentType", + "DeleteMarker", + "ETag", + "Expiration", + "Expires", + "LastModified", + "Metadata", + "MissingMeta", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "PartsCount", + "ReplicationStatus", + "RequestCharged", + "Restore", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "TagCount", + "VersionId", + "WebsiteRedirectLocation" + ] + } + }, + "Conversion": { + "UploadRequest": { + "PutObjectRequest": [ + "Bucket", + "ChecksumAlgorithm", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ExpectedBucketOwner", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "CreateMultipartRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation" + ], + "UploadPartRequest": [ + "Bucket", + "ChecksumAlgorithm", + "ExpectedBucketOwner", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "CompleteMultipartRequest": [ + "Bucket", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ExpectedBucketOwner", + "IfMatch", + "IfNoneMatch", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "AbortMultipartRequest": [ + "Bucket", + "ExpectedBucketOwner", + "Key", + "RequestPayer" + ] + }, + "CompleteMultipartResponse": { + "UploadResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "PutObjectResponse": { + "UploadResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "Size", + "VersionId" + ] + }, + "GetObjectResponse": { + "DownloadResponse": [ + "AcceptRanges", + "BucketKeyEnabled", + "CacheControl", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentLength", + "ContentRange", + "ContentType", + "DeleteMarker", + "ETag", + "Expiration", + "Expires", + "ExpiresString", + "LastModified", + "Metadata", + "MissingMeta", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "PartsCount", + "ReplicationStatus", + "RequestCharged", + "Restore", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "TagCount", + "VersionId", + "WebsiteRedirectLocation" + ] + } + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 936661a324..54c7f8dda5 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -14,45 +14,15 @@ class DownloadDirectoryRequest extends TransferRequest /** @var string */ private string $destinationDirectory; - /** @var GetObjectRequest */ - private GetObjectRequest $getObjectRequest; - - /** @var DownloadDirectoryRequestConfig */ - private DownloadDirectoryRequestConfig $config; - - /** - * @param string $sourceBucket - * @param string $destinationDirectory - * @param GetObjectRequest $getObjectRequest - * @param DownloadDirectoryRequestConfig $config - */ - public function __construct( - string $sourceBucket, - string $destinationDirectory, - GetObjectRequest $getObjectRequest, - DownloadDirectoryRequestConfig $config, - array $listeners = [], - ?TransferListener $progressTracker = null - ) { - parent::__construct($listeners, $progressTracker); - if (ArnParser::isArn($sourceBucket)) { - $sourceBucket = ArnParser::parse($sourceBucket)->getResource(); - } - - $this->sourceBucket = $sourceBucket; - $this->destinationDirectory = $destinationDirectory; - $this->getObjectRequest = $getObjectRequest; - $this->config = $config; - } + /** @var array */ + private readonly array $getObjectRequestArgs; /** * @param string $sourceBucket The bucket from where the files are going to be * downloaded from. * @param string $destinationDirectory The destination path where the downloaded * files will be placed in. - * @param array $downloadDirectoryArgs The getObject request arguments to be provided - * as part of each get object request sent to the service, except for the - * bucket and key which will be resolved internally. + * @param array $getObjectRequestArgs * @param array $config The config options for this download directory operation. * - s3_prefix: (string, optional) This parameter will be considered just if * not provided as part of the list_object_v2_args config option. @@ -90,6 +60,32 @@ public function __construct( * @param TransferListener|null $progressTracker Ideally the progress * tracker implementation provided here should be able to track multiple * transfers at once. Please see MultiProgressTracker implementation. + */ + public function __construct( + string $sourceBucket, + string $destinationDirectory, + array $getObjectRequestArgs, + array $config, + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + if (ArnParser::isArn($sourceBucket)) { + $sourceBucket = ArnParser::parse($sourceBucket)->getResource(); + } + + $this->sourceBucket = $sourceBucket; + $this->destinationDirectory = $destinationDirectory; + $this->getObjectRequestArgs = $getObjectRequestArgs; + } + + /** + * @param string $sourceBucket + * @param string $destinationDirectory + * @param array $downloadDirectoryArgs + * @param array $config + * @param array $listeners + * @param TransferListener|null $progressTracker * * @return DownloadDirectoryRequest */ @@ -104,8 +100,8 @@ public static function fromLegacyArgs( return new self( $sourceBucket, $destinationDirectory, - GetObjectRequest::fromArray($downloadDirectoryArgs), - DownloadDirectoryRequestConfig::fromArray($config), + $downloadDirectoryArgs, + $config, $listeners, $progressTracker ); @@ -128,19 +124,11 @@ public function getDestinationDirectory(): string } /** - * @return GetObjectRequest - */ - public function getGetObjectRequest(): GetObjectRequest - { - return $this->getObjectRequest; - } - - /** - * @return DownloadDirectoryRequestConfig + * @return array */ - public function getConfig(): DownloadDirectoryRequestConfig + public function getGetObjectRequestArgs(): array { - return $this->config; + return $this->getObjectRequestArgs; } /** diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php deleted file mode 100644 index 14d0490a98..0000000000 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequestConfig.php +++ /dev/null @@ -1,184 +0,0 @@ -s3Prefix = $s3Prefix; - $this->s3Delimiter = $s3Delimiter; - $this->filter = $filter; - $this->getObjectRequestCallback = $getObjectRequestCallback; - $this->failurePolicy = $failurePolicy; - $this->trackProgress = $trackProgress; - $this->targetPartSizeBytes = $targetPartSizeBytes; - $this->listObjectV2Args = $listObjectV2Args; - $this->failsWhenDestinationExists = $failsWhenDestinationExists; - } - - /** - * @param array $config - * @return DownloadDirectoryRequestConfig - */ - public static function fromArray(array $config): DownloadDirectoryRequestConfig - { - return new self( - s3Prefix: $config['s3_prefix'] ?? null, - s3Delimiter: $config['s3_delimiter'] ?? '/', - filter: $config['filter'] ?? null, - getObjectRequestCallback: $config['get_object_request_callback'] ?? null, - failurePolicy: $config['failure_policy'] ?? null, - targetPartSizeBytes: $config['target_part_size_bytes'] ?? null, - listObjectV2Args: $config['list_object_v2_args'] ?? [], - failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, - trackProgress: $config['track_progress'] ?? null - ); - } - - /** - * @return string|null - */ - public function getS3Prefix(): ?string - { - return $this->s3Prefix; - } - - /** - * @return string - */ - public function getS3Delimiter(): string - { - return $this->s3Delimiter; - } - - /** - * @return Closure|null - */ - public function getFilter(): ?Closure - { - return $this->filter; - } - - /** - * @return Closure|null - */ - public function getGetObjectRequestCallback(): ?Closure - { - return $this->getObjectRequestCallback; - } - - /** - * @return Closure|null - */ - public function getFailurePolicy(): ?Closure - { - return $this->failurePolicy; - } - - /** - * @return int|null - */ - public function getTargetPartSizeBytes(): ?int - { - return $this->targetPartSizeBytes; - } - - /** - * @return array - */ - public function getListObjectV2Args(): array - { - return $this->listObjectV2Args; - } - - - /** - * @return string|null - */ - public function getEffectivePrefix(): ?string - { - return $this->listObjectV2Args['Prefix'] ?? $this->s3Prefix; - } - - /** - * @return string - */ - public function getEffectiveDelimiter(): string - { - return $this->listObjectV2Args['Delimiter'] ?? $this->s3Delimiter; - } - - /** - * @return bool - */ - public function isFailsWhenDestinationExists(): bool - { - return $this->failsWhenDestinationExists; - } - - /** - * @return array - */ - public function toArray(): array - { - return [ - 's3_prefix' => $this->s3Prefix, - 's3_delimiter' => $this->s3Delimiter, - 'filter' => $this->filter, - 'get_object_request_callback' => $this->getObjectRequestCallback, - 'failure_policy' => $this->failurePolicy, - 'track_progress' => $this->trackProgress, - 'target_part_size_bytes' => $this->targetPartSizeBytes, - 'list_object_v2_args' => $this->listObjectV2Args, - 'fails_when_destination_exists' => $this->failsWhenDestinationExists, - ]; - } -} diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 8d90f296f6..7ee06cb6b6 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -5,56 +5,32 @@ use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\S3TransferManager; +use Aws\S3\S3Transfer\Utils\DownloadHandler; use Aws\S3\S3Transfer\Utils\FileDownloadHandler; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; final class DownloadRequest extends TransferRequest { + public static array $configKeys = [ + 'response_checksum_validation', + 'multipart_download_type', + 'track_progress' + ]; + /** @var string|array|null */ private string|array|null $source; - /** @var GetObjectRequest */ - private GetObjectRequest $getObjectRequest; - - /** @var DownloadRequestConfig */ - private DownloadRequestConfig $config; + /** @var array */ + private array $getObjectRequestArgs; /** @var DownloadHandler|null */ private ?DownloadHandler $downloadHandler; - /** - * @param string|array|null $source - * @param GetObjectRequest $getObjectRequest - * @param DownloadRequestConfig $config - * @param DownloadHandler|null $downloadHandler - * @param array $listeners - * @param TransferListener|null $progressTracker - */ - public function __construct( - string|array|null $source, - GetObjectRequest $getObjectRequest, - DownloadRequestConfig $config, - ?DownloadHandler $downloadHandler, - array $listeners = [], - ?TransferListener $progressTracker = null - ) { - parent::__construct($listeners, $progressTracker); - $this->source = $source; - $this->getObjectRequest = $getObjectRequest; - $this->config = $config; - if ($downloadHandler === null) { - $downloadHandler = new StreamDownloadHandler(); - } - $this->downloadHandler = $downloadHandler; - } - /** * @param string|array|null $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key * properties set. - * @param array $downloadRequestArgs The getObject request arguments to be provided as part - * of each get object operation, except for the bucket and key, which - * are already provided as the source. + * @param array $getObjectRequestArgs * @param array $config The configuration to be used for this operation: * - multipart_download_type: (string, optional) * Overrides the resolved value from the transfer manager config. @@ -72,8 +48,34 @@ public function __construct( * @param DownloadHandler|null $downloadHandler * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker + */ + public function __construct( + string|array|null $source, + array $getObjectRequestArgs, + array $config, + ?DownloadHandler $downloadHandler, + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + $this->source = $source; + $this->getObjectRequestArgs = $getObjectRequestArgs; + $this->config = $config; + if ($downloadHandler === null) { + $downloadHandler = new StreamDownloadHandler(); + } + $this->downloadHandler = $downloadHandler; + } + + /** + * @param string|array|null $source + * @param array $downloadRequestArgs + * @param array $config + * @param DownloadHandler|null $downloadHandler + * @param array $listeners + * @param TransferListener|null $progressTracker * - * @return static + * @return DownloadRequest */ public static function fromLegacyArgs( string | array | null $source, @@ -86,8 +88,8 @@ public static function fromLegacyArgs( { return new DownloadRequest( $source, - GetObjectRequest::fromArray($downloadRequestArgs), - DownloadRequestConfig::fromArray($config), + $downloadRequestArgs, + $config, $downloadHandler, $listeners, $progressTracker @@ -107,7 +109,7 @@ public static function fromDownloadRequestAndDownloadHandler( { return new DownloadRequest( $downloadRequest->getSource(), - $downloadRequest->getGetObjectRequest(), + $downloadRequest->getObjectRequestArgs(), $downloadRequest->getConfig(), $downloadHandler, $downloadRequest->getListeners(), @@ -124,19 +126,11 @@ public function getSource(): array|string|null } /** - * @return GetObjectRequest - */ - public function getGetObjectRequest(): GetObjectRequest - { - return $this->getObjectRequest; - } - - /** - * @return DownloadRequestConfig + * @return array */ - public function getConfig(): DownloadRequestConfig + public function getObjectRequestArgs(): array { - return $this->config; + return $this->getObjectRequestArgs; } /** @@ -156,8 +150,8 @@ public function getDownloadHandler(): DownloadHandler { public function normalizeSourceAsArray(): array { // If source is null then fall back to getObjectRequest. $source = $this->getSource() ?? [ - 'Bucket' => $this->getObjectRequest->getBucket(), - 'Key' => $this->getObjectRequest->getKey(), + 'Bucket' => $this->getObjectRequestArgs['Bucket'] ?? null, + 'Key' => $this->getObjectRequestArgs['Key'] ?? null, ]; if (is_string($source)) { $sourceAsArray = S3TransferManager::s3UriAsBucketAndKey($source); diff --git a/src/S3/S3Transfer/Models/DownloadRequestConfig.php b/src/S3/S3Transfer/Models/DownloadRequestConfig.php deleted file mode 100644 index 84581c812b..0000000000 --- a/src/S3/S3Transfer/Models/DownloadRequestConfig.php +++ /dev/null @@ -1,92 +0,0 @@ -multipartDownloadType = $multipartDownloadType; - $this->requestChecksumValidation = $requestChecksumValidation; - $this->targetPartSizeBytes = $targetPartSizeBytes; - } - - /** - * Convert the DownloadRequestConfig instance to an array - * - * @return array - */ - public function toArray(): array - { - return [ - 'multipart_download_type' => $this->multipartDownloadType, - 'request_checksum_validation' => $this->requestChecksumValidation, - 'target_part_size_bytes' => $this->targetPartSizeBytes, - 'track_progress' => $this->getTrackProgress(), // Assuming this getter exists in parent class - ]; - } - - /** - * Create a DownloadRequestConfig instance from an array - * - * @param array $data - * @return static - */ - public static function fromArray(array $data): static - { - return new self( - $data['multipart_download_type'] ?? null, - $data['request_checksum_validation'] ?? null, - $data['target_part_size_bytes'] ?? null, - $data['track_progress'] ?? null - ); - } - - /** - * @return string|null - */ - public function getMultipartDownloadType(): ?string - { - return $this->multipartDownloadType; - } - - /** - * @return string|null - */ - public function getRequestChecksumValidation(): ?string - { - return $this->requestChecksumValidation; - } - - /** - * @return int|null - */ - public function getTargetPartSizeBytes(): ?int - { - return $this->targetPartSizeBytes; - } -} diff --git a/src/S3/S3Transfer/Models/DownloadResponse.php b/src/S3/S3Transfer/Models/DownloadResponse.php deleted file mode 100644 index 6c601118ee..0000000000 --- a/src/S3/S3Transfer/Models/DownloadResponse.php +++ /dev/null @@ -1,31 +0,0 @@ -downloadDataResult; - } - - /** - * @return array - */ - public function getDownloadResponse(): array - { - return $this->downloadResponse; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadResult.php b/src/S3/S3Transfer/Models/DownloadResult.php new file mode 100644 index 0000000000..29a38f3b74 --- /dev/null +++ b/src/S3/S3Transfer/Models/DownloadResult.php @@ -0,0 +1,30 @@ +downloadDataResult = $downloadDataResult; + } + + /** + * @return mixed + */ + public function getDownloadDataResult(): mixed + { + return $this->downloadDataResult; + } +} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/GetObjectRequest.php b/src/S3/S3Transfer/Models/GetObjectRequest.php deleted file mode 100644 index 0844616026..0000000000 --- a/src/S3/S3Transfer/Models/GetObjectRequest.php +++ /dev/null @@ -1,345 +0,0 @@ -bucket = $bucket; - $this->checksumMode = $checksumMode; - $this->expectedBucketOwner = $expectedBucketOwner; - $this->ifMatch = $ifMatch; - $this->ifModifiedSince = $ifModifiedSince; - $this->ifNoneMatch = $ifNoneMatch; - $this->ifUnmodifiedSince = $ifUnmodifiedSince; - $this->key = $key; - $this->requestPayer = $requestPayer; - $this->responseCacheControl = $responseCacheControl; - $this->responseContentDisposition = $responseContentDisposition; - $this->responseContentEncoding = $responseContentEncoding; - $this->responseContentLanguage = $responseContentLanguage; - $this->responseContentType = $responseContentType; - $this->responseExpires = $responseExpires; - $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; - $this->sseCustomerKey = $sseCustomerKey; - $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; - $this->versionId = $versionId; - } - - /** - * Create an instance from an array of data - * - * @param array $data - * @return self - */ - public static function fromArray(array $data): self - { - return new self( - $data['Bucket'] ?? null, - $data['ChecksumMode'] ?? null, - $data['ExpectedBucketOwner'] ?? null, - $data['IfMatch'] ?? null, - $data['IfModifiedSince'] ?? null, - $data['IfNoneMatch'] ?? null, - $data['IfUnmodifiedSince'] ?? null, - $data['Key'] ?? null, - $data['RequestPayer'] ?? null, - $data['ResponseCacheControl'] ?? null, - $data['ResponseContentDisposition'] ?? null, - $data['ResponseContentEncoding'] ?? null, - $data['ResponseContentLanguage'] ?? null, - $data['ResponseContentType'] ?? null, - $data['ResponseExpires'] ?? null, - $data['SSECustomerAlgorithm'] ?? null, - $data['SSECustomerKey'] ?? null, - $data['SSECustomerKeyMD5'] ?? null, - $data['VersionId'] ?? null - ); - } - - /** - * @return string|null - */ - public function getBucket(): ?string - { - return $this->bucket; - } - - /** - * @return string|null - */ - public function getChecksumMode(): ?string - { - return $this->checksumMode; - } - - /** - * @return string|null - */ - public function getExpectedBucketOwner(): ?string - { - return $this->expectedBucketOwner; - } - - /** - * @return string|null - */ - public function getIfMatch(): ?string - { - return $this->ifMatch; - } - - /** - * @return string|null - */ - public function getIfModifiedSince(): ?string - { - return $this->ifModifiedSince; - } - - /** - * @return string|null - */ - public function getIfNoneMatch(): ?string - { - return $this->ifNoneMatch; - } - - /** - * @return string|null - */ - public function getIfUnmodifiedSince(): ?string - { - return $this->ifUnmodifiedSince; - } - - /** - * @return string|null - */ - public function getKey(): ?string - { - return $this->key; - } - - /** - * @return string|null - */ - public function getRequestPayer(): ?string - { - return $this->requestPayer; - } - - /** - * @return string|null - */ - public function getResponseCacheControl(): ?string - { - return $this->responseCacheControl; - } - - /** - * @return string|null - */ - public function getResponseContentDisposition(): ?string - { - return $this->responseContentDisposition; - } - - /** - * @return string|null - */ - public function getResponseContentEncoding(): ?string - { - return $this->responseContentEncoding; - } - - /** - * @return string|null - */ - public function getResponseContentLanguage(): ?string - { - return $this->responseContentLanguage; - } - - /** - * @return string|null - */ - public function getResponseContentType(): ?string - { - return $this->responseContentType; - } - - /** - * @return string|null - */ - public function getResponseExpires(): ?string - { - return $this->responseExpires; - } - - /** - * @return string|null - */ - public function getSseCustomerAlgorithm(): ?string - { - return $this->sseCustomerAlgorithm; - } - - /** - * @return string|null - */ - public function getSseCustomerKey(): ?string - { - return $this->sseCustomerKey; - } - - /** - * @return string|null - */ - public function getSseCustomerKeyMD5(): ?string - { - return $this->sseCustomerKeyMD5; - } - - /** - * @return string|null - */ - public function getVersionId(): ?string - { - return $this->versionId; - } - - /** - * Convert the object to an array format suitable for AWS S3 API request - * - * @return array Array containing AWS S3 request fields with their corresponding values - */ - public function toArray(): array - { - $array = [ - 'Bucket' => $this->bucket, - 'ChecksumMode' => $this->checksumMode, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'IfMatch' => $this->ifMatch, - 'IfModifiedSince' => $this->ifModifiedSince, - 'IfNoneMatch' => $this->ifNoneMatch, - 'IfUnmodifiedSince' => $this->ifUnmodifiedSince, - 'Key' => $this->key, - 'RequestPayer' => $this->requestPayer, - 'ResponseCacheControl' => $this->responseCacheControl, - 'ResponseContentDisposition' => $this->responseContentDisposition, - 'ResponseContentEncoding' => $this->responseContentEncoding, - 'ResponseContentLanguage' => $this->responseContentLanguage, - 'ResponseContentType' => $this->responseContentType, - 'ResponseExpires' => $this->responseExpires, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKey' => $this->sseCustomerKey, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - 'VersionId' => $this->versionId - ]; - - remove_nulls_from_array($array); - - return $array; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/GetObjectResponse.php b/src/S3/S3Transfer/Models/GetObjectResponse.php deleted file mode 100644 index ae59bb830c..0000000000 --- a/src/S3/S3Transfer/Models/GetObjectResponse.php +++ /dev/null @@ -1,629 +0,0 @@ -acceptRanges = $acceptRanges; - $this->bucketKeyEnabled = $bucketKeyEnabled; - $this->cacheControl = $cacheControl; - $this->checksumCRC32 = $checksumCRC32; - $this->checksumCRC32C = $checksumCRC32C; - $this->checksumCRC64NVME = $checksumCRC64NVME; - $this->checksumSHA1 = $checksumSHA1; - $this->checksumSHA256 = $checksumSHA256; - $this->checksumType = $checksumType; - $this->contentDisposition = $contentDisposition; - $this->contentEncoding = $contentEncoding; - $this->contentLanguage = $contentLanguage; - $this->contentLength = $contentLength; - $this->contentRange = $contentRange; - $this->contentType = $contentType; - $this->deleteMarker = $deleteMarker; - $this->eTag = $eTag; - $this->expiration = $expiration; - $this->expires = $expires; - $this->lastModified = $lastModified; - $this->metadata = $metadata; - $this->missingMeta = $missingMeta; - $this->objectLockLegalHoldStatus = $objectLockLegalHoldStatus; - $this->objectLockMode = $objectLockMode; - $this->objectLockRetainUntilDate = $objectLockRetainUntilDate; - $this->partsCount = $partsCount; - $this->replicationStatus = $replicationStatus; - $this->requestCharged = $requestCharged; - $this->restore = $restore; - $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; - $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; - $this->sseKMSKeyId = $sseKMSKeyId; - $this->serverSideEncryption = $serverSideEncryption; - $this->storageClass = $storageClass; - $this->tagCount = $tagCount; - $this->versionId = $versionId; - $this->websiteRedirectLocation = $websiteRedirectLocation; - } - - /** - * @param array $array - * @return GetObjectResponse - */ - public static function fromArray(array $array): GetObjectResponse - { - return new GetObjectResponse( - acceptRanges: $array['AcceptRanges'] ?? null, - bucketKeyEnabled: $array['BucketKeyEnabled'] ?? null, - cacheControl: $array['CacheControl'] ?? null, - checksumCRC32: $array['ChecksumCRC32'] ?? null, - checksumCRC32C: $array['ChecksumCRC32C'] ?? null, - checksumCRC64NVME: $array['ChecksumCRC64NVME'] ?? null, - checksumSHA1: $array['ChecksumSHA1'] ?? null, - checksumSHA256: $array['ChecksumSHA256'] ?? null, - checksumType: $array['ChecksumType'] ?? null, - contentDisposition: $array['ContentDisposition'] ?? null, - contentEncoding: $array['ContentEncoding'] ?? null, - contentLanguage: $array['ContentLanguage'] ?? null, - contentLength: $array['ContentLength'] ?? null, - contentRange: $array['ContentRange'] ?? null, - contentType: $array['ContentType'] ?? null, - deleteMarker: $array['DeleteMarker'] ?? null, - eTag: $array['ETag'] ?? null, - expiration: $array['Expiration'] ?? null, - expires: $array['Expires'] ?? null, - lastModified: $array['LastModified'] ?? null, - metadata: $array['@metadata'] ?? null, - missingMeta: $array['MissingMeta'] ?? null, - objectLockLegalHoldStatus: $array['ObjectLockLegalHoldStatus'] ?? null, - objectLockMode: $array['ObjectLockMode'] ?? null, - objectLockRetainUntilDate: $array['ObjectLockRetainUntilDate'] ?? null, - partsCount: $array['PartsCount'] ?? null, - replicationStatus: $array['ReplicationStatus'] ?? null, - requestCharged: $array['RequestCharged'] ?? null, - restore: $array['Restore'] ?? null, - sseCustomerAlgorithm: $array['SSECustomerAlgorithm'] ?? null, - sseCustomerKeyMD5: $array['SSECustomerKeyMD5'] ?? null, - sseKMSKeyId: $array['SSEKMSKeyId'] ?? null, - serverSideEncryption: $array['ServerSideEncryption'] ?? null, - storageClass: $array['StorageClass'] ?? null, - tagCount: $array['TagCount'] ?? null, - versionId: $array['VersionId'] ?? null, - websiteRedirectLocation: $array['WebsiteRedirectLocation'] ?? null - ); - } - - /** - * @return string|null - */ - public function getAcceptRanges(): ?string - { - return $this->acceptRanges; - } - - /** - * @return string|null - */ - public function getBucketKeyEnabled(): ?string - { - return $this->bucketKeyEnabled; - } - - /** - * @return string|null - */ - public function getCacheControl(): ?string - { - return $this->cacheControl; - } - - /** - * @return string|null - */ - public function getChecksumCRC32(): ?string - { - return $this->checksumCRC32; - } - - /** - * @return string|null - */ - public function getChecksumCRC32C(): ?string - { - return $this->checksumCRC32C; - } - - /** - * @return string|null - */ - public function getChecksumCRC64NVME(): ?string - { - return $this->checksumCRC64NVME; - } - - /** - * @return string|null - */ - public function getChecksumSHA1(): ?string - { - return $this->checksumSHA1; - } - - /** - * @return string|null - */ - public function getChecksumSHA256(): ?string - { - return $this->checksumSHA256; - } - - /** - * @return string|null - */ - public function getChecksumType(): ?string - { - return $this->checksumType; - } - - /** - * @return string|null - */ - public function getContentDisposition(): ?string - { - return $this->contentDisposition; - } - - /** - * @return string|null - */ - public function getContentEncoding(): ?string - { - return $this->contentEncoding; - } - - /** - * @return string|null - */ - public function getContentLanguage(): ?string - { - return $this->contentLanguage; - } - - /** - * @return string|null - */ - public function getContentLength(): ?string - { - return $this->contentLength; - } - - /** - * @return string|null - */ - public function getContentRange(): ?string - { - return $this->contentRange; - } - - /** - * @return string|null - */ - public function getContentType(): ?string - { - return $this->contentType; - } - - /** - * @return string|null - */ - public function getDeleteMarker(): ?string - { - return $this->deleteMarker; - } - - /** - * @return string|null - */ - public function getETag(): ?string - { - return $this->eTag; - } - - /** - * @return string|null - */ - public function getExpiration(): ?string - { - return $this->expiration; - } - - /** - * @return string|null - */ - public function getExpires(): ?string - { - return $this->expires; - } - - /** - * @return string|null - */ - public function getLastModified(): ?string - { - return $this->lastModified; - } - - /** - * @return array|null - */ - public function getMetadata(): ?array - { - return $this->metadata; - } - - /** - * @return string|null - */ - public function getMissingMeta(): ?string - { - return $this->missingMeta; - } - - /** - * @return string|null - */ - public function getObjectLockLegalHoldStatus(): ?string - { - return $this->objectLockLegalHoldStatus; - } - - /** - * @return string|null - */ - public function getObjectLockMode(): ?string - { - return $this->objectLockMode; - } - - /** - * @return string|null - */ - public function getObjectLockRetainUntilDate(): ?string - { - return $this->objectLockRetainUntilDate; - } - - /** - * @return string|null - */ - public function getPartsCount(): ?string - { - return $this->partsCount; - } - - /** - * @return string|null - */ - public function getReplicationStatus(): ?string - { - return $this->replicationStatus; - } - - /** - * @return string|null - */ - public function getRequestCharged(): ?string - { - return $this->requestCharged; - } - - /** - * @return string|null - */ - public function getRestore(): ?string - { - return $this->restore; - } - - /** - * @return string|null - */ - public function getSSECustomerAlgorithm(): ?string - { - return $this->sseCustomerAlgorithm; - } - - /** - * @return string|null - */ - public function getSSECustomerKeyMD5(): ?string - { - return $this->sseCustomerKeyMD5; - } - - /** - * @return string|null - */ - public function getSSEKMSKeyId(): ?string - { - return $this->sseKMSKeyId; - } - - /** - * @return string|null - */ - public function getServerSideEncryption(): ?string - { - return $this->serverSideEncryption; - } - - /** - * @return string|null - */ - public function getStorageClass(): ?string - { - return $this->storageClass; - } - - /** - * @return string|null - */ - public function getTagCount(): ?string - { - return $this->tagCount; - } - - /** - * @return string|null - */ - public function getVersionId(): ?string - { - return $this->versionId; - } - - /** - * @return string|null - */ - public function getWebsiteRedirectLocation(): ?string - { - return $this->websiteRedirectLocation; - } - - /** - * @return array - */ - public function toArray(): array - { - $array = [ - 'AcceptRanges' => $this->acceptRanges, - 'BucketKeyEnabled' => $this->bucketKeyEnabled, - 'CacheControl' => $this->cacheControl, - 'ChecksumCRC32' => $this->checksumCRC32, - 'ChecksumCRC32C' => $this->checksumCRC32C, - 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, - 'ChecksumSHA1' => $this->checksumSHA1, - 'ChecksumSHA256' => $this->checksumSHA256, - 'ChecksumType' => $this->checksumType, - 'ContentDisposition' => $this->contentDisposition, - 'ContentEncoding' => $this->contentEncoding, - 'ContentLanguage' => $this->contentLanguage, - 'ContentLength' => $this->contentLength, - 'ContentRange' => $this->contentRange, - 'ContentType' => $this->contentType, - 'DeleteMarker' => $this->deleteMarker, - 'ETag' => $this->eTag, - 'Expiration' => $this->expiration, - 'Expires' => $this->expires, - 'LastModified' => $this->lastModified, - 'Metadata' => $this->metadata, - 'MissingMeta' => $this->missingMeta, - 'ObjectLockLegalHoldStatus' => $this->objectLockLegalHoldStatus, - 'ObjectLockMode' => $this->objectLockMode, - 'ObjectLockRetainUntilDate' => $this->objectLockRetainUntilDate, - 'PartsCount' => $this->partsCount, - 'ReplicationStatus' => $this->replicationStatus, - 'RequestCharged' => $this->requestCharged, - 'Restore' => $this->restore, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - 'SSEKMSKeyId' => $this->sseKMSKeyId, - 'ServerSideEncryption' => $this->serverSideEncryption, - 'StorageClass' => $this->storageClass, - 'TagCount' => $this->tagCount, - 'VersionId' => $this->versionId, - 'WebsiteRedirectLocation' => $this->websiteRedirectLocation, - ]; - - remove_nulls_from_array($array); - - return $array; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/MultipartDownloaderConfig.php b/src/S3/S3Transfer/Models/MultipartDownloaderConfig.php deleted file mode 100644 index cfd3689148..0000000000 --- a/src/S3/S3Transfer/Models/MultipartDownloaderConfig.php +++ /dev/null @@ -1,81 +0,0 @@ -targetPartSizeBytes = $targetPartSizeBytes; - $this->responseChecksumValidationEnabled = $responseChecksumValidationEnabled; - $this->multipartDownloadType = $multipartDownloadType; - } - - /** - * @param array $array - * - * @return MultipartDownloaderConfig - */ - public static function fromArray(array $array): MultipartDownloaderConfig { - return new self( - $array['target_part_size_bytes'] - ?? S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, - $array['response_checksum_validation_enabled'] - ?? true, - $array['multipart_download_type'] - ?? S3TransferManagerConfig::DEFAULT_MULTIPART_DOWNLOAD_TYPE - ); - } - - /** - * @return int - */ - public function getTargetPartSizeBytes(): int - { - return $this->targetPartSizeBytes; - } - - /** - * @return bool - */ - public function getResponseChecksumValidationEnabled(): bool - { - return $this->responseChecksumValidationEnabled; - } - - /** - * @return string - */ - public function getMultipartDownloadType(): string - { - return $this->multipartDownloadType; - } - - /** - * @return array - */ - public function toArray(): array { - return [ - 'target_part_size_bytes' => $this->targetPartSizeBytes, - 'response_checksum_validation_enabled' => $this->responseChecksumValidationEnabled, - 'multipart_download_type' => $this->multipartDownloadType, - ]; - } -} diff --git a/src/S3/S3Transfer/Models/MultipartUploaderConfig.php b/src/S3/S3Transfer/Models/MultipartUploaderConfig.php deleted file mode 100644 index 43a249b3a4..0000000000 --- a/src/S3/S3Transfer/Models/MultipartUploaderConfig.php +++ /dev/null @@ -1,86 +0,0 @@ -targetPartSizeBytes = $targetPartSizeBytes; - $this->concurrency = $concurrency; - $this->requestChecksumCalculation = $requestChecksumCalculation; - } - - /** - * Create an MultipartUploaderConfig instance from an array - * - * @param array $data Array containing configuration data - * - * @return MultipartUploaderConfig - */ - public static function fromArray(array $data): MultipartUploaderConfig - { - return new self( - $data['target_part_size_bytes'] - ?? S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, - $data['concurrency'] - ?? S3TransferManagerConfig::DEFAULT_CONCURRENCY, - $data['request_checksum_calculation'] - ?? S3TransferManagerConfig::DEFAULT_REQUEST_CHECKSUM_CALCULATION - ); - } - - /** - * @return int - */ - public function getTargetPartSizeBytes(): int { - return $this->targetPartSizeBytes; - } - - /** - * @return int - */ - public function getConcurrency(): int { - return $this->concurrency; - } - - /** - * @return string - */ - public function getRequestChecksumCalculation(): string { - return $this->requestChecksumCalculation; - } - - /** - * Convert the configuration to an array - * - * @return array - */ - public function toArray(): array - { - return [ - 'target_part_size_bytes' => $this->targetPartSizeBytes, - 'concurrency' => $this->concurrency, - 'request_checksum_calculation' => $this->requestChecksumCalculation, - ]; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/PutObjectRequest.php b/src/S3/S3Transfer/Models/PutObjectRequest.php deleted file mode 100644 index c1e5cc526c..0000000000 --- a/src/S3/S3Transfer/Models/PutObjectRequest.php +++ /dev/null @@ -1,777 +0,0 @@ -acl = $acl; - $this->bucket = $bucket; - $this->bucketKeyEnabled = $bucketKeyEnabled; - $this->cacheControl = $cacheControl; - $this->checksumAlgorithm = $checksumAlgorithm; - $this->contentDisposition = $contentDisposition; - $this->contentEncoding = $contentEncoding; - $this->contentLanguage = $contentLanguage; - $this->contentType = $contentType; - $this->expectedBucketOwner = $expectedBucketOwner; - $this->expires = $expires; - $this->grantFullControl = $grantFullControl; - $this->grantRead = $grantRead; - $this->grantReadACP = $grantReadACP; - $this->grantWriteACP = $grantWriteACP; - $this->key = $key; - $this->metadata = $metadata; - $this->objectLockLegalHoldStatus = $objectLockLegalHoldStatus; - $this->objectLockMode = $objectLockMode; - $this->objectLockRetainUntilDate = $objectLockRetainUntilDate; - $this->requestPayer = $requestPayer; - $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; - $this->sseCustomerKey = $sseCustomerKey; - $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; - $this->ssekmsEncryptionContext = $ssekmsEncryptionContext; - $this->ssekmsKeyId = $ssekmsKeyId; - $this->serverSideEncryption = $serverSideEncryption; - $this->storageClass = $storageClass; - $this->tagging = $tagging; - $this->websiteRedirectLocation = $websiteRedirectLocation; - $this->checksumCRC32 = $checksumCRC32; - $this->checksumCRC32C = $checksumCRC32C; - $this->checksumCRC64NVME = $checksumCRC64NVME; - $this->checksumSHA1 = $checksumSHA1; - $this->checksumSHA256 = $checksumSHA256; - $this->ifMatch = $ifMatch; - $this->ifNoneMatch = $ifNoneMatch; - } - - /** - * @param array $array - * - * @return PutObjectRequest - */ - public static function fromArray(array $array): PutObjectRequest - { - return new self( - $array['ACL'] ?? null, - $array['Bucket'] ?? null, - $array['BucketKeyEnabled'] ?? null, - $array['CacheControl'] ?? null, - $array['ChecksumAlgorithm'] ?? null, - $array['ContentDisposition'] ?? null, - $array['ContentEncoding'] ?? null, - $array['ContentLanguage'] ?? null, - $array['ContentType'] ?? null, - $array['ExpectedBucketOwner'] ?? null, - $array['Expires'] ?? null, - $array['GrantFullControl'] ?? null, - $array['GrantRead'] ?? null, - $array['GrantReadACP'] ?? null, - $array['GrantWriteACP'] ?? null, - $array['Key'] ?? null, - $array['Metadata'] ?? null, - $array['ObjectLockLegalHoldStatus'] ?? null, - $array['ObjectLockMode'] ?? null, - $array['ObjectLockRetainUntilDate'] ?? null, - $array['RequestPayer'] ?? null, - $array['SSECustomerAlgorithm'] ?? null, - $array['SSECustomerKey'] ?? null, - $array['SSECustomerKeyMD5'] ?? null, - $array['SSEKMSEncryptionContext'] ?? null, - $array['SSEKMSKeyId'] ?? null, - $array['ServerSideEncryption'] ?? null, - $array['StorageClass'] ?? null, - $array['Tagging'] ?? null, - $array['WebsiteRedirectLocation'] ?? null, - $array['ChecksumCRC32'] ?? null, - $array['ChecksumCRC32C'] ?? null, - $array['ChecksumCRC64NVME'] ?? null, - $array['ChecksumSHA1'] ?? null, - $array['ChecksumSHA256'] ?? null, - $array['IfMatch'] ?? null, - $array['IfNoneMatch'] ?? null - ); - } - - /** - * @return string|null - */ - public function getAcl(): ?string - { - return $this->acl; - } - - /** - * @return string|null - */ - public function getBucket(): ?string - { - return $this->bucket; - } - - /** - * @return bool|null - */ - public function getBucketKeyEnabled(): ?bool - { - return $this->bucketKeyEnabled; - } - - /** - * @return string|null - */ - public function getCacheControl(): ?string - { - return $this->cacheControl; - } - - /** - * @return string|null - */ - public function getChecksumAlgorithm(): ?string - { - return $this->checksumAlgorithm; - } - - /** - * @return string|null - */ - public function getContentDisposition(): ?string - { - return $this->contentDisposition; - } - - /** - * @return string|null - */ - public function getContentEncoding(): ?string - { - return $this->contentEncoding; - } - - /** - * @return string|null - */ - public function getContentLanguage(): ?string - { - return $this->contentLanguage; - } - - /** - * @return string|null - */ - public function getContentType(): ?string - { - return $this->contentType; - } - - /** - * @return string|null - */ - public function getExpectedBucketOwner(): ?string - { - return $this->expectedBucketOwner; - } - - /** - * @return string|null - */ - public function getExpires(): ?string - { - return $this->expires; - } - - /** - * @return string|null - */ - public function getGrantFullControl(): ?string - { - return $this->grantFullControl; - } - - /** - * @return string|null - */ - public function getGrantRead(): ?string - { - return $this->grantRead; - } - - /** - * @return string|null - */ - public function getGrantReadACP(): ?string - { - return $this->grantReadACP; - } - - /** - * @return string|null - */ - public function getGrantWriteACP(): ?string - { - return $this->grantWriteACP; - } - - /** - * @return string|null - */ - public function getKey(): ?string - { - return $this->key; - } - - /** - * @return string|null - */ - public function getMetadata(): ?string - { - return $this->metadata; - } - - /** - * @return string|null - */ - public function getObjectLockLegalHoldStatus(): ?string - { - return $this->objectLockLegalHoldStatus; - } - - /** - * @return string|null - */ - public function getObjectLockMode(): ?string - { - return $this->objectLockMode; - } - - /** - * @return string|null - */ - public function getObjectLockRetainUntilDate(): ?string - { - return $this->objectLockRetainUntilDate; - } - - /** - * @return string|null - */ - public function getRequestPayer(): ?string - { - return $this->requestPayer; - } - - /** - * @return string|null - */ - public function getSseCustomerAlgorithm(): ?string - { - return $this->sseCustomerAlgorithm; - } - - /** - * @return string|null - */ - public function getSseCustomerKey(): ?string - { - return $this->sseCustomerKey; - } - - /** - * @return string|null - */ - public function getSseCustomerKeyMD5(): ?string - { - return $this->sseCustomerKeyMD5; - } - - /** - * @return string|null - */ - public function getSsekmsEncryptionContext(): ?string - { - return $this->ssekmsEncryptionContext; - } - - /** - * @return string|null - */ - public function getSsekmsKeyId(): ?string - { - return $this->ssekmsKeyId; - } - - /** - * @return string|null - */ - public function getServerSideEncryption(): ?string - { - return $this->serverSideEncryption; - } - - /** - * @return string|null - */ - public function getStorageClass(): ?string - { - return $this->storageClass; - } - - /** - * @return string|null - */ - public function getTagging(): ?string - { - return $this->tagging; - } - - /** - * @return string|null - */ - public function getWebsiteRedirectLocation(): ?string - { - return $this->websiteRedirectLocation; - } - - /** - * @return string|null - */ - public function getChecksumCRC32(): ?string - { - return $this->checksumCRC32; - } - - /** - * @return string|null - */ - public function getChecksumCRC32C(): ?string - { - return $this->checksumCRC32C; - } - - /** - * @return string|null - */ - public function getChecksumCRC64NVME(): ?string - { - return $this->checksumCRC64NVME; - } - - /** - * @return string|null - */ - public function getChecksumSHA1(): ?string - { - return $this->checksumSHA1; - } - - /** - * @return string|null - */ - public function getChecksumSHA256(): ?string - { - return $this->checksumSHA256; - } - - /** - * @return string|null - */ - public function getIfMatch(): ?string - { - return $this->ifMatch; - } - - /** - * @return string|null - */ - public function getIfNoneMatch(): ?string - { - return $this->ifNoneMatch; - } - - /** - * Convert to single object request array - * - * @return array - */ - public function toSingleObjectRequest(): array - { - $requestArgs = [ - 'Bucket' => $this->bucket, - 'ChecksumAlgorithm' => $this->checksumAlgorithm, - 'ChecksumCRC32' => $this->checksumCRC32, - 'ChecksumCRC32C' => $this->checksumCRC32C, - 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, - 'ChecksumSHA1' => $this->checksumSHA1, - 'ChecksumSHA256' => $this->checksumSHA256, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'Key' => $this->key, - 'RequestPayer' => $this->requestPayer, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKey' => $this->sseCustomerKey, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - ]; - - remove_nulls_from_array($requestArgs); - - return $requestArgs; - } - - /** - * Convert to create multipart request array - * - * @return array - */ - public function toCreateMultipartRequest(): array - { - $requestArgs = [ - 'ACL' => $this->acl, - 'Bucket' => $this->bucket, - 'BucketKeyEnabled' => $this->bucketKeyEnabled, - 'CacheControl' => $this->cacheControl, - 'ChecksumAlgorithm' => $this->checksumAlgorithm, - 'ContentDisposition' => $this->contentDisposition, - 'ContentEncoding' => $this->contentEncoding, - 'ContentLanguage' => $this->contentLanguage, - 'ContentType' => $this->contentType, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'Expires' => $this->expires, - 'GrantFullControl' => $this->grantFullControl, - 'GrantRead' => $this->grantRead, - 'GrantReadACP' => $this->grantReadACP, - 'GrantWriteACP' => $this->grantWriteACP, - 'Key' => $this->key, - 'Metadata' => $this->metadata, - 'ObjectLockLegalHoldStatus' => $this->objectLockLegalHoldStatus, - 'ObjectLockMode' => $this->objectLockMode, - 'ObjectLockRetainUntilDate' => $this->objectLockRetainUntilDate, - 'RequestPayer' => $this->requestPayer, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKey' => $this->sseCustomerKey, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - 'SSEKMSEncryptionContext' => $this->ssekmsEncryptionContext, - 'SSEKMSKeyId' => $this->ssekmsKeyId, - 'ServerSideEncryption' => $this->serverSideEncryption, - 'StorageClass' => $this->storageClass, - 'Tagging' => $this->tagging, - 'WebsiteRedirectLocation' => $this->websiteRedirectLocation, - ]; - - remove_nulls_from_array($requestArgs); - - return $requestArgs; - } - - /** - * Convert to upload part request array - * - * @return array - */ - public function toUploadPartRequest(): array - { - $requestArgs = [ - 'Bucket' => $this->bucket, - 'ChecksumAlgorithm' => $this->checksumAlgorithm, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'Key' => $this->key, - 'RequestPayer' => $this->requestPayer, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKey' => $this->sseCustomerKey, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - ]; - - remove_nulls_from_array($requestArgs); - - return $requestArgs; - } - - /** - * Convert to complete multipart upload request array - * - * @return array - */ - public function toCompleteMultipartUploadRequest(): array - { - $requestArgs = [ - 'Bucket' => $this->bucket, - 'ChecksumCRC32' => $this->checksumCRC32, - 'ChecksumCRC32C' => $this->checksumCRC32C, - 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, - 'ChecksumSHA1' => $this->checksumSHA1, - 'ChecksumSHA256' => $this->checksumSHA256, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'IfMatch' => $this->ifMatch, - 'IfNoneMatch' => $this->ifNoneMatch, - 'Key' => $this->key, - 'RequestPayer' => $this->requestPayer, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKey' => $this->sseCustomerKey, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - ]; - - remove_nulls_from_array($requestArgs); - - return $requestArgs; - } - - /** - * Convert to abort multipart upload request array - * - * @return array - */ - public function toAbortMultipartRequest(): array - { - $requestArgs = [ - 'Bucket' => $this->bucket, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'Key' => $this->key, - 'RequestPayer' => $this->requestPayer, - ]; - - remove_nulls_from_array($requestArgs); - - return $requestArgs; - } - - /** - * Convert the object to an array - * - * @return array - */ - public function toArray(): array - { - $array = [ - 'ACL' => $this->acl, - 'Bucket' => $this->bucket, - 'BucketKeyEnabled' => $this->bucketKeyEnabled, - 'CacheControl' => $this->cacheControl, - 'ChecksumAlgorithm' => $this->checksumAlgorithm, - 'ContentDisposition' => $this->contentDisposition, - 'ContentEncoding' => $this->contentEncoding, - 'ContentLanguage' => $this->contentLanguage, - 'ContentType' => $this->contentType, - 'ExpectedBucketOwner' => $this->expectedBucketOwner, - 'Expires' => $this->expires, - 'GrantFullControl' => $this->grantFullControl, - 'GrantRead' => $this->grantRead, - 'GrantReadACP' => $this->grantReadACP, - 'GrantWriteACP' => $this->grantWriteACP, - 'Key' => $this->key, - 'Metadata' => $this->metadata, - 'ObjectLockLegalHoldStatus' => $this->objectLockLegalHoldStatus, - 'ObjectLockMode' => $this->objectLockMode, - 'ObjectLockRetainUntilDate' => $this->objectLockRetainUntilDate, - 'RequestPayer' => $this->requestPayer, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKey' => $this->sseCustomerKey, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - 'SSEKMSEncryptionContext' => $this->ssekmsEncryptionContext, - 'SSEKMSKeyId' => $this->ssekmsKeyId, - 'ServerSideEncryption' => $this->serverSideEncryption, - 'StorageClass' => $this->storageClass, - 'Tagging' => $this->tagging, - 'WebsiteRedirectLocation' => $this->websiteRedirectLocation, - 'ChecksumCRC32' => $this->checksumCRC32, - 'ChecksumCRC32C' => $this->checksumCRC32C, - 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, - 'ChecksumSHA1' => $this->checksumSHA1, - 'ChecksumSHA256' => $this->checksumSHA256, - 'IfMatch' => $this->ifMatch, - 'IfNoneMatch' => $this->ifNoneMatch, - ]; - - remove_nulls_from_array($array); - - return $array; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/PutObjectResponse.php b/src/S3/S3Transfer/Models/PutObjectResponse.php deleted file mode 100644 index 0c33ee8cd4..0000000000 --- a/src/S3/S3Transfer/Models/PutObjectResponse.php +++ /dev/null @@ -1,320 +0,0 @@ -bucketKeyEnabled = $bucketKeyEnabled; - $this->checksumCRC32 = $checksumCRC32; - $this->checksumCRC32C = $checksumCRC32C; - $this->checksumCRC64NVME = $checksumCRC64NVME; - $this->checksumSHA1 = $checksumSHA1; - $this->checksumSHA256 = $checksumSHA256; - $this->checksumType = $checksumType; - $this->eTag = $eTag; - $this->expiration = $expiration; - $this->requestCharged = $requestCharged; - $this->sseCustomerAlgorithm = $sseCustomerAlgorithm; - $this->sseCustomerKeyMD5 = $sseCustomerKeyMD5; - $this->ssekmsEncryptionContext = $ssekmsEncryptionContext; - $this->ssekmsKeyId = $ssekmsKeyId; - $this->serverSideEncryption = $serverSideEncryption; - $this->size = $size; - $this->versionId = $versionId; - } - - /** - * Create an instance from an array of data - * - * @param array $data - * @return self - */ - public static function fromArray(array $data): self - { - return new self( - $data['BucketKeyEnabled'] ?? null, - $data['ChecksumCRC32'] ?? null, - $data['ChecksumCRC32C'] ?? null, - $data['ChecksumCRC64NVME'] ?? null, - $data['ChecksumSHA1'] ?? null, - $data['ChecksumSHA256'] ?? null, - $data['ChecksumType'] ?? null, - $data['ETag'] ?? null, - $data['Expiration'] ?? null, - $data['RequestCharged'] ?? null, - $data['SSECustomerAlgorithm'] ?? null, - $data['SSECustomerKeyMD5'] ?? null, - $data['SSEKMSEncryptionContext'] ?? null, - $data['SSEKMSKeyId'] ?? null, - $data['ServerSideEncryption'] ?? null, - $data['Size'] ?? null, - $data['VersionId'] ?? null - ); - } - - /** - * @return bool|null - */ - public function getBucketKeyEnabled(): ?bool - { - return $this->bucketKeyEnabled; - } - - /** - * @return string|null - */ - public function getChecksumCRC32(): ?string - { - return $this->checksumCRC32; - } - - /** - * @return string|null - */ - public function getChecksumCRC32C(): ?string - { - return $this->checksumCRC32C; - } - - /** - * @return string|null - */ - public function getChecksumCRC64NVME(): ?string - { - return $this->checksumCRC64NVME; - } - - /** - * @return string|null - */ - public function getChecksumSHA1(): ?string - { - return $this->checksumSHA1; - } - - /** - * @return string|null - */ - public function getChecksumSHA256(): ?string - { - return $this->checksumSHA256; - } - - /** - * @return string|null - */ - public function getChecksumType(): ?string - { - return $this->checksumType; - } - - /** - * @return string|null - */ - public function getETag(): ?string - { - return $this->eTag; - } - - /** - * @return string|null - */ - public function getExpiration(): ?string - { - return $this->expiration; - } - - /** - * @return string|null - */ - public function getRequestCharged(): ?string - { - return $this->requestCharged; - } - - /** - * @return string|null - */ - public function getSseCustomerAlgorithm(): ?string - { - return $this->sseCustomerAlgorithm; - } - - /** - * @return string|null - */ - public function getSseCustomerKeyMD5(): ?string - { - return $this->sseCustomerKeyMD5; - } - - /** - * @return string|null - */ - public function getSsekmsEncryptionContext(): ?string - { - return $this->ssekmsEncryptionContext; - } - - /** - * @return string|null - */ - public function getSsekmsKeyId(): ?string - { - return $this->ssekmsKeyId; - } - - /** - * @return string|null - */ - public function getServerSideEncryption(): ?string - { - return $this->serverSideEncryption; - } - - /** - * @return int|null - */ - public function getSize(): ?int - { - return $this->size; - } - - /** - * @return string|null - */ - public function getVersionId(): ?string - { - return $this->versionId; - } - - /** - * Convert the object to an array format suitable for multipart upload response - * - * @return array Array containing AWS S3 response fields with their corresponding values - */ - public function toMultipartUploadResponse(): array { - $array = [ - 'BucketKeyEnabled' => $this->bucketKeyEnabled, - 'ChecksumCRC32' => $this->checksumCRC32, - 'ChecksumCRC32C' => $this->checksumCRC32C, - 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, - 'ChecksumSHA1' => $this->checksumSHA1, - 'ChecksumSHA256' => $this->checksumSHA256, - 'ChecksumType' => $this->checksumType, - 'ETag' => $this->eTag, - 'Expiration' => $this->expiration, - 'RequestCharged' => $this->requestCharged, - 'SSEKMSKeyId' => $this->ssekmsKeyId, - 'ServerSideEncryption' => $this->serverSideEncryption, - 'VersionId' => $this->versionId - ]; - - remove_nulls_from_array($array); - - return $array; - } - - /** - * Convert the object to an array format suitable for single upload response - * - * @return array Array containing AWS S3 response fields with their corresponding values - */ - public function toSingleUploadResponse(): array { - $array = [ - 'BucketKeyEnabled' => $this->bucketKeyEnabled, - 'ChecksumCRC32' => $this->checksumCRC32, - 'ChecksumCRC32C' => $this->checksumCRC32C, - 'ChecksumCRC64NVME' => $this->checksumCRC64NVME, - 'ChecksumSHA1' => $this->checksumSHA1, - 'ChecksumSHA256' => $this->checksumSHA256, - 'ChecksumType' => $this->checksumType, - 'ETag' => $this->eTag, - 'Expiration' => $this->expiration, - 'RequestCharged' => $this->requestCharged, - 'SSECustomerAlgorithm' => $this->sseCustomerAlgorithm, - 'SSECustomerKeyMD5' => $this->sseCustomerKeyMD5, - 'SSEKMSEncryptionContext' => $this->ssekmsEncryptionContext, - 'SSEKMSKeyId' => $this->ssekmsKeyId, - 'ServerSideEncryption' => $this->serverSideEncryption, - 'Size' => $this->size, - 'VersionId' => $this->versionId - ]; - - remove_nulls_from_array($array); - - return $array; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php index 53bc161a00..7f7d98bab1 100644 --- a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -5,9 +5,9 @@ class S3TransferManagerConfig { public const DEFAULT_TARGET_PART_SIZE_BYTES = 8388608; // 8MB - private const DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES = 16777216; // 16MB + public const DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES = 16777216; // 16MB public const DEFAULT_REQUEST_CHECKSUM_CALCULATION = 'when_supported'; - private const DEFAULT_RESPONSE_CHECKSUM_VALIDATION = 'when_supported'; + public const DEFAULT_RESPONSE_CHECKSUM_VALIDATION = 'when_supported'; public const DEFAULT_MULTIPART_DOWNLOAD_TYPE = 'part'; public const DEFAULT_CONCURRENCY = 5; private const DEFAULT_TRACK_PROGRESS = false; @@ -91,7 +91,7 @@ public static function fromArray(array $config): self { ?? self::DEFAULT_TARGET_PART_SIZE_BYTES, $config['multipart_upload_threshold_bytes'] ?? self::DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES, - 'request_checksum_calculation' + $config['request_checksum_calculation'] ?? self::DEFAULT_REQUEST_CHECKSUM_CALCULATION, $config['response_checksum_validation'] ?? self::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, diff --git a/src/S3/S3Transfer/Models/TransferRequest.php b/src/S3/S3Transfer/Models/TransferRequest.php index 3a470c4d74..cd1109ab3d 100644 --- a/src/S3/S3Transfer/Models/TransferRequest.php +++ b/src/S3/S3Transfer/Models/TransferRequest.php @@ -6,6 +6,9 @@ abstract class TransferRequest { + public static array $configKeys = [ + 'track_progress' + ]; /** @var array */ protected array $listeners; @@ -13,16 +16,22 @@ abstract class TransferRequest /** @var TransferListener|null */ protected ?TransferListener $progressTracker; + /** @var array */ + protected array $config; + /** * @param array $listeners * @param TransferListener|null $progressTracker + * @param array $config */ public function __construct( array $listeners, - ?TransferListener $progressTracker + ?TransferListener $progressTracker, + array $config ) { $this->listeners = $listeners; $this->progressTracker = $progressTracker; + $this->config = $config; } /** @@ -44,4 +53,24 @@ public function getProgressTracker(): ?TransferListener { return $this->progressTracker; } -} \ No newline at end of file + + /** + * @return array + */ + public function getConfig(): array { + return $this->config; + } + + /** + * @param array $defaultConfig + * + * @return void + */ + public function updateConfigWithDefaults(array $defaultConfig): void { + foreach (static::$configKeys as $key) { + if (empty($this->config[$key])) { + $this->config[$key] = $defaultConfig[$key]; + } + } + } +} diff --git a/src/S3/S3Transfer/Models/TransferRequestConfig.php b/src/S3/S3Transfer/Models/TransferRequestConfig.php deleted file mode 100644 index 40e950a1f2..0000000000 --- a/src/S3/S3Transfer/Models/TransferRequestConfig.php +++ /dev/null @@ -1,31 +0,0 @@ -trackProgress = $trackProgress; - } - - /** - * @return bool|null - */ - public function getTrackProgress(): ?bool - { - return $this->trackProgress; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index 1f91247c3f..5e65406a89 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -14,36 +14,33 @@ class UploadDirectoryRequest extends TransferRequest /** @var string */ private string $targetBucket; - /** @var PutObjectRequest */ - private PutObjectRequest $putObjectRequest; - - /** @var UploadDirectoryRequestConfig */ - private UploadDirectoryRequestConfig $config; + /** @var array */ + private readonly array $putObjectRequestArgs; /** * @param string $sourceDirectory * @param string $targetBucket - * @param PutObjectRequest $putObjectRequest - * @param UploadDirectoryRequestConfig $config + * @param array $putObjectRequestArgs + * @param array $config * @param array $listeners * @param TransferListener|null $progressTracker */ public function __construct( string $sourceDirectory, string $targetBucket, - PutObjectRequest $putObjectRequest, - UploadDirectoryRequestConfig $config, - array $listeners, - ?TransferListener $progressTracker + array $putObjectRequestArgs, + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null ) { - parent::__construct($listeners, $progressTracker); + parent::__construct($listeners, $progressTracker, $config); $this->sourceDirectory = $sourceDirectory; if (ArnParser::isArn($targetBucket)) { $targetBucket = ArnParser::parse($targetBucket)->getResource(); } $this->targetBucket = $targetBucket; - $this->putObjectRequest = $putObjectRequest; + $this->putObjectRequestArgs = $putObjectRequestArgs; $this->config = $config; } @@ -68,8 +65,8 @@ public static function fromLegacyArgs( return new self( $sourceDirectory, $targetBucket, - PutObjectRequest::fromArray($uploadDirectoryRequestArgs), - UploadDirectoryRequestConfig::fromArray($config), + $uploadDirectoryRequestArgs, + $config, $listeners, $progressTracker ); @@ -92,19 +89,11 @@ public function getTargetBucket(): string } /** - * @return PutObjectRequest - */ - public function getPutObjectRequest(): PutObjectRequest - { - return $this->putObjectRequest; - } - - /** - * @return UploadDirectoryRequestConfig + * @return array */ - public function getConfig(): UploadDirectoryRequestConfig + public function getPutObjectRequestArgs(): array { - return $this->config; + return $this->putObjectRequestArgs; } /** diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php b/src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php deleted file mode 100644 index 7d52d83903..0000000000 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequestConfig.php +++ /dev/null @@ -1,167 +0,0 @@ -followSymbolicLinks = $followSymbolicLinks; - $this->recursive = $recursive; - $this->s3Prefix = $s3Prefix; - $this->filter = $filter; - $this->s3Delimiter = $s3Delimiter; - $this->putObjectRequestCallback = $putObjectRequestCallback; - $this->failurePolicy = $failurePolicy; - } - - /* - * @param array $config The config options for this request that are: - * - follow_symbolic_links: (bool, optional, defaulted to false) - * - recursive: (bool, optional, defaulted to false) - * - s3_prefix: (string, optional, defaulted to null) - * - filter: (Closure(SplFileInfo|string), optional) - * By default an instance of SplFileInfo will be provided, however - * you can annotate the parameter with a string type and by doing - * so you will get the full path of the file. - * - s3_delimiter: (string, optional, defaulted to `/`) - * - put_object_request_callback: (Closure, optional) A callback function - * to be invoked right before the request initiates and that will receive - * as parameter the request arguments for each upload request. - * - failure_policy: (Closure, optional) A function that will be invoked - * on an upload failure and that will receive as parameters: - * - $requestArgs: (array) The arguments for the request that originated - * the failure. - * - $uploadDirectoryRequestArgs: (array) The arguments for the upload - * directory request. - * - $reason: (Throwable) The exception that originated the request failure. - * - $uploadDirectoryResponse: (UploadDirectoryResponse) The upload response - * to that point in the upload process. - * - track_progress: (bool, optional) To override the default option for - * enabling progress tracking. If this option is resolved as true and - * a progressTracker parameter is not provided then, a default implementation - * will be resolved. - */ - public static function fromArray(array $array): UploadDirectoryRequestConfig { - return new self( - $array['follow_symbolic_links'] ?? false, - $array['recursive'] ?? false, - $array['s3_prefix'] ?? null, - $array['filter'] ?? null, - $array['s3_delimiter'] ?? '/', - $array['failure_policy'] ?? null, - $array['track_progress'] ?? false - ); - } - - /** - * @return bool - */ - public function isFollowSymbolicLinks(): bool - { - return $this->followSymbolicLinks; - } - - /** - * @return bool - */ - public function isRecursive(): bool - { - return $this->recursive; - } - - /** - * @return string|null - */ - public function getS3Prefix(): ?string - { - return $this->s3Prefix; - } - - /** - * @return Closure|null - */ - public function getFilter(): ?Closure - { - return $this->filter; - } - - /** - * @return string - */ - public function getS3Delimiter(): string - { - return $this->s3Delimiter; - } - - /** - * @return Closure|null - */ - public function getPutObjectRequestCallback(): ?Closure { - return $this->putObjectRequestCallback; - } - - /** - * @return Closure|null - */ - public function getFailurePolicy(): ?Closure - { - return $this->failurePolicy; - } - - /** - * @return array - */ - public function toArray(): array { - return [ - 'follow_symbolic_links' => $this->followSymbolicLinks, - 'recursive' => $this->recursive, - 's3_prefix' => $this->s3Prefix, - 'filter' => $this->filter, - 's3_delimiter' => $this->s3Delimiter, - 'failure_policy' => $this->failurePolicy, - 'track_progress' => $this->trackProgress, - ]; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 809a3a6521..3ca9303f03 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -8,38 +8,23 @@ class UploadRequest extends TransferRequest { + public static array $configKeys = [ + 'multipart_upload_threshold_bytes', + 'target_part_size_bytes', + 'track_progress', + 'concurrency', + 'request_checksum_calculation' + ]; + /** @var StreamInterface|string */ private StreamInterface | string $source; - /** @var PutObjectRequest */ - private PutObjectRequest $putObjectRequest; - - /** @var UploadRequestConfig */ - private UploadRequestConfig $config; - - /** - * @param StreamInterface|string $source - * @param PutObjectRequest $putObjectRequest - * @param UploadRequestConfig $config - * @param array $listeners - * @param TransferListener|null $progressTracker - */ - public function __construct( - StreamInterface|string $source, - PutObjectRequest $putObjectRequest, - UploadRequestConfig $config, - array $listeners = [], - ?TransferListener $progressTracker = null - ) { - parent::__construct($listeners, $progressTracker); - $this->source = $source; - $this->putObjectRequest = $putObjectRequest; - $this->config = $config; - } + /** @var array */ + private array $putObjectRequestArgs; /** * @param string|StreamInterface $source - * @param array $requestArgs The putObject request arguments. + * @param array $putObjectRequestArgs The putObject request arguments. * Required parameters would be: * - Bucket: (string, required) * - Key: (string, required) @@ -53,20 +38,45 @@ public function __construct( * a progressTracker parameter is not provided then, a default implementation * will be resolved. This option is intended to make the operation to use * a default progress tracker implementation when $progressTracker is null. + * - concurrency: (int, optional) + * - request_checksum_calculation: (string, optional, defaulted to `when_supported`) * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker * + */ + public function __construct( + StreamInterface|string $source, + array $putObjectRequestArgs, + array $config, + array $listeners = [], + ?TransferListener $progressTracker = null + ) { + parent::__construct($listeners, $progressTracker, $config); + $this->source = $source; + $this->putObjectRequestArgs = $putObjectRequestArgs; + } + + /** + * @param string|StreamInterface $source + * @param array $putObjectRequestArgs + * @param array $config + * @param array $listeners + * @param TransferListener|null $progressTracker + * * @return UploadRequest */ - public static function fromLegacyArgs(string | StreamInterface $source, - array $requestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null): UploadRequest { + public static function fromLegacyArgs( + string | StreamInterface $source, + array $putObjectRequestArgs = [], + array $config = [], + array $listeners = [], + ?TransferListener $progressTracker = null + ): UploadRequest + { return new UploadRequest( $source, - PutObjectRequest::fromArray($requestArgs), - UploadRequestConfig::fromArray($config), + $putObjectRequestArgs, + $config, $listeners, $progressTracker ); @@ -85,15 +95,11 @@ public function getSource(): StreamInterface|string /** * Get the put object request. * - * @return PutObjectRequest + * @return array */ - public function getPutObjectRequest(): PutObjectRequest + public function getPutObjectRequestArgs(): array { - return $this->putObjectRequest; - } - - public function getConfig(): UploadRequestConfig { - return $this->config; + return $this->putObjectRequestArgs; } /** @@ -121,8 +127,8 @@ public function validateRequiredParameters( ): void { $requiredParametersWithArgs = [ - 'Bucket' => $this->getPutObjectRequest()->getBucket(), - 'Key' => $this->getPutObjectRequest()->getKey() + 'Bucket' => $this->putObjectRequestArgs['Bucket'] ?? null, + 'Key' => $this->putObjectRequestArgs['Key'] ?? null, ]; foreach ($requiredParametersWithArgs as $key => $value) { if (empty($value)) { diff --git a/src/S3/S3Transfer/Models/UploadRequestConfig.php b/src/S3/S3Transfer/Models/UploadRequestConfig.php deleted file mode 100644 index cedb0e6c5a..0000000000 --- a/src/S3/S3Transfer/Models/UploadRequestConfig.php +++ /dev/null @@ -1,120 +0,0 @@ -multipartUploadThresholdBytes = $multipartUploadThresholdBytes; - $this->targetPartSizeBytes = $targetPartSizeBytes; - $this->concurrency = $concurrency; - $this->requestChecksumCalculation = $requestChecksumCalculation; - } - - /** - * Create an UploadConfig instance from an array - * - * @param array $data Array containing configuration data - * - * @return self - */ - public static function fromArray(array $data): self - { - return new self( - $data['multipart_upload_threshold_bytes'] ?? null, - $data['target_part_size_bytes'] ?? null, - $data['track_progress'] ?? null, - $data['concurrency'] ?? null, - $data['request_checksum_calculation'] ?? null, - ); - } - - /** - * Get the multipart upload threshold in bytes - * - * @return int|null - */ - public function getMultipartUploadThresholdBytes(): ?int - { - return $this->multipartUploadThresholdBytes; - } - - /** - * Get the target part size in bytes - * - * @return int|null - */ - public function getTargetPartSizeBytes(): ?int - { - return $this->targetPartSizeBytes; - } - - /** - * @return int|null - */ - public function getConcurrency(): ?int - { - return $this->concurrency; - } - - /** - * @return string|null - */ - public function getRequestChecksumCalculation(): ?string - { - return $this->requestChecksumCalculation; - } - - /** - * Convert the configuration to an array - * - * @return array - */ - public function toArray(): array - { - return [ - 'multipart_upload_threshold_bytes' => $this->multipartUploadThresholdBytes, - 'target_part_size_bytes' => $this->targetPartSizeBytes, - 'track_progress' => $this->trackProgress, - 'concurrency' => $this->concurrency, - 'request_checksum_calculation' => $this->requestChecksumCalculation, - ]; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadResponse.php b/src/S3/S3Transfer/Models/UploadResponse.php deleted file mode 100644 index fe4f7bc89d..0000000000 --- a/src/S3/S3Transfer/Models/UploadResponse.php +++ /dev/null @@ -1,21 +0,0 @@ -uploadResponse = $uploadResponse; - } - - public function getUploadResponse(): array - { - return $this->uploadResponse; - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/UploadResult.php b/src/S3/S3Transfer/Models/UploadResult.php new file mode 100644 index 0000000000..a1f0fabf60 --- /dev/null +++ b/src/S3/S3Transfer/Models/UploadResult.php @@ -0,0 +1,16 @@ +getObjectRequest = $getObjectRequest; + $this->getObjectRequestArgs = $getObjectRequestArgs; + $this->validateConfig($config); $this->config = $config; $this->downloadHandler = $downloadHandler; $this->currentPartNo = $currentPartNo; @@ -93,6 +92,24 @@ public function __construct( $this->listenerNotifier = $listenerNotifier; } + private function validateConfig(array &$config): void { + if (!isset($config['target_part_size_bytes'])) { + $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; + } + + if (!isset($config['response_checksum_validation'])) { + $config['response_checksum_validation'] = S3TransferManagerConfig::DEFAULT_RESPONSE_CHECKSUM_VALIDATION; + } + } + + /** + * @return array + */ + public function getConfig(): array + { + return $this->config; + } + /** * @return int */ @@ -126,9 +143,9 @@ public function getCurrentSnapshot(): TransferProgressSnapshot } /** - * @return DownloadResponse + * @return DownloadResult */ - public function download(): DownloadResponse { + public function download(): DownloadResult { return $this->promise()->wait(); } @@ -170,11 +187,12 @@ public function promise(): PromiseInterface $this->downloadComplete(); // Return response - yield Create::promiseFor(new DownloadResponse( + $result = $initialRequestResult->toArray(); + unset($result['Body']); + + yield Create::promiseFor(new DownloadResult( $this->downloadHandler->getHandlerResult(), - GetObjectResponse::fromArray( - $initialRequestResult->toArray() - )->toArray(), + $result, )); } catch (\Throwable $e) { $this->downloadFailed($e); @@ -314,7 +332,7 @@ private function downloadFailed(\Throwable $reason): void ); $this->listenerNotifier?->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequest->toArray(), + TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); @@ -345,7 +363,7 @@ private function partDownloadCompleted( ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->bytesTransferred([ - TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequest->toArray(), + TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -379,7 +397,7 @@ private function downloadComplete(): void ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->transferComplete([ - TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequest->toArray(), + TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } diff --git a/src/S3/S3Transfer/MultipartDownloaderInitial.php b/src/S3/S3Transfer/MultipartDownloaderInitial.php index 8da9fc8176..e866bbd294 100644 --- a/src/S3/S3Transfer/MultipartDownloaderInitial.php +++ b/src/S3/S3Transfer/MultipartDownloaderInitial.php @@ -5,7 +5,7 @@ use Aws\CommandInterface; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; @@ -124,9 +124,9 @@ public function getCurrentSnapshot(): TransferProgressSnapshot } /** - * @return DownloadResponse + * @return DownloadResult */ - public function download(): DownloadResponse { + public function download(): DownloadResult { return $this->promise()->wait(); } @@ -185,7 +185,7 @@ public function promise(): PromiseInterface $this->downloadComplete(); unset($result['Body']); - yield Create::promiseFor(new DownloadResponse( + yield Create::promiseFor(new DownloadResult( $this->stream, $result['@metadata'] ?? [] )); diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 30912ef368..71262ccf6b 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -6,10 +6,7 @@ use Aws\PhpHash; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; -use Aws\S3\S3Transfer\Models\PutObjectRequest; -use Aws\S3\S3Transfer\Models\PutObjectResponse; -use Aws\S3\S3Transfer\Models\UploadResponse; +use Aws\S3\S3Transfer\Models\UploadResult; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use GuzzleHttp\Promise\Create; @@ -37,8 +34,8 @@ class MultipartUploader extends AbstractMultipartUploader public function __construct( S3ClientInterface $s3Client, - PutObjectRequest $putObjectRequest, - MultipartUploaderConfig $config, + array $putObjectRequestArgs, + array $config, string | StreamInterface $source, ?string $uploadId = null, array $parts = [], @@ -48,7 +45,7 @@ public function __construct( { parent::__construct( $s3Client, - $putObjectRequest, + $putObjectRequestArgs, $config, $uploadId, $parts, @@ -57,6 +54,7 @@ public function __construct( ); $this->body = $this->parseBody($source); $this->calculatedObjectSize = 0; + $this->evaluateCustomChecksum(); } /** @@ -77,7 +75,7 @@ private function parseBody( } $body = new LazyOpenStream($source, 'r'); // To make sure the resource is closed. - $this->deferFns[] = function () use ($body) { + $this->onCompletionCallbacks[] = function () use ($body) { $body->close(); }; } elseif ($source instanceof StreamInterface) { @@ -91,25 +89,52 @@ private function parseBody( return $body; } + /** + * @return void + */ + private function evaluateCustomChecksum(): void { + // Evaluation for custom provided checksums + $checksumName = self::filterChecksum($this->putObjectRequestArgs); + if ($checksumName !== null) { + $this->requestChecksum = $this->putObjectRequestArgs[$checksumName]; + $this->requestChecksumAlgorithm = str_replace( + 'Checksum', + '', + $checksumName + ); + } else { + $this->requestChecksum = null; + $this->requestChecksumAlgorithm = null; + } + } + protected function processMultipartOperation(): PromiseInterface { + $uploadPartCommandArgs = $this->putObjectRequestArgs; $this->calculatedObjectSize = 0; $partSize = $this->calculatePartSize(); $partsCount = ceil($this->getTotalSize() / $partSize); $commands = []; $partNo = count($this->parts); - $uploadPartCommandArgs = $this->putObjectRequest->toUploadPartRequest(); $uploadPartCommandArgs['UploadId'] = $this->uploadId; // Customer provided checksum $hashBody = false; if ($this->requestChecksum !== null) { // To avoid default calculation $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; - } elseif ($this->requestChecksumAlgorithm === self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM) { + unset($uploadPartCommandArgs['Checksum'. ucfirst($this->requestChecksumAlgorithm)]); + } elseif ($this->requestChecksumAlgorithm !== null) { + // Normalize algorithm name + $algoName = strtolower($this->requestChecksumAlgorithm); + if ($algoName === self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM) { + $algoName = 'crc32b'; + } + $hashBody = true; - $this->hashContext = hash_init('crc32b'); + $this->hashContext = hash_init($algoName); // To avoid default calculation $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + unset($uploadPartCommandArgs['Checksum'. ucfirst($this->requestChecksumAlgorithm)]); } while (!$this->body->eof()) { @@ -163,7 +188,7 @@ protected function processMultipartOperation(): PromiseInterface $this->s3Client, $commands, [ - 'concurrency' => $this->config->getConcurrency(), + 'concurrency' => $this->config['concurrency'], 'fulfilled' => function (ResultInterface $result, $index) use ($commands) { $command = $commands[$index]; @@ -201,14 +226,12 @@ protected function getTotalSize(): int /** * @param ResultInterface $result * - * @return UploadResponse + * @return UploadResult */ - protected function createResponse(ResultInterface $result): UploadResponse + protected function createResponse(ResultInterface $result): UploadResult { - return new UploadResponse( - PutObjectResponse::fromArray( - $result->toArray() - )->toMultipartUploadResponse() + return new UploadResult( + $result->toArray() ); } @@ -226,4 +249,29 @@ private function decorateWithHashes(StreamInterface $stream, array &$data): Stre $data['ContentSHA256'] = bin2hex($result); }); } + + /** + * Filters a provided checksum if one was provided. + * + * @param array $requestArgs + * + * @return string | null + */ + private static function filterChecksum(array $requestArgs):? string + { + static $algorithms = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + foreach ($algorithms as $algorithm) { + if (isset($requestArgs[$algorithm])) { + return $algorithm; + } + } + + return null; + } } diff --git a/src/S3/S3Transfer/MultipartUploaderInitial.php b/src/S3/S3Transfer/MultipartUploaderInitial.php index 612e64797a..316583525b 100644 --- a/src/S3/S3Transfer/MultipartUploaderInitial.php +++ b/src/S3/S3Transfer/MultipartUploaderInitial.php @@ -7,7 +7,7 @@ use Aws\PhpHash; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Models\UploadResponse; +use Aws\S3\S3Transfer\Models\UploadResult; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; @@ -147,9 +147,9 @@ public function getCurrentSnapshot(): ?TransferProgressSnapshot } /** - * @return UploadResponse + * @return UploadResult */ - public function upload(): UploadResponse { + public function upload(): UploadResult { return $this->promise()->wait(); } @@ -164,7 +164,7 @@ public function promise(): PromiseInterface yield $this->uploadParts(); $result = yield $this->completeMultipartUpload(); yield Create::promiseFor( - new UploadResponse($result->toArray()) + new UploadResult($result->toArray()) ); } catch (Throwable $e) { $this->uploadFailed($e); diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index 95b717ce5e..8058ccaeed 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -25,9 +25,9 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = $this->getObjectRequest->toArray(); + $nextRequestArgs = $this->getObjectRequestArgs; $nextRequestArgs['PartNumber'] = $this->currentPartNo; - if ($this->config->getResponseChecksumValidationEnabled()) { + if ($this->config['response_checksum_validation'] === 'when_supported') { $nextRequestArgs['ChecksumMode'] = 'ENABLED'; } diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index d4e96b6a36..5cf1e9f67f 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -21,8 +21,8 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = $this->getObjectRequest->toArray(); - $partSize = $this->config->getTargetPartSizeBytes(); + $nextRequestArgs = $this->getObjectRequestArgs; + $partSize = $this->config['target_part_size_bytes']; $from = ($this->currentPartNo - 1) * $partSize; $to = ($this->currentPartNo * $partSize) - 1; @@ -32,7 +32,7 @@ protected function nextCommand(): CommandInterface $nextRequestArgs['Range'] = "bytes=$from-$to"; - if ($this->config->getResponseChecksumValidationEnabled()) { + if ($this->config['response_checksum_validation'] === 'when_supported') { $nextRequestArgs['ChecksumMode'] = 'ENABLED'; } @@ -62,7 +62,7 @@ protected function computeObjectDimensions(ResultInterface $result): void ); } - $partSize = $this->config->getTargetPartSizeBytes(); + $partSize = $this->config['target_part_size_bytes']; if ($this->objectSizeInBytes > $partSize) { $this->objectPartsCount = intval( ceil($this->objectSizeInBytes / $partSize) diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 55e32cb69b..2b30421644 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,31 +2,24 @@ namespace Aws\S3\S3Transfer; -use Aws\Arn\ArnParser; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; use Aws\S3\S3Transfer\Models\DownloadFileRequest; -use Aws\S3\S3Transfer\Models\DownloadHandler; use Aws\S3\S3Transfer\Models\DownloadRequest; -use Aws\S3\S3Transfer\Models\DownloadResponse; -use Aws\S3\S3Transfer\Models\GetObjectRequest; -use Aws\S3\S3Transfer\Models\MultipartDownloaderConfig; -use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; -use Aws\S3\S3Transfer\Models\PutObjectRequest; -use Aws\S3\S3Transfer\Models\PutObjectResponse; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; use Aws\S3\S3Transfer\Models\UploadRequest; -use Aws\S3\S3Transfer\Models\UploadResponse; +use Aws\S3\S3Transfer\Models\UploadResult; use Aws\S3\S3Transfer\Progress\MultiProgressTracker; use Aws\S3\S3Transfer\Progress\SingleProgressTracker; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; +use Aws\S3\S3Transfer\Utils\DownloadHandler; use FilesystemIterator; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; @@ -49,13 +42,18 @@ class S3TransferManager * @param S3ClientInterface | null $s3Client If provided as null then, * a default client will be created where its region will be the one * resolved from either the default from the config or the provided. - * @param S3TransferManagerConfig|null $config + * @param array|S3TransferManagerConfig|null $config */ public function __construct( ?S3ClientInterface $s3Client = null, - ?S3TransferManagerConfig $config = null + array|S3TransferManagerConfig|null $config = null ) { - $this->config = $config ?? S3TransferManagerConfig::fromArray([]); + if ($config === null || is_array($config)) { + $this->config = S3TransferManagerConfig::fromArray($config ?? []); + } else { + $this->config = $config; + } + if ($s3Client === null) { $this->s3Client = $this->defaultS3Client(); } else { @@ -92,12 +90,16 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface // Valid required parameters $uploadRequest->validateRequiredParameters(); + $uploadRequest->updateConfigWithDefaults( + $this->config->toArray() + ); + $config = $uploadRequest->getConfig(); // Validate progress tracker $progressTracker = $uploadRequest->getProgressTracker(); if ($progressTracker === null - && ($config->getTrackProgress() + && ($config['track_progress'] ?? $this->config->isTrackProgress())) { $progressTracker = new SingleProgressTracker(); } @@ -111,7 +113,7 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface $listenerNotifier = new TransferListenerNotifier($listeners); // Validate multipart upload threshold - $mupThreshold = $config->getMultipartUploadThresholdBytes() + $mupThreshold = $config['multipart_upload_threshold_bytes'] ?? $this->config->getMultipartUploadThresholdBytes(); if ($mupThreshold < AbstractMultipartUploader::PART_MIN_SIZE) { throw new InvalidArgumentException( @@ -129,7 +131,7 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface return $this->trySingleUpload( $uploadRequest->getSource(), - $uploadRequest->getPutObjectRequest()->toSingleObjectRequest(), + $uploadRequest->getPutObjectRequestArgs(), $listenerNotifier ); } @@ -145,26 +147,49 @@ public function uploadDirectory( { $uploadDirectoryRequest->validateSourceDirectory(); $targetBucket = $uploadDirectoryRequest->getTargetBucket(); + + $uploadDirectoryRequest->updateConfigWithDefaults( + $this->config->toArray() + ); + $config = $uploadDirectoryRequest->getConfig(); $progressTracker = $uploadDirectoryRequest->getProgressTracker(); if ($progressTracker === null - && ($config->getTrackProgress() ?? $this->config->isTrackProgress())) { + && ($config['track_progress'] ?? $this->config->isTrackProgress())) { $progressTracker = new MultiProgressTracker(); } $filter = null; - if ($config->getFilter() !== null) { - $filter = $config->getFilter(); + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new InvalidArgumentException( + "The provided config `filter` must be callable." + ); + } + + $filter = $config['filter']; } $putObjectRequestCallback = null; - if ($config->getPutObjectRequestCallback() !== null) { - $putObjectRequestCallback = $config->getPutObjectRequestCallback(); + if (isset($config['put_object_request_callback'])) { + if (!is_callable($config['put_object_request_callback'])) { + throw new InvalidArgumentException( + "The provided config `put_object_request_callback` must be callable." + ); + } + + $putObjectRequestCallback = $config['put_object_request_callback']; } $failurePolicyCallback = null; - if ($config->getFailurePolicy() !== null) { - $failurePolicyCallback = $config->getFailurePolicy(); + if (isset($config['failure_policy'])) { + if (!is_callable($config['failure_policy'])) { + throw new InvalidArgumentException( + "The provided config `failure_policy` must be callable." + ); + } + + $failurePolicyCallback = $config['failure_policy']; } $sourceDirectory = $uploadDirectoryRequest->getSourceDirectory(); @@ -172,11 +197,11 @@ public function uploadDirectory( $sourceDirectory ); $dirIterator->setFlags(FilesystemIterator::SKIP_DOTS); - if ($config->isFollowSymbolicLinks()) { + if ($config['follow_symbolic_links'] ?? false) { $dirIterator->setFlags(FilesystemIterator::FOLLOW_SYMLINKS); } - if ($config->isRecursive()) { + if ($config['recursive'] ?? false) { $dirIterator = new RecursiveIteratorIterator($dirIterator); } @@ -191,12 +216,12 @@ function ($file) use ($filter) { } ); - $prefix = $config->getS3Prefix() ?? ''; + $prefix = $config['s3_prefix'] ?? ''; if ($prefix !== '' && !str_ends_with($prefix, '/')) { $prefix .= '/'; } - $delimiter = $config->getS3Delimiter(); + $delimiter = $config['s3_delimiter'] ?? '/'; $promises = []; $objectsUploaded = 0; $objectsFailed = 0; @@ -214,11 +239,10 @@ function ($file) use ($filter) { $delimiter, $objectKey ); - $putObjectRequestArgs = [ - ...$uploadDirectoryRequest->getPutObjectRequest()->toArray(), - 'Bucket' => $targetBucket, - 'Key' => $objectKey - ]; + $putObjectRequestArgs = $uploadDirectoryRequest->getPutObjectRequestArgs(); + $putObjectRequestArgs['Bucket'] = $targetBucket; + $putObjectRequestArgs['Key'] = $objectKey; + if ($putObjectRequestCallback !== null) { $putObjectRequestCallback($putObjectRequestArgs); } @@ -227,14 +251,14 @@ function ($file) use ($filter) { UploadRequest::fromLegacyArgs( $file, $putObjectRequestArgs, - $config->toArray(), + $config, array_map( function ($listener) { return clone $listener; }, $uploadDirectoryRequest->getListeners() ), $progressTracker ) - )->then(function (UploadResponse $response) use (&$objectsUploaded) { + )->then(function (UploadResult $response) use (&$objectsUploaded) { $objectsUploaded++; return $response; @@ -269,8 +293,8 @@ function ($listener) { return clone $listener; }, }); } - return Each::ofLimitAll($promises, $this->config['concurrency']) - ->then(function ($_) use ($objectsUploaded, $objectsFailed) { + return Each::ofLimitAll($promises, $this->config->getConcurrency()) + ->then(function ($_) use (&$objectsUploaded, &$objectsFailed) { return new UploadDirectoryResponse($objectsUploaded, $objectsFailed); }); } @@ -283,29 +307,14 @@ function ($listener) { return clone $listener; }, public function download(DownloadRequest $downloadRequest): PromiseInterface { $sourceArgs = $downloadRequest->normalizeSourceAsArray(); - $getObjectRequest = $downloadRequest->getGetObjectRequest(); - $config = [ - 'response_checksum_validation_enabled' => false - ]; - if (empty($getObjectRequest->getChecksumMode())) { - $requestChecksumValidation = - $downloadRequest->getConfig()->getRequestChecksumValidation() - ?? $this->config->getRequestChecksumCalculation(); + $getObjectRequestArgs = $downloadRequest->getObjectRequestArgs(); - if ($requestChecksumValidation === 'when_supported') { - $config['response_checksum_validation_enabled'] = true; - } - } else { - $config['response_checksum_validation_enabled'] = true; - } + $downloadRequest->updateConfigWithDefaults($this->config->toArray()); - $config['multipart_download_type'] = $downloadRequest->getConfig() - ->getMultipartDownloadType() ?? $this->config->getMultipartDownloadType(); + $config = $downloadRequest->getConfig(); $progressTracker = $downloadRequest->getProgressTracker(); - if ($progressTracker === null - && ($downloadRequest->getConfig()->getTrackProgress() - ?? $this->getConfig()->isTrackProgress())) { + if ($progressTracker === null && $config['track_progress']) { $progressTracker = new SingleProgressTracker(); } @@ -318,14 +327,13 @@ public function download(DownloadRequest $downloadRequest): PromiseInterface $listenerNotifier = new TransferListenerNotifier($listeners); // Assign source - $getObjectRequestArray = $getObjectRequest->toArray(); foreach ($sourceArgs as $key => $value) { - $getObjectRequestArray[$key] = $value; + $getObjectRequestArgs[$key] = $value; } return $this->tryMultipartDownload( - GetObjectRequest::fromArray($getObjectRequestArray), - MultipartDownloaderConfig::fromArray($config), + $getObjectRequestArgs, + $config, $downloadRequest->getDownloadHandler(), $listenerNotifier, ); @@ -356,33 +364,43 @@ public function downloadDirectory( $destinationDirectory = $downloadDirectoryRequest->getDestinationDirectory(); $sourceBucket = $downloadDirectoryRequest->getSourceBucket(); $progressTracker = $downloadDirectoryRequest->getProgressTracker(); + + $downloadDirectoryRequest->updateConfigWithDefaults( + $this->config->toArray() + ); $config = $downloadDirectoryRequest->getConfig(); - if ($progressTracker === null - && ($config->getTrackProgress() ?? $this->config->isTrackProgress())) { + if ($progressTracker === null && $config['track_progress']) { $progressTracker = new MultiProgressTracker(); } $listArgs = [ 'Bucket' => $sourceBucket, - ] + ($config->getListObjectV2Args()); - $s3Prefix = $config->getEffectivePrefix(); - if ($s3Prefix !== null) { + ] + ($config['list_object_v2_args'] ?? []); + $s3Prefix = $config['s3_prefix'] ?? null; + if (empty($listArgs['Prefix']) && $s3Prefix !== null) { $listArgs['Prefix'] = $s3Prefix; } - $listArgs['Delimiter'] = $listArgs['Delimiter'] ?? null; + $listArgs['Delimiter'] = $listArgs['Delimiter'] + ?? $config['s3_delimiter'] ?? null; $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) ->search('Contents[].Key'); - $filter = $config->getFilter(); - if ($filter !== null) { + if (isset($config['filter'])) { + if (!is_callable($config['filter'])) { + throw new InvalidArgumentException( + "The provided config `filter` must be callable." + ); + } + + $filter = $config['filter']; $objects = filter($objects, function (string $key) use ($filter) { return call_user_func($filter, $key) && !str_ends_with($key, "/"); }); } else { - $objects = filter($objects, function (string $key) use ($filter) { + $objects = filter($objects, function (string $key) { return !str_ends_with($key, "/"); }); } @@ -390,14 +408,27 @@ public function downloadDirectory( $objects = map($objects, function (string $key) use ($sourceBucket) { return self::formatAsS3URI($sourceBucket, $key); }); + $getObjectRequestCallback = null; - if ($config->getGetObjectRequestCallback() !== null) { - $getObjectRequestCallback = $config->getGetObjectRequestCallback(); + if (isset($config['get_object_request_callback'])) { + if (!is_callable($config['get_object_request_callback'])) { + throw new InvalidArgumentException( + "The provided config `get_object_request_callback` must be callable." + ); + } + + $getObjectRequestCallback = $config['get_object_request_callback']; } $failurePolicyCallback = null; - if ($config->getFailurePolicy() !== null) { - $failurePolicyCallback = $config->getFailurePolicy(); + if (isset($config['failure_policy'])) { + if (!is_callable($config['failure_policy'])) { + throw new InvalidArgumentException( + "The provided config `failure_policy` must be callable." + ); + } + + $failurePolicyCallback = $config['failure_policy']; } $promises = []; @@ -415,7 +446,7 @@ public function downloadDirectory( ); } - $requestArgs = $downloadDirectoryRequest->getGetObjectRequest()->toArray(); + $requestArgs = $downloadDirectoryRequest->getGetObjectRequestArgs(); foreach ($bucketAndKeyArray as $key => $value) { $requestArgs[$key] = $value; } @@ -426,12 +457,12 @@ public function downloadDirectory( $promises[] = $this->downloadFile( new DownloadFileRequest( $destinationFile, - $config->isFailsWhenDestinationExists(), - DownloadRequest::fromLegacyArgs( + $config['fails_when_destination_exists'] ?? false, + new DownloadRequest( null, $requestArgs, [ - 'target_part_size_bytes' => $config->getTargetPartSizeBytes() ?? 0, + 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, ], null, array_map( @@ -488,26 +519,26 @@ function ($listener) { return clone $listener; }, /** * Tries an object multipart download. * - * @param GetObjectRequest $getObjectRequest - * @param MultipartDownloaderConfig $config + * @param array $getObjectRequestArgs + * @param array $config * @param DownloadHandler $downloadHandler * @param TransferListenerNotifier|null $listenerNotifier * * @return PromiseInterface */ private function tryMultipartDownload( - GetObjectRequest $getObjectRequest, - MultipartDownloaderConfig $config, + array $getObjectRequestArgs, + array $config, DownloadHandler $downloadHandler, ?TransferListenerNotifier $listenerNotifier = null, ): PromiseInterface { $downloaderClassName = MultipartDownloader::chooseDownloaderClass( - $config->getMultipartDownloadType() + $config['multipart_download_type'] ); $multipartDownloader = new $downloaderClassName( $this->s3Client, - $getObjectRequest, + $getObjectRequestArgs, $config, $downloadHandler, listenerNotifier: $listenerNotifier, @@ -524,7 +555,7 @@ private function tryMultipartDownload( * @return PromiseInterface */ private function trySingleUpload( - string | StreamInterface $source, + string|StreamInterface $source, array $requestArgs, ?TransferListenerNotifier $listenerNotifier = null ): PromiseInterface { @@ -578,10 +609,8 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { ] ); - return new UploadResponse( - PutObjectResponse::fromArray( - $result->toArray() - )->toSingleUploadResponse() + return new UploadResult( + $result->toArray() ); } )->otherwise(function ($reason) use ($objectSize, $requestArgs, $listenerNotifier) { @@ -605,7 +634,7 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { return $this->s3Client->executeAsync($command) ->then(function ($result) { - return new UploadResponse($result->toArray()); + return new UploadResult($result->toArray()); }); } @@ -621,10 +650,8 @@ private function tryMultipartUpload( ): PromiseInterface { return (new MultipartUploader( $this->s3Client, - $uploadRequest->getPutObjectRequest(), - MultipartUploaderConfig::fromArray( - $uploadRequest->getConfig()->toArray() - ), + $uploadRequest->getPutObjectRequestArgs(), + $uploadRequest->getConfig(), $uploadRequest->getSource(), listenerNotifier: $listenerNotifier, ))->promise(); @@ -665,7 +692,7 @@ private function requiresMultipartUpload( private function defaultS3Client(): S3ClientInterface { return new S3Client([ - 'region' => $this->config['region'], + 'region' => $this->config->getDefaultRegion(), ]); } diff --git a/src/S3/S3Transfer/Models/DownloadHandler.php b/src/S3/S3Transfer/Utils/DownloadHandler.php similarity index 92% rename from src/S3/S3Transfer/Models/DownloadHandler.php rename to src/S3/S3Transfer/Utils/DownloadHandler.php index f5337abfab..c887d68f26 100644 --- a/src/S3/S3Transfer/Models/DownloadHandler.php +++ b/src/S3/S3Transfer/Utils/DownloadHandler.php @@ -1,6 +1,6 @@ download([ 'Bucket' => self::getResourceName(), 'Key' => $filename, - ])->then(function (DownloadResponse $response) use ($filename) { + ])->then(function (DownloadResult $response) use ($filename) { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; file_put_contents($fullFilePath, $response->getData()->getContents()); })->wait(); @@ -381,7 +381,7 @@ public function iDownloadTheObjectWithNameByUsingTheMultipartDownloadType($filen [], [ 'multipart_download_type' => $download_type, - ])->then(function (DownloadResponse $response) use ($filename) { + ])->then(function (DownloadResult $response) use ($filename) { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; file_put_contents($fullFilePath, $response->getData()->getContents()); })->wait(); diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 2675f2b2d4..682275953e 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -5,12 +5,15 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; -use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\DownloadResult; +use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\PartGetMultipartDownloader; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\DownloadHandler; +use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; @@ -28,7 +31,7 @@ class MultipartDownloaderTest extends TestCase public function testChooseDownloaderClass(): void { $multipartDownloadTypes = [ MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, - MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, + MultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, ]; foreach ($multipartDownloadTypes as $multipartDownloadType => $class) { $resolvedClass = MultipartDownloader::chooseDownloaderClass($multipartDownloadType); @@ -57,8 +60,8 @@ public function testChooseDownloaderClassThrowsExceptionForInvalidType(): void public function testConstants(): void { $this->assertEquals('GetObject', MultipartDownloader::GET_OBJECT_COMMAND); - $this->assertEquals('partGet', MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER); - $this->assertEquals('rangeGet', MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER); + $this->assertEquals('part', MultipartDownloader::PART_GET_MULTIPART_DOWNLOADER); + $this->assertEquals('ranged', MultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER); } /** @@ -114,17 +117,17 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $s3Client, $requestArgs, [], + new StreamDownloadHandler(), 0, 0, 0, '', null, - null, - $listenerNotifier + $listenerNotifier, ); $response = $multipartDownloader->promise()->wait(); - $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertInstanceOf(DownloadResult::class, $response); } /** @@ -167,10 +170,10 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $s3Client, $requestArgs, [], + new StreamDownloadHandler(), 0, 0, 0, - '', null, null, $listenerNotifier @@ -218,16 +221,67 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $s3Client, $requestArgs, [], + new StreamDownloadHandler(), 0, 0, 0, - '', null, null, $listenerNotifier ); $response = $multipartDownloader->promise()->wait(); - $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertInstanceOf(DownloadResult::class, $response); + } + + /** + * @return void + */ + public function testConfigIsSetToDefaultValues(): void { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $multipartDownloader = new PartGetMultipartDownloader( + $mockClient, + [], + [], + new StreamDownloadHandler(), + ); + $config = $multipartDownloader->getConfig(); + $this->assertEquals( + S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES, + $config['target_part_size_bytes'] + ); + $this->assertEquals( + S3TransferManagerConfig::DEFAULT_RESPONSE_CHECKSUM_VALIDATION, + $config['response_checksum_validation'] + ); + } + + /** + * @return void + */ + public function testCustomConfigIsSet(): void { + $mockClient = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $multipartDownloader = new PartGetMultipartDownloader( + $mockClient, + [], + [ + 'target_part_size_bytes' => 1024 * 1024 * 10, + 'response_checksum_validation' => 'when_required', + ], + new StreamDownloadHandler(), + ); + $config = $multipartDownloader->getConfig(); + $this->assertEquals( + 1024 * 1024 * 10, + $config['target_part_size_bytes'] + ); + $this->assertEquals( + 'when_required', + $config['response_checksum_validation'] + ); } } \ No newline at end of file diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index e77ee6a230..bce9d6b423 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -9,21 +9,16 @@ use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\AbstractMultipartUploader; use Aws\S3\S3Transfer\Exceptions\S3TransferException; -use Aws\S3\S3Transfer\Models\MultipartUploaderConfig; -use Aws\S3\S3Transfer\Models\PutObjectRequest; -use Aws\S3\S3Transfer\Models\UploadRequestConfig; -use Aws\S3\S3Transfer\Models\UploadResponse; +use Aws\S3\S3Transfer\Models\UploadResult; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\Test\TestsUtility; use GuzzleHttp\Promise\Create; -use GuzzleHttp\Psr7\NoSeekStream; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; -use Psr\Http\Message\StreamInterface; class MultipartUploaderTest extends TestCase { @@ -110,19 +105,15 @@ public function testMultipartUpload( try { $multipartUploader = new MultipartUploader( $s3Client, - PutObjectRequest::fromArray( - $requestArgs, - ), - MultipartUploaderConfig::fromArray( - $config - ), + $requestArgs, + $config, $source ); - /** @var UploadResponse $response */ + /** @var UploadResult $response */ $response = $multipartUploader->promise()->wait(); $snapshot = $multipartUploader->getCurrentSnapshot(); - $this->assertInstanceOf(UploadResponse::class, $response); + $this->assertInstanceOf(UploadResult::class, $response); $this->assertCount($expected['parts'], $multipartUploader->getParts()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); @@ -145,7 +136,9 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'target_part_size_bytes' => 10240000 + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' ], 'expected' => [ 'succeed' => true, @@ -160,7 +153,9 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'target_part_size_bytes' => 10240000 + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' ], 'expected' => [ 'succeed' => true, @@ -175,7 +170,9 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'target_part_size_bytes' => 10240000 + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' ], 'expected' => [ 'succeed' => true, @@ -190,7 +187,9 @@ public function multipartUploadProvider(): array { ], 'command_args' => [], 'config' => [ - 'target_part_size_bytes' => 10240000 + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' ], 'expected' => [ 'succeed' => true, @@ -207,7 +206,9 @@ public function multipartUploadProvider(): array { 'ChecksumCRC32' => 'FooChecksum', ], 'config' => [ - 'target_part_size_bytes' => 10240000 + 'target_part_size_bytes' => 10240000, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' ], 'expected' => [ 'succeed' => true, @@ -286,12 +287,12 @@ public function testValidatePartSize( new MultipartUploader( $this->getMultipartUploadS3Client(), - PutObjectRequest::fromArray( - ['Bucket' => 'test-bucket', 'Key' => 'test-key'] - ), - MultipartUploaderConfig::fromArray([ - 'target_part_size_bytes' => $partSize - ]), + ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + [ + 'target_part_size_bytes' => $partSize, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], Utils::streamFor('') ); } @@ -356,11 +357,15 @@ public function testInvalidSourceStringThrowsException( try { new MultipartUploader( $this->getMultipartUploadS3Client(), - PutObjectRequest::fromArray([ + ([ 'Bucket' => 'test-bucket', 'Key' => 'test-key' ]), - MultipartUploaderConfig::fromArray([]), + [ + 'target_part_size_bytes' => 1024 * 1024 * 5, + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], $source ); } finally { @@ -446,11 +451,12 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $multipartUploader = new MultipartUploader( $s3Client, - PutObjectRequest::fromArray($requestArgs), - MultipartUploaderConfig::fromArray([ + $requestArgs, + [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ]), + 'request_checksum_calculation' => 'when_supported' + ], $stream, null, [], @@ -459,7 +465,7 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void ); $response = $multipartUploader->promise()->wait(); - $this->assertInstanceOf(UploadResponse::class, $response); + $this->assertInstanceOf(UploadResult::class, $response); } /** @@ -504,10 +510,12 @@ public function testMultipartOperationsAreCalled(): void { $multipartUploader = new MultipartUploader( $s3Client, - PutObjectRequest::fromArray($requestArgs), - MultipartUploaderConfig::fromArray([ + $requestArgs, + [ + 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ]), + 'request_checksum_calculation' => 'when_supported' + ], $stream ); @@ -598,18 +606,20 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) try { $multipartUploader = new MultipartUploader( $s3Client, - PutObjectRequest::fromArray($requestArgs), - MultipartUploaderConfig::fromArray([ + $requestArgs, + [ + 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 3, - ]), + 'request_checksum_calculation' => 'when_supported' + ], $source, ); - /** @var UploadResponse $response */ + /** @var UploadResult $response */ $response = $multipartUploader->promise()->wait(); foreach ($operationsCalled as $key => $value) { $this->assertTrue($value, 'Operation {' . $key . '} was not called'); } - $this->assertInstanceOf(UploadResponse::class, $response); + $this->assertInstanceOf(UploadResult::class, $response); } finally { foreach ($cleanUpFns as $fn) { $fn(); @@ -728,10 +738,12 @@ public function testMultipartUploadAbort() { try { $multipartUploader = new MultipartUploader( $s3Client, - PutObjectRequest::fromArray($requestArgs), - MultipartUploaderConfig::fromArray([ - 'concurrency' => 3, - ]), + $requestArgs, + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 1, + 'request_checksum_calculation' => 'when_supported' + ], $source, ); $multipartUploader->promise()->wait(); @@ -788,11 +800,12 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $multipartUploader = new MultipartUploader( $s3Client, - PutObjectRequest::fromArray($requestArgs), - MultipartUploaderConfig::fromArray([ + $requestArgs, + [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ]), + 'request_checksum_calculation' => 'when_supported' + ], $stream, null, [], @@ -839,11 +852,11 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $multipartUploader = new MultipartUploader( $s3Client, - PUtObjectRequest::fromArray($requestArgs), - MultipartUploaderConfig::fromArray([ + $requestArgs, + [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, - ]), + ], $stream, null, [], @@ -852,6 +865,6 @@ public function testTransferListenerNotifierWithEmptyListeners(): void ); $response = $multipartUploader->promise()->wait(); - $this->assertInstanceOf(UploadResponse::class, $response); + $this->assertInstanceOf(UploadResult::class, $response); } } \ No newline at end of file diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index b5ca7685f4..f37417cc26 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -5,8 +5,9 @@ use Aws\Command; use Aws\Result; use Aws\S3\S3Client; -use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\PartGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; @@ -73,13 +74,14 @@ public function testPartGetMultipartDownloader( ], [ 'minimum_part_size' => $targetPartSize, - ] + ], + new StreamDownloadHandler() ); - /** @var DownloadResponse $response */ + /** @var DownloadResult $response */ $response = $downloader->promise()->wait(); $snapshot = $downloader->getCurrentSnapshot(); - $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertInstanceOf(DownloadResult::class, $response); $this->assertEquals($objectKey, $snapshot->getIdentifier()); $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); @@ -143,7 +145,9 @@ public function testNextCommandIncrementsPartNumber(): void [ 'Bucket' => 'TestBucket', 'Key' => 'TestKey', - ] + ], + [], + new StreamDownloadHandler() ); // Use reflection to test the protected nextCommand method @@ -177,7 +181,9 @@ public function testComputeObjectDimensions(): void [ 'Bucket' => 'TestBucket', 'Key' => 'TestKey', - ] + ], + [], + new StreamDownloadHandler() ); // Use reflection to test the protected computeObjectDimensions method diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index 6a17928095..9d6c89fe4f 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -6,8 +6,9 @@ use Aws\Result; use Aws\S3\S3Client; use Aws\S3\S3Transfer\Exceptions\S3TransferException; -use Aws\S3\S3Transfer\Models\DownloadResponse; +use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; +use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; @@ -73,14 +74,15 @@ public function testRangeGetMultipartDownloader( 'Key' => $objectKey, ], [ - 'minimum_part_size' => $targetPartSize, - ] + 'target_part_size_bytes' => $targetPartSize, + ], + new StreamDownloadHandler() ); - /** @var DownloadResponse $response */ + /** @var DownloadResult $response */ $response = $downloader->promise()->wait(); $snapshot = $downloader->getCurrentSnapshot(); - $this->assertInstanceOf(DownloadResponse::class, $response); + $this->assertInstanceOf(DownloadResult::class, $response); $this->assertEquals($objectKey, $snapshot->getIdentifier()); $this->assertEquals($objectSizeInBytes, $snapshot->getTotalBytes()); $this->assertEquals($objectSizeInBytes, $snapshot->getTransferredBytes()); @@ -123,30 +125,6 @@ public function rangeGetMultipartDownloaderProvider(): array { ]; } - /** - * Tests constructor throws exception when minimum_part_size is not provided. - * - * @return void - */ - public function testConstructorThrowsExceptionWithoutMinimumPartSize(): void - { - $mockClient = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->getMock(); - - $this->expectException(S3TransferException::class); - $this->expectExceptionMessage('You must provide a valid minimum part size in bytes'); - - new RangeGetMultipartDownloader( - $mockClient, - [ - 'Bucket' => 'TestBucket', - 'Key' => 'TestKey', - ], - [] // Missing minimum_part_size - ); - } - /** * Tests nextCommand method generates correct range headers. * @@ -171,8 +149,9 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void 'Key' => 'TestKey', ], [ - 'minimum_part_size' => $partSize, - ] + 'target_part_size_bytes' => $partSize, + ], + new StreamDownloadHandler() ); // Use reflection to test the protected nextCommand method @@ -191,38 +170,6 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void $this->assertEquals(2, $downloader->getCurrentPartNo()); } - /** - * Tests computeObjectDimensions method with known object size. - * - * @return void - */ - public function testComputeObjectDimensionsWithKnownSize(): void - { - $mockClient = $this->getMockBuilder(S3Client::class) - ->disableOriginalConstructor() - ->getMock(); - - $objectSize = 5120; - $partSize = 1024; - $downloader = new RangeGetMultipartDownloader( - $mockClient, - [ - 'Bucket' => 'TestBucket', - 'Key' => 'TestKey', - ], - [ - 'minimum_part_size' => $partSize, - ], - 0, // currentPartNo - 0, // objectPartsCount - $objectSize // objectSizeInBytes - known at construction - ); - - // With known object size, parts count should be calculated during construction - $this->assertEquals(5, $downloader->getObjectPartsCount()); - $this->assertEquals($objectSize, $downloader->getObjectSizeInBytes()); - } - /** * Tests computeObjectDimensions method for single part download. * @@ -243,7 +190,8 @@ public function testComputeObjectDimensionsForSinglePart(): void ], [ 'minimum_part_size' => $partSize, - ] + ], + new StreamDownloadHandler(), ); // Use reflection to test the protected computeObjectDimensions method @@ -289,6 +237,7 @@ public function testNextCommandIncludesIfMatchWhenETagPresent(): void [ 'minimum_part_size' => 1024, ], + new StreamDownloadHandler(), 0, // currentPartNo 0, // objectPartsCount 0, // objectSizeInBytes diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 5207ec1e7f..9fac0f9dfd 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -8,9 +8,15 @@ use Aws\HandlerList; use Aws\Result; use Aws\S3\S3Client; +use Aws\S3\S3Transfer\AbstractMultipartUploader; use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; +use Aws\S3\S3Transfer\Models\DownloadRequest; +use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; +use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; @@ -34,35 +40,35 @@ public function testDefaultConfigIsSet(): void $manager = new S3TransferManager(); $this->assertArrayHasKey( 'target_part_size_bytes', - $manager->getConfig() + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( 'multipart_upload_threshold_bytes', - $manager->getConfig() + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( - 'checksum_validation_enabled', - $manager->getConfig() + 'request_checksum_calculation', + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( - 'checksum_algorithm', - $manager->getConfig() + 'response_checksum_validation', + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( 'multipart_download_type', - $manager->getConfig() + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( 'concurrency', - $manager->getConfig() + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( 'track_progress', - $manager->getConfig() + $manager->getConfig()->toArray() ); $this->assertArrayHasKey( - 'region', - $manager->getConfig() + 'default_region', + $manager->getConfig()->toArray() ); $this->assertInstanceOf( S3Client::class, @@ -80,23 +86,24 @@ public function testCustomConfigIsSet(): void [ 'target_part_size_bytes' => 1024, 'multipart_upload_threshold_bytes' => 1024, - 'checksum_validation_enabled' => false, + 'request_checksum_calculation' => 'when_required', + 'response_checksum_validation' => 'when_required', 'checksum_algorithm' => 'sha256', 'multipart_download_type' => 'partGet', 'concurrency' => 20, 'track_progress' => true, - 'region' => 'us-west-1', + 'default_region' => 'us-west-1', ] ); - $config = $manager->getConfig(); + $config = $manager->getConfig()->toArray(); $this->assertEquals(1024, $config['target_part_size_bytes']); $this->assertEquals(1024, $config['multipart_upload_threshold_bytes']); - $this->assertFalse($config['checksum_validation_enabled']); - $this->assertEquals('sha256', $config['checksum_algorithm']); + $this->assertEquals('when_required', $config['request_checksum_calculation']); + $this->assertEquals('when_required', $config['response_checksum_validation']); $this->assertEquals('partGet', $config['multipart_download_type']); $this->assertEquals(20, $config['concurrency']); $this->assertTrue($config['track_progress']); - $this->assertEquals('us-west-1', $config['region']); + $this->assertEquals('us-west-1', $config['default_region']); } /** @@ -108,13 +115,18 @@ public function testUploadExpectsAReadableSource(): void $this->expectExceptionMessage("Please provide a valid readable file path or a valid stream as source."); $manager = new S3TransferManager(); $manager->upload( - "noreadablefile", + UploadRequest::fromLegacyArgs( + "noreadablefile" + ), )->wait(); } /** * @dataProvider uploadBucketAndKeyProvider * + * @param array $bucketKeyArgs + * @param string $missingProperty + * * @return void */ public function testUploadFailsWhenBucketAndKeyAreNotProvided( @@ -126,8 +138,10 @@ public function testUploadFailsWhenBucketAndKeyAreNotProvided( $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The `$missingProperty` parameter must be provided as part of the request arguments."); $manager->upload( - Utils::streamFor(), - $bucketKeyArgs + uploadRequest::fromLegacyArgs( + Utils::streamFor(), + $bucketKeyArgs + ) )->wait(); } @@ -162,14 +176,16 @@ public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void . "must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE); $manager = new S3TransferManager(); $manager->upload( - Utils::streamFor(), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE - 1 - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor(), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE - 1 + ] + ) )->wait(); } @@ -191,20 +207,22 @@ public function testDoesMultipartUploadWhenApplicable(): void $transferListener->expects($this->exactly($expectedPartCount)) ->method('bytesTransferred'); $manager->upload( - Utils::streamFor( - str_repeat("#", MultipartUploader::PART_MIN_SIZE * $expectedPartCount) - ), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'part_size' => MultipartUploader::PART_MIN_SIZE, - 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, - ], - [ - $transferListener, - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'target_part_size_bytes' => MultipartUploader::PART_MIN_SIZE, + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + ) )->wait(); } @@ -221,19 +239,21 @@ public function testDoesSingleUploadWhenApplicable(): void $transferListener->expects($this->once()) ->method('bytesTransferred'); $manager->upload( - Utils::streamFor( - str_repeat("#", MultipartUploader::PART_MIN_SIZE - 1) - ), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, - ], - [ - $transferListener, - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor( + str_repeat("#", MultipartUploader::PART_MIN_SIZE - 1) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => MultipartUploader::PART_MIN_SIZE, + ], + [ + $transferListener, + ] + ) )->wait(); } @@ -251,21 +271,23 @@ public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void $transferListener->expects($this->exactly($expectedPartCount)) ->method('bytesTransferred'); $manager->upload( - Utils::streamFor( - str_repeat("#", $manager->getConfig()['multipart_upload_threshold_bytes']) - ), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'part_size' => intval( - $manager->getConfig()['multipart_upload_threshold_bytes'] / $expectedPartCount + UploadRequest::fromLegacyArgs( + Utils::streamFor( + str_repeat("#", $manager->getConfig()->toArray()['multipart_upload_threshold_bytes']) ), - ], - [ - $transferListener, - ] + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'target_part_size_bytes' => intval( + $manager->getConfig()->toArray()['multipart_upload_threshold_bytes'] / $expectedPartCount + ), + ], + [ + $transferListener, + ] + ) )->wait(); } @@ -303,20 +325,22 @@ public function testUploadUsesCustomMupThreshold( $expectedIncrementalPartSize += $expectedPartSize; }); $manager->upload( - Utils::streamFor( - str_repeat("#", $expectedPartSize * $expectedPartCount) - ), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'multipart_upload_threshold_bytes' => $mupThreshold, - 'part_size' => $expectedPartSize, - ], - [ - $transferListener, - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $mupThreshold, + 'target_part_size_bytes' => $expectedPartSize, + ], + [ + $transferListener, + ] + ) )->wait(); if ($isMultipartUpload) { $this->assertGreaterThan(1, $expectedPartCount); @@ -330,13 +354,13 @@ public function uploadUsesCustomMupThresholdProvider(): array { return [ 'mup_threshold_multipart_upload' => [ - 'mup_threshold' => 1024 * 1024 * 7, + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 7, 'expected_part_count' => 3, 'expected_part_size' => 1024 * 1024 * 7, 'is_multipart_upload' => true, ], 'mup_threshold_single_upload' => [ - 'mup_threshold' => 1024 * 1024 * 7, + 'multipart_upload_threshold_bytes' => 1024 * 1024 * 7, 'expected_part_count' => 1, 'expected_part_size' => 1024 * 1024 * 5, 'is_multipart_upload' => false, @@ -358,19 +382,21 @@ public function testUploadUsesTransferManagerConfigDefaultTargetPartSize(): void $transferListener->expects($this->exactly($expectedPartCount)) ->method('bytesTransferred'); $manager->upload( - Utils::streamFor( - str_repeat("#", $manager->getConfig()['target_part_size_bytes'] * $expectedPartCount) - ), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'multipart_upload_threshold_bytes' => $manager->getConfig()['target_part_size_bytes'], - ], - [ - $transferListener, - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor( + str_repeat("#", $manager->getConfig()->toArray()['target_part_size_bytes'] * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'multipart_upload_threshold_bytes' => $manager->getConfig()->toArray()['target_part_size_bytes'], + ], + [ + $transferListener, + ] + ) )->wait(); } @@ -403,20 +429,22 @@ public function testUploadUsesCustomPartSize(): void ->method('bytesTransferred'); $manager->upload( - Utils::streamFor( - str_repeat("#", $expectedPartSize * $expectedPartCount) - ), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - [ - 'part_size' => $expectedPartSize, - 'multipart_upload_threshold_bytes' => $expectedPartSize, - ], - [ - $transferListener, - ] + UploadRequest::fromLegacyArgs( + Utils::streamFor( + str_repeat("#", $expectedPartSize * $expectedPartCount) + ), + [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ], + [ + 'target_part_size_bytes' => $expectedPartSize, + 'multipart_upload_threshold_bytes' => $expectedPartSize, + ], + [ + $transferListener, + ] + ) )->wait(); } @@ -427,8 +455,8 @@ public function testUploadUsesDefaultChecksumAlgorithm(): void { $manager = new S3TransferManager(); $this->testUploadResolvedChecksum( - [], // No checksum provided - $manager->getConfig()['checksum_algorithm'] // default checksum algo + null, // No checksum provided + AbstractMultipartUploader::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM, ); } @@ -444,7 +472,7 @@ public function testUploadUsesCustomChecksumAlgorithm( ): void { $this->testUploadResolvedChecksum( - ['checksum_algorithm' => $checksumAlgorithm], + $checksumAlgorithm, $checksumAlgorithm ); } @@ -471,13 +499,13 @@ public function uploadUsesCustomChecksumAlgorithmProvider(): array } /** - * @param array $config + * @param string|null $checksumAlgorithm * @param string $expectedChecksum * * @return void */ private function testUploadResolvedChecksum( - array $config, + ?string $checksumAlgorithm, string $expectedChecksum ): void { $client = $this->getS3ClientMock([ @@ -487,10 +515,14 @@ private function testUploadResolvedChecksum( ) use ( $expectedChecksum ) { - $this->assertEquals( - strtoupper($expectedChecksum), - strtoupper($args['ChecksumAlgorithm']) - ); + if ($commandName !== 'CompleteMultipartUpload') { + $this->assertEquals( + strtoupper($expectedChecksum), + strtoupper($args['ChecksumAlgorithm']) + ); + } else { + $this->assertTrue(true); + } return new Command($commandName, $args); }, @@ -498,16 +530,22 @@ private function testUploadResolvedChecksum( return Create::promiseFor(new Result([])); } ]); + $putObjectRequestArgs = [ + 'Bucket' => 'Bucket', + 'Key' => 'Key', + ]; + if ($checksumAlgorithm !== null) { + $putObjectRequestArgs['ChecksumAlgorithm'] = $checksumAlgorithm; + } + $manager = new S3TransferManager( $client, ); $manager->upload( - Utils::streamFor(), - [ - 'Bucket' => 'Bucket', - 'Key' => 'Key', - ], - $config + UploadRequest::fromLegacyArgs( + Utils::streamFor(), + $putObjectRequestArgs, + ) )->wait(); } @@ -537,8 +575,10 @@ public function testUploadDirectoryValidatesProvidedDirectory( $this->getS3ClientMock(), ); $manager->uploadDirectory( - $directory, - "Bucket", + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + ) )->wait(); // Clean up resources if ($isDirectoryValid) { @@ -556,13 +596,18 @@ public function uploadDirectoryValidatesProvidedDirectoryProvider(): array mkdir($validDirectory, 0777, true); } + $invalidDirectory = sys_get_temp_dir() . "/invalid-directory-test"; + if (is_dir($invalidDirectory)) { + rmdir($invalidDirectory); + } + return [ 'valid_directory' => [ 'directory' => $validDirectory, 'is_valid_directory' => true, ], 'invalid_directory' => [ - 'directory' => 'invalid-directory', + 'directory' => $invalidDirectory, 'is_valid_directory' => false, ] ]; @@ -575,7 +620,7 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage( - 'The parameter $config[\'filter\'] must be callable' + 'The provided config `filter` must be callable' ); $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { @@ -591,12 +636,14 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void $client, ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'filter' => 'invalid_filter', - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'filter' => 'invalid_filter', + ] + ) )->wait(); } finally { rmdir($directory); @@ -644,21 +691,23 @@ public function testUploadDirectoryFileFilter(): void ); $calledTimes = 0; $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'filter' => function (string $objectKey) { - return str_ends_with($objectKey, "-valid.txt"); - }, - 'put_object_request_callback' => function ($requestArgs) use (&$calledTimes) { - $this->assertStringContainsString( - 'valid.txt', - $requestArgs["Key"] - ); - $calledTimes++; - } - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'filter' => function (string $objectKey) { + return str_ends_with($objectKey, "-valid.txt"); + }, + 'put_object_request_callback' => function ($requestArgs) use (&$calledTimes) { + $this->assertStringContainsString( + 'valid.txt', + $requestArgs["Key"] + ); + $calledTimes++; + } + ] + ) )->wait(); $this->assertEquals($validFilesCount, $calledTimes); } finally { @@ -711,12 +760,14 @@ public function testUploadDirectoryRecursive(): void $client, ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + ] + ) )->wait(); foreach ($objectKeys as $key => $validated) { $this->assertTrue($validated); @@ -773,12 +824,14 @@ public function testUploadDirectoryNonRecursive(): void $client, ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'recursive' => false, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'recursive' => false, + ] + ) )->wait(); $subDirPrefix = str_replace($directory . "/", "", $subDirectory); foreach ($objectKeys as $key => $validated) { @@ -856,13 +909,15 @@ public function testUploadDirectoryFollowsSymbolicLink(): void // First lets make sure that when follows_symbolic_link is false // the directory in the link will not be traversed. $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - 'follow_symbolic_links' => false, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => false, + ] + ) )->wait(); foreach ($objectKeys as $key => $validated) { if (str_contains($key, "symlink")) { @@ -875,13 +930,15 @@ public function testUploadDirectoryFollowsSymbolicLink(): void // Now let's enable follow_symbolic_links and all files should have // been considered, included the ones in the symlink directory. $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'recursive' => true, - 'follow_symbolic_links' => true, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'recursive' => true, + 'follow_symbolic_links' => true, + ] + ) )->wait(); foreach ($objectKeys as $key => $validated) { $this->assertTrue($validated, "Key {$key} should have been considered"); @@ -938,12 +995,14 @@ public function testUploadDirectoryUsesProvidedPrefix(): void $client, ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 's3_prefix' => $s3Prefix - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix + ] + ) )->wait(); foreach ($objectKeys as $key => $validated) { @@ -1001,13 +1060,15 @@ public function testUploadDirectoryUsesProvidedDelimiter(): void $client, ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 's3_prefix' => $s3Prefix, - 's3_delimiter' => $s3Delimiter, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 's3_prefix' => $s3Prefix, + 's3_delimiter' => $s3Delimiter, + ] + ) )->wait(); foreach ($objectKeys as $key => $validated) { @@ -1027,7 +1088,7 @@ public function testUploadDirectoryUsesProvidedDelimiter(): void public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("The parameter \$config['put_object_request_callback'] must be callable."); + $this->expectExceptionMessage("The provided config `put_object_request_callback` must be callable."); $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); @@ -1038,12 +1099,14 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi $client, ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'put_object_request_callback' => false, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'put_object_request_callback' => false, + ] + ) )->wait(); } finally { rmdir($directory); @@ -1086,17 +1149,19 @@ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void ); $called = 0; $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'put_object_request_callback' => function ( - &$requestArgs - ) use (&$called) { - $requestArgs["FooParameter"] = "Test"; - $called++; - }, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'put_object_request_callback' => function ( + &$requestArgs + ) use (&$called) { + $requestArgs["FooParameter"] = "Test"; + $called++; + }, + ] + ) )->wait(); $this->assertEquals(count($files), $called); } finally { @@ -1145,39 +1210,41 @@ public function testUploadDirectoryUsesFailurePolicy(): void ); $called = false; $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'failure_policy' => function ( - array $requestArgs, - array $uploadDirectoryRequestArgs, - \Throwable $reason, - UploadDirectoryResponse $uploadDirectoryResponse - ) use ($directory, &$called) { - $called = true; - $this->assertEquals( - $directory, - $uploadDirectoryRequestArgs["source_directory"] - ); - $this->assertEquals( - "Bucket", - $uploadDirectoryRequestArgs["bucket_to"] - ); - $this->assertEquals( - "Failed uploading second file", - $reason->getMessage() - ); - $this->assertEquals( - 1, - $uploadDirectoryResponse->getObjectsUploaded() - ); - $this->assertEquals( - 1, - $uploadDirectoryResponse->getObjectsFailed() - ); - }, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + UploadDirectoryResponse $uploadDirectoryResponse + ) use ($directory, &$called) { + $called = true; + $this->assertEquals( + $directory, + $uploadDirectoryRequestArgs["source_directory"] + ); + $this->assertEquals( + "Bucket", + $uploadDirectoryRequestArgs["bucket_to"] + ); + $this->assertEquals( + "Failed uploading second file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsUploaded() + ); + $this->assertEquals( + 1, + $uploadDirectoryResponse->getObjectsFailed() + ); + }, + ] + ) )->wait(); $this->assertTrue($called); } finally { @@ -1195,7 +1262,7 @@ public function testUploadDirectoryUsesFailurePolicy(): void public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("The parameter \$config['failure_policy'] must be callable."); + $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); $directory = sys_get_temp_dir() . "/upload-directory-test"; if (!is_dir($directory)) { mkdir($directory, 0777, true); @@ -1206,12 +1273,14 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void $client ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [ - 'failure_policy' => false, - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [ + 'failure_policy' => false, + ] + ) )->wait(); } finally { rmdir($directory); @@ -1249,10 +1318,12 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi $client ); $manager->uploadDirectory( - $directory, - "Bucket", - [], - ['s3_delimiter' => $s3Delimiter] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + ['s3_delimiter' => $s3Delimiter] + ) )->wait(); } finally { foreach ($files as $file) { @@ -1303,13 +1374,15 @@ public function testUploadDirectoryTracksMultipleFiles(): void $objectKeys[$snapshot->getIdentifier()] = true; }); $manager->uploadDirectory( - $directory, - "Bucket", - [], - [], - [ - $transferListener - ] + UploadDirectoryRequest::fromLegacyArgs( + $directory, + "Bucket", + [], + [], + [ + $transferListener + ] + ) )->wait(); foreach ($objectKeys as $key => $validated) { $this->assertTrue( @@ -1339,7 +1412,9 @@ public function testDownloadFailsOnInvalidS3UriSource(): void $client ); $manager->download( - $invalidS3Uri + DownloadRequest::fromLegacyArgs( + $invalidS3Uri + ) ); } @@ -1363,7 +1438,7 @@ public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( $client ); $manager->download( - $sourceAsArray + DownloadRequest::fromLegacyArgs($sourceAsArray) ); } @@ -1377,13 +1452,13 @@ public function downloadFailsWhenSourceAsArrayMissesBucketOrKeyPropertyProvider( 'source' => [ 'Bucket' => 'bucket', ], - 'expected_exception' => "A valid key must be provided." + 'expected_exception' => "`Key` is required but not provided" ], 'missing_bucket' => [ 'source' => [ 'Key' => 'key', ], - 'expected_exception' => "A valid bucket must be provided." + 'expected_exception' => "`Bucket` is required but not provided" ] ]; } @@ -1417,7 +1492,7 @@ public function testDownloadWorksWithS3UriAsSource(): void $client ); $manager->download( - $sourceAsArray, + DownloadRequest::fromLegacyArgs($sourceAsArray) )->wait(); $this->assertTrue($called); } @@ -1451,7 +1526,9 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void $client ); $manager->download( - $sourceAsS3Uri, + DownloadRequest::fromLegacyArgs( + $sourceAsS3Uri + ), )->wait(); $this->assertTrue($called); } @@ -1483,16 +1560,11 @@ public function testDownloadAppliesChecksumMode( $called = true; if ($expectedChecksumMode) { $this->assertEquals( - 'enabled', + 'ENABLED', $command['ChecksumMode'], ); } else { - if (isset($command['ChecksumMode'])) { - $this->assertEquals( - 'disabled', - $command['ChecksumMode'], - ); - } + $this->assertArrayNotHasKey('ChecksumMode', $command); } if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { @@ -1510,9 +1582,11 @@ public function testDownloadAppliesChecksumMode( $transferManagerConfig, ); $manager->download( - "s3://bucket/key", - $downloadArgs, - $downloadConfig + DownloadRequest::fromLegacyArgs( + "s3://bucket/key", + $downloadArgs, + $downloadConfig + ) )->wait(); $this->assertTrue($called); } @@ -1529,13 +1603,11 @@ public function downloadAppliesChecksumProvider(): array 'download_args' => [ 'PartNumber' => 1 ], - 'expected_checksum_mode' => S3TransferManager::getDefaultConfig()[ - 'checksum_validation_enabled' - ], + 'expected_checksum_mode' => true, ], 'checksum_mode_enabled_by_transfer_manager_config' => [ 'transfer_manager_config' => [ - 'checksum_validation_enabled' => true + 'response_checksum_validation' => 'when_supported' ], 'download_config' => [], 'download_args' => [ @@ -1545,7 +1617,7 @@ public function downloadAppliesChecksumProvider(): array ], 'checksum_mode_disabled_by_transfer_manager_config' => [ 'transfer_manager_config' => [ - 'checksum_validation_enabled' => false + 'response_checksum_validation' => 'when_required' ], 'download_config' => [], 'download_args' => [ @@ -1556,7 +1628,7 @@ public function downloadAppliesChecksumProvider(): array 'checksum_mode_enabled_by_download_config' => [ 'transfer_manager_config' => [], 'download_config' => [ - 'checksum_validation_enabled' => true + 'response_checksum_validation' => 'when_supported' ], 'download_args' => [ 'PartNumber' => 1 @@ -1566,7 +1638,7 @@ public function downloadAppliesChecksumProvider(): array 'checksum_mode_disabled_by_download_config' => [ 'transfer_manager_config' => [], 'download_config' => [ - 'checksum_validation_enabled' => false + 'response_checksum_validation' => 'when_required' ], 'download_args' => [ 'PartNumber' => 1 @@ -1575,10 +1647,10 @@ public function downloadAppliesChecksumProvider(): array ], 'checksum_mode_download_config_overrides_transfer_manager_config' => [ 'transfer_manager_config' => [ - 'checksum_validation_enabled' => false + 'response_checksum_validation' => 'when_required' ], 'download_config' => [ - 'checksum_validation_enabled' => true + 'response_checksum_validation' => 'when_supported' ], 'download_args' => [ 'PartNumber' => 1 @@ -1588,68 +1660,6 @@ public function downloadAppliesChecksumProvider(): array ]; } - /** - * @param array $downloadArgs - * - * @dataProvider singleDownloadWhenPartNumberOrRangeArePresentProvider - * - * @return void - */ - public function testDoesSingleDownloadWhenPartNumberOrRangeArePresent( - array $downloadArgs, - ): void - { - $calledOnce = false; - $client = $this->getS3ClientMock([ - 'executeAsync' => function (CommandInterface $command) use (&$calledOnce) { - if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { - if ($calledOnce) { - $this->fail(MultipartDownloader::GET_OBJECT_COMMAND . " should have been called once."); - } - - $calledOnce = true; - return Create::promiseFor(new Result([ - 'PartsCount' => 2, - 'ContentRange' => 10240000, - 'Body' => Utils::streamFor( - str_repeat("*", 1024 * 1024 * 20) - ), - '@metadata' => [] - ])); - } else { - $this->fail("Unexpected command execution `" . $command->getName() . "`."); - } - } - ]); - $manager = new S3TransferManager( - $client, - ); - $manager->download( - "s3://bucket/key", - $downloadArgs, - )->wait(); - $this->assertTrue($calledOnce); - } - - /** - * @return array - */ - public function singleDownloadWhenPartNumberOrRangeArePresentProvider(): array - { - return [ - 'part_number_present' => [ - 'download_args' => [ - 'PartNumber' => 1 - ] - ], - 'range_present' => [ - 'download_args' => [ - 'Range' => '100-1024' - ] - ] - ]; - } - /** * @param string $multipartDownloadType * @param string $expectedParameter @@ -1684,9 +1694,11 @@ public function testDownloadChoosesMultipartDownloadType( $client, ); $manager->download( - "s3://bucket/key", - [], - ['multipart_download_type' => $multipartDownloadType] + DownloadRequest::fromLegacyArgs( + "s3://bucket/key", + [], + ['multipart_download_type' => $multipartDownloadType] + ) )->wait(); $this->assertTrue($calledOnce); } @@ -1702,7 +1714,7 @@ public function downloadChoosesMultipartDownloadTypeProvider(): array 'expected_parameter' => 'PartNumber' ], 'range_get_multipart_download' => [ - 'multipart_download_type' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, + 'multipart_download_type' => MultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER, 'expected_parameter' => 'Range' ] ]; @@ -1711,11 +1723,12 @@ public function downloadChoosesMultipartDownloadTypeProvider(): array /** * @param int $minimumPartSize * @param int $objectSize - * @param array $expectedPartsSize + * @param array $expectedRangeSizes + * + * @return void * * @dataProvider rangeGetMultipartDownloadMinimumPartSizeProvider * - * @return void */ public function testRangeGetMultipartDownloadMinimumPartSize( int $minimumPartSize, @@ -1741,7 +1754,8 @@ public function testRangeGetMultipartDownloadMinimumPartSize( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), - 'ContentRange' => $objectSize, + 'ContentRange' => "0-$objectSize/$objectSize", + 'ETag' => 'TestEtag', '@metadata' => [] ])); } @@ -1750,12 +1764,14 @@ public function testRangeGetMultipartDownloadMinimumPartSize( $client, ); $manager->download( - "s3://bucket/key", - [], - [ - 'multipart_download_type' => MultipartDownloader::RANGE_GET_MULTIPART_DOWNLOADER, - 'minimum_part_size' => $minimumPartSize, - ] + DownloadRequest::fromLegacyArgs( + "s3://bucket/key", + [], + [ + 'multipart_download_type' => MultipartDownloader::RANGED_GET_MULTIPART_DOWNLOADER, + 'target_part_size_bytes' => $minimumPartSize, + ] + ) )->wait(); $this->assertEquals(count($expectedRangeSizes), $calledTimes); } @@ -1808,19 +1824,51 @@ public function rangeGetMultipartDownloadMinimumPartSizeProvider(): array /** * @return void */ - public function testDownloadDirectoryValidatesDestinationDirectory(): void + public function testDownloadDirectoryCreatesDestinationDirectory(): void { - $destinationDirectory = "invalid-directory"; - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("Destination directory `$destinationDirectory` MUST exists."); - $client = $this->getS3ClientMock(); - $manager = new S3TransferManager( - $client, - ); - $manager->downloadDirectory( - "Bucket", - $destinationDirectory - ); + $destinationDirectory = sys_get_temp_dir() . DIRECTORY_SEPARATOR . uniqid(); + if (is_dir($destinationDirectory)) { + rmdir($destinationDirectory); + } + + try { + $client = $this->getS3ClientMock([ + 'getApi' => function () { + $service = $this->getMockBuilder(Service::class) + ->disableOriginalConstructor() + ->onlyMethods(["getPaginatorConfig"]) + ->getMock(); + $service->method('getPaginatorConfig') + ->willReturn([ + 'input_token' => null, + 'output_token' => null, + 'limit_key' => null, + 'result_key' => null, + 'more_results' => null, + ]); + + return $service; + }, + 'getHandlerList' => function () { + return new HandlerList(); + }, + 'executeAsync' => function (CommandInterface $command) { + return Create::promiseFor(new Result([])); + } + ]); + $manager = new S3TransferManager( + $client, + ); + $manager->downloadDirectory( + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory + ) + )->wait(); + $this->assertFileExists($destinationDirectory); + } finally { + rmdir($destinationDirectory); + } } /** @@ -1884,10 +1932,12 @@ public function testDownloadDirectoryAppliesS3Prefix( $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - $config + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + $config + ) )->wait(); $this->assertTrue($called); @@ -1990,10 +2040,12 @@ public function testDownloadDirectoryAppliesDelimiter( $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - $config + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + $config + ) )->wait(); $this->assertTrue($called); @@ -2041,7 +2093,7 @@ public function downloadDirectoryAppliesDelimiterProvider(): array public function testDownloadDirectoryFailsOnInvalidFilter(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("The parameter \$config['filter'] must be callable."); + $this->expectExceptionMessage("The provided config `filter` must be callable."); $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); @@ -2079,10 +2131,12 @@ public function testDownloadDirectoryFailsOnInvalidFilter(): void $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - ['filter' => false] + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + ['filter' => false] + ) )->wait(); $this->assertTrue($called); } finally { @@ -2096,7 +2150,7 @@ public function testDownloadDirectoryFailsOnInvalidFilter(): void public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage("The parameter \$config['failure_policy'] must be callable."); + $this->expectExceptionMessage("The provided config `failure_policy` must be callable."); $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { mkdir($destinationDirectory, 0777, true); @@ -2134,10 +2188,12 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - ['failure_policy' => false] + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => false] + ) )->wait(); $this->assertTrue($called); } finally { @@ -2188,33 +2244,35 @@ public function testDownloadDirectoryUsesFailurePolicy(): void $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - ['failure_policy' => function ( - array $requestArgs, - array $uploadDirectoryRequestArgs, - \Throwable $reason, - DownloadDirectoryResponse $downloadDirectoryResponse - ) use ($destinationDirectory, &$called) { - $called = true; - $this->assertEquals( - $destinationDirectory, - $uploadDirectoryRequestArgs['destination_directory'] - ); - $this->assertEquals( - "Failed downloading file", - $reason->getMessage() - ); - $this->assertEquals( - 1, - $downloadDirectoryResponse->getObjectsDownloaded() - ); - $this->assertEquals( - 1, - $downloadDirectoryResponse->getObjectsFailed() - ); - }] + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + ['failure_policy' => function ( + array $requestArgs, + array $uploadDirectoryRequestArgs, + \Throwable $reason, + DownloadDirectoryResponse $downloadDirectoryResponse + ) use ($destinationDirectory, &$called) { + $called = true; + $this->assertEquals( + $destinationDirectory, + $uploadDirectoryRequestArgs['destination_directory'] + ); + $this->assertEquals( + "Failed downloading file", + $reason->getMessage() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsDownloaded() + ); + $this->assertEquals( + 1, + $downloadDirectoryResponse->getObjectsFailed() + ); + }] + ) )->wait(); $this->assertTrue($called); } finally { @@ -2292,10 +2350,12 @@ public function testDownloadDirectoryAppliesFilter( $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - ['filter' => $filter] + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + ['filter' => $filter] + ) )->wait(); $this->assertTrue($called); @@ -2410,7 +2470,7 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage( - "The parameter \$config['get_object_request_callback'] must be callable." + "The provided config `get_object_request_callback` must be callable." ); $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; if (!is_dir($destinationDirectory)) { @@ -2454,10 +2514,12 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [], - ['get_object_request_callback' => false] + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [], + ['get_object_request_callback' => false] + ) )->wait(); } finally { rmdir($destinationDirectory); @@ -2525,12 +2587,14 @@ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void ); }; $manager->downloadDirectory( - "Bucket", - $destinationDirectory, - [ - 'CustomParameter' => 'CustomParameterValue' - ], - ['get_object_request_callback' => $getObjectRequestCallback] + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + [ + 'CustomParameter' => 'CustomParameterValue' + ], + ['get_object_request_callback' => $getObjectRequestCallback] + ) )->wait(); $this->assertTrue($called); } finally { @@ -2615,8 +2679,10 @@ public function testDownloadDirectoryCreateFiles( $client, ); $manager->downloadDirectory( - "Bucket", - $destinationDirectory, + DownloadDirectoryRequest::fromLegacyArgs( + "Bucket", + $destinationDirectory, + ) )->wait(); $this->assertTrue($called); foreach ($expectedFileKeys as $key) { @@ -2749,12 +2815,14 @@ public function testFailsWhenKeyResolvesOutsideTargetDirectory( $client, ); $manager->downloadDirectory( - $bucket, - $fullDirectoryPath, - [], - [ - 's3_prefix' => $prefix, - ] + DownloadDirectoryRequest::fromLegacyArgs( + $bucket, + $fullDirectoryPath, + [], + [ + 's3_prefix' => $prefix, + ] + ) )->wait(); $this->assertTrue($called); } finally { From 153227b1193bc43bfa75c53cbaacd920384a66ca Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 17 Jul 2025 14:23:58 -0700 Subject: [PATCH 36/62] chore: minor update - Remove model files. - Update function to use lambda syntax. --- src/S3/S3Transfer/Data/tm-model-schema.json | 304 -------------------- src/S3/S3Transfer/Data/tm-model.json | 287 ------------------ src/S3/S3Transfer/S3TransferManager.php | 4 +- 3 files changed, 2 insertions(+), 593 deletions(-) delete mode 100644 src/S3/S3Transfer/Data/tm-model-schema.json delete mode 100644 src/S3/S3Transfer/Data/tm-model.json diff --git a/src/S3/S3Transfer/Data/tm-model-schema.json b/src/S3/S3Transfer/Data/tm-model-schema.json deleted file mode 100644 index 11c90ef03e..0000000000 --- a/src/S3/S3Transfer/Data/tm-model-schema.json +++ /dev/null @@ -1,304 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://aws.amazon.com/schemas/s3-transfer-manager/tm-model.json", - "title": "S3 Transfer Manager Model Configuration", - "description": "Configuration schema for S3 Transfer Manager field mappings and conversions between request/response types", - "type": "object", - "required": ["Definition", "Conversion"], - "properties": { - "Definition": { - "type": "object", - "description": "Defines the base field mappings for S3 Transfer Manager operations", - "required": ["UploadRequest", "UploadResponse", "DownloadRequest", "DownloadResponse"], - "properties": { - "UploadRequest": { - "type": "object", - "description": "Field mappings for upload request operations", - "required": ["PutObjectRequest"], - "properties": { - "PutObjectRequest": { - "type": "array", - "description": "List of fields available in PutObjectRequest that should be added to UploadRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - }, - "UploadResponse": { - "type": "object", - "description": "Field mappings for upload response operations", - "required": ["PutObjectResponse"], - "properties": { - "PutObjectResponse": { - "type": "array", - "description": "List of fields available in PutObjectResponse that should be added to UploadResponse", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - }, - "DownloadRequest": { - "type": "object", - "description": "Field mappings for download request operations", - "required": ["GetObjectRequest"], - "properties": { - "GetObjectRequest": { - "type": "array", - "description": "List of fields available in GetObjectRequest that should be added to DownloadRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - }, - "DownloadResponse": { - "type": "object", - "description": "Field mappings for download response operations", - "required": ["GetObjectResponse"], - "properties": { - "GetObjectResponse": { - "type": "array", - "description": "List of fields available in GetObjectResponse that should be added to DownloadResponse", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - }, - "Conversion": { - "type": "object", - "description": "Defines field conversion mappings between different S3 operation types", - "required": ["UploadRequest", "CompleteMultipartResponse", "PutObjectResponse", "GetObjectResponse"], - "properties": { - "UploadRequest": { - "type": "object", - "description": "Field conversion mappings for upload request operations", - "required": ["PutObjectRequest", "CreateMultipartRequest", "UploadPartReuqest", "CompleteMultipartRequest", "AbortMultipartRequest"], - "properties": { - "PutObjectRequest": { - "type": "array", - "description": "Fields to copy when converting UploadRequest to PutObjectRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - }, - "CreateMultipartRequest": { - "type": "array", - "description": "Fields to copy when converting UploadRequest to CreateMultipartRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - }, - "UploadPartReuqest": { - "type": "array", - "description": "Fields to copy when converting UploadRequest to UploadPartRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - }, - "CompleteMultipartRequest": { - "type": "array", - "description": "Fields to convert from UploadRequest to CompleteMultipartRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - }, - "AbortMultipartRequest": { - "type": "array", - "description": "Fields to copy when converting UploadRequest to AbortMultipartRequest", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - }, - "CompleteMultipartResponse": { - "type": "object", - "description": "Field conversion mappings for complete multipart response operations", - "required": ["UploadResponse"], - "properties": { - "UploadResponse": { - "type": "array", - "description": "Fields to copy when converting CompleteMultipartResponse to UploadResponse", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - }, - "PutObjectResponse": { - "type": "object", - "description": "Field conversion mappings for put object response operations", - "required": ["UploadResponse"], - "properties": { - "UploadResponse": { - "type": "array", - "description": "Fields to copy when converting PutObjectResponse to UploadResponse", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - }, - "GetObjectResponse": { - "type": "object", - "description": "Field conversion mappings for get object response operations", - "required": ["DowloadResponse"], - "properties": { - "DowloadResponse": { - "type": "array", - "description": "Fields to copy when converting GetObjectResponse to DownloadResponse", - "items": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$" - }, - "uniqueItems": true, - "minItems": 1 - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false, - "definitions": { - "S3FieldName": { - "type": "string", - "pattern": "^[A-Za-z][A-Za-z0-9]*$", - "description": "Valid S3 API field name following PascalCase convention" - }, - "S3FieldList": { - "type": "array", - "items": { - "$ref": "#/definitions/S3FieldName" - }, - "uniqueItems": true, - "minItems": 1, - "description": "List of unique S3 API field names" - } - }, - "examples": [ - { - "Definition": { - "UploadRequest": { - "PutObjectRequest": [ - "Bucket", - "Key", - "ContentType", - "Metadata" - ] - }, - "UploadResponse": { - "PutObjectResponse": [ - "ETag", - "VersionId" - ] - }, - "DownloadRequest": { - "GetObjectRequest": [ - "Bucket", - "Key", - "Range" - ] - }, - "DownloadResponse": { - "GetObjectResponse": [ - "ContentLength", - "ContentType", - "ETag" - ] - } - }, - "Conversion": { - "UploadRequest": { - "PutObjectRequest": [ - "Bucket", - "Key" - ], - "CreateMultipartRequest": [ - "Bucket", - "Key", - "ContentType" - ], - "UploadPartReuqest": [ - "Bucket", - "Key" - ], - "CompleteMultipartRequest": [ - "Bucket", - "Key" - ], - "AbortMultipartRequest": [ - "Bucket", - "Key" - ] - }, - "CompleteMultipartResponse": { - "UploadResponse": [ - "ETag", - "VersionId" - ] - }, - "PutObjectResponse": { - "UploadResponse": [ - "ETag", - "VersionId" - ] - }, - "GetObjectResponse": { - "DowloadResponse": [ - "ContentLength", - "ContentType" - ] - } - } - } - ] -} diff --git a/src/S3/S3Transfer/Data/tm-model.json b/src/S3/S3Transfer/Data/tm-model.json deleted file mode 100644 index 02aa71cc6d..0000000000 --- a/src/S3/S3Transfer/Data/tm-model.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "Definition": { - "UploadRequest": { - "PutObjectRequest": [ - "ACL", - "Bucket", - "BucketKeyEnabled", - "CacheControl", - "ChecksumAlgorithm", - "ContentDisposition", - "ContentEncoding", - "ContentLanguage", - "ContentType", - "ExpectedBucketOwner", - "Expires", - "GrantFullControl", - "GrantRead", - "GrantReadACP", - "GrantWriteACP", - "Key", - "Metadata", - "ObjectLockLegalHoldStatus", - "ObjectLockMode", - "ObjectLockRetainUntilDate", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5", - "SSEKMSEncryptionContext", - "SSEKMSKeyId", - "ServerSideEncryption", - "StorageClass", - "Tagging", - "WebsiteRedirectLocation", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME" - ] - }, - "UploadResponse": { - "PutObjectResponse": [ - "BucketKeyEnabled", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ChecksumType", - "ETag", - "Expiration", - "RequestCharged", - "SSEKMSKeyId", - "ServerSideEncryption", - "VersionId" - ] - }, - "DownloadRequest": { - "GetObjectRequest": [ - "Bucket", - "ChecksumMode", - "ExpectedBucketOwner", - "IfMatch", - "IfModifiedSince", - "IfNoneMatch", - "IfUnmodifiedSince", - "Key", - "PartNumber", - "Range", - "RequestPayer", - "ResponseCacheControl", - "ResponseContentDisposition", - "ResponseContentEncoding", - "ResponseContentLanguage", - "ResponseContentType", - "ResponseExpires", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5", - "VersionId" - ] - }, - "DownloadResponse": { - "GetObjectResponse": [ - "AcceptRanges", - "BucketKeyEnabled", - "CacheControl", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ChecksumType", - "ContentDisposition", - "ContentEncoding", - "ContentLanguage", - "ContentLength", - "ContentRange", - "ContentType", - "DeleteMarker", - "ETag", - "Expiration", - "Expires", - "LastModified", - "Metadata", - "MissingMeta", - "ObjectLockLegalHoldStatus", - "ObjectLockMode", - "ObjectLockRetainUntilDate", - "PartsCount", - "ReplicationStatus", - "RequestCharged", - "Restore", - "SSECustomerAlgorithm", - "SSECustomerKeyMD5", - "SSEKMSKeyId", - "ServerSideEncryption", - "StorageClass", - "TagCount", - "VersionId", - "WebsiteRedirectLocation" - ] - } - }, - "Conversion": { - "UploadRequest": { - "PutObjectRequest": [ - "Bucket", - "ChecksumAlgorithm", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ExpectedBucketOwner", - "Key", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5" - ], - "CreateMultipartRequest": [ - "ACL", - "Bucket", - "BucketKeyEnabled", - "CacheControl", - "ChecksumAlgorithm", - "ContentDisposition", - "ContentEncoding", - "ContentLanguage", - "ContentType", - "ExpectedBucketOwner", - "Expires", - "GrantFullControl", - "GrantRead", - "GrantReadACP", - "GrantWriteACP", - "Key", - "Metadata", - "ObjectLockLegalHoldStatus", - "ObjectLockMode", - "ObjectLockRetainUntilDate", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5", - "SSEKMSEncryptionContext", - "SSEKMSKeyId", - "ServerSideEncryption", - "StorageClass", - "Tagging", - "WebsiteRedirectLocation" - ], - "UploadPartRequest": [ - "Bucket", - "ChecksumAlgorithm", - "ExpectedBucketOwner", - "Key", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5" - ], - "CompleteMultipartRequest": [ - "Bucket", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ExpectedBucketOwner", - "IfMatch", - "IfNoneMatch", - "Key", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5" - ], - "AbortMultipartRequest": [ - "Bucket", - "ExpectedBucketOwner", - "Key", - "RequestPayer" - ] - }, - "CompleteMultipartResponse": { - "UploadResponse": [ - "BucketKeyEnabled", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ChecksumType", - "ETag", - "Expiration", - "RequestCharged", - "SSEKMSKeyId", - "ServerSideEncryption", - "VersionId" - ] - }, - "PutObjectResponse": { - "UploadResponse": [ - "BucketKeyEnabled", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ChecksumType", - "ETag", - "Expiration", - "RequestCharged", - "SSECustomerAlgorithm", - "SSECustomerKeyMD5", - "SSEKMSEncryptionContext", - "SSEKMSKeyId", - "ServerSideEncryption", - "Size", - "VersionId" - ] - }, - "GetObjectResponse": { - "DownloadResponse": [ - "AcceptRanges", - "BucketKeyEnabled", - "CacheControl", - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ChecksumType", - "ContentDisposition", - "ContentEncoding", - "ContentLanguage", - "ContentLength", - "ContentRange", - "ContentType", - "DeleteMarker", - "ETag", - "Expiration", - "Expires", - "ExpiresString", - "LastModified", - "Metadata", - "MissingMeta", - "ObjectLockLegalHoldStatus", - "ObjectLockMode", - "ObjectLockRetainUntilDate", - "PartsCount", - "ReplicationStatus", - "RequestCharged", - "Restore", - "SSECustomerAlgorithm", - "SSECustomerKeyMD5", - "SSEKMSKeyId", - "ServerSideEncryption", - "StorageClass", - "TagCount", - "VersionId", - "WebsiteRedirectLocation" - ] - } - } -} \ No newline at end of file diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 2b30421644..4308ea44e0 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -253,7 +253,7 @@ function ($file) use ($filter) { $putObjectRequestArgs, $config, array_map( - function ($listener) { return clone $listener; }, + fn($listener) => clone $listener, $uploadDirectoryRequest->getListeners() ), $progressTracker @@ -466,7 +466,7 @@ public function downloadDirectory( ], null, array_map( - function ($listener) { return clone $listener; }, + fn($listener) => clone $listener, $downloadDirectoryRequest->getListeners() ), $progressTracker, From 31dbd198834c6c990bb3410bcce6630380e4eda5 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 17 Jul 2025 14:34:26 -0700 Subject: [PATCH 37/62] chore: add empty lines at the end --- src/S3/S3Transfer/Exceptions/FileDownloadException.php | 2 +- src/S3/S3Transfer/Exceptions/ProgressTrackerException.php | 2 +- src/S3/S3Transfer/Exceptions/S3TransferException.php | 2 +- src/S3/S3Transfer/Models/DownloadDirectoryRequest.php | 2 +- src/S3/S3Transfer/Models/DownloadDirectoryResponse.php | 2 +- src/S3/S3Transfer/Models/DownloadFileRequest.php | 2 +- src/S3/S3Transfer/Models/DownloadRequest.php | 2 +- src/S3/S3Transfer/Models/DownloadResult.php | 2 +- src/S3/S3Transfer/Models/S3TransferManagerConfig.php | 2 +- src/S3/S3Transfer/Models/UploadDirectoryRequest.php | 2 +- src/S3/S3Transfer/Models/UploadDirectoryResponse.php | 2 +- src/S3/S3Transfer/Models/UploadRequest.php | 2 +- src/S3/S3Transfer/Models/UploadResult.php | 2 +- src/S3/S3Transfer/MultipartDownloaderInitial.php | 2 +- src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php | 2 +- src/S3/S3Transfer/Progress/ConsoleProgressBar.php | 2 +- src/S3/S3Transfer/Progress/MultiProgressBarFormat.php | 2 +- src/S3/S3Transfer/Progress/MultiProgressTracker.php | 2 +- src/S3/S3Transfer/Progress/PlainProgressBarFormat.php | 2 +- src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php | 2 +- src/S3/S3Transfer/Progress/ProgressBarFormat.php | 2 +- src/S3/S3Transfer/Progress/ProgressBarInterface.php | 2 +- src/S3/S3Transfer/Progress/ProgressTrackerInterface.php | 2 +- src/S3/S3Transfer/Progress/SingleProgressTracker.php | 2 +- src/S3/S3Transfer/Progress/TransferListener.php | 2 +- src/S3/S3Transfer/Progress/TransferListenerNotifier.php | 2 +- src/S3/S3Transfer/Progress/TransferProgressBarFormat.php | 2 +- src/S3/S3Transfer/Progress/TransferProgressSnapshot.php | 2 +- src/S3/S3Transfer/RangeGetMultipartDownloader.php | 2 +- src/S3/S3Transfer/Utils/DownloadHandler.php | 2 +- src/S3/S3Transfer/Utils/FileDownloadHandler.php | 2 +- src/S3/S3Transfer/Utils/StreamDownloadHandler.php | 2 +- 32 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/S3/S3Transfer/Exceptions/FileDownloadException.php b/src/S3/S3Transfer/Exceptions/FileDownloadException.php index 9cad8fa555..1c67691975 100644 --- a/src/S3/S3Transfer/Exceptions/FileDownloadException.php +++ b/src/S3/S3Transfer/Exceptions/FileDownloadException.php @@ -5,4 +5,4 @@ class FileDownloadException extends \RuntimeException { -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php index 66d2a90cbd..df00709e57 100644 --- a/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php +++ b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php @@ -5,4 +5,4 @@ class ProgressTrackerException extends \RuntimeException { -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Exceptions/S3TransferException.php b/src/S3/S3Transfer/Exceptions/S3TransferException.php index be02bea08e..1ffbd6426b 100644 --- a/src/S3/S3Transfer/Exceptions/S3TransferException.php +++ b/src/S3/S3Transfer/Exceptions/S3TransferException.php @@ -4,4 +4,4 @@ class S3TransferException extends \RuntimeException { -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 54c7f8dda5..0999c96f41 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -148,4 +148,4 @@ public function validateDestinationDirectory(): void ); } } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php index d493630e16..c1d87e5e28 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php @@ -44,4 +44,4 @@ public function __toString(): string $this->objectsFailed ); } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php index 9ad8568776..b66ff6e378 100644 --- a/src/S3/S3Transfer/Models/DownloadFileRequest.php +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -60,4 +60,4 @@ public function getDownloadRequest(): DownloadRequest { return $this->downloadRequest; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 7ee06cb6b6..942bf4fd4a 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -174,4 +174,4 @@ public function normalizeSourceAsArray(): array { return $sourceAsArray; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/DownloadResult.php b/src/S3/S3Transfer/Models/DownloadResult.php index 29a38f3b74..bb4ab69910 100644 --- a/src/S3/S3Transfer/Models/DownloadResult.php +++ b/src/S3/S3Transfer/Models/DownloadResult.php @@ -27,4 +27,4 @@ public function getDownloadDataResult(): mixed { return $this->downloadDataResult; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php index 7f7d98bab1..5b2730b30d 100644 --- a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -183,4 +183,4 @@ public function toArray(): array { 'default_region' => $this->defaultRegion, ]; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index 5e65406a89..f10bc6e37d 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -109,4 +109,4 @@ public function validateSourceDirectory(): void ); } } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/UploadDirectoryResponse.php b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php index 867442986c..1cf9663128 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php @@ -35,4 +35,4 @@ public function getObjectsFailed(): int { return $this->objectsFailed; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 3ca9303f03..432cb21d0c 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -143,4 +143,4 @@ public function validateRequiredParameters( } } } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Models/UploadResult.php b/src/S3/S3Transfer/Models/UploadResult.php index a1f0fabf60..2f053ea1d3 100644 --- a/src/S3/S3Transfer/Models/UploadResult.php +++ b/src/S3/S3Transfer/Models/UploadResult.php @@ -13,4 +13,4 @@ public function __construct(array $data) { parent::__construct($data); } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/MultipartDownloaderInitial.php b/src/S3/S3Transfer/MultipartDownloaderInitial.php index e866bbd294..a0c99b2ccb 100644 --- a/src/S3/S3Transfer/MultipartDownloaderInitial.php +++ b/src/S3/S3Transfer/MultipartDownloaderInitial.php @@ -385,4 +385,4 @@ public static function chooseDownloaderClass( ) }; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php index 3041f2a71e..2063c444e6 100644 --- a/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/ColoredTransferProgressBarFormat.php @@ -44,4 +44,4 @@ protected function getFormatDefaultParameterValues(): array 'color_code' => ColoredTransferProgressBarFormat::BLACK_COLOR_CODE, ]; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/ConsoleProgressBar.php b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php index 6fe118b167..4e4c869464 100644 --- a/src/S3/S3Transfer/Progress/ConsoleProgressBar.php +++ b/src/S3/S3Transfer/Progress/ConsoleProgressBar.php @@ -100,4 +100,4 @@ public function render(): string return $this->progressBarFormat->format(); } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php index 428a55b71d..e080cee32f 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/MultiProgressBarFormat.php @@ -37,4 +37,4 @@ protected function getFormatDefaultParameterValues(): array { return []; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php index 67895465bc..6cb63d684b 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressTracker.php +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -220,4 +220,4 @@ public function showProgress(): void ) ); } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php index 0e077d6e5e..f75f6ca89e 100644 --- a/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/PlainProgressBarFormat.php @@ -25,4 +25,4 @@ protected function getFormatDefaultParameterValues(): array { return []; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php b/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php index 87f0fea51c..9ac5807fc1 100644 --- a/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php +++ b/src/S3/S3Transfer/Progress/ProgressBarFactoryInterface.php @@ -5,4 +5,4 @@ interface ProgressBarFactoryInterface { public function __invoke(): ProgressBarInterface; -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/ProgressBarFormat.php b/src/S3/S3Transfer/Progress/ProgressBarFormat.php index 8e54675f4d..37b3e4e63a 100644 --- a/src/S3/S3Transfer/Progress/ProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/ProgressBarFormat.php @@ -85,4 +85,4 @@ abstract public function getFormatParameters(): array; * @return array */ abstract protected function getFormatDefaultParameterValues(): array; -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/ProgressBarInterface.php b/src/S3/S3Transfer/Progress/ProgressBarInterface.php index ed8de193c2..b64545eb59 100644 --- a/src/S3/S3Transfer/Progress/ProgressBarInterface.php +++ b/src/S3/S3Transfer/Progress/ProgressBarInterface.php @@ -28,4 +28,4 @@ public function getPercentCompleted(): int; * @return ProgressBarFormat */ public function getProgressBarFormat(): ProgressBarFormat; -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/ProgressTrackerInterface.php b/src/S3/S3Transfer/Progress/ProgressTrackerInterface.php index 3a9a8567ab..a4a39dedc0 100644 --- a/src/S3/S3Transfer/Progress/ProgressTrackerInterface.php +++ b/src/S3/S3Transfer/Progress/ProgressTrackerInterface.php @@ -10,4 +10,4 @@ interface ProgressTrackerInterface * @return void */ public function showProgress(): void; -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 75c986f2a5..5800eb6aa3 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -227,4 +227,4 @@ public function showProgress(): void )); fflush($this->output); } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/TransferListener.php b/src/S3/S3Transfer/Progress/TransferListener.php index 1a75d1c9b7..d121a2761e 100644 --- a/src/S3/S3Transfer/Progress/TransferListener.php +++ b/src/S3/S3Transfer/Progress/TransferListener.php @@ -48,4 +48,4 @@ public function transferComplete(array $context): void {} * @return void */ public function transferFail(array $context): void {} -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php index d4f455171c..de136fcecf 100644 --- a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php +++ b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php @@ -78,4 +78,4 @@ public function transferFail(array $context): void $listener->transferFail($context); } } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php index 2737a1b513..9e3c3c729a 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/TransferProgressBarFormat.php @@ -35,4 +35,4 @@ protected function getFormatDefaultParameterValues(): array { return []; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index 148357d43d..d42397a14c 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -92,4 +92,4 @@ public function getReason(): Throwable|string|null } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index 5cf1e9f67f..26c98df4cd 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -73,4 +73,4 @@ protected function computeObjectDimensions(ResultInterface $result): void $this->currentPartNo = 1; } } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Utils/DownloadHandler.php b/src/S3/S3Transfer/Utils/DownloadHandler.php index c887d68f26..8dd7f00a97 100644 --- a/src/S3/S3Transfer/Utils/DownloadHandler.php +++ b/src/S3/S3Transfer/Utils/DownloadHandler.php @@ -15,4 +15,4 @@ abstract class DownloadHandler extends TransferListener * @return mixed */ public abstract function getHandlerResult(): mixed; -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php index d649a15501..8cb95613f4 100644 --- a/src/S3/S3Transfer/Utils/FileDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -162,4 +162,4 @@ public function getHandlerResult(): string { return $this->destination; } -} \ No newline at end of file +} diff --git a/src/S3/S3Transfer/Utils/StreamDownloadHandler.php b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php index 12296e2f23..0df9e769bf 100644 --- a/src/S3/S3Transfer/Utils/StreamDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php @@ -80,4 +80,4 @@ public function getHandlerResult(): StreamInterface { return $this->stream; } -} \ No newline at end of file +} From aeed758b959f47f3921fa7c26be98b157b8f4d2e Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 17 Jul 2025 14:39:31 -0700 Subject: [PATCH 38/62] chore: remove unused implementations - Removed old MultipartUploader and DownloaderImplementation which were suffixed with [class]Initial.php - Remove empty space. --- .../S3Transfer/MultipartDownloaderInitial.php | 388 ------------ src/S3/S3Transfer/MultipartUploader.php | 5 +- .../S3Transfer/MultipartUploaderInitial.php | 570 ------------------ 3 files changed, 2 insertions(+), 961 deletions(-) delete mode 100644 src/S3/S3Transfer/MultipartDownloaderInitial.php delete mode 100644 src/S3/S3Transfer/MultipartUploaderInitial.php diff --git a/src/S3/S3Transfer/MultipartDownloaderInitial.php b/src/S3/S3Transfer/MultipartDownloaderInitial.php deleted file mode 100644 index a0c99b2ccb..0000000000 --- a/src/S3/S3Transfer/MultipartDownloaderInitial.php +++ /dev/null @@ -1,388 +0,0 @@ -requestArgs = $requestArgs; - $this->currentPartNo = $currentPartNo; - $this->objectPartsCount = $objectPartsCount; - $this->objectSizeInBytes = $objectSizeInBytes; - $this->eTag = $eTag; - if ($stream === null) { - $this->stream = Utils::streamFor( - fopen('php://temp', 'w+') - ); - } else { - $this->stream = $stream; - // Position at the end of the stream - $this->stream->seek($stream->getSize()); - } - $this->currentSnapshot = $currentSnapshot; - $this->listenerNotifier = $listenerNotifier; - } - - /** - * @return int - */ - public function getCurrentPartNo(): int - { - return $this->currentPartNo; - } - - /** - * @return int - */ - public function getObjectPartsCount(): int - { - return $this->objectPartsCount; - } - - /** - * @return int - */ - public function getObjectSizeInBytes(): int - { - return $this->objectSizeInBytes; - } - - /** - * @return TransferProgressSnapshot - */ - public function getCurrentSnapshot(): TransferProgressSnapshot - { - return $this->currentSnapshot; - } - - /** - * @return DownloadResult - */ - public function download(): DownloadResult { - return $this->promise()->wait(); - } - - /** - * Returns that resolves a multipart download operation, - * or to a rejection in case of any failures. - * - * @return PromiseInterface - */ - public function promise(): PromiseInterface - { - return Coroutine::of(function () { - $this->downloadInitiated($this->requestArgs); - $result = ['@metadata'=>[]]; - try { - $result = yield $this->s3Client->executeAsync($this->nextCommand()) - ->then(function (ResultInterface $result) { - // Calculate object size and parts count. - $this->computeObjectDimensions($result); - // Trigger first part completed - $this->partDownloadCompleted($result); - - return $result; - })->otherwise(function ($reason) { - $this->partDownloadFailed($reason); - - throw $reason; - }); - } catch (\Throwable $e) { - $this->downloadFailed($e); - // TODO: yield transfer exception modeled with a transfer failed response. - yield Create::rejectionFor($e); - } - - while ($this->currentPartNo < $this->objectPartsCount) { - try { - yield $this->s3Client->executeAsync($this->nextCommand()) - ->then(function ($result) { - $this->partDownloadCompleted($result); - - return $result; - })->otherwise(function ($reason) { - $this->partDownloadFailed($reason); - - throw $reason; - }); - } catch (\Throwable $reason) { - $this->downloadFailed($reason); - // TODO: yield transfer exception modeled with a transfer failed response. - yield Create::rejectionFor($reason); - } - - } - - // Transfer completed - $this->downloadComplete(); - - unset($result['Body']); - yield Create::promiseFor(new DownloadResult( - $this->stream, - $result['@metadata'] ?? [] - )); - }); - } - - /** - * Returns the next command for fetching the next object part. - * - * @return CommandInterface - */ - abstract protected function nextCommand() : CommandInterface; - - /** - * Compute the object dimensions, such as size and parts count. - * - * @param ResultInterface $result - * - * @return void - */ - abstract protected function computeObjectDimensions(ResultInterface $result): void; - - /** - * Calculates the object size dynamically. - * - * @param $sizeSource - * - * @return int - */ - protected function computeObjectSize($sizeSource): int - { - if (is_int($sizeSource)) { - return (int) $sizeSource; - } - - if (empty($sizeSource)) { - return 0; - } - - // For extracting the object size from the ContentRange header value. - if (preg_match(self::OBJECT_SIZE_REGEX, $sizeSource, $matches)) { - return $matches[1]; - } - - throw new \RuntimeException('Invalid source size format'); - } - - /** - * Main purpose of this method is to propagate - * the download-initiated event to listeners, but - * also it does some computation regarding internal states - * that need to be maintained. - * - * @param array $commandArgs - * - * @return void - */ - private function downloadInitiated(array $commandArgs): void - { - if ($this->currentSnapshot === null) { - $this->currentSnapshot = new TransferProgressSnapshot( - $commandArgs['Key'], - 0, - $this->objectSizeInBytes - ); - } else { - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse() - ); - } - - $this->listenerNotifier?->transferInitiated([ - TransferListener::REQUEST_ARGS_KEY => $commandArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); - } - - /** - * Propagates download-failed event to listeners. - * - * @param \Throwable $reason - * - * @return void - */ - private function downloadFailed(\Throwable $reason): void - { - // Event already propagated. - if ($this->currentSnapshot->getReason() !== null) { - return; - } - - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $reason - ); - $this->stream->close(); - $this->listenerNotifier?->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - 'reason' => $reason, - ]); - } - - /** - * Propagates part-download-completed to listeners. - * It also does some computation in order to maintain internal states. - * In this specific method we move each part content into an accumulative - * stream, which is meant to hold the full object content once the download - * is completed. - * - * @param ResultInterface $result - * - * @return void - */ - private function partDownloadCompleted( - ResultInterface $result - ): void - { - $partDownloadBytes = $result['ContentLength']; - if (isset($result['ETag'])) { - $this->eTag = $result['ETag']; - } - Utils::copyToStream($result['Body'], $this->stream); - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partDownloadBytes, - $this->objectSizeInBytes, - $result->toArray() - ); - $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->bytesTransferred([ - TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); - } - - /** - * Propagates part-download-failed event to listeners. - * - * @param \Throwable $reason - * - * @return void - */ - private function partDownloadFailed( - \Throwable $reason, - ): void - { - $this->downloadFailed($reason); - } - - /** - * Propagates object-download-completed event to listeners. - * It also resets the pointer of the stream to the first position, - * so that the stream is ready to be consumed once returned. - * - * @return void - */ - private function downloadComplete(): void - { - $this->stream->rewind(); - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->objectSizeInBytes, - $this->currentSnapshot->getResponse() - ); - $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->transferComplete([ - TransferListener::REQUEST_ARGS_KEY => $this->requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); - } - - /** - * @param mixed $multipartDownloadType - * - * @return string - */ - public static function chooseDownloaderClass( - string $multipartDownloadType - ): string - { - return match ($multipartDownloadType) { - MultipartDownloaderInitial::PART_GET_MULTIPART_DOWNLOADER => PartGetMultipartDownloader::class, - MultipartDownloaderInitial::RANGE_GET_MULTIPART_DOWNLOADER => RangeGetMultipartDownloader::class, - default => throw new \InvalidArgumentException( - "The config value for `multipart_download_type` must be one of:\n" - . "\t* " . MultipartDownloaderInitial::PART_GET_MULTIPART_DOWNLOADER - ."\n" - . "\t* " . MultipartDownloaderInitial::RANGE_GET_MULTIPART_DOWNLOADER - ) - }; - } -} diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 71262ccf6b..b5399690e5 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -150,9 +150,8 @@ protected function processMultipartOperation(): PromiseInterface hash_update($this->hashContext, $read); } - $partBody = Utils::streamFor( - $read - ); + $partBody = Utils::streamFor($read); + $uploadPartCommandArgs['PartNumber'] = $partNo; $uploadPartCommandArgs['ContentLength'] = $partBody->getSize(); // Attach body diff --git a/src/S3/S3Transfer/MultipartUploaderInitial.php b/src/S3/S3Transfer/MultipartUploaderInitial.php deleted file mode 100644 index 316583525b..0000000000 --- a/src/S3/S3Transfer/MultipartUploaderInitial.php +++ /dev/null @@ -1,570 +0,0 @@ -s3Client = $s3Client; - $this->createMultipartArgs = $createMultipartArgs; - $this->validateConfig($config); - $this->config = $config; - $this->body = $this->parseBody($source); - $this->uploadId = $uploadId; - $this->parts = $parts; - $this->currentSnapshot = $currentSnapshot; - $this->listenerNotifier = $listenerNotifier; - } - - /** - * @param array $config - * - * @return void - */ - private function validateConfig(array &$config): void - { - if (isset($config['part_size'])) { - if ($config['part_size'] < self::PART_MIN_SIZE - || $config['part_size'] > self::PART_MAX_SIZE) { - throw new \InvalidArgumentException( - "The config `part_size` value must be between " - . self::PART_MIN_SIZE . " and " . self::PART_MAX_SIZE - . " but ${config['part_size']} given." - ); - } - } else { - $config['part_size'] = self::PART_MIN_SIZE; - } - } - - /** - * @return string|null - */ - public function getUploadId(): ?string - { - return $this->uploadId; - } - - /** - * @return array - */ - public function getParts(): array - { - return $this->parts; - } - - /** - * @return int - */ - public function getCalculatedObjectSize(): int - { - return $this->calculatedObjectSize; - } - - /** - * @return TransferProgressSnapshot|null - */ - public function getCurrentSnapshot(): ?TransferProgressSnapshot - { - return $this->currentSnapshot; - } - - /** - * @return UploadResult - */ - public function upload(): UploadResult { - return $this->promise()->wait(); - } - - /** - * @return PromiseInterface - */ - public function promise(): PromiseInterface - { - return Coroutine::of(function () { - try { - yield $this->createMultipartUpload(); - yield $this->uploadParts(); - $result = yield $this->completeMultipartUpload(); - yield Create::promiseFor( - new UploadResult($result->toArray()) - ); - } catch (Throwable $e) { - $this->uploadFailed($e); - yield Create::rejectionFor($e); - } finally { - $this->callDeferredFns(); - } - }); - } - - /** - * @return PromiseInterface - */ - private function createMultipartUpload(): PromiseInterface - { - $requestArgs = [...$this->createMultipartArgs]; - $checksum = $this->filterChecksum($requestArgs); - // Customer provided checksum - if ($checksum !== null) { - $requestArgs['ChecksumType'] = 'FULL_OBJECT'; - $requestArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); - $requestArgs['@context']['request_checksum_calculation'] = 'when_required'; - unset($requestArgs[$checksum]); - } - - $this->uploadInitiated($requestArgs); - $command = $this->s3Client->getCommand( - 'CreateMultipartUpload', - $requestArgs - ); - - return $this->s3Client->executeAsync($command) - ->then(function (ResultInterface $result) { - $this->uploadId = $result['UploadId']; - - return $result; - }); - } - - /** - * @return PromiseInterface - */ - private function uploadParts(): PromiseInterface - { - $this->calculatedObjectSize = 0; - $partSize = $this->config['part_size']; - $commands = []; - $partNo = count($this->parts); - $baseUploadPartCommandArgs = [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, - ]; - // Customer provided checksum - $checksum = $this->filterChecksum($this->createMultipartArgs); - if ($checksum !== null) { - unset($baseUploadPartCommandArgs['ChecksumAlgorithm']); - unset($baseUploadPartCommandArgs[$checksum]); - $baseUploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; - } - - while (!$this->body->eof()) { - $partNo++; - $read = $this->body->read($partSize); - // To make sure we do not create an empty part when - // we already reached the end of file. - if (empty($read) && $this->body->eof()) { - break; - } - - $partBody = Utils::streamFor( - $read - ); - $uploadPartCommandArgs = [ - ...$baseUploadPartCommandArgs, - 'PartNumber' => $partNo, - 'ContentLength' => $partBody->getSize(), - ]; - - // To get `requestArgs` when notifying the bytesTransfer listeners. - $uploadPartCommandArgs['requestArgs'] = [...$uploadPartCommandArgs]; - // Attach body - $uploadPartCommandArgs['Body'] = $this->decorateWithHashes( - $partBody, - $uploadPartCommandArgs - ); - $command = $this->s3Client->getCommand('UploadPart', $uploadPartCommandArgs); - $commands[] = $command; - $this->calculatedObjectSize += $partBody->getSize(); - if ($partNo > self::PART_MAX_NUM) { - return Create::rejectionFor( - "The max number of parts has been exceeded. " . - "Max = " . self::PART_MAX_NUM - ); - } - } - - return (new CommandPool( - $this->s3Client, - $commands, - [ - 'concurrency' => $this->config['concurrency'], - 'fulfilled' => function (ResultInterface $result, $index) - use ($commands) { - $command = $commands[$index]; - $this->collectPart( - $result, - $command - ); - // Part Upload Completed Event - $this->partUploadCompleted( - $command['ContentLength'], - $command['requestArgs'] - ); - }, - 'rejected' => function (Throwable $e) { - $this->partUploadFailed($e); - - throw $e; - } - ] - ))->promise(); - } - - /** - * @return PromiseInterface - */ - private function completeMultipartUpload(): PromiseInterface - { - $this->sortParts(); - $completeMultipartUploadArgs = [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, - 'MpuObjectSize' => $this->calculatedObjectSize, - 'MultipartUpload' => [ - 'Parts' => $this->parts, - ] - ]; - $checksum = $this->filterChecksum($completeMultipartUploadArgs); - // Customer provided checksum - if ($checksum !== null) { - $completeMultipartUploadArgs['ChecksumAlgorithm'] = str_replace('Checksum', '', $checksum); - $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; - $completeMultipartUploadArgs['@context']['request_checksum_calculation'] = 'when_required'; - } - - $command = $this->s3Client->getCommand( - 'CompleteMultipartUpload', - $completeMultipartUploadArgs - ); - - return $this->s3Client->executeAsync($command) - ->then(function (ResultInterface $result) { - $this->uploadCompleted($result); - - return $result; - }); - } - - /** - * @return PromiseInterface - */ - private function abortMultipartUpload(): PromiseInterface - { - $command = $this->s3Client->getCommand('AbortMultipartUpload', [ - ...$this->createMultipartArgs, - 'UploadId' => $this->uploadId, - ]); - - return $this->s3Client->executeAsync($command); - } - - /** - * @param ResultInterface $result - * @param CommandInterface $command - * - * @return void - */ - private function collectPart( - ResultInterface $result, - CommandInterface $command, - ): void - { - $checksumResult = $command->getName() === 'UploadPart' - ? $result - : $result[$command->getName() . 'Result']; - $partData = [ - 'PartNumber' => $command['PartNumber'], - 'ETag' => $result['ETag'], - ]; - if (isset($command['ChecksumAlgorithm'])) { - $checksumMemberName = 'Checksum' . strtoupper($command['ChecksumAlgorithm']); - $partData[$checksumMemberName] = $checksumResult[$checksumMemberName] ?? null; - } - - $this->parts[] = $partData; - } - - /** - * @return void - */ - private function sortParts(): void - { - usort($this->parts, function($partOne, $partTwo) { - return $partOne['PartNumber'] <=> $partTwo['PartNumber']; - }); - } - - /** - * @param string|StreamInterface $source - * - * @return StreamInterface - */ - private function parseBody(string | StreamInterface $source): StreamInterface - { - if (is_string($source)) { - // Make sure the files exists - if (!is_readable($source)) { - throw new \InvalidArgumentException( - "The source for this upload must be either a readable file path or a valid stream." - ); - } - $body = new LazyOpenStream($source, 'r'); - // To make sure the resource is closed. - $this->deferFns[] = function () use ($body) { - $body->close(); - }; - } elseif ($source instanceof StreamInterface) { - $body = $source; - } else { - throw new \InvalidArgumentException( - "The source must be a valid string file path or a StreamInterface." - ); - } - - return $body; - } - - /** - * @param array $requestArgs - * - * @return void - */ - private function uploadInitiated(array $requestArgs): void - { - if ($this->currentSnapshot === null) { - $this->currentSnapshot = new TransferProgressSnapshot( - $requestArgs['Key'], - 0, - $this->body->getSize(), - ); - } else { - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $this->currentSnapshot->getReason(), - ); - } - - $this->listenerNotifier?->transferInitiated([ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot - ]); - } - - /** - * @param Throwable $reason - * - * @return void - */ - private function uploadFailed(Throwable $reason): void { - // Event has been already propagated - if ($this->currentSnapshot->getReason() !== null) { - return; - } - - $this->currentSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $reason - ); - - if (!empty($this->uploadId)) { - $this->abortMultipartUpload()->wait(); - } - $this->listenerNotifier?->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - 'reason' => $reason, - ]); - } - - /** - * @param ResultInterface $result - * - * @return void - */ - private function uploadCompleted(ResultInterface $result): void { - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes(), - $this->currentSnapshot->getTotalBytes(), - $result->toArray(), - $this->currentSnapshot->getReason(), - ); - $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->transferComplete([ - TransferListener::REQUEST_ARGS_KEY => $this->createMultipartArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - ]); - } - - /** - * @param int $partCompletedBytes - * @param array $requestArgs - * - * @return void - */ - private function partUploadCompleted( - int $partCompletedBytes, - array $requestArgs - ): void - { - $newSnapshot = new TransferProgressSnapshot( - $this->currentSnapshot->getIdentifier(), - $this->currentSnapshot->getTransferredBytes() + $partCompletedBytes, - $this->currentSnapshot->getTotalBytes(), - $this->currentSnapshot->getResponse(), - $this->currentSnapshot->getReason(), - ); - $this->currentSnapshot = $newSnapshot; - $this->listenerNotifier?->bytesTransferred([ - TransferListener::REQUEST_ARGS_KEY => $requestArgs, - TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, - $this->currentSnapshot - ]); - } - - /** - * @param Throwable $reason - * - * @return void - */ - private function partUploadFailed(Throwable $reason): void - { - $this->uploadFailed($reason); - } - - /** - * @return void - */ - private function callDeferredFns(): void - { - foreach ($this->deferFns as $fn) { - $fn(); - } - - $this->deferFns = []; - } - - /** - * Filters a provided checksum if one was provided. - * - * @param array $requestArgs - * - * @return string | null - */ - private function filterChecksum(array $requestArgs):? string - { - static $algorithms = [ - 'ChecksumCRC32', - 'ChecksumCRC32C', - 'ChecksumCRC64NVME', - 'ChecksumSHA1', - 'ChecksumSHA256', - ]; - foreach ($algorithms as $algorithm) { - if (isset($requestArgs[$algorithm])) { - return $algorithm; - } - } - - return null; - } - - /** - * @param StreamInterface $stream - * @param array $data - * - * @return StreamInterface - */ - private function decorateWithHashes(StreamInterface $stream, array &$data): StreamInterface - { - // Decorate source with a hashing stream - $hash = new PhpHash('sha256'); - return new HashingStream($stream, $hash, function ($result) use (&$data) { - $data['ContentSHA256'] = bin2hex($result); - }); - } -} From 26d2469a4f4049318c074c71e43a203fba9a262f Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 17 Jul 2025 15:02:59 -0700 Subject: [PATCH 39/62] fix: object key should be normalized When a s3 prefix is provided and the object key contains the delimiter then, the prefix should be stripped off from the object key. --- src/S3/S3Transfer/S3TransferManager.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 4308ea44e0..7e13dec920 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -376,13 +376,13 @@ public function downloadDirectory( $listArgs = [ 'Bucket' => $sourceBucket, ] + ($config['list_object_v2_args'] ?? []); + $s3Prefix = $config['s3_prefix'] ?? null; if (empty($listArgs['Prefix']) && $s3Prefix !== null) { $listArgs['Prefix'] = $s3Prefix; } - $listArgs['Delimiter'] = $listArgs['Delimiter'] - ?? $config['s3_delimiter'] ?? null; + $listArgs['Delimiter'] = $listArgs['Delimiter'] ?? null; $objects = $this->s3Client ->getPaginator('ListObjectsV2', $listArgs) @@ -434,9 +434,18 @@ public function downloadDirectory( $promises = []; $objectsDownloaded = 0; $objectsFailed = 0; + $s3Delimiter = $config['s3_delimiter'] ?? '/'; foreach ($objects as $object) { $bucketAndKeyArray = self::s3UriAsBucketAndKey($object); $objectKey = $bucketAndKeyArray['Key']; + if ($s3Prefix !== null && str_contains($objectKey, $s3Delimiter)) { + if (!str_ends_with($s3Prefix, $s3Delimiter)) { + $s3Prefix = $s3Prefix.$s3Delimiter; + } + + $objectKey = substr($objectKey, strlen($s3Prefix)); + } + $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { throw new S3TransferException( From 10928caaa777819230e8bada373051fd96ff9450 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 18 Jul 2025 08:19:14 -0700 Subject: [PATCH 40/62] fix: minor logic and test fix - when the directory separator is different from the s3 delimiter then the separator from the object key is replaced with the os directory separator. - Update the tests to validate object key destination correctly whe using the download directory API. --- src/S3/S3Transfer/S3TransferManager.php | 14 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 138 +++++++++++++++--- 2 files changed, 127 insertions(+), 25 deletions(-) diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 7e13dec920..ff3d9a87a1 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -446,12 +446,20 @@ public function downloadDirectory( $objectKey = substr($objectKey, strlen($s3Prefix)); } + // CONVERT THE KEY DIR SEPARATOR TO OS BASED DIR SEPARATOR + if (DIRECTORY_SEPARATOR !== $s3Delimiter) { + $objectKey = str_replace( + $s3Delimiter, + DIRECTORY_SEPARATOR, + $objectKey + ); + } + $destinationFile = $destinationDirectory . DIRECTORY_SEPARATOR . $objectKey; if ($this->resolvesOutsideTargetDirectory($destinationFile, $objectKey)) { throw new S3TransferException( - "Cannot download key ' . $objectKey - . ', its relative path resolves outside the' - . ' parent directory" + "Cannot download key $objectKey " + ."its relative path resolves outside the parent directory." ); } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 9fac0f9dfd..97161d4431 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -2752,16 +2752,28 @@ public function downloadDirectoryCreateFilesProvider(): array } /** + * @param string|null $prefix + * @param string|null $delimiter * @param array $objects - * - * @dataProvider failsWhenKeyResolvesOutsideTargetDirectoryProvider + * @param array $expectedOutput * * @return void + * @dataProvider failsWhenKeyResolvesOutsideTargetDirectoryProvider */ - public function testFailsWhenKeyResolvesOutsideTargetDirectory( - string $prefix, + public function testResolvesOutsideTargetDirectory( + ?string $prefix, + ?string $delimiter, array $objects, + array $expectedOutput ) { + if ($expectedOutput['success'] === false) { + $this->expectException(S3TransferException::class); + $this->expectExceptionMessageMatches( + '/Cannot download key [^\s]+ its relative path' + .' resolves outside the parent directory\./' + ); + } + $bucket = "test-bucket"; $directory = "test-directory"; try { @@ -2770,7 +2782,6 @@ public function testFailsWhenKeyResolvesOutsideTargetDirectory( TestsUtility::cleanUpDir($fullDirectoryPath); } mkdir($fullDirectoryPath, 0777, true); - $this->expectException(S3TransferException::class); $called = false; $client = $this->getS3ClientMock([ 'executeAsync' => function (CommandInterface $command) use ( @@ -2821,10 +2832,19 @@ public function testFailsWhenKeyResolvesOutsideTargetDirectory( [], [ 's3_prefix' => $prefix, + 's3_delimiter' => $delimiter, ] ) )->wait(); $this->assertTrue($called); + // Validate the expected file output + if ($expectedOutput['success']) { + $this->assertFileExists( + $fullDirectoryPath + . DIRECTORY_SEPARATOR + . $expectedOutput['filename'] + ); + } } finally { TestsUtility::cleanUpDir($directory); } @@ -2833,46 +2853,120 @@ public function testFailsWhenKeyResolvesOutsideTargetDirectory( /** * @return array */ - public function failsWhenKeyResolvesOutsideTargetDirectoryProvider(): array { + public function resolvesOutsideTargetDirectoryProvider(): array { return [ - 'resolves_outside_target_directory_1' => [ - 'prefix' => 'foo-objects/', + 'download_directory_1_linux' => [ + 'prefix' => null, + 'delimiter' => null, 'objects' => [ [ - 'Key' => '../outside/key1.txt' + 'Key' => '2023/Jan/1.png' ], ], + 'expected_output' => [ + 'success' => true, + 'filename' => '2023/Jan/1.png', + ] ], - 'resolves_outside_target_directory_2' => [ - 'prefix' => 'foo-objects/', + 'download_directory_2' => [ + 'prefix' => '2023/Jan/', + 'delimiter' => null, 'objects' => [ [ - 'Key' => '../../foo/key2.txt' + 'Key' => '2023/Jan/1.png' ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '1.png', ] ], - 'resolves_outside_target_directory_3' => [ - 'prefix' => 'buzz/', + 'download_directory_3' => [ + 'prefix' => '2023/Jan', + 'delimiter' => null, 'objects' => [ [ - 'Key' => '..//inner//key3.txt' + 'Key' => '2023/Jan/1.png' ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '1.png', ] ], - 'resolves_outside_target_directory_4' => [ - 'prefix' => 'test/', + 'download_directory_4' => [ + 'prefix' => null, + 'delimiter' => '-', 'objects' => [ [ - 'Key' => './../../key4.txt' + 'Key' => '2023-Jan-1.png' ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '2023/Jan/1.png', ] ], - 'resolves_outside_target_directory_5' => [ - 'prefix' => 'test/', + 'download_directory_5' => [ + 'prefix' => null, + 'delimiter' => '-', 'objects' => [ [ - 'Key' => './../another_dir/.././key1.txt', - ], + 'Key' => '2023-Jan-.png' + ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => '2023/Jan/.png', + ] + ], + 'download_directory_6' => [ + 'prefix' => '2023', + 'delimiter' => '-', + 'objects' => [ + [ + 'Key' => '2023/Jan-1.png' + ] + ], + 'expected_output' => [ + 'success' => true, + 'filename' => 'Jan/1.png', + ] + ], + 'download_directory_7_fails' => [ + 'prefix' => null, + 'delimiter' => null, + 'objects' => [ + [ + 'Key' => '../2023/Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => false, + ] + ], + 'download_directory_9_fails' => [ + 'prefix' => null, + 'delimiter' => null, + 'objects' => [ + [ + 'Key' => 'foo/../2023/../../Jan/1.png' + ] + ], + 'expected_output' => [ + 'success' => false, + ] + ], + 'download_directory_10_fails' => [ + 'prefix' => null, + 'delimiter' => null, + 'objects' => [ + [ + 'Key' => '../test-2/object.dat' + ] + ], + 'expected_output' => [ + 'success' => false, ] ], ]; From 632ece964ad44137c5640a3552f649a9a9e07767 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 18 Jul 2025 09:13:07 -0700 Subject: [PATCH 41/62] fix: fix s3 delimiter test The delimiter for the list object request in a download directory must be defaulted to null unless explicitly provided in the list object v2 args. --- tests/S3/S3Transfer/S3TransferManagerTest.php | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 97161d4431..4f8b846b92 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -1980,16 +1980,16 @@ public function downloadDirectoryAppliesS3PrefixProvider(): array } /** - * @param array $config - * @param string $expectedS3Delimiter + * @param string|null $delimiter + * @param string|null $expectedS3Delimiter * * @dataProvider downloadDirectoryAppliesDelimiterProvider * * @return void */ public function testDownloadDirectoryAppliesDelimiter( - array $config, - string $expectedS3Delimiter + ?string $delimiter, + ?string $expectedS3Delimiter ): void { $destinationDirectory = sys_get_temp_dir() . "/download-directory-test"; @@ -2010,7 +2010,7 @@ public function testDownloadDirectoryAppliesDelimiter( $listObjectsCalled = true; $this->assertEquals( $expectedS3Delimiter, - $command['Delimiter'] + $command['Delimiter'] ?? null ); } @@ -2036,6 +2036,13 @@ public function testDownloadDirectoryAppliesDelimiter( return new HandlerList(); } ]); + $config = []; + if ($delimiter !== null) { + $config['list_object_v2_args'] = [ + 'Delimiter' => $delimiter, + ]; + } + $manager = new S3TransferManager( $client, ); @@ -2061,28 +2068,17 @@ public function testDownloadDirectoryAppliesDelimiter( public function downloadDirectoryAppliesDelimiterProvider(): array { return [ - 's3_delimiter_from_config' => [ - 'config' => [ - 's3_delimiter' => 'FooDelimiter', - ], + 's3_delimiter_1' => [ + 'Delimiter' => 'FooDelimiter', 'expected_s3_delimiter' => 'FooDelimiter' ], - 's3_delimiter_from_list_object_v2_args' => [ - 'config' => [ - 'list_object_v2_args' => [ - 'Delimiter' => 'DelimiterFromArgs' - ], - ], - 'expected_s3_delimiter' => 'DelimiterFromArgs' + 's3_delimiter_2' => [ + 'Delimiter' => 'FooDelimiter2', + 'expected_s3_delimiter' => 'FooDelimiter2' ], - 's3_delimiter_from_config_is_ignored_when_present_in_list_object_args' => [ - 'config' => [ - 's3_delimiter' => 'TestDelimiter', - 'list_object_v2_args' => [ - 'Delimiter' => 'DelimiterFromArgs' - ], - ], - 'expected_s3_delimiter' => 'DelimiterFromArgs' + 's3_delimiter_4_defaulted_to_null' => [ + 'Delimiter' => null, + 'expected_s3_delimiter' => null ], ]; } From 6efcc0afe0439874e31d23301eead819b8e2a511 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 18 Jul 2025 09:33:38 -0700 Subject: [PATCH 42/62] fix: wrong data provider name used The data provider for testResolvesOutsideTargetDirectory was renamed to `resolvesOutsideTargetDirectoryProvider`. --- tests/S3/S3Transfer/S3TransferManagerTest.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 4f8b846b92..f0eb079086 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -13,7 +13,6 @@ use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; use Aws\S3\S3Transfer\Models\DownloadRequest; -use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; use Aws\S3\S3Transfer\Models\UploadRequest; @@ -2754,7 +2753,7 @@ public function downloadDirectoryCreateFilesProvider(): array * @param array $expectedOutput * * @return void - * @dataProvider failsWhenKeyResolvesOutsideTargetDirectoryProvider + * @dataProvider resolvesOutsideTargetDirectoryProvider */ public function testResolvesOutsideTargetDirectory( ?string $prefix, From b0c5f75a0790fbae4ccbfde5edba8876e6d32ace Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 18 Jul 2025 10:51:36 -0700 Subject: [PATCH 43/62] chore: addressed some styling --- src/S3/S3Transfer/AbstractMultipartUploader.php | 3 +-- .../S3Transfer/Exceptions/FileDownloadException.php | 5 +---- .../Exceptions/ProgressTrackerException.php | 5 +---- src/S3/S3Transfer/Exceptions/S3TransferException.php | 4 +--- src/S3/S3Transfer/Models/DownloadDirectoryRequest.php | 3 ++- src/S3/S3Transfer/Models/DownloadFileRequest.php | 3 +-- src/S3/S3Transfer/Models/DownloadRequest.php | 6 ++++-- src/S3/S3Transfer/Models/S3TransferManagerConfig.php | 3 ++- src/S3/S3Transfer/Models/TransferRequest.php | 6 ++++-- src/S3/S3Transfer/Models/UploadDirectoryRequest.php | 3 ++- src/S3/S3Transfer/MultipartDownloader.php | 9 ++++++--- src/S3/S3Transfer/MultipartUploader.php | 11 +++++++---- src/S3/S3Transfer/Progress/MultiProgressTracker.php | 6 +++--- src/S3/S3Transfer/Progress/ProgressBarFormat.php | 6 ++++-- src/S3/S3Transfer/Progress/SingleProgressTracker.php | 6 +++--- .../S3Transfer/Progress/TransferListenerNotifier.php | 3 ++- .../S3Transfer/Progress/TransferProgressSnapshot.php | 4 +--- src/S3/S3Transfer/S3TransferManager.php | 6 ++++-- src/S3/S3Transfer/Utils/FileDownloadHandler.php | 6 ++++-- 19 files changed, 53 insertions(+), 45 deletions(-) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 4e3a89e51e..32919fbcbb 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -86,8 +86,7 @@ public function __construct array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null, - ) - { + ) { $this->s3Client = $s3Client; $this->putObjectRequestArgs = $putObjectRequestArgs; $this->validateConfig($config); diff --git a/src/S3/S3Transfer/Exceptions/FileDownloadException.php b/src/S3/S3Transfer/Exceptions/FileDownloadException.php index 1c67691975..a80cde583a 100644 --- a/src/S3/S3Transfer/Exceptions/FileDownloadException.php +++ b/src/S3/S3Transfer/Exceptions/FileDownloadException.php @@ -2,7 +2,4 @@ namespace Aws\S3\S3Transfer\Exceptions; -class FileDownloadException extends \RuntimeException -{ - -} +class FileDownloadException extends \RuntimeException {} diff --git a/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php index df00709e57..2034a40fb8 100644 --- a/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php +++ b/src/S3/S3Transfer/Exceptions/ProgressTrackerException.php @@ -2,7 +2,4 @@ namespace Aws\S3\S3Transfer\Exceptions; -class ProgressTrackerException extends \RuntimeException -{ - -} +class ProgressTrackerException extends \RuntimeException {} diff --git a/src/S3/S3Transfer/Exceptions/S3TransferException.php b/src/S3/S3Transfer/Exceptions/S3TransferException.php index 1ffbd6426b..d95d262aab 100644 --- a/src/S3/S3Transfer/Exceptions/S3TransferException.php +++ b/src/S3/S3Transfer/Exceptions/S3TransferException.php @@ -2,6 +2,4 @@ namespace Aws\S3\S3Transfer\Exceptions; -class S3TransferException extends \RuntimeException -{ -} +class S3TransferException extends \RuntimeException {} diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 0999c96f41..c3066dc4d4 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -96,7 +96,8 @@ public static function fromLegacyArgs( array $config = [], array $listeners = [], ?TransferListener $progressTracker = null, - ): DownloadDirectoryRequest { + ): DownloadDirectoryRequest + { return new self( $sourceBucket, $destinationDirectory, diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php index b66ff6e378..fa6e4cc0a3 100644 --- a/src/S3/S3Transfer/Models/DownloadFileRequest.php +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -24,8 +24,7 @@ public function __construct( string $destination, bool $failsWhenDestinationExists, DownloadRequest $downloadRequest - ) - { + ) { $this->destination = $destination; $this->failsWhenDestinationExists = $failsWhenDestinationExists; $this->downloadRequest = DownloadRequest::fromDownloadRequestAndDownloadHandler( diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 942bf4fd4a..9fe03f7a99 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -136,7 +136,8 @@ public function getObjectRequestArgs(): array /** * @return DownloadHandler */ - public function getDownloadHandler(): DownloadHandler { + public function getDownloadHandler(): DownloadHandler + { return $this->downloadHandler; } @@ -147,7 +148,8 @@ public function getDownloadHandler(): DownloadHandler { * * @return array */ - public function normalizeSourceAsArray(): array { + public function normalizeSourceAsArray(): array + { // If source is null then fall back to getObjectRequest. $source = $this->getSource() ?? [ 'Bucket' => $this->getObjectRequestArgs['Bucket'] ?? null, diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php index 5b2730b30d..3bef77fe26 100644 --- a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -171,7 +171,8 @@ public function getDefaultRegion(): string /** * @return array */ - public function toArray(): array { + public function toArray(): array + { return [ 'target_part_size_bytes' => $this->targetPartSizeBytes, 'multipart_upload_threshold_bytes' => $this->multipartUploadThresholdBytes, diff --git a/src/S3/S3Transfer/Models/TransferRequest.php b/src/S3/S3Transfer/Models/TransferRequest.php index cd1109ab3d..014abef912 100644 --- a/src/S3/S3Transfer/Models/TransferRequest.php +++ b/src/S3/S3Transfer/Models/TransferRequest.php @@ -57,7 +57,8 @@ public function getProgressTracker(): ?TransferListener /** * @return array */ - public function getConfig(): array { + public function getConfig(): array + { return $this->config; } @@ -66,7 +67,8 @@ public function getConfig(): array { * * @return void */ - public function updateConfigWithDefaults(array $defaultConfig): void { + public function updateConfigWithDefaults(array $defaultConfig): void + { foreach (static::$configKeys as $key) { if (empty($this->config[$key])) { $this->config[$key] = $defaultConfig[$key]; diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index f10bc6e37d..afd79288dd 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -61,7 +61,8 @@ public static function fromLegacyArgs( array $config = [], array $listeners = [], ?TransferListener $progressTracker = null, - ): UploadDirectoryRequest { + ): UploadDirectoryRequest + { return new self( $sourceDirectory, $targetBucket, diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 1b1798851b..b11bafe80c 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -92,7 +92,8 @@ public function __construct( $this->listenerNotifier = $listenerNotifier; } - private function validateConfig(array &$config): void { + private function validateConfig(array &$config): void + { if (!isset($config['target_part_size_bytes'])) { $config['target_part_size_bytes'] = S3TransferManagerConfig::DEFAULT_TARGET_PART_SIZE_BYTES; } @@ -145,7 +146,8 @@ public function getCurrentSnapshot(): TransferProgressSnapshot /** * @return DownloadResult */ - public function download(): DownloadResult { + public function download(): DownloadResult + { return $this->promise()->wait(); } @@ -206,7 +208,8 @@ public function promise(): PromiseInterface * * @return PromiseInterface */ - protected function initialRequest(): PromiseInterface { + protected function initialRequest(): PromiseInterface + { $command = $this->nextCommand(); // Notify download initiated $this->downloadInitiated($command->toArray()); diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index b5399690e5..63947ae9ed 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -41,8 +41,7 @@ public function __construct( array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null, - ) - { + ) { parent::__construct( $s3Client, $putObjectRequestArgs, @@ -92,7 +91,8 @@ private function parseBody( /** * @return void */ - private function evaluateCustomChecksum(): void { + private function evaluateCustomChecksum(): void + { // Evaluation for custom provided checksums $checksumName = self::filterChecksum($this->putObjectRequestArgs); if ($checksumName !== null) { @@ -240,7 +240,10 @@ protected function createResponse(ResultInterface $result): UploadResult * * @return StreamInterface */ - private function decorateWithHashes(StreamInterface $stream, array &$data): StreamInterface + private function decorateWithHashes( + StreamInterface $stream, + array &$data + ): StreamInterface { // Decorate source with a hashing stream $hash = new PhpHash('sha256'); diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php index 6cb63d684b..4357b4b36c 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressTracker.php +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -23,8 +23,8 @@ final class MultiProgressTracker extends TransferListener implements ProgressTra /** @var int */ private int $failed; - /** @var ProgressBarFactoryInterface | Closure | null */ - private readonly ProgressBarFactoryInterface | Closure | null $progressBarFactory; + /** @var ProgressBarFactoryInterface|Closure|null */ + private readonly ProgressBarFactoryInterface|Closure|null $progressBarFactory; /** * @param array $singleProgressTrackers @@ -40,7 +40,7 @@ public function __construct( int $transferCount = 0, int $completed = 0, int $failed = 0, - ProgressBarFactoryInterface | Closure | null $progressBarFactory = null + ProgressBarFactoryInterface|Closure|null $progressBarFactory = null ) { $this->singleProgressTrackers = $singleProgressTrackers; diff --git a/src/S3/S3Transfer/Progress/ProgressBarFormat.php b/src/S3/S3Transfer/Progress/ProgressBarFormat.php index 37b3e4e63a..562ae19818 100644 --- a/src/S3/S3Transfer/Progress/ProgressBarFormat.php +++ b/src/S3/S3Transfer/Progress/ProgressBarFormat.php @@ -19,7 +19,8 @@ public function __construct( $this->args = $args; } - public function getArgs(): array { + public function getArgs(): array + { return $this->args; } @@ -54,7 +55,8 @@ public function setArg(string $key, mixed $value): void /** * @return string */ - public function format(): string { + public function format(): string + { $parameters = $this->getFormatParameters(); $defaultParameterValues = $this->getFormatDefaultParameterValues(); foreach ($parameters as $param) { diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 5800eb6aa3..5a5dd2b2a4 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -7,7 +7,8 @@ /** * To track single object transfers. */ -final class SingleProgressTracker extends TransferListener implements ProgressTrackerInterface +final class SingleProgressTracker extends TransferListener + implements ProgressTrackerInterface { /** @var ProgressBarInterface */ private ProgressBarInterface $progressBar; @@ -37,8 +38,7 @@ public function __construct( bool $clear = true, ?TransferProgressSnapshot $currentSnapshot = null, bool $showProgressOnUpdate = true - ) - { + ) { $this->progressBar = $progressBar; if (get_resource_type($output) !== 'stream') { throw new \InvalidArgumentException("The type for $output must be a stream"); diff --git a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php index de136fcecf..1f5bb65313 100644 --- a/src/S3/S3Transfer/Progress/TransferListenerNotifier.php +++ b/src/S3/S3Transfer/Progress/TransferListenerNotifier.php @@ -27,7 +27,8 @@ public function __construct(array $listeners = []) * * @return void */ - public function addListener(TransferListener $listener): void { + public function addListener(TransferListener $listener): void + { $this->listeners[] = $listener; } diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index d42397a14c..aa95ad0695 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -65,7 +65,7 @@ public function getTotalBytes(): int /** * @return array|null */ - public function getResponse(): array | null + public function getResponse(): array|null { return $this->response; } @@ -90,6 +90,4 @@ public function getReason(): Throwable|string|null { return $this->reason; } - - } diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index ff3d9a87a1..05b8970d2e 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -575,7 +575,8 @@ private function trySingleUpload( string|StreamInterface $source, array $requestArgs, ?TransferListenerNotifier $listenerNotifier = null - ): PromiseInterface { + ): PromiseInterface + { if (is_string($source) && is_readable($source)) { $requestArgs['SourceFile'] = $source; $objectSize = filesize($source); @@ -664,7 +665,8 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { private function tryMultipartUpload( UploadRequest $uploadRequest, ?TransferListenerNotifier $listenerNotifier = null, - ): PromiseInterface { + ): PromiseInterface + { return (new MultipartUploader( $this->s3Client, $uploadRequest->getPutObjectRequestArgs(), diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php index 8cb95613f4..d6bcd2a59b 100644 --- a/src/S3/S3Transfer/Utils/FileDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -25,8 +25,10 @@ class FileDownloadHandler extends DownloadHandler * @param string $destination * @param bool $failsWhenDestinationExists */ - public function __construct(string $destination, bool $failsWhenDestinationExists) - { + public function __construct( + string $destination, + bool $failsWhenDestinationExists + ) { $this->destination = $destination; $this->failsWhenDestinationExists = $failsWhenDestinationExists; $this->temporaryDestination = ""; From 44f6ff411c76dd57ab819a4aac0a68567b72dbf6 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 18 Jul 2025 10:55:12 -0700 Subject: [PATCH 44/62] chore: update argument name In AbstractMultipartUploader the parameter used to be putObjectRequestArgs, but it makes more sense to name it requestArgs. --- .../S3Transfer/AbstractMultipartUploader.php | 18 +++++++++--------- src/S3/S3Transfer/MultipartUploader.php | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 32919fbcbb..0d35ae2afe 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -30,7 +30,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface protected readonly S3ClientInterface $s3Client; /** @var array @ */ - protected readonly array $putObjectRequestArgs; + protected readonly array $requestArgs; /** @var array @ */ protected readonly array $config; @@ -67,7 +67,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface /** * @param S3ClientInterface $s3Client - * @param array $putObjectRequestArgs + * @param array $requestArgs * @param array $config * - target_part_size_bytes: (int, optional) * - request_checksum_calculation: (string, optional) @@ -80,7 +80,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface public function __construct ( S3ClientInterface $s3Client, - array $putObjectRequestArgs, + array $requestArgs, array $config, ?string $uploadId = null, array $parts = [], @@ -88,7 +88,7 @@ public function __construct ?TransferListenerNotifier $listenerNotifier = null, ) { $this->s3Client = $s3Client; - $this->putObjectRequestArgs = $putObjectRequestArgs; + $this->requestArgs = $requestArgs; $this->validateConfig($config); $this->config = $config; $this->uploadId = $uploadId; @@ -176,7 +176,7 @@ public function promise(): PromiseInterface */ protected function createMultipartUpload(): PromiseInterface { - $createMultipartUploadArgs = $this->putObjectRequestArgs; + $createMultipartUploadArgs = $this->requestArgs; if ($this->requestChecksum !== null) { $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; @@ -206,7 +206,7 @@ protected function createMultipartUpload(): PromiseInterface protected function completeMultipartUpload(): PromiseInterface { $this->sortParts(); - $completeMultipartUploadArgs = $this->putObjectRequestArgs; + $completeMultipartUploadArgs = $this->requestArgs; $completeMultipartUploadArgs['UploadId'] = $this->uploadId; $completeMultipartUploadArgs['MultipartUpload'] = [ 'Parts' => $this->parts @@ -237,7 +237,7 @@ protected function completeMultipartUpload(): PromiseInterface */ protected function abortMultipartUpload(): PromiseInterface { - $abortMultipartUploadArgs = $this->putObjectRequestArgs; + $abortMultipartUploadArgs = $this->requestArgs; $abortMultipartUploadArgs['UploadId'] = $this->uploadId; $command = $this->s3Client->getCommand( 'AbortMultipartUpload', @@ -349,7 +349,7 @@ protected function operationCompleted(ResultInterface $result): void $this->listenerNotifier?->transferComplete([ TransferListener::REQUEST_ARGS_KEY => - $this->putObjectRequestArgs, + $this->requestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot ]); } @@ -392,7 +392,7 @@ protected function operationFailed(Throwable $reason): void $this->listenerNotifier?->transferFail([ TransferListener::REQUEST_ARGS_KEY => - $this->putObjectRequestArgs, + $this->requestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 63947ae9ed..58132df6ee 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -34,7 +34,7 @@ class MultipartUploader extends AbstractMultipartUploader public function __construct( S3ClientInterface $s3Client, - array $putObjectRequestArgs, + array $requestArgs, array $config, string | StreamInterface $source, ?string $uploadId = null, @@ -44,7 +44,7 @@ public function __construct( ) { parent::__construct( $s3Client, - $putObjectRequestArgs, + $requestArgs, $config, $uploadId, $parts, @@ -94,9 +94,9 @@ private function parseBody( private function evaluateCustomChecksum(): void { // Evaluation for custom provided checksums - $checksumName = self::filterChecksum($this->putObjectRequestArgs); + $checksumName = self::filterChecksum($this->requestArgs); if ($checksumName !== null) { - $this->requestChecksum = $this->putObjectRequestArgs[$checksumName]; + $this->requestChecksum = $this->requestArgs[$checksumName]; $this->requestChecksumAlgorithm = str_replace( 'Checksum', '', @@ -110,7 +110,7 @@ private function evaluateCustomChecksum(): void protected function processMultipartOperation(): PromiseInterface { - $uploadPartCommandArgs = $this->putObjectRequestArgs; + $uploadPartCommandArgs = $this->requestArgs; $this->calculatedObjectSize = 0; $partSize = $this->calculatePartSize(); $partsCount = ceil($this->getTotalSize() / $partSize); From 83ccd9ba8cd9ee39771d776cf3c554b600ea3232 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 22 Jul 2025 10:20:01 -0700 Subject: [PATCH 45/62] chore: make config optional --- src/S3/S3Transfer/Models/DownloadDirectoryRequest.php | 2 +- src/S3/S3Transfer/Models/DownloadRequest.php | 4 ++-- src/S3/S3Transfer/Models/UploadRequest.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index c3066dc4d4..824eeb8b8c 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -65,7 +65,7 @@ public function __construct( string $sourceBucket, string $destinationDirectory, array $getObjectRequestArgs, - array $config, + array $config = [], array $listeners = [], ?TransferListener $progressTracker = null ) { diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 9fe03f7a99..d3444c681f 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -52,8 +52,8 @@ final class DownloadRequest extends TransferRequest public function __construct( string|array|null $source, array $getObjectRequestArgs, - array $config, - ?DownloadHandler $downloadHandler, + array $config = [], + ?DownloadHandler $downloadHandler = null, array $listeners = [], ?TransferListener $progressTracker = null ) { diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 432cb21d0c..1fcc694ec9 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -47,7 +47,7 @@ class UploadRequest extends TransferRequest public function __construct( StreamInterface|string $source, array $putObjectRequestArgs, - array $config, + array $config = [], array $listeners = [], ?TransferListener $progressTracker = null ) { From 039a67bc9c29b56c43b07be703526b160c410cc4 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 23 Jul 2025 05:31:20 -0700 Subject: [PATCH 46/62] chore: minor refactor and fix - Make config arguments optional - Make getObjectRequest/putObjectRequest argument optional - Delete file if exists when using the file download handler. --- src/S3/S3Transfer/Models/DownloadRequest.php | 8 ++++---- src/S3/S3Transfer/Models/S3TransferManagerConfig.php | 2 +- src/S3/S3Transfer/Models/UploadDirectoryRequest.php | 2 +- src/S3/S3Transfer/Models/UploadRequest.php | 2 +- src/S3/S3Transfer/Utils/FileDownloadHandler.php | 2 ++ 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index d3444c681f..894cf7c000 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -34,14 +34,14 @@ final class DownloadRequest extends TransferRequest * @param array $config The configuration to be used for this operation: * - multipart_download_type: (string, optional) * Overrides the resolved value from the transfer manager config. - * - checksum_validation_enabled: (bool, optional) Overrides the resolved + * - response_checksum_validation: (string, optional) Overrides the resolved * value from transfer manager config for whether checksum validation * should be done. This option will be considered just if ChecksumMode * is not present in the request args. * - track_progress: (bool) Overrides the config option set in the transfer * manager instantiation to decide whether transfer progress should be * tracked. - * - minimum_part_size: (int) The minimum part size in bytes to be used + * - target_part_size_bytes: (int) The part size in bytes to be used * in a range multipart download. If this parameter is not provided * then it fallbacks to the transfer manager `target_part_size_bytes` * config value. @@ -51,11 +51,11 @@ final class DownloadRequest extends TransferRequest */ public function __construct( string|array|null $source, - array $getObjectRequestArgs, + array $getObjectRequestArgs = [], array $config = [], ?DownloadHandler $downloadHandler = null, array $listeners = [], - ?TransferListener $progressTracker = null + ?TransferListener $progressTracker = null ) { parent::__construct($listeners, $progressTracker, $config); $this->source = $source; diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php index 3bef77fe26..0b50e2eb6c 100644 --- a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -74,7 +74,7 @@ public function __construct( * The threshold to decided whether a multipart upload is needed. * - request_checksum_calculation: (string, default=`when_supported`) * To decide whether a checksum validation will be applied to the response. - * - request_checksum_validation: (string, default=`when_supported`) + * - response_checksum_validation: (string, default=`when_supported`) * - multipart_download_type: (string, default='part') * The download type to be used in a multipart download. * - concurrency: (int, default=5) diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index afd79288dd..b87074fa4b 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -28,7 +28,7 @@ class UploadDirectoryRequest extends TransferRequest public function __construct( string $sourceDirectory, string $targetBucket, - array $putObjectRequestArgs, + array $putObjectRequestArgs = [], array $config = [], array $listeners = [], ?TransferListener $progressTracker = null diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 1fcc694ec9..78393ceffd 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -38,7 +38,7 @@ class UploadRequest extends TransferRequest * a progressTracker parameter is not provided then, a default implementation * will be resolved. This option is intended to make the operation to use * a default progress tracker implementation when $progressTracker is null. - * - concurrency: (int, optional) + * - concurrency: (int, optional) To override default value for concurrency. * - request_checksum_calculation: (string, optional, defaulted to `when_supported`) * @param TransferListener[]|null $listeners * @param TransferListener|null $progressTracker diff --git a/src/S3/S3Transfer/Utils/FileDownloadHandler.php b/src/S3/S3Transfer/Utils/FileDownloadHandler.php index d6bcd2a59b..ded7c154ce 100644 --- a/src/S3/S3Transfer/Utils/FileDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/FileDownloadHandler.php @@ -114,6 +114,8 @@ public function transferComplete(array $context): void throw new FileDownloadException( "The destination '$this->destination' already exists." ); + } else { + unlink($this->destination); } } From 50715d1e411de13f74324dea96e932a099f266d4 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 23 Jul 2025 10:22:02 -0700 Subject: [PATCH 47/62] chore: make parameter optional - Make getObjectRequestArgs optional in download directory operation. - Add config key in docs --- src/S3/S3Transfer/Models/DownloadDirectoryRequest.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 824eeb8b8c..29ab9fb15e 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -47,7 +47,7 @@ class DownloadDirectoryRequest extends TransferRequest * - track_progress: (bool, optional) Overrides the config option set * in the transfer manager instantiation to decide whether transfer * progress should be tracked. - * - minimum_part_size: (int, optional) The minimum part size in bytes + * - target_part_size_bytes: (int, optional) The part size in bytes * to be used in a range multipart download. * - list_object_v2_args: (array, optional) The arguments to be included * as part of the listObjectV2 request in order to fetch the objects to @@ -55,6 +55,8 @@ class DownloadDirectoryRequest extends TransferRequest * - MaxKeys: (int) Sets the maximum number of keys returned in the response. * - Prefix: (string) To limit the response to keys that begin with the * specified prefix. + * - fails_when_destination_exists: (bool) Whether to fail when a destination + * file exists. * @param TransferListener[] $listeners The listeners for watching * transfer events. Each listener will be cloned per file upload. * @param TransferListener|null $progressTracker Ideally the progress @@ -64,7 +66,7 @@ class DownloadDirectoryRequest extends TransferRequest public function __construct( string $sourceBucket, string $destinationDirectory, - array $getObjectRequestArgs, + array $getObjectRequestArgs = [], array $config = [], array $listeners = [], ?TransferListener $progressTracker = null From 3034ad8c513885c81b78ff3888323f2afc81d6f1 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 27 Jul 2025 18:10:58 -0700 Subject: [PATCH 48/62] chore: make model classes final --- src/S3/S3Transfer/Models/DownloadDirectoryRequest.php | 6 +++--- src/S3/S3Transfer/Models/DownloadDirectoryResponse.php | 2 +- src/S3/S3Transfer/Models/DownloadResult.php | 2 +- src/S3/S3Transfer/Models/S3TransferManagerConfig.php | 2 +- src/S3/S3Transfer/Models/UploadDirectoryRequest.php | 6 +++--- src/S3/S3Transfer/Models/UploadDirectoryResponse.php | 2 +- src/S3/S3Transfer/Models/UploadRequest.php | 8 ++++---- src/S3/S3Transfer/Models/UploadResult.php | 2 +- src/S3/S3Transfer/MultipartUploader.php | 10 ++++++++++ src/S3/S3Transfer/Utils/StreamDownloadHandler.php | 2 +- 10 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 29ab9fb15e..627b16c315 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -6,7 +6,7 @@ use Aws\S3\S3Transfer\Progress\TransferListener; use InvalidArgumentException; -class DownloadDirectoryRequest extends TransferRequest +final class DownloadDirectoryRequest extends TransferRequest { /** @var string */ private string $sourceBucket; @@ -89,7 +89,7 @@ public function __construct( * @param array $listeners * @param TransferListener|null $progressTracker * - * @return DownloadDirectoryRequest + * @return self */ public static function fromLegacyArgs( string $sourceBucket, @@ -98,7 +98,7 @@ public static function fromLegacyArgs( array $config = [], array $listeners = [], ?TransferListener $progressTracker = null, - ): DownloadDirectoryRequest + ): self { return new self( $sourceBucket, diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php index c1d87e5e28..1da220e316 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Models; -class DownloadDirectoryResponse +final class DownloadDirectoryResponse { /** @var int */ private int $objectsDownloaded; diff --git a/src/S3/S3Transfer/Models/DownloadResult.php b/src/S3/S3Transfer/Models/DownloadResult.php index bb4ab69910..64152faaf9 100644 --- a/src/S3/S3Transfer/Models/DownloadResult.php +++ b/src/S3/S3Transfer/Models/DownloadResult.php @@ -4,7 +4,7 @@ use Aws\Result; -class DownloadResult extends Result +final class DownloadResult extends Result { private readonly mixed $downloadDataResult; diff --git a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php index 0b50e2eb6c..695371cc62 100644 --- a/src/S3/S3Transfer/Models/S3TransferManagerConfig.php +++ b/src/S3/S3Transfer/Models/S3TransferManagerConfig.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Models; -class S3TransferManagerConfig +final class S3TransferManagerConfig { public const DEFAULT_TARGET_PART_SIZE_BYTES = 8388608; // 8MB public const DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES = 16777216; // 16MB diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index b87074fa4b..81df2e648c 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -6,7 +6,7 @@ use Aws\S3\S3Transfer\Progress\TransferListener; use InvalidArgumentException; -class UploadDirectoryRequest extends TransferRequest +final class UploadDirectoryRequest extends TransferRequest { /** @var string */ private string $sourceDirectory; @@ -52,7 +52,7 @@ public function __construct( * @param array $listeners * @param TransferListener|null $progressTracker * - * @return UploadDirectoryRequest + * @return self */ public static function fromLegacyArgs( string $sourceDirectory, @@ -61,7 +61,7 @@ public static function fromLegacyArgs( array $config = [], array $listeners = [], ?TransferListener $progressTracker = null, - ): UploadDirectoryRequest + ): self { return new self( $sourceDirectory, diff --git a/src/S3/S3Transfer/Models/UploadDirectoryResponse.php b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php index 1cf9663128..93ce6e00d0 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryResponse.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Models; -class UploadDirectoryResponse +final class UploadDirectoryResponse { /** @var int */ private int $objectsUploaded; diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 78393ceffd..f2419c1421 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -6,7 +6,7 @@ use InvalidArgumentException; use Psr\Http\Message\StreamInterface; -class UploadRequest extends TransferRequest +final class UploadRequest extends TransferRequest { public static array $configKeys = [ 'multipart_upload_threshold_bytes', @@ -63,7 +63,7 @@ public function __construct( * @param array $listeners * @param TransferListener|null $progressTracker * - * @return UploadRequest + * @return self */ public static function fromLegacyArgs( string | StreamInterface $source, @@ -71,9 +71,9 @@ public static function fromLegacyArgs( array $config = [], array $listeners = [], ?TransferListener $progressTracker = null - ): UploadRequest + ): self { - return new UploadRequest( + return new self( $source, $putObjectRequestArgs, $config, diff --git a/src/S3/S3Transfer/Models/UploadResult.php b/src/S3/S3Transfer/Models/UploadResult.php index 2f053ea1d3..30029082be 100644 --- a/src/S3/S3Transfer/Models/UploadResult.php +++ b/src/S3/S3Transfer/Models/UploadResult.php @@ -4,7 +4,7 @@ use Aws\Result; -class UploadResult extends Result +final class UploadResult extends Result { /** * @param array $data diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 58132df6ee..c03a82dddb 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -56,6 +56,16 @@ public function __construct( $this->evaluateCustomChecksum(); } + /** + * Sync upload method. + * + * @return UploadResult + */ + public function upload(): UploadResult + { + return $this->promise()->wait(); + } + /** * @param string|StreamInterface $source * diff --git a/src/S3/S3Transfer/Utils/StreamDownloadHandler.php b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php index 0df9e769bf..01ac1b97bd 100644 --- a/src/S3/S3Transfer/Utils/StreamDownloadHandler.php +++ b/src/S3/S3Transfer/Utils/StreamDownloadHandler.php @@ -26,7 +26,7 @@ public function __construct(?StreamInterface $stream = null) */ public function transferInitiated(array $context): void { - if ($this->stream === null) { + if (is_null($this->stream)) { $this->stream = Utils::streamFor( fopen('php://temp', 'w+') ); From cfe4ab533399eb967e0d6a843f9f661e32407c4d Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Tue, 29 Jul 2025 10:01:29 -0700 Subject: [PATCH 49/62] chore: make classes final and refactor tests - Make data model final - Refactor test cases to use correct parameters for S3TransferManager APIs --- features/s3Transfer/s3TransferManager.feature | 94 +++++++++---------- .../Models/DownloadDirectoryRequest.php | 2 +- ...sponse.php => DownloadDirectoryResult.php} | 4 +- src/S3/S3Transfer/Models/DownloadRequest.php | 14 +-- ...Response.php => UploadDirectoryResult.php} | 2 +- src/S3/S3Transfer/Models/UploadRequest.php | 2 +- src/S3/S3Transfer/MultipartDownloader.php | 1 - src/S3/S3Transfer/MultipartUploader.php | 18 ++-- .../S3Transfer/PartGetMultipartDownloader.php | 1 - .../Progress/MultiProgressTracker.php | 4 - src/S3/S3Transfer/S3TransferManager.php | 12 +-- tests/S3/S3Transfer/S3TransferManagerTest.php | 8 +- 12 files changed, 78 insertions(+), 84 deletions(-) rename src/S3/S3Transfer/Models/{DownloadDirectoryResponse.php => DownloadDirectoryResult.php} (87%) rename src/S3/S3Transfer/Models/{UploadDirectoryResponse.php => UploadDirectoryResult.php} (95%) diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature index e86a3cdd65..6e0acea944 100644 --- a/features/s3Transfer/s3TransferManager.feature +++ b/features/s3Transfer/s3TransferManager.feature @@ -14,10 +14,10 @@ Feature: S3 Transfer Manager Then the file should exist in the test bucket and its content should be Examples: - | filename | content | - | myfile-test-1-1.txt | Test content #1 | - | myfile-test-1-2.txt | Test content #2 | - | myfile-test-1-3.txt | Test content #3 | + | filename | content | + | myfile-test-1-1.txt | Test content #1 | + | myfile-test-1-2.txt | Test content #2 | + | myfile-test-1-3.txt | Test content #3 | Scenario Outline: Successfully does a single upload from a stream @@ -25,93 +25,93 @@ Feature: S3 Transfer Manager When I do the upload to a test bucket with key Then the object , once downloaded from the test bucket, should match the content Examples: - | content | key | - | "This is a test text - 1" | myfile-test-2-1.txt | - | "This is a test text - 2" | myfile-test-2-2.txt | - | "This is a test text - 3" | myfile-test-2-3.txt | + | content | key | + | "This is a test text - 1" | myfile-test-2-1.txt | + | "This is a test text - 2" | myfile-test-2-2.txt | + | "This is a test text - 3" | myfile-test-2-3.txt | Scenario Outline: Successfully do multipart object upload from file Given I have a file with name where its content's size is When I do upload this file with name with the specified part size of Then the object with name should have a total of parts and its size must be Examples: - | filename | filesize | partsize | partnum | - | myfile-test-3-1.txt | 10485760 | 5242880 | 2 | - | myfile-test-3-2.txt | 24117248 | 5242880 | 5 | - | myfile-test-3-3.txt | 24117248 | 8388608 | 3 | + | filename | filesize | partsize | partnum | + | myfile-test-3-1.txt | 10485760 | 5242880 | 2 | + | myfile-test-3-2.txt | 24117248 | 5242880 | 5 | + | myfile-test-3-3.txt | 24117248 | 8388608 | 3 | Scenario Outline: Successfully do multipart object upload from streams - Given I have want to upload a stream of size + Given I want to upload a stream of size When I do upload this stream with name and the specified part size of Then the object with name should have a total of parts and its size must be Examples: - | filename | filesize | partsize | partnum | - | myfile-test-4-1.txt | 10485760 | 5242880 | 2 | - | myfile-test-4-2.txt | 24117248 | 5242880 | 5 | - | myfile-test-4-3.txt | 24117248 | 8388608 | 3 | + | filename | filesize | partsize | partnum | + | myfile-test-4-1.txt | 10485760 | 5242880 | 2 | + | myfile-test-4-2.txt | 24117248 | 5242880 | 5 | + | myfile-test-4-3.txt | 24117248 | 8388608 | 3 | Scenario Outline: Does single object upload with custom checksum Given I have a file with name and its content is When I upload this file with name by providing a custom checksum algorithm Then the checksum from the object with name should be equals to the calculation of the object content with the checksum algorithm Examples: - | filename | content | checksum_algorithm | - | myfile-test-5-1.txt | This is a test file content #1 | crc32 | - | myfile-test-5-2.txt | This is a test file content #2 | crc32c | - | myfile-test-5-3.txt | This is a test file content #3 | sha256 | - | myfile-test-5-4.txt | This is a test file content #4 | sha1 | + | filename | content | checksum_algorithm | + | myfile-test-5-1.txt | This is a test file content #1 | crc32 | + | myfile-test-5-2.txt | This is a test file content #2 | crc32c | + | myfile-test-5-3.txt | This is a test file content #3 | sha256 | + | myfile-test-5-4.txt | This is a test file content #4 | sha1 | Scenario Outline: Does single object download Given I have an object in S3 with name and its content is When I do a download of the object with name Then the object with name should have been downloaded and its content should be Examples: - | filename | content | - | myfile-test-6-1.txt | This is a test file content #1 | - | myfile-test-6-2.txt | This is a test file content #2 | - | myfile-test-6-3.txt | This is a test file content #3 | + | filename | content | + | myfile-test-6-1.txt | This is a test file content #1 | + | myfile-test-6-2.txt | This is a test file content #2 | + | myfile-test-6-3.txt | This is a test file content #3 | Scenario Outline: Successfully does multipart object download Given I have an object in S3 with name and its size is When I download the object with name by using the multipart download type Then the content size for the object with name should be Examples: - | filename | filesize | download_type | - | myfile-test-7-1.txt | 20971520 | rangeGet | - | myfile-test-7-2.txt | 28311552 | rangeGet | - | myfile-test-7-3.txt | 12582912 | rangeGet | - | myfile-test-7-4.txt | 20971520 | partGet | - | myfile-test-7-5.txt | 28311552 | partGet | - | myfile-test-7-6.txt | 12582912 | partGet | + | filename | filesize | download_type | + | myfile-test-7-1.txt | 20971520 | ranged | + | myfile-test-7-2.txt | 28311552 | ranged | + | myfile-test-7-3.txt | 12582912 | ranged | + | myfile-test-7-4.txt | 20971520 | part | + | myfile-test-7-5.txt | 28311552 | part | + | myfile-test-7-6.txt | 12582912 | part | Scenario Outline: Successfully does directory upload Given I have a directory with files that I want to upload When I upload this directory Then the files from this directory where its count should be should exist in the bucket Examples: - | directory | numfile | - | directory-test-1-1 | 10 | - | directory-test-1-2 | 3 | - | directory-test-1-3 | 25 | - | directory-test-1-4 | 1 | + | directory | numfile | + | directory-test-1-1 | 10 | + | directory-test-1-2 | 3 | + | directory-test-1-3 | 25 | + | directory-test-1-4 | 1 | Scenario Outline: Successfully does a directory download Given I have a total of objects in a bucket prefixed with When I download all of them into the directory Then the objects should exist as files within the directory Examples: - | numfile | directory | - | 15 | directory-test-2-1 | - | 12 | directory-test-2-2 | - | 1 | directory-test-2-3 | - | 30 | directory-test-2-4 | + | numfile | directory | + | 15 | directory-test-2-1 | + | 12 | directory-test-2-2 | + | 1 | directory-test-2-3 | + | 30 | directory-test-2-4 | Scenario Outline: Abort a multipart upload Given I am uploading the file with size When I upload the file using multipart upload and fails at part number Then The multipart upload should have been aborted for file Examples: - | file | size | partNumberFail | - | abort-file-1.txt | 1024 * 1024 * 20 | 3 | - | abort-file-2.txt | 1024 * 1024 * 40 | 5 | - | abort-file-3.txt | 1024 * 1024 * 10 | 1 | \ No newline at end of file + | file | size | partNumberFail | + | abort-file-1.txt | 20971520 | 3 | + | abort-file-2.txt | 41943040 | 5 | + | abort-file-3.txt | 10485760 | 1 | \ No newline at end of file diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 627b16c315..8eeba9f1a9 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -42,7 +42,7 @@ final class DownloadDirectoryRequest extends TransferRequest * - $downloadDirectoryRequestArgs: (array) The arguments for the download * directory request. * - $reason: (Throwable) The exception that originated the request failure. - * - $downloadDirectoryResponse: (DownloadDirectoryResponse) The download response + * - $downloadDirectoryResponse: (DownloadDirectoryResult) The download response * to that point in the upload process. * - track_progress: (bool, optional) Overrides the config option set * in the transfer manager instantiation to decide whether transfer diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php similarity index 87% rename from src/S3/S3Transfer/Models/DownloadDirectoryResponse.php rename to src/S3/S3Transfer/Models/DownloadDirectoryResult.php index 1da220e316..4ab54669e7 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryResult.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Models; -final class DownloadDirectoryResponse +final class DownloadDirectoryResult { /** @var int */ private int $objectsDownloaded; @@ -39,7 +39,7 @@ public function getObjectsFailed(): int public function __toString(): string { return sprintf( - "DownloadDirectoryResponse: %d objects downloaded, %d objects failed", + "DownloadDirectoryResult: %d objects downloaded, %d objects failed", $this->objectsDownloaded, $this->objectsFailed ); diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 894cf7c000..6379b916a4 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -75,18 +75,18 @@ public function __construct( * @param array $listeners * @param TransferListener|null $progressTracker * - * @return DownloadRequest + * @return self */ public static function fromLegacyArgs( - string | array | null $source, + string|array|null $source, array $downloadRequestArgs = [], array $config = [], ?DownloadHandler $downloadHandler = null, array $listeners = [], ?TransferListener $progressTracker = null, - ): DownloadRequest + ): self { - return new DownloadRequest( + return new self( $source, $downloadRequestArgs, $config, @@ -100,14 +100,14 @@ public static function fromLegacyArgs( * @param DownloadRequest $downloadRequest * @param FileDownloadHandler $downloadHandler * - * @return DownloadRequest + * @return self */ public static function fromDownloadRequestAndDownloadHandler( DownloadRequest $downloadRequest, FileDownloadHandler $downloadHandler - ): DownloadRequest + ): self { - return new DownloadRequest( + return new self( $downloadRequest->getSource(), $downloadRequest->getObjectRequestArgs(), $downloadRequest->getConfig(), diff --git a/src/S3/S3Transfer/Models/UploadDirectoryResponse.php b/src/S3/S3Transfer/Models/UploadDirectoryResult.php similarity index 95% rename from src/S3/S3Transfer/Models/UploadDirectoryResponse.php rename to src/S3/S3Transfer/Models/UploadDirectoryResult.php index 93ce6e00d0..2c99f8e0d4 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryResponse.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryResult.php @@ -2,7 +2,7 @@ namespace Aws\S3\S3Transfer\Models; -final class UploadDirectoryResponse +final class UploadDirectoryResult { /** @var int */ private int $objectsUploaded; diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index f2419c1421..25c5befc84 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -66,7 +66,7 @@ public function __construct( * @return self */ public static function fromLegacyArgs( - string | StreamInterface $source, + string|StreamInterface $source, array $putObjectRequestArgs = [], array $config = [], array $listeners = [], diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index b11bafe80c..11512f7399 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -1,5 +1,4 @@ output, self::CLEAR_ASCII_CODE); $percentsSum = 0; - /** - * @var $_ - * @var SingleProgressTracker $progressTracker - */ foreach ($this->singleProgressTrackers as $_ => $progressTracker) { $progressTracker->showProgress(); $percentsSum += $progressTracker->getProgressBar()->getPercentCompleted(); diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 05b8970d2e..ad1e81c0f0 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -6,12 +6,12 @@ use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; -use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; +use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadFileRequest; use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; -use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Models\UploadResult; use Aws\S3\S3Transfer\Progress\MultiProgressTracker; @@ -280,7 +280,7 @@ function ($file) use ($filter) { "bucket_to" => $targetBucket, ], $reason, - new UploadDirectoryResponse( + new UploadDirectoryResult( $objectsUploaded, $objectsFailed ) @@ -295,7 +295,7 @@ function ($file) use ($filter) { return Each::ofLimitAll($promises, $this->config->getConcurrency()) ->then(function ($_) use (&$objectsUploaded, &$objectsFailed) { - return new UploadDirectoryResponse($objectsUploaded, $objectsFailed); + return new UploadDirectoryResult($objectsUploaded, $objectsFailed); }); } @@ -511,7 +511,7 @@ public function downloadDirectory( "bucket" => $sourceBucket, ], $reason, - new DownloadDirectoryResponse( + new DownloadDirectoryResult( $objectsDownloaded, $objectsFailed ) @@ -526,7 +526,7 @@ public function downloadDirectory( return Each::ofLimitAll($promises, $this->config->getConcurrency()) ->then(function ($_) use (&$objectsFailed, &$objectsDownloaded) { - return new DownloadDirectoryResponse( + return new DownloadDirectoryResult( $objectsDownloaded, $objectsFailed ); diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index f0eb079086..97ec644994 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -11,10 +11,10 @@ use Aws\S3\S3Transfer\AbstractMultipartUploader; use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; -use Aws\S3\S3Transfer\Models\DownloadDirectoryResponse; +use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; -use Aws\S3\S3Transfer\Models\UploadDirectoryResponse; +use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\MultipartUploader; @@ -1218,7 +1218,7 @@ public function testUploadDirectoryUsesFailurePolicy(): void array $requestArgs, array $uploadDirectoryRequestArgs, \Throwable $reason, - UploadDirectoryResponse $uploadDirectoryResponse + UploadDirectoryResult $uploadDirectoryResponse ) use ($directory, &$called) { $called = true; $this->assertEquals( @@ -2247,7 +2247,7 @@ public function testDownloadDirectoryUsesFailurePolicy(): void array $requestArgs, array $uploadDirectoryRequestArgs, \Throwable $reason, - DownloadDirectoryResponse $downloadDirectoryResponse + DownloadDirectoryResult $downloadDirectoryResponse ) use ($destinationDirectory, &$called) { $called = true; $this->assertEquals( From f4c42e0eb07bf5ec7474e6487431e4e6dbabed89 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Wed, 30 Jul 2025 06:07:13 -0700 Subject: [PATCH 50/62] chore: fix and reformat integ test - Fix S3TransferManagetContext.php to use the correct parameter expected in the different APIs exposed by S3TranserManager - Refactor some formatting styling. --- tests/Integ/S3TransferManagerContext.php | 360 ++++++++++++++++------- 1 file changed, 248 insertions(+), 112 deletions(-) diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index b0bab0d188..836bb71d49 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -3,8 +3,12 @@ namespace Aws\Test\Integ; use Aws\S3\ApplyChecksumMiddleware; +use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; +use Aws\S3\S3Transfer\Models\DownloadFileRequest; +use Aws\S3\S3Transfer\Models\DownloadRequest; use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; +use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadRequest; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; @@ -74,6 +78,7 @@ public function afterScenarioRuns(): void { // Clean up data holders $this->stream?->close(); + $this->stream = null; } /** @@ -95,7 +100,7 @@ public function iUploadTheFileToATestBucketUsingTheS3TransferManager($filename): self::getSdk()->createS3() ); $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $fullFilePath, [ 'Bucket' => self::getResourceName(), @@ -108,7 +113,10 @@ public function iUploadTheFileToATestBucketUsingTheS3TransferManager($filename): /** * @Then /^the file (.*) should exist in the test bucket and its content should be (.*)$/ */ - public function theFileShouldExistInTheTestBucketAndItsContentShouldBe($filename, $content): void + public function theFileShouldExistInTheTestBucketAndItsContentShouldBe( + $filename, + $content + ): void { $client = self::getSdk()->createS3(); $response = $client->getObject([ @@ -137,7 +145,7 @@ public function iDoTheUploadToATestBucketWithKey($key): void self::getSdk()->createS3() ); $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $this->stream, [ 'Bucket' => self::getResourceName(), @@ -150,7 +158,10 @@ public function iDoTheUploadToATestBucketWithKey($key): void /** * @Then /^the object (.*), once downloaded from the test bucket, should match the content (.*)$/ */ - public function theObjectOnceDownloadedFromTheTestBucketShouldMatchTheContent($key, $content): void + public function theObjectOnceDownloadedFromTheTestBucketShouldMatchTheContent( + $key, + $content + ): void { $client = self::getSdk()->createS3(); $response = $client->getObject([ @@ -165,66 +176,75 @@ public function theObjectOnceDownloadedFromTheTestBucketShouldMatchTheContent($k /** * @Given /^I have a file with name (.*) where its content's size is (.*)$/ */ - public function iHaveAFileWithNameWhereItsContentSSizeIs($filename, $filesize): void + public function iHaveAFileWithNameWhereItsContentSSizeIs( + $filename, + $filesize + ): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; - file_put_contents($fullFilePath, str_repeat('a', $filesize)); + file_put_contents($fullFilePath, str_repeat('a', (int)$filesize)); } /** * @When /^I do upload this file with name (.*) with the specified part size of (.*)$/ */ - public function iDoUploadThisFileWithNameWithTheSpecifiedPartSizeOf($filename, $partsize): void + public function iDoUploadThisFileWithNameWithTheSpecifiedPartSizeOf( + $filename, + $partsize + ): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), S3TransferManagerConfig::fromArray([ - 'multipart_upload_threshold_bytes' => $partsize, + 'multipart_upload_threshold_bytes' => (int)$partsize, ]) ); $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $fullFilePath, [ 'Bucket' => self::getResourceName(), 'Key' => $filename, ], [ - 'part_size' => intval($partsize), + 'target_part_size_bytes' => (int)$partsize, ] ) )->wait(); } /** - * @Given /^I have want to upload a stream of size (.*)$/ + * @Given /^I want to upload a stream of size (.*)$/ */ - public function iHaveWantToUploadAStreamOfSize($filesize): void + public function iWantToUploadAStreamOfSize($filesize): void { - $this->stream = Utils::streamFor(str_repeat('a', $filesize)); + $this->stream = Utils::streamFor(str_repeat('a', (int)$filesize)); } /** * @When /^I do upload this stream with name (.*) and the specified part size of (.*)$/ */ - public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf($filename, $partsize): void + public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf( + $filename, + $partsize + ): void { $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), S3TransferManagerConfig::fromArray([ - 'multipart_upload_threshold_bytes' => $partsize, + 'multipart_upload_threshold_bytes' => (int)$partsize, ]) ); $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $this->stream, [ 'Bucket' => self::getResourceName(), 'Key' => $filename, ], [ - 'part_size' => intval($partsize), + 'target_part_size_bytes' => (int)$partsize, ] ) )->wait(); @@ -233,7 +253,11 @@ public function iDoUploadThisStreamWithNameAndTheSpecifiedPartSizeOf($filename, /** * @Then /^the object with name (.*) should have a total of (.*) parts and its size must be (.*)$/ */ - public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe($filename, $partnum, $filesize): void + public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe( + $filename, + $partnum, + $filesize + ): void { $partNo = 1; $s3Client = self::getSdk()->createS3(); @@ -243,10 +267,10 @@ public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe($filena 'PartNumber' => $partNo ]); Assert::assertEquals(206, $response['@metadata']['statusCode']); - Assert::assertEquals($partnum, $response['PartsCount']); + Assert::assertEquals((int)$partnum, $response['PartsCount']); $contentLength = $response['@metadata']['headers']['content-length']; $partNo++; - while ($partNo <= $partnum) { + while ($partNo <= (int)$partnum) { $response = $s3Client->headObject([ 'Bucket' => self::getResourceName(), 'Key' => $filename, @@ -256,7 +280,7 @@ public function theObjectWithNameShouldHaveATotalOfPartsAndItsSizeMustBe($filena $partNo++; } - Assert::assertEquals($filesize, $contentLength); + Assert::assertEquals((int)$filesize, $contentLength); } /** @@ -271,22 +295,23 @@ public function iHaveAFileWithNameAndItsContentIs($filename, $content): void /** * @When /^I upload this file with name (.*) by providing a custom checksum algorithm (.*)$/ */ - public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm($filename, $checksum_algorithm): void + public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm( + $filename, + $checksumAlgorithm + ): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), ); $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $fullFilePath, [ 'Bucket' => self::getResourceName(), 'Key' => $filename, + 'ChecksumAlgorithm' => $checksumAlgorithm, ], - [ - 'checksum_algorithm' => $checksum_algorithm, - ] ) )->wait(); } @@ -294,7 +319,10 @@ public function iUploadThisFileWithNameByProvidingACustomChecksumAlgorithm($file /** * @Then /^the checksum from the object with name (.*) should be equals to the calculation of the object content with the checksum algorithm (.*)$/ */ - public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOfTheObjectContentWithTheChecksumAlgorithm($filename, $checksum_algorithm): void + public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOfTheObjectContentWithTheChecksumAlgorithm( + $filename, + $checksumAlgorithm + ): void { $s3Client = self::getSdk()->createS3(); $response = $s3Client->getObject([ @@ -306,17 +334,20 @@ public function theChecksumFromTheObjectWithNameShouldBeEqualsToTheCalculationOf Assert::assertEquals(200, $response['@metadata']['statusCode']); Assert::assertEquals( ApplyChecksumMiddleware::getEncodedValue( - $checksum_algorithm, + $checksumAlgorithm, $response['Body'] ), - $response['Checksum' . strtoupper($checksum_algorithm)] + $response['Checksum' . strtoupper($checksumAlgorithm)] ); } /** * @Given /^I have an object in S3 with name (.*) and its content is (.*)$/ */ - public function iHaveAnObjectInS3withNameAndItsContentIs($filename, $content): void + public function iHaveAnObjectInS3withNameAndItsContentIs( + $filename, + $content + ): void { $client = self::getSdk()->createS3(); $client->putObject([ @@ -331,22 +362,27 @@ public function iHaveAnObjectInS3withNameAndItsContentIs($filename, $content): v */ public function iDoADownloadOfTheObjectWithName($filename): void { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), ); - $s3TransferManager->download([ - 'Bucket' => self::getResourceName(), - 'Key' => $filename, - ])->then(function (DownloadResult $response) use ($filename) { - $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; - file_put_contents($fullFilePath, $response->getData()->getContents()); - })->wait(); + $s3TransferManager->downloadFile(new DownloadFileRequest( + $fullFilePath, + false, + new DownloadRequest([ + 'Bucket' => self::getResourceName(), + 'Key' => $filename, + ]) + ))->wait(); } /** * @Then /^the object with name (.*) should have been downloaded and its content should be (.*)$/ */ - public function theObjectWithNameShouldHaveBeenDownloadedAndItsContentShouldBe($filename, $content): void + public function theObjectWithNameShouldHaveBeenDownloadedAndItsContentShouldBe( + $filename, + $content + ): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; Assert::assertFileExists($fullFilePath); @@ -356,65 +392,86 @@ public function theObjectWithNameShouldHaveBeenDownloadedAndItsContentShouldBe($ /** * @Given /^I have an object in S3 with name (.*) and its size is (.*)$/ */ - public function iHaveAnObjectInS3withNameAndItsSizeIs($filename, $filesize): void + public function iHaveAnObjectInS3withNameAndItsSizeIs( + $filename, + $filesize + ): void { $client = self::getSdk()->createS3(); $client->putObject([ 'Bucket' => self::getResourceName(), 'Key' => $filename, - 'Body' => str_repeat('*', $filesize), + 'Body' => str_repeat('*', (int)$filesize), ]); } /** * @When /^I download the object with name (.*) by using the (.*) multipart download type$/ */ - public function iDownloadTheObjectWithNameByUsingTheMultipartDownloadType($filename, $download_type): void + public function iDownloadTheObjectWithNameByUsingTheMultipartDownloadType( + $filename, + $downloadType + ): void { + $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), ); - $s3TransferManager->download([ - 'Bucket' => self::getResourceName(), - 'Key' => $filename, - ], - [], - [ - 'multipart_download_type' => $download_type, - ])->then(function (DownloadResult $response) use ($filename) { - $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; - file_put_contents($fullFilePath, $response->getData()->getContents()); - })->wait(); + $s3TransferManager->downloadFile( + new DownloadFileRequest( + $fullFilePath, + false, + new DownloadRequest( + [ + 'Bucket' => self::getResourceName(), + 'Key' => $filename + ], + [], + [ + 'multipart_download_type' => $downloadType, + ] + ) + ) + )->wait(); } /** * @Then /^the content size for the object with name (.*) should be (.*)$/ */ - public function theContentSizeForTheObjectWithNameShouldBe($filename, $filesize): void + public function theContentSizeForTheObjectWithNameShouldBe( + $filename, + $filesize + ): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $filename; Assert::assertFileExists($fullFilePath); - Assert::assertEquals($filesize, filesize($fullFilePath)); + Assert::assertEquals((int)$filesize, filesize($fullFilePath)); } /** * @Given /^I have a directory (.*) with (.*) files that I want to upload$/ */ - public function iHaveADirectoryWithFilesThatIWantToUpload($directory, $numfile): void + public function iHaveADirectoryWithFilesThatIWantToUpload( + $directory, + $numfile + ): void { $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; if (!is_dir($fullDirectoryPath)) { mkdir($fullDirectoryPath, 0777, true); } - for ($i = 0; $i < $numfile - 1; $i++) { + $count = (int)$numfile; + for ($i = 0; $i < $count - 1; $i++) { $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt"; file_put_contents($fullFilePath, "This is a test file content #" . ($i + 1)); } - // 1 extra for multipart upload - $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . ($i + 1) . ".txt"; - file_put_contents($fullFilePath, str_repeat('*', 1024 * 1024 * 15)); + // Create one large file for multipart upload testing + if ($count > 0) { + $fullFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . "file" . $count . ".txt"; + file_put_contents($fullFilePath, str_repeat('*', 1024 * 1024 * 15)); + } } /** @@ -427,29 +484,67 @@ public function iUploadThisDirectory($directory): void self::getSdk()->createS3(), ); $s3TransferManager->uploadDirectory( - $fullDirectoryPath, - self::getResourceName(), + new UploadDirectoryRequest( + $fullDirectoryPath, + self::getResourceName(), + ) )->wait(); } /** * @Then /^the files from this directory (.*) where its count should be (.*) should exist in the bucket$/ */ - public function theFilesFromThisDirectoryWhereItsCountShouldBeShouldExistInTheBucket($directory, $numfile): void + public function theFilesFromThisDirectoryWhereItsCountShouldBeShouldExistInTheBucket( + $directory, + $numfile + ): void { $s3Client = self::getSdk()->createS3(); - $objects = $s3Client->getPaginator('ListObjectsV2', [ - 'Bucket' => self::getResourceName(), - 'Prefix' => $directory . DIRECTORY_SEPARATOR, - ]); - $count = 0; - foreach ($objects as $object) { - $fullObjectPath = self::$tempDir . DIRECTORY_SEPARATOR . $object; - Assert::assertFileExists($fullObjectPath); - $count++; + $localDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + $localFiles = array_diff( + scandir($localDirectoryPath), + ['..', '.'] + ); + $uploadedCount = 0; + + foreach ($localFiles as $fileName) { + $localFilePath = $localDirectoryPath . DIRECTORY_SEPARATOR . $fileName; + + if (!is_file($localFilePath)) { + continue; + } + + $s3Key = $directory . DIRECTORY_SEPARATOR . $fileName; + + try { + // Verify the object exists in S3 + $response = $s3Client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $s3Key, + ]); + + Assert::assertEquals(200, $response['@metadata']['statusCode']); + + $localContent = file_get_contents($localFilePath); + $s3Content = $response['Body']->getContents(); + + Assert::assertEquals( + $localContent, + $s3Content, + "Content mismatch for file: {$fileName}" + ); + + $uploadedCount++; + } catch (\Exception $e) { + Assert::fail("Failed to verify S3 object {$s3Key}: " . $e->getMessage()); + } } - - Assert::assertEquals($numfile, $count); + + Assert::assertEquals( + (int)$numfile, + $uploadedCount, + "Expected {$numfile} files but found {$uploadedCount} uploaded files" + ); } /** @@ -460,9 +555,11 @@ public function iHaveATotalOfObjectsInABucketPrefixedWith($numfile, $directory): $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), ); - for ($i = 0; $i < $numfile - 1; $i++) { + + $numFileInt = (int)$numfile; + for ($i = 0; $i < $numFileInt; $i++) { $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor("This is a test file content #" . ($i + 1)), [ 'Bucket' => self::getResourceName(), @@ -479,17 +576,23 @@ public function iHaveATotalOfObjectsInABucketPrefixedWith($numfile, $directory): public function iDownloadAllOfThemIntoTheDirectory($directory): void { $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; + if (!is_dir($fullDirectoryPath)) { + mkdir($fullDirectoryPath, 0777, true); + } + $s3TransferManager = new S3TransferManager( self::getSdk()->createS3(), ); $s3TransferManager->downloadDirectory( - self::getResourceName(), - $fullDirectoryPath, - [], - [ - 's3_prefix' => $directory . DIRECTORY_SEPARATOR, - ] - ); + new DownloadDirectoryRequest( + self::getResourceName(), + $fullDirectoryPath, + [], + [ + 's3_prefix' => $directory . DIRECTORY_SEPARATOR, + ] + ) + )->wait(); } /** @@ -502,17 +605,39 @@ public function theObjectsShouldExistsAsFilesWithinTheDirectory( { $fullDirectoryPath = self::$tempDir . DIRECTORY_SEPARATOR . $directory; $s3Client = self::getSdk()->createS3(); + + // Get list of objects from S3 $objects = $s3Client->getPaginator('ListObjectsV2', [ 'Bucket' => self::getResourceName(), 'Prefix' => $directory . DIRECTORY_SEPARATOR, ]); + $count = 0; - foreach ($objects as $object) { - Assert::assertFileExists($fullDirectoryPath . DIRECTORY_SEPARATOR . $object); - $count++; + foreach ($objects as $page) { + if (isset($page['Contents'])) { + foreach ($page['Contents'] as $object) { + $key = $object['Key']; + $fileName = basename($key); + $localFilePath = $fullDirectoryPath . DIRECTORY_SEPARATOR . $fileName; + + // Verify the file was downloaded locally + Assert::assertFileExists($localFilePath); + + // Verify content matches S3 object + $s3Response = $s3Client->getObject([ + 'Bucket' => self::getResourceName(), + 'Key' => $key, + ]); + $s3Content = $s3Response['Body']->getContents(); + $localContent = file_get_contents($localFilePath); + Assert::assertEquals($s3Content, $localContent); + + $count++; + } + } } - Assert::assertEquals($numfile, $count); + Assert::assertEquals((int)$numfile, $count); } /** @@ -521,7 +646,7 @@ public function theObjectsShouldExistsAsFilesWithinTheDirectory( public function iAmUploadingTheFileWithSize($file, $size): void { $fullFilePath = self::$tempDir . DIRECTORY_SEPARATOR . $file; - file_put_contents($fullFilePath, str_repeat('*', $size)); + file_put_contents($fullFilePath, str_repeat('*', (int)$size)); } /** @@ -536,7 +661,7 @@ public function iUploadTheFileUsingMultipartUploadAndFailsAtPartNumber( $s3TransferManager = new S3TransferManager( self::getSdk()->createS3() ); - $transferListener = new class($partNumberFail) extends TransferListener { + $transferListener = new class((int)$partNumberFail) extends TransferListener { private int $partNumber; private int $partNumberFail; @@ -555,6 +680,7 @@ public function bytesTransferred(array $context): void } } }; + // To make sure transferFail is called $testCase = new class extends TestCase {}; $transferListener2 = $testCase->getMockBuilder( @@ -563,20 +689,31 @@ public function bytesTransferred(array $context): void $transferListener2->expects($testCase->once())->method('transferInitiated'); $transferListener2->expects($testCase->once())->method('transferFail'); - $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( - $fullFilePath, - [ - 'Bucket' => self::getResourceName(), - 'Key' => $file, - ], - [], - [ - $transferListener, - $transferListener2 - ] - ) - )->wait(); + try { + $s3TransferManager->upload( + UploadRequest::fromLegacyArgs( + $fullFilePath, + [ + 'Bucket' => self::getResourceName(), + 'Key' => $file, + ], + [], + [ + $transferListener, + $transferListener2 + ] + ) + )->wait(); + + // If we reach here, the test should fail because exception was expected + Assert::fail("Expected RuntimeException was not thrown"); + + } catch (\RuntimeException $exception) { + Assert::assertEquals( + "Transfer failed at part number {$partNumberFail} failed", + $exception->getMessage(), + ); + } } /** @@ -588,20 +725,19 @@ public function theMultipartUploadShouldHaveBeenAbortedForFile($file): void $inProgressMultipartUploads = $client->listMultipartUploads([ 'Bucket' => self::getResourceName(), ]); - // Make sure that, if there are in progress multipart upload - // it is not for the file being uploaded in this test. - $multipartUploadCount = count($inProgressMultipartUploads); - if ($multipartUploadCount > 0) { - $multipartUploadCount = 0; - foreach ($inProgressMultipartUploads as $inProgressMultipartUpload) { - if ($inProgressMultipartUpload['Key'] === $file) { + + // Make sure that, if there are in progress multipart uploads, + // none are for the file being uploaded in this test. + $multipartUploadCount = 0; + if (isset($inProgressMultipartUploads['Uploads'])) { + foreach ($inProgressMultipartUploads['Uploads'] as $upload) { + if ($upload['Key'] === $file) { $multipartUploadCount++; } } - - Assert::assertEquals(0, $multipartUploadCount); } - Assert::assertEquals(0, $multipartUploadCount); + Assert::assertEquals(0, $multipartUploadCount, + "Expected no in-progress multipart uploads for file: {$file}"); } } \ No newline at end of file From 9fb03f199bdad210241885a64d97290184ddd0bd Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 31 Jul 2025 08:07:07 -0700 Subject: [PATCH 51/62] chore: address some styling suggestions - Added empty line at the end of files. - Add a more descriptive documentation in $failsWhenDestinationExists. - Remove unnecessary break line. --- src/S3/S3Transfer/AbstractMultipartUploader.php | 10 +++++----- src/S3/S3Transfer/Models/DownloadFileRequest.php | 7 ++++++- tests/S3/S3Transfer/MultipartDownloaderTest.php | 2 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 2 +- tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php | 2 +- .../S3Transfer/Progress/MultiProgressTrackerTest.php | 2 +- tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php | 2 +- .../S3Transfer/Progress/SingleProgressTrackerTest.php | 2 +- .../Progress/TransferListenerNotifierTest.php | 2 +- .../Progress/TransferProgressSnapshotTest.php | 2 +- .../S3/S3Transfer/RangeGetMultipartDownloaderTest.php | 2 +- 11 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 0d35ae2afe..f226d1f375 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -262,8 +262,7 @@ protected function sortParts(): void * @param CommandInterface $command * @return void */ - protected function collectPart - ( + protected function collectPart( ResultInterface $result, CommandInterface $command ): void @@ -293,8 +292,7 @@ protected function collectPart * @param callable $rejectedCallback * @return PromiseInterface */ - protected function createCommandPool - ( + protected function createCommandPool( array $commands, callable $fulfilledCallback, callable $rejectedCallback @@ -430,7 +428,9 @@ protected function partCompleted( protected function callOnCompletionCallbacks(): void { foreach ($this->onCompletionCallbacks as $fn) { - $fn(); + if (is_callable($fn)) { + call_user_func($fn); + } } $this->onCompletionCallbacks = []; diff --git a/src/S3/S3Transfer/Models/DownloadFileRequest.php b/src/S3/S3Transfer/Models/DownloadFileRequest.php index fa6e4cc0a3..d71e2ead5b 100644 --- a/src/S3/S3Transfer/Models/DownloadFileRequest.php +++ b/src/S3/S3Transfer/Models/DownloadFileRequest.php @@ -9,7 +9,12 @@ final class DownloadFileRequest /** @var string */ private string $destination; - /** @var bool */ + /** + * To decide whether an error should be raised + * if the destination file exists. + * + * @var bool + */ private bool $failsWhenDestinationExists; /** @var DownloadRequest */ diff --git a/tests/S3/S3Transfer/MultipartDownloaderTest.php b/tests/S3/S3Transfer/MultipartDownloaderTest.php index 682275953e..c9fa4fee99 100644 --- a/tests/S3/S3Transfer/MultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/MultipartDownloaderTest.php @@ -284,4 +284,4 @@ public function testCustomConfigIsSet(): void { $config['response_checksum_validation'] ); } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index bce9d6b423..43f352656a 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -867,4 +867,4 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $response = $multipartUploader->promise()->wait(); $this->assertInstanceOf(UploadResult::class, $response); } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index f37417cc26..7b26ffa9d0 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -200,4 +200,4 @@ public function testComputeObjectDimensions(): void $this->assertEquals(5, $downloader->getObjectPartsCount()); $this->assertEquals(2048, $downloader->getObjectSizeInBytes()); } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php index b3096e7e24..debe0276c1 100644 --- a/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/MultiProgressTrackerTest.php @@ -730,4 +730,4 @@ public function multiProgressTrackerProvider(): array ] ]; } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php index 42fa59fc35..e89f7e0615 100644 --- a/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php +++ b/tests/S3/S3Transfer/Progress/ProgressBarFormatTest.php @@ -150,4 +150,4 @@ public function progressBarFormatProvider(): array ], ]; } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php index 586675c2f1..738ee878a8 100644 --- a/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php +++ b/tests/S3/S3Transfer/Progress/SingleProgressTrackerTest.php @@ -297,4 +297,4 @@ public function singleProgressTrackingProvider(): array ] ]; } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php index 7ccd168ede..c0c90d1ea6 100644 --- a/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php +++ b/tests/S3/S3Transfer/Progress/TransferListenerNotifierTest.php @@ -36,4 +36,4 @@ public function testListenerNotifier(): void { $listenerNotifier->transferComplete([]); $listenerNotifier->transferFail([]); } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php index c0033324bb..5a003a8159 100644 --- a/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php +++ b/tests/S3/S3Transfer/Progress/TransferProgressSnapshotTest.php @@ -81,4 +81,4 @@ public function ratioTransferredProvider(): array ], ]; } -} \ No newline at end of file +} diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index 9d6c89fe4f..941e969494 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -252,4 +252,4 @@ public function testNextCommandIncludesIfMatchWhenETagPresent(): void $command = $nextCommandMethod->invoke($downloader); $this->assertEquals($eTag, $command['IfMatch']); } -} \ No newline at end of file +} From 42682660dae021701bf5c7883a0dc2d2828c507b Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 4 Aug 2025 13:14:11 -0700 Subject: [PATCH 52/62] chore: address PR suggestions - Remove spaces between union type definitions - Add documentation for config parameters in UploadDirectoryRequest. - Add missing new lines in a few places. --- features/s3Transfer/s3TransferManager.feature | 2 +- .../S3Transfer/AbstractMultipartUploader.php | 7 +++--- .../Models/UploadDirectoryRequest.php | 25 +++++++++++++++---- src/S3/S3Transfer/Models/UploadRequest.php | 2 +- src/S3/S3Transfer/MultipartDownloader.php | 10 +++++--- src/S3/S3Transfer/MultipartUploader.php | 4 +-- .../Progress/MultiProgressTracker.php | 2 +- .../Progress/SingleProgressTracker.php | 2 +- .../Progress/TransferProgressSnapshot.php | 13 +++++----- src/S3/S3Transfer/S3TransferManager.php | 6 ++--- tests/TestsUtility.php | 2 +- 11 files changed, 47 insertions(+), 28 deletions(-) diff --git a/features/s3Transfer/s3TransferManager.feature b/features/s3Transfer/s3TransferManager.feature index 6e0acea944..f2a3c8b8ee 100644 --- a/features/s3Transfer/s3TransferManager.feature +++ b/features/s3Transfer/s3TransferManager.feature @@ -114,4 +114,4 @@ Feature: S3 Transfer Manager | file | size | partNumberFail | | abort-file-1.txt | 20971520 | 3 | | abort-file-2.txt | 41943040 | 5 | - | abort-file-3.txt | 10485760 | 1 | \ No newline at end of file + | abort-file-3.txt | 10485760 | 1 | diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index f226d1f375..183eccffe3 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -54,14 +54,14 @@ abstract class AbstractMultipartUploader implements PromisorInterface /** * This will be used for custom or default checksum. * - * @var string | null + * @var string|null */ protected ?string $requestChecksum; /** * This will be used for custom or default checksum. * - * @var string | null + * @var string|null */ protected ?string $requestChecksumAlgorithm; @@ -77,8 +77,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface * @param TransferProgressSnapshot|null $currentSnapshot * @param TransferListenerNotifier|null $listenerNotifier */ - public function __construct - ( + public function __construct( S3ClientInterface $s3Client, array $requestArgs, array $config, diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index 81df2e648c..b3ca3ec1d3 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -18,12 +18,27 @@ final class UploadDirectoryRequest extends TransferRequest private readonly array $putObjectRequestArgs; /** - * @param string $sourceDirectory - * @param string $targetBucket - * @param array $putObjectRequestArgs + * @param string $sourceDirectory The source directory to upload. + * @param string $targetBucket The name of the bucket to upload objects to. + * @param array $putObjectRequestArgs The extract arguments to be passed in + * each upload request. * @param array $config - * @param array $listeners - * @param TransferListener|null $progressTracker + * - follow_symbolic_links: (boolean, optional) Whether to follow symbolic links when + * traversing the file tree. + * - recursive: (boolean, optional) Whether to upload directories recursively. + * - s3_prefix: (string, optional) The S3 key prefix to use for each object. + * If not provided, files will be uploaded to the root of the bucket. + * - filter: (callable, optional) A callback to allow users to filter out unwanted files. + * It is invoked for each file. An example implementation is a predicate + * that takes a file and returns a boolean indicating whether this file + * should be uploaded. + * - s3_delimiter: The S3 delimiter. A delimiter causes a list operation + * to roll up all the keys that share a common prefix into a single summary list result. + * - put_object_request_callback: (callable, optional) A callback mechanism + * to allow customers to update individual putObjectRequest that the S3 Transfer Manager generates. + * - failure_policy: (callable, optional) The failure policy to handle failed requests. + * @param array $listeners For listening to transfer events such as transferInitiated. + * @param TransferListener|null $progressTracker For showing progress in transfers. */ public function __construct( string $sourceDirectory, diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 25c5befc84..6f7b37e8e5 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -17,7 +17,7 @@ final class UploadRequest extends TransferRequest ]; /** @var StreamInterface|string */ - private StreamInterface | string $source; + private StreamInterface|string $source; /** @var array */ private array $putObjectRequestArgs; diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 11512f7399..1f9d481727 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -11,6 +11,7 @@ use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\S3\S3Transfer\Progress\TransferProgressSnapshot; use Aws\S3\S3Transfer\Utils\DownloadHandler; +use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; use GuzzleHttp\Promise\Coroutine; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\PromiseInterface; @@ -44,7 +45,7 @@ abstract class MultipartDownloader implements PromisorInterface /** @var string|null */ protected ?string $eTag; - /** @var TransferListenerNotifier | null */ + /** @var TransferListenerNotifier|null */ private readonly ?TransferListenerNotifier $listenerNotifier; /** Tracking Members */ @@ -54,7 +55,7 @@ abstract class MultipartDownloader implements PromisorInterface * @param S3ClientInterface $s3Client * @param array $getObjectRequestArgs * @param array $config - * @param DownloadHandler $downloadHandler + * @param ?DownloadHandler $downloadHandler * @param int $currentPartNo * @param int $objectPartsCount * @param int $objectSizeInBytes @@ -66,7 +67,7 @@ public function __construct( protected readonly S3ClientInterface $s3Client, array $getObjectRequestArgs, array $config, - DownloadHandler $downloadHandler, + ?DownloadHandler $downloadHandler, int $currentPartNo = 0, int $objectPartsCount = 0, int $objectSizeInBytes = 0, @@ -77,6 +78,9 @@ public function __construct( $this->getObjectRequestArgs = $getObjectRequestArgs; $this->validateConfig($config); $this->config = $config; + if ($downloadHandler === null) { + $downloadHandler = new StreamDownloadHandler(); + } $this->downloadHandler = $downloadHandler; $this->currentPartNo = $currentPartNo; $this->objectPartsCount = $objectPartsCount; diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 8962d3af66..d807bf7e91 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -79,7 +79,7 @@ public function upload(): UploadResult * @return StreamInterface */ private function parseBody( - string | StreamInterface $source + string|StreamInterface $source ): StreamInterface { if (is_string($source)) { @@ -274,7 +274,7 @@ private function decorateWithHashes( * * @param array $requestArgs * - * @return string | null + * @return string|null */ private static function filterChecksum(array $requestArgs):? string { diff --git a/src/S3/S3Transfer/Progress/MultiProgressTracker.php b/src/S3/S3Transfer/Progress/MultiProgressTracker.php index 0aec1fc33a..c722e546cf 100644 --- a/src/S3/S3Transfer/Progress/MultiProgressTracker.php +++ b/src/S3/S3Transfer/Progress/MultiProgressTracker.php @@ -94,7 +94,7 @@ public function getFailed(): int /** * @return ProgressBarFactoryInterface|Closure|null */ - public function getProgressBarFactory(): ProgressBarFactoryInterface | Closure | null + public function getProgressBarFactory(): ProgressBarFactoryInterface|Closure|null { return $this->progressBarFactory; } diff --git a/src/S3/S3Transfer/Progress/SingleProgressTracker.php b/src/S3/S3Transfer/Progress/SingleProgressTracker.php index 5a5dd2b2a4..1e7e64946c 100644 --- a/src/S3/S3Transfer/Progress/SingleProgressTracker.php +++ b/src/S3/S3Transfer/Progress/SingleProgressTracker.php @@ -19,7 +19,7 @@ final class SingleProgressTracker extends TransferListener /** @var bool */ private bool $clear; - /** @var TransferProgressSnapshot | null */ + /** @var TransferProgressSnapshot|null */ private ?TransferProgressSnapshot $currentSnapshot; /** @var bool */ diff --git a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php index aa95ad0695..b1507f1e42 100644 --- a/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php +++ b/src/S3/S3Transfer/Progress/TransferProgressSnapshot.php @@ -15,24 +15,25 @@ class TransferProgressSnapshot /** @var int */ private int $totalBytes; - /** @var array | null */ - private array | null $response; + /** @var array|null */ + private array|null $response; - /** @var Throwable | string | null */ - private Throwable | string | null $reason; + /** @var Throwable|string|null */ + private Throwable|string|null $reason; /** * @param string $identifier * @param int $transferredBytes * @param int $totalBytes - * @param array | null $response + * @param array|null $response + * @param Throwable|string|null $reason */ public function __construct( string $identifier, int $transferredBytes, int $totalBytes, ?array $response = null, - Throwable | string | null $reason = null, + Throwable|string|null $reason = null, ) { $this->identifier = $identifier; $this->transferredBytes = $transferredBytes; diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index ad1e81c0f0..a8ff508b57 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -39,7 +39,7 @@ class S3TransferManager private S3TransferManagerConfig $config; /** - * @param S3ClientInterface | null $s3Client If provided as null then, + * @param S3ClientInterface|null $s3Client If provided as null then, * a default client will be created where its region will be the one * resolved from either the default from the config or the provided. * @param array|S3TransferManagerConfig|null $config @@ -574,7 +574,7 @@ private function tryMultipartDownload( private function trySingleUpload( string|StreamInterface $source, array $requestArgs, - ?TransferListenerNotifier $listenerNotifier = null + ?TransferListenerNotifier $listenerNotifier = null ): PromiseInterface { if (is_string($source) && is_readable($source)) { @@ -683,7 +683,7 @@ private function tryMultipartUpload( * @return bool */ private function requiresMultipartUpload( - string | StreamInterface $source, + string|StreamInterface $source, int $mupThreshold ): bool { diff --git a/tests/TestsUtility.php b/tests/TestsUtility.php index 81a43b3f1a..7c552d4d8b 100644 --- a/tests/TestsUtility.php +++ b/tests/TestsUtility.php @@ -34,4 +34,4 @@ public static function cleanUpDir($dirPath): void rmdir($dirPath); } -} \ No newline at end of file +} From 59a8aa4af7bf3bb5bf4abfa13289fc7a6f523ef9 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 4 Aug 2025 13:17:31 -0700 Subject: [PATCH 53/62] chore: null as default --- src/S3/S3Transfer/MultipartDownloader.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 1f9d481727..0e13c531c8 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -67,7 +67,7 @@ public function __construct( protected readonly S3ClientInterface $s3Client, array $getObjectRequestArgs, array $config, - ?DownloadHandler $downloadHandler, + ?DownloadHandler $downloadHandler = null, int $currentPartNo = 0, int $objectPartsCount = 0, int $objectSizeInBytes = 0, From 45ac2896abba83baaaa171f5d4d1b4e055aa07d1 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 10 Aug 2025 20:05:05 -0700 Subject: [PATCH 54/62] chore: use UploadRequest construct --- src/S3/S3Transfer/S3TransferManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index a8ff508b57..ef935f18f5 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -248,7 +248,7 @@ function ($file) use ($filter) { } $promises[] = $this->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $file, $putObjectRequestArgs, $config, From 20ada7167d308f88d750acfdfc02bd81864e7592 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Sun, 17 Aug 2025 20:10:05 -0700 Subject: [PATCH 55/62] chore: address PR feedback - Rename upload and download request args. - Refactor tests to clean up resources specifically in the finally block instead of using cleanUpFns. - Add Throwable and Result type. - Make parameter optional by adding a default value. --- .../S3Transfer/AbstractMultipartUploader.php | 35 ++-- .../Models/DownloadDirectoryRequest.php | 41 +--- src/S3/S3Transfer/Models/DownloadRequest.php | 43 +--- .../Models/UploadDirectoryRequest.php | 41 +--- src/S3/S3Transfer/Models/UploadRequest.php | 42 +--- src/S3/S3Transfer/MultipartDownloader.php | 16 +- src/S3/S3Transfer/MultipartUploader.php | 2 +- .../S3Transfer/PartGetMultipartDownloader.php | 2 +- .../RangeGetMultipartDownloader.php | 2 +- src/S3/S3Transfer/S3TransferManager.php | 52 ++--- tests/Integ/S3TransferManagerContext.php | 2 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 183 +++++++++++------- tests/S3/S3Transfer/S3TransferManagerTest.php | 89 ++++----- 13 files changed, 245 insertions(+), 305 deletions(-) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 183eccffe3..abc08aa0bb 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -6,6 +6,7 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; +use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; @@ -25,6 +26,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface public const PART_MAX_SIZE = 5 * 1024 * 1024 * 1024; // 5 GiB public const PART_MAX_NUM = 10000; public const DEFAULT_CHECKSUM_CALCULATION_ALGORITHM = 'crc32'; + private const CHECKSUM_TYPE_FULL_OBJECT = 'FULL_OBJECT'; /** @var S3ClientInterface */ protected readonly S3ClientInterface $s3Client; @@ -80,7 +82,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface public function __construct( S3ClientInterface $s3Client, array $requestArgs, - array $config, + array $config = [], ?string $uploadId = null, array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, @@ -157,9 +159,9 @@ public function promise(): PromiseInterface { return Coroutine::of(function () { try { - yield $this->createMultipartUpload(); + yield $this->createMultipartOperation(); yield $this->processMultipartOperation(); - $result = yield $this->completeMultipartUpload(); + $result = yield $this->completeMultipartOperation(); yield Create::promiseFor($this->createResponse($result)); } catch (Throwable $e) { $this->operationFailed($e); @@ -173,19 +175,30 @@ public function promise(): PromiseInterface /** * @return PromiseInterface */ - protected function createMultipartUpload(): PromiseInterface + protected function createMultipartOperation(): PromiseInterface { $createMultipartUploadArgs = $this->requestArgs; if ($this->requestChecksum !== null) { - $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $createMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; } elseif ($this->config['request_checksum_calculation'] === 'when_supported') { $this->requestChecksumAlgorithm = $createMultipartUploadArgs['ChecksumAlgorithm'] ?? self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM; - $createMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $createMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; } - + + // Make sure algorithm with full object is a supported one + if (($createMultipartUploadArgs['ChecksumType'] ?? '') === self::CHECKSUM_TYPE_FULL_OBJECT) { + if (stripos($this->requestChecksumAlgorithm, 'crc') !== 0) { + return Create::rejectionFor( + new S3TransferException( + "Full object checksum algorithm must be `CRC` family base." + ) + ); + } + } + $this->operationInitiated($createMultipartUploadArgs); $command = $this->s3Client->getCommand( 'CreateMultipartUpload', @@ -202,7 +215,7 @@ protected function createMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - protected function completeMultipartUpload(): PromiseInterface + protected function completeMultipartOperation(): PromiseInterface { $this->sortParts(); $completeMultipartUploadArgs = $this->requestArgs; @@ -213,7 +226,7 @@ protected function completeMultipartUpload(): PromiseInterface $completeMultipartUploadArgs['MpuObjectSize'] = $this->getTotalSize(); if ($this->requestChecksum !== null) { - $completeMultipartUploadArgs['ChecksumType'] = 'FULL_OBJECT'; + $completeMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $completeMultipartUploadArgs[ 'Checksum' . ucfirst($this->requestChecksumAlgorithm) ] = $this->requestChecksum; @@ -234,7 +247,7 @@ protected function completeMultipartUpload(): PromiseInterface /** * @return PromiseInterface */ - protected function abortMultipartUpload(): PromiseInterface + protected function abortMultipartOperation(): PromiseInterface { $abortMultipartUploadArgs = $this->requestArgs; $abortMultipartUploadArgs['UploadId'] = $this->uploadId; @@ -384,7 +397,7 @@ protected function operationFailed(Throwable $reason): void "Multipart Upload with id: " . $this->uploadId . " failed", E_USER_WARNING ); - $this->abortMultipartUpload()->wait(); + $this->abortMultipartOperation()->wait(); } $this->listenerNotifier?->transferFail([ diff --git a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php index 8eeba9f1a9..6c4a9bf366 100644 --- a/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/DownloadDirectoryRequest.php @@ -15,14 +15,14 @@ final class DownloadDirectoryRequest extends TransferRequest private string $destinationDirectory; /** @var array */ - private readonly array $getObjectRequestArgs; + private readonly array $downloadRequestArgs; /** * @param string $sourceBucket The bucket from where the files are going to be * downloaded from. * @param string $destinationDirectory The destination path where the downloaded * files will be placed in. - * @param array $getObjectRequestArgs + * @param array $downloadRequestArgs * @param array $config The config options for this download directory operation. * - s3_prefix: (string, optional) This parameter will be considered just if * not provided as part of the list_object_v2_args config option. @@ -66,7 +66,7 @@ final class DownloadDirectoryRequest extends TransferRequest public function __construct( string $sourceBucket, string $destinationDirectory, - array $getObjectRequestArgs = [], + array $downloadRequestArgs = [], array $config = [], array $listeners = [], ?TransferListener $progressTracker = null @@ -78,36 +78,7 @@ public function __construct( $this->sourceBucket = $sourceBucket; $this->destinationDirectory = $destinationDirectory; - $this->getObjectRequestArgs = $getObjectRequestArgs; - } - - /** - * @param string $sourceBucket - * @param string $destinationDirectory - * @param array $downloadDirectoryArgs - * @param array $config - * @param array $listeners - * @param TransferListener|null $progressTracker - * - * @return self - */ - public static function fromLegacyArgs( - string $sourceBucket, - string $destinationDirectory, - array $downloadDirectoryArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null, - ): self - { - return new self( - $sourceBucket, - $destinationDirectory, - $downloadDirectoryArgs, - $config, - $listeners, - $progressTracker - ); + $this->downloadRequestArgs = $downloadRequestArgs; } /** @@ -129,9 +100,9 @@ public function getDestinationDirectory(): string /** * @return array */ - public function getGetObjectRequestArgs(): array + public function getDownloadRequestArgs(): array { - return $this->getObjectRequestArgs; + return $this->downloadRequestArgs; } /** diff --git a/src/S3/S3Transfer/Models/DownloadRequest.php b/src/S3/S3Transfer/Models/DownloadRequest.php index 6379b916a4..40983e1c0f 100644 --- a/src/S3/S3Transfer/Models/DownloadRequest.php +++ b/src/S3/S3Transfer/Models/DownloadRequest.php @@ -21,7 +21,7 @@ final class DownloadRequest extends TransferRequest private string|array|null $source; /** @var array */ - private array $getObjectRequestArgs; + private array $downloadRequestArgs; /** @var DownloadHandler|null */ private ?DownloadHandler $downloadHandler; @@ -30,7 +30,7 @@ final class DownloadRequest extends TransferRequest * @param string|array|null $source The object to be downloaded from S3. * It can be either a string with a S3 URI or an array with a Bucket and Key * properties set. - * @param array $getObjectRequestArgs + * @param array $downloadRequestArgs * @param array $config The configuration to be used for this operation: * - multipart_download_type: (string, optional) * Overrides the resolved value from the transfer manager config. @@ -51,7 +51,7 @@ final class DownloadRequest extends TransferRequest */ public function __construct( string|array|null $source, - array $getObjectRequestArgs = [], + array $downloadRequestArgs = [], array $config = [], ?DownloadHandler $downloadHandler = null, array $listeners = [], @@ -59,7 +59,7 @@ public function __construct( ) { parent::__construct($listeners, $progressTracker, $config); $this->source = $source; - $this->getObjectRequestArgs = $getObjectRequestArgs; + $this->downloadRequestArgs = $downloadRequestArgs; $this->config = $config; if ($downloadHandler === null) { $downloadHandler = new StreamDownloadHandler(); @@ -67,35 +67,6 @@ public function __construct( $this->downloadHandler = $downloadHandler; } - /** - * @param string|array|null $source - * @param array $downloadRequestArgs - * @param array $config - * @param DownloadHandler|null $downloadHandler - * @param array $listeners - * @param TransferListener|null $progressTracker - * - * @return self - */ - public static function fromLegacyArgs( - string|array|null $source, - array $downloadRequestArgs = [], - array $config = [], - ?DownloadHandler $downloadHandler = null, - array $listeners = [], - ?TransferListener $progressTracker = null, - ): self - { - return new self( - $source, - $downloadRequestArgs, - $config, - $downloadHandler, - $listeners, - $progressTracker - ); - } - /** * @param DownloadRequest $downloadRequest * @param FileDownloadHandler $downloadHandler @@ -130,7 +101,7 @@ public function getSource(): array|string|null */ public function getObjectRequestArgs(): array { - return $this->getObjectRequestArgs; + return $this->downloadRequestArgs; } /** @@ -152,8 +123,8 @@ public function normalizeSourceAsArray(): array { // If source is null then fall back to getObjectRequest. $source = $this->getSource() ?? [ - 'Bucket' => $this->getObjectRequestArgs['Bucket'] ?? null, - 'Key' => $this->getObjectRequestArgs['Key'] ?? null, + 'Bucket' => $this->downloadRequestArgs['Bucket'] ?? null, + 'Key' => $this->downloadRequestArgs['Key'] ?? null, ]; if (is_string($source)) { $sourceAsArray = S3TransferManager::s3UriAsBucketAndKey($source); diff --git a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php index b3ca3ec1d3..69386163fc 100644 --- a/src/S3/S3Transfer/Models/UploadDirectoryRequest.php +++ b/src/S3/S3Transfer/Models/UploadDirectoryRequest.php @@ -15,12 +15,12 @@ final class UploadDirectoryRequest extends TransferRequest private string $targetBucket; /** @var array */ - private readonly array $putObjectRequestArgs; + private readonly array $uploadRequestArgs; /** * @param string $sourceDirectory The source directory to upload. * @param string $targetBucket The name of the bucket to upload objects to. - * @param array $putObjectRequestArgs The extract arguments to be passed in + * @param array $uploadRequestArgs The extract arguments to be passed in * each upload request. * @param array $config * - follow_symbolic_links: (boolean, optional) Whether to follow symbolic links when @@ -43,7 +43,7 @@ final class UploadDirectoryRequest extends TransferRequest public function __construct( string $sourceDirectory, string $targetBucket, - array $putObjectRequestArgs = [], + array $uploadRequestArgs = [], array $config = [], array $listeners = [], ?TransferListener $progressTracker = null @@ -55,39 +55,10 @@ public function __construct( $targetBucket = ArnParser::parse($targetBucket)->getResource(); } $this->targetBucket = $targetBucket; - $this->putObjectRequestArgs = $putObjectRequestArgs; + $this->uploadRequestArgs = $uploadRequestArgs; $this->config = $config; } - /** - * @param string $sourceDirectory - * @param string $targetBucket - * @param array $uploadDirectoryRequestArgs - * @param array $config - * @param array $listeners - * @param TransferListener|null $progressTracker - * - * @return self - */ - public static function fromLegacyArgs( - string $sourceDirectory, - string $targetBucket, - array $uploadDirectoryRequestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null, - ): self - { - return new self( - $sourceDirectory, - $targetBucket, - $uploadDirectoryRequestArgs, - $config, - $listeners, - $progressTracker - ); - } - /** * @return string */ @@ -107,9 +78,9 @@ public function getTargetBucket(): string /** * @return array */ - public function getPutObjectRequestArgs(): array + public function getUploadRequestArgs(): array { - return $this->putObjectRequestArgs; + return $this->uploadRequestArgs; } /** diff --git a/src/S3/S3Transfer/Models/UploadRequest.php b/src/S3/S3Transfer/Models/UploadRequest.php index 6f7b37e8e5..a2ad002fd6 100644 --- a/src/S3/S3Transfer/Models/UploadRequest.php +++ b/src/S3/S3Transfer/Models/UploadRequest.php @@ -20,11 +20,11 @@ final class UploadRequest extends TransferRequest private StreamInterface|string $source; /** @var array */ - private array $putObjectRequestArgs; + private array $uploadRequestArgs; /** * @param string|StreamInterface $source - * @param array $putObjectRequestArgs The putObject request arguments. + * @param array $uploadRequestArgs The putObject request arguments. * Required parameters would be: * - Bucket: (string, required) * - Key: (string, required) @@ -46,40 +46,14 @@ final class UploadRequest extends TransferRequest */ public function __construct( StreamInterface|string $source, - array $putObjectRequestArgs, + array $uploadRequestArgs, array $config = [], array $listeners = [], ?TransferListener $progressTracker = null ) { parent::__construct($listeners, $progressTracker, $config); $this->source = $source; - $this->putObjectRequestArgs = $putObjectRequestArgs; - } - - /** - * @param string|StreamInterface $source - * @param array $putObjectRequestArgs - * @param array $config - * @param array $listeners - * @param TransferListener|null $progressTracker - * - * @return self - */ - public static function fromLegacyArgs( - string|StreamInterface $source, - array $putObjectRequestArgs = [], - array $config = [], - array $listeners = [], - ?TransferListener $progressTracker = null - ): self - { - return new self( - $source, - $putObjectRequestArgs, - $config, - $listeners, - $progressTracker - ); + $this->uploadRequestArgs = $uploadRequestArgs; } /** @@ -97,9 +71,9 @@ public function getSource(): StreamInterface|string * * @return array */ - public function getPutObjectRequestArgs(): array + public function getUploadRequestArgs(): array { - return $this->putObjectRequestArgs; + return $this->uploadRequestArgs; } /** @@ -127,8 +101,8 @@ public function validateRequiredParameters( ): void { $requiredParametersWithArgs = [ - 'Bucket' => $this->putObjectRequestArgs['Bucket'] ?? null, - 'Key' => $this->putObjectRequestArgs['Key'] ?? null, + 'Bucket' => $this->uploadRequestArgs['Bucket'] ?? null, + 'Key' => $this->uploadRequestArgs['Key'] ?? null, ]; foreach ($requiredParametersWithArgs as $key => $value) { if (empty($value)) { diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 0e13c531c8..80ebbb5279 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -25,7 +25,7 @@ abstract class MultipartDownloader implements PromisorInterface private const OBJECT_SIZE_REGEX = "/\/(\d+)$/"; /** @var array */ - protected readonly array $getObjectRequestArgs; + protected readonly array $downloadRequestArgs; /** @var array */ protected readonly array $config; @@ -53,7 +53,7 @@ abstract class MultipartDownloader implements PromisorInterface /** * @param S3ClientInterface $s3Client - * @param array $getObjectRequestArgs + * @param array $downloadRequestArgs * @param array $config * @param ?DownloadHandler $downloadHandler * @param int $currentPartNo @@ -65,8 +65,8 @@ abstract class MultipartDownloader implements PromisorInterface */ public function __construct( protected readonly S3ClientInterface $s3Client, - array $getObjectRequestArgs, - array $config, + array $downloadRequestArgs = [], + array $config = [], ?DownloadHandler $downloadHandler = null, int $currentPartNo = 0, int $objectPartsCount = 0, @@ -75,7 +75,7 @@ public function __construct( ?TransferProgressSnapshot $currentSnapshot = null, ?TransferListenerNotifier $listenerNotifier = null ) { - $this->getObjectRequestArgs = $getObjectRequestArgs; + $this->downloadRequestArgs = $downloadRequestArgs; $this->validateConfig($config); $this->config = $config; if ($downloadHandler === null) { @@ -338,7 +338,7 @@ private function downloadFailed(\Throwable $reason): void ); $this->listenerNotifier?->transferFail([ - TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequestArgs, + TransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, 'reason' => $reason, ]); @@ -369,7 +369,7 @@ private function partDownloadCompleted( ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->bytesTransferred([ - TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequestArgs, + TransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } @@ -403,7 +403,7 @@ private function downloadComplete(): void ); $this->currentSnapshot = $newSnapshot; $this->listenerNotifier?->transferComplete([ - TransferListener::REQUEST_ARGS_KEY => $this->getObjectRequestArgs, + TransferListener::REQUEST_ARGS_KEY => $this->downloadRequestArgs, TransferListener::PROGRESS_SNAPSHOT_KEY => $this->currentSnapshot, ]); } diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index d807bf7e91..5bd6615d36 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -42,8 +42,8 @@ class MultipartUploader extends AbstractMultipartUploader public function __construct( S3ClientInterface $s3Client, array $requestArgs, - array $config, string|StreamInterface $source, + array $config = [], ?string $uploadId = null, array $parts = [], ?TransferProgressSnapshot $currentSnapshot = null, diff --git a/src/S3/S3Transfer/PartGetMultipartDownloader.php b/src/S3/S3Transfer/PartGetMultipartDownloader.php index da59c86c38..205a914408 100644 --- a/src/S3/S3Transfer/PartGetMultipartDownloader.php +++ b/src/S3/S3Transfer/PartGetMultipartDownloader.php @@ -24,7 +24,7 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = $this->getObjectRequestArgs; + $nextRequestArgs = $this->downloadRequestArgs; $nextRequestArgs['PartNumber'] = $this->currentPartNo; if ($this->config['response_checksum_validation'] === 'when_supported') { $nextRequestArgs['ChecksumMode'] = 'ENABLED'; diff --git a/src/S3/S3Transfer/RangeGetMultipartDownloader.php b/src/S3/S3Transfer/RangeGetMultipartDownloader.php index 26c98df4cd..56395fadde 100644 --- a/src/S3/S3Transfer/RangeGetMultipartDownloader.php +++ b/src/S3/S3Transfer/RangeGetMultipartDownloader.php @@ -21,7 +21,7 @@ protected function nextCommand(): CommandInterface $this->currentPartNo++; } - $nextRequestArgs = $this->getObjectRequestArgs; + $nextRequestArgs = $this->downloadRequestArgs; $partSize = $this->config['target_part_size_bytes']; $from = ($this->currentPartNo - 1) * $partSize; $to = ($this->currentPartNo * $partSize) - 1; diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index ef935f18f5..402f89ee11 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -2,6 +2,7 @@ namespace Aws\S3\S3Transfer; +use Aws\ResultInterface; use Aws\S3\S3Client; use Aws\S3\S3ClientInterface; use Aws\S3\S3Transfer\Exceptions\S3TransferException; @@ -27,6 +28,7 @@ use Psr\Http\Message\StreamInterface; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; +use Throwable; use function Aws\filter; use function Aws\map; @@ -131,7 +133,7 @@ public function upload(UploadRequest $uploadRequest): PromiseInterface return $this->trySingleUpload( $uploadRequest->getSource(), - $uploadRequest->getPutObjectRequestArgs(), + $uploadRequest->getUploadRequestArgs(), $listenerNotifier ); } @@ -239,7 +241,7 @@ function ($file) use ($filter) { $delimiter, $objectKey ); - $putObjectRequestArgs = $uploadDirectoryRequest->getPutObjectRequestArgs(); + $putObjectRequestArgs = $uploadDirectoryRequest->getUploadRequestArgs(); $putObjectRequestArgs['Bucket'] = $targetBucket; $putObjectRequestArgs['Key'] = $objectKey; @@ -262,7 +264,7 @@ function ($file) use ($filter) { $objectsUploaded++; return $response; - })->otherwise(function ($reason) use ( + })->otherwise(function (Throwable $reason) use ( $targetBucket, $sourceDirectory, $failurePolicyCallback, @@ -294,7 +296,7 @@ function ($file) use ($filter) { } return Each::ofLimitAll($promises, $this->config->getConcurrency()) - ->then(function ($_) use (&$objectsUploaded, &$objectsFailed) { + ->then(function () use (&$objectsUploaded, &$objectsFailed) { return new UploadDirectoryResult($objectsUploaded, $objectsFailed); }); } @@ -463,7 +465,7 @@ public function downloadDirectory( ); } - $requestArgs = $downloadDirectoryRequest->getGetObjectRequestArgs(); + $requestArgs = $downloadDirectoryRequest->getDownloadRequestArgs(); foreach ($bucketAndKeyArray as $key => $value) { $requestArgs[$key] = $value; } @@ -473,27 +475,27 @@ public function downloadDirectory( $promises[] = $this->downloadFile( new DownloadFileRequest( - $destinationFile, - $config['fails_when_destination_exists'] ?? false, - new DownloadRequest( - null, - $requestArgs, - [ + destination: $destinationFile, + failsWhenDestinationExists: ['fails_when_destination_exists'] ?? false, + downloadRequest: new DownloadRequest( + source: $sourceBucket, + downloadRequestArgs: $requestArgs, + config: [ 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, ], - null, - array_map( + downloadHandler: null, + listeners: array_map( fn($listener) => clone $listener, $downloadDirectoryRequest->getListeners() ), - $progressTracker, + progressTracker: $progressTracker, ) ), )->then(function () use ( &$objectsDownloaded ) { $objectsDownloaded++; - })->otherwise(function ($reason) use ( + })->otherwise(function (Throwable $reason) use ( $sourceBucket, $destinationDirectory, $failurePolicyCallback, @@ -525,7 +527,7 @@ public function downloadDirectory( } return Each::ofLimitAll($promises, $this->config->getConcurrency()) - ->then(function ($_) use (&$objectsFailed, &$objectsDownloaded) { + ->then(function () use (&$objectsFailed, &$objectsDownloaded) { return new DownloadDirectoryResult( $objectsDownloaded, $objectsFailed @@ -603,7 +605,8 @@ private function trySingleUpload( $command = $this->s3Client->getCommand('PutObject', $requestArgs); return $this->s3Client->executeAsync($command)->then( - function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { + function (ResultInterface $result) + use ($objectSize, $listenerNotifier, $requestArgs) { $listenerNotifier->bytesTransferred( [ TransferListener::REQUEST_ARGS_KEY => $requestArgs, @@ -631,7 +634,8 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { $result->toArray() ); } - )->otherwise(function ($reason) use ($objectSize, $requestArgs, $listenerNotifier) { + )->otherwise(function (Throwable $reason) + use ($objectSize, $requestArgs, $listenerNotifier) { $listenerNotifier->transferFail( [ TransferListener::REQUEST_ARGS_KEY => $requestArgs, @@ -651,7 +655,7 @@ function ($result) use ($objectSize, $listenerNotifier, $requestArgs) { $command = $this->s3Client->getCommand('PutObject', $requestArgs); return $this->s3Client->executeAsync($command) - ->then(function ($result) { + ->then(function (ResultInterface $result) { return new UploadResult($result->toArray()); }); } @@ -669,9 +673,9 @@ private function tryMultipartUpload( { return (new MultipartUploader( $this->s3Client, - $uploadRequest->getPutObjectRequestArgs(), - $uploadRequest->getConfig(), + $uploadRequest->getUploadRequestArgs(), $uploadRequest->getSource(), + $uploadRequest->getConfig(), listenerNotifier: $listenerNotifier, ))->promise(); } @@ -759,12 +763,12 @@ public static function s3UriAsBucketAndKey(string $uri): array } /** - * @param $bucket - * @param $key + * @param string $bucket + * @param string $key * * @return string */ - private static function formatAsS3URI($bucket, $key): string + private static function formatAsS3URI(string $bucket, string $key): string { return "s3://$bucket/$key"; } diff --git a/tests/Integ/S3TransferManagerContext.php b/tests/Integ/S3TransferManagerContext.php index 836bb71d49..da98a82778 100644 --- a/tests/Integ/S3TransferManagerContext.php +++ b/tests/Integ/S3TransferManagerContext.php @@ -691,7 +691,7 @@ public function bytesTransferred(array $context): void try { $s3TransferManager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( $fullFilePath, [ 'Bucket' => self::getResourceName(), diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 43f352656a..c9e3635286 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -14,11 +14,13 @@ use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; use Aws\Test\TestsUtility; +use Generator; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\StreamInterface; class MultipartUploaderTest extends TestCase { @@ -73,7 +75,7 @@ public function testMultipartUpload( 'Bucket' => 'FooBucket', ...$commandArgs ]; - $cleanUpFns = []; + $tempDir = null; if ($sourceConfig['type'] === 'stream') { $source = Utils::streamFor( str_repeat('*', $sourceConfig['size']) @@ -89,25 +91,16 @@ public function testMultipartUpload( } $source = $tempDir . DIRECTORY_SEPARATOR . 'temp-file.txt'; file_put_contents($source, str_repeat('*', $sourceConfig['size'])); - $cleanUpFns[] = function () use ($tempDir, $source) { - TestsUtility::cleanUpDir($tempDir); - }; } else { $this->fail("Unsupported Source type"); } - if ($sourceConfig['type'] !== 'file') { - $cleanUpFns[] = function () use ($source) { - $source->close(); - }; - } - try { $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $source, $config, - $source ); /** @var UploadResult $response */ $response = $multipartUploader->promise()->wait(); @@ -118,8 +111,12 @@ public function testMultipartUpload( $this->assertEquals($expected['bytesUploaded'], $snapshot->getTransferredBytes()); $this->assertEquals($expected['bytesUploaded'], $snapshot->getTotalBytes()); } finally { - foreach ($cleanUpFns as $fn) { - $fn(); + if ($source instanceof StreamInterface) { + $source->close(); + } + + if (!is_null($tempDir)) { + TestsUtility::cleanUpDir($tempDir); } } } @@ -288,12 +285,12 @@ public function testValidatePartSize( new MultipartUploader( $this->getMultipartUploadS3Client(), ['Bucket' => 'test-bucket', 'Key' => 'test-key'], + Utils::streamFor(), [ 'target_part_size_bytes' => $partSize, 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], - Utils::streamFor('') ); } @@ -334,7 +331,7 @@ public function testInvalidSourceStringThrowsException( bool $expectError ): void { - $cleanUpFns = []; + $tempDir = null; if ($expectError) { $this->expectException(\InvalidArgumentException::class); $this->expectExceptionMessage( @@ -349,9 +346,6 @@ public function testInvalidSourceStringThrowsException( $source = $tempDir . DIRECTORY_SEPARATOR . $source; file_put_contents($source, 'foo'); - $cleanUpFns[] = function () use ($tempDir) { - TestsUtility::cleanUpDir($tempDir); - }; } try { @@ -361,16 +355,16 @@ public function testInvalidSourceStringThrowsException( 'Bucket' => 'test-bucket', 'Key' => 'test-key' ]), + $source, [ 'target_part_size_bytes' => 1024 * 1024 * 5, 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' - ], - $source + ] ); } finally { - foreach ($cleanUpFns as $cleanUpFn) { - $cleanUpFn(); + if (!is_null($tempDir)) { + TestsUtility::cleanUpDir($tempDir); } } } @@ -452,12 +446,12 @@ public function testTransferListenerNotifierNotifiesListenersOnSuccess(): void $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $stream, [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], - $stream, null, [], null, @@ -511,12 +505,12 @@ public function testMultipartOperationsAreCalled(): void { $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $stream, [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' - ], - $stream + ] ); $multipartUploader->promise()->wait(); @@ -574,7 +568,7 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) 'Bucket' => 'FooBucket', ...$checksumConfig, ]; - $cleanUpFns = []; + $tempDir = null; if ($sourceConfig['type'] === 'stream') { $source = Utils::streamFor( str_repeat($sourceConfig['char'], $sourceConfig['size']) @@ -590,29 +584,20 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) } $source = $tempDir . DIRECTORY_SEPARATOR . 'temp-file.txt'; file_put_contents($source, str_repeat($sourceConfig['char'], $sourceConfig['size'])); - $cleanUpFns[] = function () use ($tempDir, $source) { - TestsUtility::cleanUpDir($tempDir); - }; } else { $this->fail("Unsupported Source type"); } - if ($sourceConfig['type'] !== 'file') { - $cleanUpFns[] = function () use ($source) { - $source->close(); - }; - } - try { $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $source, [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 3, 'request_checksum_calculation' => 'when_supported' - ], - $source, + ] ); /** @var UploadResult $response */ $response = $multipartUploader->promise()->wait(); @@ -621,8 +606,12 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) } $this->assertInstanceOf(UploadResult::class, $response); } finally { - foreach ($cleanUpFns as $fn) { - $fn(); + if ($source instanceof StreamInterface) { + $source->close(); + } + + if (!is_null($tempDir)) { + TestsUtility::cleanUpDir($tempDir); } } } @@ -632,37 +621,6 @@ function (callable $handler) use (&$operationsCalled, $expectedOperationHeaders) */ public function multipartUploadWithCustomChecksumProvider(): array { return [ - 'custom_checksum_sha256_1' => [ - 'source_config' => [ - 'type' => 'stream', - 'size' => 1024 * 1024 * 20, - 'char' => '*' - ], - 'checksum_config' => [ - 'ChecksumSHA256' => '0c58gNl31EVxhClRWw5+WHiAUp2B3/3g1zQDCvY4bmQ=', - ], - 'expected_operation_headers' => [ - 'CreateMultipartUpload' => [ - 'has' => [ - 'x-amz-checksum-algorithm' => 'SHA256', - 'x-amz-checksum-type' => 'FULL_OBJECT' - ] - ], - 'UploadPart' => [ - 'has_not' => [ - 'x-amz-checksum-algorithm', - 'x-amz-checksum-type', - 'x-amz-checksum-sha256' - ] - ], - 'CompleteMultipartUpload' => [ - 'has' => [ - 'x-amz-checksum-sha256' => '0c58gNl31EVxhClRWw5+WHiAUp2B3/3g1zQDCvY4bmQ=', - 'x-amz-checksum-type' => 'FULL_OBJECT', - ], - ] - ] - ], 'custom_checksum_crc32_1' => [ 'source_config' => [ 'type' => 'stream', @@ -739,12 +697,12 @@ public function testMultipartUploadAbort() { $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $source, [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' - ], - $source, + ] ); $multipartUploader->promise()->wait(); } finally { @@ -801,12 +759,12 @@ public function testTransferListenerNotifierNotifiesListenersOnFailure(): void $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $stream, [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, 'request_checksum_calculation' => 'when_supported' ], - $stream, null, [], null, @@ -853,11 +811,11 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $multipartUploader = new MultipartUploader( $s3Client, $requestArgs, + $stream, [ 'target_part_size_bytes' => 5242880, // 5MB 'concurrency' => 1, ], - $stream, null, [], null, @@ -867,4 +825,81 @@ public function testTransferListenerNotifierWithEmptyListeners(): void $response = $multipartUploader->promise()->wait(); $this->assertInstanceOf(UploadResult::class, $response); } + + /** + * This test makes sure that when full object checksum type is resolved + * then, if a custom algorithm provide is not CRC family then it should fail. + * + * @param array $checksumConfig + * @param bool $expectsError + * + * @dataProvider fullObjectChecksumWorksJustWithCRCProvider + * + * @return void + */ + public function testFullObjectChecksumWorksJustWithCRC( + array $checksumConfig, + bool $expectsError + ): void { + $s3Client = $this->getMultipartUploadS3Client(); + $requestArgs = [ + 'Key' => 'FooKey', + 'Bucket' => 'FooBucket', + ...$checksumConfig, + ]; + + try { + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + Utils::streamFor(''), + [ + 'target_part_size_bytes' => 5242880, // 5MB + 'concurrency' => 3, + 'request_checksum_calculation' => 'when_supported' + ] + ); + $response = $multipartUploader->promise()->wait(); + if ($expectsError) { + $this->fail("An expected exception has not been raised"); + } else { + $this->assertInstanceOf(UploadResult::class, $response); + } + } catch (S3TransferException $exception) { + if ($expectsError) { + $this->assertEquals( + "Full object checksum algorithm must be `CRC` family base.", + $exception->getMessage() + ); + } else { + $this->fail("An exception has been thrown when not expected"); + } + } + } + + /** + * @return Generator + */ + public function fullObjectChecksumWorksJustWithCRCProvider(): Generator { + yield 'sha_256_should_fail' => [ + 'checksum_config' => [ + 'ChecksumSHA256' => '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=' + ], + 'expects_error' => true, + ]; + + yield 'sha_1_should_fail' => [ + 'checksum_config' => [ + 'ChecksumSHA1' => '2jmj7l5rSw0yVb/vlWAYkK/YBwk=' + ], + 'expects_error' => true, + ]; + + yield 'crc32_should_fail' => [ + 'checksum_config' => [ + 'ChecksumCRC32' => 'AAAAAA==' + ], + 'expects_error' => false, + ]; + } } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 97ec644994..e5808429b5 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -114,8 +114,9 @@ public function testUploadExpectsAReadableSource(): void $this->expectExceptionMessage("Please provide a valid readable file path or a valid stream as source."); $manager = new S3TransferManager(); $manager->upload( - UploadRequest::fromLegacyArgs( - "noreadablefile" + new UploadRequest( + "noreadablefile", + [] ), )->wait(); } @@ -137,7 +138,7 @@ public function testUploadFailsWhenBucketAndKeyAreNotProvided( $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage("The `$missingProperty` parameter must be provided as part of the request arguments."); $manager->upload( - uploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor(), $bucketKeyArgs ) @@ -175,7 +176,7 @@ public function testUploadFailsWhenMultipartThresholdIsLessThanMinSize(): void . "must be greater than or equal to " . MultipartUploader::PART_MIN_SIZE); $manager = new S3TransferManager(); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor(), [ 'Bucket' => 'Bucket', @@ -206,7 +207,7 @@ public function testDoesMultipartUploadWhenApplicable(): void $transferListener->expects($this->exactly($expectedPartCount)) ->method('bytesTransferred'); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor( str_repeat("#", MultipartUploader::PART_MIN_SIZE * $expectedPartCount) ), @@ -238,7 +239,7 @@ public function testDoesSingleUploadWhenApplicable(): void $transferListener->expects($this->once()) ->method('bytesTransferred'); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor( str_repeat("#", MultipartUploader::PART_MIN_SIZE - 1) ), @@ -270,7 +271,7 @@ public function testUploadUsesTransferManagerConfigDefaultMupThreshold(): void $transferListener->expects($this->exactly($expectedPartCount)) ->method('bytesTransferred'); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor( str_repeat("#", $manager->getConfig()->toArray()['multipart_upload_threshold_bytes']) ), @@ -324,7 +325,7 @@ public function testUploadUsesCustomMupThreshold( $expectedIncrementalPartSize += $expectedPartSize; }); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor( str_repeat("#", $expectedPartSize * $expectedPartCount) ), @@ -381,7 +382,7 @@ public function testUploadUsesTransferManagerConfigDefaultTargetPartSize(): void $transferListener->expects($this->exactly($expectedPartCount)) ->method('bytesTransferred'); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor( str_repeat("#", $manager->getConfig()->toArray()['target_part_size_bytes'] * $expectedPartCount) ), @@ -428,7 +429,7 @@ public function testUploadUsesCustomPartSize(): void ->method('bytesTransferred'); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor( str_repeat("#", $expectedPartSize * $expectedPartCount) ), @@ -541,7 +542,7 @@ private function testUploadResolvedChecksum( $client, ); $manager->upload( - UploadRequest::fromLegacyArgs( + new UploadRequest( Utils::streamFor(), $putObjectRequestArgs, ) @@ -574,7 +575,7 @@ public function testUploadDirectoryValidatesProvidedDirectory( $this->getS3ClientMock(), ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", ) @@ -635,7 +636,7 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void $client, ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -690,7 +691,7 @@ public function testUploadDirectoryFileFilter(): void ); $calledTimes = 0; $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -759,7 +760,7 @@ public function testUploadDirectoryRecursive(): void $client, ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -823,7 +824,7 @@ public function testUploadDirectoryNonRecursive(): void $client, ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -908,7 +909,7 @@ public function testUploadDirectoryFollowsSymbolicLink(): void // First lets make sure that when follows_symbolic_link is false // the directory in the link will not be traversed. $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -929,7 +930,7 @@ public function testUploadDirectoryFollowsSymbolicLink(): void // Now let's enable follow_symbolic_links and all files should have // been considered, included the ones in the symlink directory. $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -994,7 +995,7 @@ public function testUploadDirectoryUsesProvidedPrefix(): void $client, ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1059,7 +1060,7 @@ public function testUploadDirectoryUsesProvidedDelimiter(): void $client, ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1098,7 +1099,7 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi $client, ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1148,7 +1149,7 @@ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void ); $called = 0; $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1209,7 +1210,7 @@ public function testUploadDirectoryUsesFailurePolicy(): void ); $called = false; $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1272,7 +1273,7 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void $client ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1317,7 +1318,7 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi $client ); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1373,7 +1374,7 @@ public function testUploadDirectoryTracksMultipleFiles(): void $objectKeys[$snapshot->getIdentifier()] = true; }); $manager->uploadDirectory( - UploadDirectoryRequest::fromLegacyArgs( + new UploadDirectoryRequest( $directory, "Bucket", [], @@ -1411,7 +1412,7 @@ public function testDownloadFailsOnInvalidS3UriSource(): void $client ); $manager->download( - DownloadRequest::fromLegacyArgs( + new DownloadRequest( $invalidS3Uri ) ); @@ -1437,7 +1438,7 @@ public function testDownloadFailsWhenSourceAsArrayMissesBucketOrKeyProperty( $client ); $manager->download( - DownloadRequest::fromLegacyArgs($sourceAsArray) + new DownloadRequest($sourceAsArray) ); } @@ -1491,7 +1492,7 @@ public function testDownloadWorksWithS3UriAsSource(): void $client ); $manager->download( - DownloadRequest::fromLegacyArgs($sourceAsArray) + new DownloadRequest($sourceAsArray) )->wait(); $this->assertTrue($called); } @@ -1525,7 +1526,7 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void $client ); $manager->download( - DownloadRequest::fromLegacyArgs( + new DownloadRequest( $sourceAsS3Uri ), )->wait(); @@ -1581,7 +1582,7 @@ public function testDownloadAppliesChecksumMode( $transferManagerConfig, ); $manager->download( - DownloadRequest::fromLegacyArgs( + new DownloadRequest( "s3://bucket/key", $downloadArgs, $downloadConfig @@ -1693,7 +1694,7 @@ public function testDownloadChoosesMultipartDownloadType( $client, ); $manager->download( - DownloadRequest::fromLegacyArgs( + new DownloadRequest( "s3://bucket/key", [], ['multipart_download_type' => $multipartDownloadType] @@ -1763,7 +1764,7 @@ public function testRangeGetMultipartDownloadMinimumPartSize( $client, ); $manager->download( - DownloadRequest::fromLegacyArgs( + new DownloadRequest( "s3://bucket/key", [], [ @@ -1859,7 +1860,7 @@ public function testDownloadDirectoryCreatesDestinationDirectory(): void $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory ) @@ -1931,7 +1932,7 @@ public function testDownloadDirectoryAppliesS3Prefix( $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2046,7 +2047,7 @@ public function testDownloadDirectoryAppliesDelimiter( $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2126,7 +2127,7 @@ public function testDownloadDirectoryFailsOnInvalidFilter(): void $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2183,7 +2184,7 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2239,7 +2240,7 @@ public function testDownloadDirectoryUsesFailurePolicy(): void $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2345,7 +2346,7 @@ public function testDownloadDirectoryAppliesFilter( $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2509,7 +2510,7 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [], @@ -2582,7 +2583,7 @@ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void ); }; $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, [ @@ -2674,7 +2675,7 @@ public function testDownloadDirectoryCreateFiles( $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( "Bucket", $destinationDirectory, ) @@ -2821,7 +2822,7 @@ public function testResolvesOutsideTargetDirectory( $client, ); $manager->downloadDirectory( - DownloadDirectoryRequest::fromLegacyArgs( + new DownloadDirectoryRequest( $bucket, $fullDirectoryPath, [], From e7842eda029b0dc1bab97e3235ce2c653fb0754a Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Mon, 18 Aug 2025 07:45:01 -0700 Subject: [PATCH 56/62] chore: minor tests fix - Source should have been provided as null when the bucket and key are already provided as part of the download request args. - When checksum type will be resolved to FULL_OBJECT the only supported algorithms are CRC family. --- src/S3/S3Transfer/S3TransferManager.php | 4 ++-- tests/S3/S3Transfer/S3TransferManagerTest.php | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 402f89ee11..8dfc29ad08 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -476,9 +476,9 @@ public function downloadDirectory( $promises[] = $this->downloadFile( new DownloadFileRequest( destination: $destinationFile, - failsWhenDestinationExists: ['fails_when_destination_exists'] ?? false, + failsWhenDestinationExists: $config['fails_when_destination_exists'] ?? false, downloadRequest: new DownloadRequest( - source: $sourceBucket, + source: null, // Source has been provided in the request args downloadRequestArgs: $requestArgs, config: [ 'target_part_size_bytes' => $config['target_part_size_bytes'] ?? 0, diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index e5808429b5..016efd3c76 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -483,12 +483,6 @@ public function testUploadUsesCustomChecksumAlgorithm( public function uploadUsesCustomChecksumAlgorithmProvider(): array { return [ - 'checksum_sha256' => [ - 'checksum_algorithm' => 'sha256', - ], - 'checksum_sha1' => [ - 'checksum_algorithm' => 'sha1', - ], 'checksum_crc32c' => [ 'checksum_algorithm' => 'crc32c', ], @@ -2272,7 +2266,11 @@ public function testDownloadDirectoryUsesFailurePolicy(): void )->wait(); $this->assertTrue($called); } finally { - unlink($destinationDirectory . '/file1.txt'); + $file = $destinationDirectory . '/file1.txt'; + if (file_exists($file)) { + unlink($file); + } + rmdir($destinationDirectory); } } From 7c52a2f563f7f0d56b041332cc6e121ffb3447e0 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 28 Aug 2025 08:11:30 -0700 Subject: [PATCH 57/62] enhancement: tests and fixes - Fix checksum parameter in the request should have been ChecksumCRC32 and no ChecksumCrc32. - Validates next part fetched in a multipart download operation is sequentially correct. - Fix checksum calculation in multipart parts processing. It should have been base64 encoded and the hash_final should have been the binary. - Prevents parts uploading when there was a failure and its running with concurrency. For example, when running in concurrency parts may not be uploaded in order and a failure in one part was not preventing other parts from being uploaded. This fix prevents any part from being uploaded after a failure happened. - Forces multipart_download_type parameter to be lowercase, even when provided as upper case. - Add tests to validate input fields are copied in the different multipart upload operations. - Add tests to validate IfMatch is present in subsequent part or range get requests. - Add test runners for upload and download for modeled test cases. --- .../S3Transfer/AbstractMultipartUploader.php | 2 +- src/S3/S3Transfer/MultipartDownloader.php | 12 +- src/S3/S3Transfer/MultipartUploader.php | 19 +- src/S3/S3Transfer/S3TransferManager.php | 2 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 419 +++++++++- .../PartGetMultipartDownloaderTest.php | 108 +++ .../RangeGetMultipartDownloaderTest.php | 110 ++- tests/S3/S3Transfer/S3TransferManagerTest.php | 587 +++++++++++++- .../test-cases/download-single-object.json | 582 ++++++++++++++ .../test-cases/upload-single-object.json | 746 ++++++++++++++++++ 10 files changed, 2572 insertions(+), 15 deletions(-) create mode 100644 tests/S3/S3Transfer/test-cases/download-single-object.json create mode 100644 tests/S3/S3Transfer/test-cases/upload-single-object.json diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index abc08aa0bb..d7ad802525 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -228,7 +228,7 @@ protected function completeMultipartOperation(): PromiseInterface if ($this->requestChecksum !== null) { $completeMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $completeMultipartUploadArgs[ - 'Checksum' . ucfirst($this->requestChecksumAlgorithm) + 'Checksum' . strtoupper($this->requestChecksumAlgorithm) ] = $this->requestChecksum; } diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 80ebbb5279..1b74ef90ea 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -65,7 +65,7 @@ abstract class MultipartDownloader implements PromisorInterface */ public function __construct( protected readonly S3ClientInterface $s3Client, - array $downloadRequestArgs = [], + array $downloadRequestArgs, array $config = [], ?DownloadHandler $downloadHandler = null, int $currentPartNo = 0, @@ -168,7 +168,7 @@ public function promise(): PromiseInterface $prevPartNo = $this->currentPartNo - 1; while ($this->currentPartNo < $this->objectPartsCount) { // To prevent infinite loops - if ($prevPartNo === $this->currentPartNo) { + if ($prevPartNo !== $this->currentPartNo - 1) { throw new S3TransferException( "Current part `$this->currentPartNo` MUST increment." ); @@ -188,6 +188,12 @@ public function promise(): PromiseInterface }); } + if ($this->currentPartNo !== $this->objectPartsCount) { + throw new S3TransferException( + "Expected number of parts `$this->objectPartsCount` to have been transferred but got `$this->currentPartNo`." + ); + } + // Transfer completed $this->downloadComplete(); @@ -222,7 +228,7 @@ protected function initialRequest(): PromiseInterface // Compute object dimensions such as parts count and object size $this->computeObjectDimensions($result); - // If a multipart is likely to happen then save the ETag + // If there are more than one part then save the ETag if ($this->objectPartsCount > 1) { $this->eTag = $result['ETag']; } diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 5bd6615d36..6aeb308cb8 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -119,6 +119,9 @@ private function evaluateCustomChecksum(): void '', $checksumName ); + $this->requestChecksumAlgorithm = strtolower( + $this->requestChecksumAlgorithm + ); } else { $this->requestChecksum = null; $this->requestChecksumAlgorithm = null; @@ -139,7 +142,7 @@ protected function processMultipartOperation(): PromiseInterface if ($this->requestChecksum !== null) { // To avoid default calculation $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; - unset($uploadPartCommandArgs['Checksum'. ucfirst($this->requestChecksumAlgorithm)]); + unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); } elseif ($this->requestChecksumAlgorithm !== null) { // Normalize algorithm name $algoName = strtolower($this->requestChecksumAlgorithm); @@ -151,7 +154,7 @@ protected function processMultipartOperation(): PromiseInterface $this->hashContext = hash_init($algoName); // To avoid default calculation $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; - unset($uploadPartCommandArgs['Checksum'. ucfirst($this->requestChecksumAlgorithm)]); + unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); } while (!$this->body->eof()) { @@ -197,7 +200,12 @@ protected function processMultipartOperation(): PromiseInterface } if ($hashBody) { - $this->requestChecksum = hash_final($this->hashContext); + $this->requestChecksum = base64_encode( + hash_final( + $this->hashContext, + true + ) + ); } return (new CommandPool( @@ -207,6 +215,11 @@ protected function processMultipartOperation(): PromiseInterface 'concurrency' => $this->config['concurrency'], 'fulfilled' => function (ResultInterface $result, $index) use ($commands) { + // To make sure we don't continue when a failure occurred + if ($this->currentSnapshot->getReason() !== null) { + throw $this->currentSnapshot->getReason(); + } + $command = $commands[$index]; $this->collectPart( $result, diff --git a/src/S3/S3Transfer/S3TransferManager.php b/src/S3/S3Transfer/S3TransferManager.php index 8dfc29ad08..b57930b6b9 100644 --- a/src/S3/S3Transfer/S3TransferManager.php +++ b/src/S3/S3Transfer/S3TransferManager.php @@ -553,7 +553,7 @@ private function tryMultipartDownload( ): PromiseInterface { $downloaderClassName = MultipartDownloader::chooseDownloaderClass( - $config['multipart_download_type'] + strtolower($config['multipart_download_type']) ); $multipartDownloader = new $downloaderClassName( $this->s3Client, diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index c9e3635286..7c377cbaeb 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -633,7 +633,7 @@ public function multipartUploadWithCustomChecksumProvider(): array { 'expected_operation_headers' => [ 'CreateMultipartUpload' => [ 'has' => [ - 'x-amz-checksum-algorithm' => 'CRC32', + 'x-amz-checksum-algorithm' => 'crc32', 'x-amz-checksum-type' => 'FULL_OBJECT' ] ], @@ -840,7 +840,8 @@ public function testTransferListenerNotifierWithEmptyListeners(): void public function testFullObjectChecksumWorksJustWithCRC( array $checksumConfig, bool $expectsError - ): void { + ): void + { $s3Client = $this->getMultipartUploadS3Client(); $requestArgs = [ 'Key' => 'FooKey', @@ -902,4 +903,418 @@ public function fullObjectChecksumWorksJustWithCRCProvider(): Generator { 'expects_error' => false, ]; } + + /** + * @param array $sourceConfig + * @param array $requestArgs + * @param array $expectedInputArgs + * @param bool $expectsError + * @param int|null $errorOnPartNumber + * @return void + * @dataProvider inputArgumentsPerOperationProvider + */ + public function testInputArgumentsPerOperation( + array $sourceConfig, + array $requestArgs, + array $expectedInputArgs, + bool $expectsError, + ?int $errorOnPartNumber = null + ): void + { + try { + $calledCommands = array_map(function () { + return 1; + }, $expectedInputArgs); + $this->assertNotEmpty( + $calledCommands, + "Expected input arguments should not be empty" + ); + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method( + 'getCommand' + )->willReturnCallback( + function ($commandName, $args) + use (&$calledCommands, $expectedInputArgs) { + if (isset($expectedInputArgs[$commandName])) { + $calledCommands[$commandName] = 0; + $expected = $expectedInputArgs[$commandName]; + foreach ($expected as $key => $value) { + $this->assertArrayHasKey($key, $args); + $this->assertEquals( + $value, + $args[$key] + ); + } + } + + return new Command($commandName, $args); + }); + $s3Client->method('executeAsync') + ->willReturnCallback(function ($command) + use ($errorOnPartNumber, $expectsError) { + if ($command->getName() === 'UploadPart') { + if ($expectsError && $command['PartNumber'] === $errorOnPartNumber) { + return Create::rejectionFor( + new S3TransferException('Upload failed') + ); + } + + return Create::promiseFor(new Result([])); + } + + return match ($command->getName()) { + 'CreateMultipartUpload' => Create::promiseFor(new Result([ + 'UploadId' => 'FooUploadId', + ])), + 'CompleteMultipartUpload', + 'AbortMultipartUpload', + 'PutObject' => Create::promiseFor(new Result([])), + default => null, + }; + }); + $source = Utils::streamFor( + str_repeat( + $sourceConfig['char'], + $sourceConfig['size'] + ) + ); + $multipartUploader = new MultipartUploader( + $s3Client, + $requestArgs, + Utils::streamFor($source) + ); + $multipartUploader->upload(); + foreach ($calledCommands as $key => $value) { + $this->assertEquals( + 0, + $value, + "$key not called" + ); + } + $this->assertFalse( + $expectsError, + "Expected error while uploading" + ); + } catch (S3TransferException $exception) { + $this->assertTrue( + $expectsError, + "Unexpected error while uploading" . "\n" . $exception->getMessage() + ); + } + } + + /** + * @return Generator + */ + public function inputArgumentsPerOperationProvider(): Generator + { + yield 'test_input_fields_are_copied_without_custom_checksums' => [ + // Source config to generate a stub body + 'source_config' => [ + 'size' => 1024 * 1024 * 10, + 'char' => '#' + ], + 'request_args' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'expected_input_args' => [ + 'CreateMultipartUpload' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'crc32', + ], + 'UploadPart' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + 'CompleteMultipartUpload' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumCRC32' => 'b71d0814', // From default algorithm used + ], + ], + 'expects_error' => false, + ]; + + yield 'test_input_fields_are_copied_with_custom_checksum_crc32' => [ + // Source config to generate a stub body + 'source_config' => [ + 'size' => 1024 * 1024 * 10, + 'char' => '#' + ], + 'request_args' => [ + 'ChecksumCRC32' => 'tx0IFA==', + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'expected_input_args' => [ + 'CreateMultipartUpload' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'crc32', + ], + 'UploadPart' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + 'CompleteMultipartUpload' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumCRC32' => 'tx0IFA==', // From default algorithm used + ], + ], + 'expects_error' => false, + ]; + + yield 'test_input_fields_are_copied_with_error' => [ + // Source config to generate a stub body + 'source_config' => [ + 'size' => 1024 * 1024 * 10, + 'char' => '#' + ], + 'request_args' => [ + 'ChecksumCRC32' => 'tx0IFA==', + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + ], + 'expected_input_args' => [ + 'CreateMultipartUpload' => [ + "ACL" => 'private', + "Bucket" => 'test-bucket', + "BucketKeyEnabled" => 'test-bucket-key-enabled', + "CacheControl" => 'test-cache-control', + "ContentDisposition" => 'test-content-disposition', + "ContentEncoding" => 'test-content-encoding', + "ContentLanguage" => 'test-content-language', + "ContentType" => 'test-content-type', + "ExpectedBucketOwner" => 'test-bucket-owner', + "Expires" => 'test-expires', + "GrantFullControl" => 'test-grant-control', + "GrantRead" => 'test-grant-control', + "GrantReadACP" => 'test-grant-control', + "GrantWriteACP" => 'test-grant-control', + "Key" => 'test-key', + "Metadata" => [ + 'metadata-1' => 'test-metadata-1', + 'metadata-2' => 'test-metadata-2', + ], + "ObjectLockLegalHoldStatus" => 'test-object-lock-legal-hold', + "ObjectLockMode" => 'test-object-lock-mode', + "ObjectLockRetainUntilDate" => 'test-object-lock-retain-until', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + "SSEKMSEncryptionContext" => 'test-sse-kms-encryption-context', + "SSEKMSKeyId" => 'test-sse-kms-key-id', + "ServerSideEncryption" => 'test-server-side-encryption', + "StorageClass" => 'test-storage-class', + "Tagging" => 'test-tagging', + "WebsiteRedirectLocation" => 'test-website-redirect-location', + 'ChecksumType' => 'FULL_OBJECT', + 'ChecksumAlgorithm' => 'crc32', + ], + 'UploadPart' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', + "SSECustomerKey" => 'test-sse-customer-key', + "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', + ], + 'AbortMultipartUpload' => [ + "Bucket" => 'test-bucket', + "UploadId" => "FooUploadId", // Fixed from test + "ExpectedBucketOwner" => 'test-bucket-owner', + "Key" => 'test-key', + "RequestPayer" => 'test-request-payer', + ], + ], + 'expects_error' => true, + 'error_on_part_number' => 2 + ]; + } } diff --git a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php index 7b26ffa9d0..c1d3b4d50f 100644 --- a/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/PartGetMultipartDownloaderTest.php @@ -8,6 +8,7 @@ use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\PartGetMultipartDownloader; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; +use Generator; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; @@ -200,4 +201,111 @@ public function testComputeObjectDimensions(): void $this->assertEquals(5, $downloader->getObjectPartsCount()); $this->assertEquals(2048, $downloader->getObjectSizeInBytes()); } + + /** + * Test IfMatch is properly called in each part get operation. + * + * @param int $objectSizeInBytes + * @param int $targetPartSize + * @param string $eTag + * + * @dataProvider ifMatchIsPresentInEachPartRequestAfterFirstProvider + * + * @return void + */ + public function testIfMatchIsPresentInEachRangeRequestAfterFirst( + int $objectSizeInBytes, + int $targetPartSize, + string $eTag + ): void + { + $firstRequestCalled = false; + $ifMatchCalledTimes = 0; + $partsCount = ceil($objectSizeInBytes / $targetPartSize); + $remainingToTransfer = $objectSizeInBytes; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) + use ($eTag, &$ifMatchCalledTimes) { + if (isset($args['IfMatch'])) { + $ifMatchCalledTimes++; + $this->assertEquals( + $eTag, + $args['IfMatch'] + ); + } + + return new Command($commandName, $args); + }); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $eTag, + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer, + &$firstRequestCalled + ) { + $firstRequestCalled = true; + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'PartsCount' => $partsCount, + 'PartNumber' => $command['PartNumber'], + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength, + 'ETag' => $eTag, + ])); + }); + $requestArgs = [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ]; + $partGetMultipartDownloader = new PartGetMultipartDownloader( + $s3Client, + $requestArgs, + [ + 'target_part_size_bytes' => $targetPartSize, + ] + ); + $partGetMultipartDownloader->download(); + $this->assertTrue($firstRequestCalled); + $this->assertEquals( + $partsCount - 1, + $ifMatchCalledTimes + ); + } + + /** + * @return Generator + */ + public function ifMatchIsPresentInEachPartRequestAfterFirstProvider(): Generator + { + yield 'multipart_download_with_3_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 20, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag1234', + ]; + + yield 'multipart_download_with_2_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 16, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + + yield 'multipart_download_with_5_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 40, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + } } diff --git a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php index 941e969494..8849b5c37f 100644 --- a/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php +++ b/tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php @@ -9,6 +9,7 @@ use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\RangeGetMultipartDownloader; use Aws\S3\S3Transfer\Utils\StreamDownloadHandler; +use Generator; use GuzzleHttp\Promise\Create; use GuzzleHttp\Psr7\Utils; use PHPUnit\Framework\TestCase; @@ -157,7 +158,6 @@ public function testNextCommandGeneratesCorrectRangeHeaders(): void // Use reflection to test the protected nextCommand method $reflection = new \ReflectionClass($downloader); $nextCommandMethod = $reflection->getMethod('nextCommand'); - $nextCommandMethod->setAccessible(true); // First call should create range 0-1023 $command1 = $nextCommandMethod->invoke($downloader); @@ -197,7 +197,6 @@ public function testComputeObjectDimensionsForSinglePart(): void // Use reflection to test the protected computeObjectDimensions method $reflection = new \ReflectionClass($downloader); $computeObjectDimensionsMethod = $reflection->getMethod('computeObjectDimensions'); - $computeObjectDimensionsMethod->setAccessible(true); // Simulate object smaller than part size $result = new Result([ @@ -215,6 +214,7 @@ public function testComputeObjectDimensionsForSinglePart(): void * Tests nextCommand method includes IfMatch header when ETag is present. * * @return void + * @throws \ReflectionException */ public function testNextCommandIncludesIfMatchWhenETagPresent(): void { @@ -247,9 +247,113 @@ public function testNextCommandIncludesIfMatchWhenETagPresent(): void // Use reflection to test the protected nextCommand method $reflection = new \ReflectionClass($downloader); $nextCommandMethod = $reflection->getMethod('nextCommand'); - $nextCommandMethod->setAccessible(true); $command = $nextCommandMethod->invoke($downloader); $this->assertEquals($eTag, $command['IfMatch']); } + + /** + * Test IfMatch is properly called in each part get operation. + * + * @param int $objectSizeInBytes + * @param int $targetPartSize + * @param string $eTag + * + * @dataProvider ifMatchIsPresentInEachRangeRequestAfterFirstProvider + * + * @return void + */ + public function testIfMatchIsPresentInEachRangeRequestAfterFirst( + int $objectSizeInBytes, + int $targetPartSize, + string $eTag + ): void + { + $firstRequestCalled = false; + $ifMatchCalledTimes = 0; + $partsCount = ceil($objectSizeInBytes / $targetPartSize); + $remainingToTransfer = $objectSizeInBytes; + $s3Client = $this->getMockBuilder(S3Client::class) + ->disableOriginalConstructor() + ->getMock(); + $s3Client->method('getCommand') + ->willReturnCallback(function ($commandName, $args) + use ($eTag, &$ifMatchCalledTimes) { + if (isset($args['IfMatch'])) { + $ifMatchCalledTimes++; + $this->assertEquals( + $eTag, + $args['IfMatch'] + ); + } + + return new Command($commandName, $args); + }); + $s3Client->method('executeAsync') + -> willReturnCallback(function ($command) + use ( + $eTag, + $objectSizeInBytes, + $partsCount, + $targetPartSize, + &$remainingToTransfer, + &$firstRequestCalled + ) { + $firstRequestCalled = true; + $currentPartLength = min( + $targetPartSize, + $remainingToTransfer + ); + $from = $objectSizeInBytes - $remainingToTransfer; + $to = $from + $currentPartLength; + $remainingToTransfer -= $currentPartLength; + return Create::promiseFor(new Result([ + 'Body' => Utils::streamFor('Foo'), + 'ContentRange' => "bytes $from-$to/$objectSizeInBytes", + 'ContentLength' => $currentPartLength, + 'ETag' => $eTag, + ])); + }); + $requestArgs = [ + 'Bucket' => 'TestBucket', + 'Key' => 'TestKey', + ]; + $rangeGetMultipartDownloader = new RangeGetMultipartDownloader( + $s3Client, + $requestArgs, + [ + 'target_part_size_bytes' => $targetPartSize, + ] + ); + $rangeGetMultipartDownloader->download(); + $this->assertTrue($firstRequestCalled); + $this->assertEquals( + $partsCount - 1, + $ifMatchCalledTimes + ); + } + + /** + * @return Generator + */ + public function ifMatchIsPresentInEachRangeRequestAfterFirstProvider(): Generator + { + yield 'multipart_download_with_3_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 20, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag1234', + ]; + + yield 'multipart_download_with_2_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 16, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + + yield 'multipart_download_with_5_parts_1' => [ + 'object_size_in_bytes' => 1024 * 1024 * 40, + 'target_part_size_bytes' => 8 * 1024 * 1024, + 'eTag' => 'ETag12345678', + ]; + } } diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 016efd3c76..3e508b47ce 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -7,15 +7,19 @@ use Aws\CommandInterface; use Aws\HandlerList; use Aws\Result; +use Aws\S3\ApplyChecksumMiddleware; +use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; use Aws\S3\S3Transfer\AbstractMultipartUploader; use Aws\S3\S3Transfer\Exceptions\S3TransferException; use Aws\S3\S3Transfer\Models\DownloadDirectoryRequest; use Aws\S3\S3Transfer\Models\DownloadDirectoryResult; use Aws\S3\S3Transfer\Models\DownloadRequest; +use Aws\S3\S3Transfer\Models\DownloadResult; use Aws\S3\S3Transfer\Models\UploadDirectoryRequest; use Aws\S3\S3Transfer\Models\UploadDirectoryResult; use Aws\S3\S3Transfer\Models\UploadRequest; +use Aws\S3\S3Transfer\Models\UploadResult; use Aws\S3\S3Transfer\MultipartDownloader; use Aws\S3\S3Transfer\MultipartUploader; use Aws\S3\S3Transfer\Progress\TransferListener; @@ -24,13 +28,33 @@ use Aws\Test\TestsUtility; use Closure; use Exception; +use Generator; +use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Promise\Create; +use GuzzleHttp\Promise\RejectedPromise; +use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Utils; use InvalidArgumentException; use PHPUnit\Framework\TestCase; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\StreamInterface; class S3TransferManagerTest extends TestCase { + private const DOWNLOAD_BASE_CASES = __DIR__ . '/test-cases/download-single-object.json'; + private const UPLOAD_BASE_CASES = __DIR__ . '/test-cases/upload-single-object.json'; + private static array $multipartUploadBodyTemplates = [ + 'CreateMultipartUpload' => << + + {Bucket} + {Key} + {UploadId} + +EOF, + ]; + /** * @return void */ @@ -2759,7 +2783,8 @@ public function testResolvesOutsideTargetDirectory( ?string $delimiter, array $objects, array $expectedOutput - ) { + ): void + { if ($expectedOutput['success'] === false) { $this->expectException(S3TransferException::class); $this->expectExceptionMessageMatches( @@ -2847,7 +2872,8 @@ public function testResolvesOutsideTargetDirectory( /** * @return array */ - public function resolvesOutsideTargetDirectoryProvider(): array { + public function resolvesOutsideTargetDirectoryProvider(): array + { return [ 'download_directory_1_linux' => [ 'prefix' => null, @@ -2966,6 +2992,563 @@ public function resolvesOutsideTargetDirectoryProvider(): array { ]; } + /** + * @param string $testId + * @param array $config + * @param array $requestArgs + * @param array $expectations + * @param array $outcomes + * + * @return void + * @dataProvider modeledDownloadCasesProvider + * + */ + public function testModeledCasesForDownload( + string $testId, + array $config, + array $requestArgs, + array $expectations, + array $outcomes + ): void + { + $testsToSkip = [ + "Test download with part GET - validation failure when part count mismatch" => true, + ]; + if ($testsToSkip[$testId] ?? false) { + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); + } + + // Outcomes has only one item for now + $outcome = $outcomes[0]; + // Standardize config + $this->parseConfigFromCamelCaseToSnakeCase($config); + // Standardize request + $this->parseRequestArgsFromCamelCaseToPascalCase($requestArgs); + + if (isset($config['multipart_download_type']) && $config['multipart_download_type'] === 'RANGE') { + $config['multipart_download_type'] = 'RANGED'; + } + + // Operational values + $totalBytesReceived = 0; + $totalPartsReceived = 0; + // Mock client to validate expected requests + $s3Client = $this->getS3ClientWithSequentialResponses( + array_map(function ($expectation) { + $operation = $expectation['request']['operation']; + + return array_merge( + $expectation['response'], + ['operation' => $operation] + ); + }, $expectations), + function ( + string $operation, + array|string|null $body, + ?array &$headers + ): StreamInterface + { + $fixedBody = Utils::streamFor( + str_repeat( + '*', + $headers['Content-Length'] + ) + ); + + if (isset($headers['ChecksumAlgorithm'])) { + // Checksum injection when expected to succeed at checksum validation + // This is needed because the checksum in the test is wrong + $algorithm = strtolower($headers['ChecksumAlgorithm']); + $checksumValue = ApplyChecksumMiddleware::getEncodedValue( + $algorithm, + $fixedBody + ); + $headers['Checksum'.strtoupper($algorithm)] = $checksumValue; + $fixedBody->rewind(); + } + + // If body was provided then we override the fixed one + if ($body !== null) { + $fixedBody = Utils::streamFor($body); + } + + return $fixedBody; + }, + ); + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + try { + $response = $s3TransferManager->download( + new DownloadRequest( + [ + 'Bucket' => 'test-bucket', + 'Key' => 'test-key', + ], + downloadRequestArgs: $requestArgs, + config: $config, + listeners: [ + new class($totalBytesReceived, $totalPartsReceived) + extends TransferListener { + private int $totalBytesReceived; + private int $totalPartsReceived; + + public function __construct( + int &$totalBytesReceived, + int &$totalPartsReceived + ) + { + $this->totalBytesReceived =& $totalBytesReceived; + $this->totalPartsReceived =& $totalPartsReceived; + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void { + $snapshot = $context[ + TransferListener::PROGRESS_SNAPSHOT_KEY + ]; + $this->totalBytesReceived = $snapshot->getTransferredBytes(); + $this->totalPartsReceived++; + } + } + ] + ) + )->wait(); + $this->assertEquals( + "success", + $outcome['result'], + "Operation should have failed at this point" + ); + $this->assertInstanceOf( + DownloadResult::class, + $response, + ); + if (isset($outcome['totalBytes'])) { + $this->assertEquals( + $outcome['totalBytes'], + $totalBytesReceived + ); + } + if (isset($outcome['totalParts'])) { + $this->assertEquals( + $outcome['totalParts'], + $totalPartsReceived + ); + } + if (isset($outcome['checksumValidated'])) { + $this->assertArrayHasKey( + 'ChecksumValidated', + $response + ); + $this->assertEquals( + $outcome['checksumAlgorithm'], + $response['ChecksumValidated'] + ); + } + } catch (S3TransferException | S3Exception $e) { + $this->assertEquals( + "error", + $outcome['result'], + "Operation did not expect a failure" + ); + + $this->assertTrue( + $this->assertEachWordMatchesTheErrorMessage( + $outcome['errorMessage'], + $e->getMessage() + ) + ); + } + } + + /** + * @param string $testId + * @param array $config + * @param array $requestArgs + * @param array $expectations + * @param array $outcomes + * + * @return void + * @dataProvider modeledUploadCasesProvider + * + */ + public function testModeledCasesForUpload( + string $testId, + array $config, + array $requestArgs, + array $expectations, + array $outcomes + ): void + { + $testsToSkip = [ + "Test upload with multipart upload - validation failure when part size mismatch" => true, + "Test upload with multipart upload - validation failure when part count mismatch" => true + ]; + if ($testsToSkip[$testId] ?? false) { + $this->markTestSkipped( + "The test `" . $testId . "` is not supported yet." + ); + } + + // Outcomes has only one item for now + $outcome = $outcomes[0]; + // Standardize config + $this->parseConfigFromCamelCaseToSnakeCase($config); + // Standardize request + $this->parseRequestArgsFromCamelCaseToPascalCase($requestArgs); + + // Operational values + $contentLength = $requestArgs['ContentLength']; + $totalBytesReceived = 0; + $totalPartsReceived = 0; + // Mock client to validate expected requests + $s3Client = $this->getS3ClientWithSequentialResponses( + array_map(function ($expectation) { + $operation = $expectation['request']['operation']; + + return array_merge( + $expectation['response'], + ['operation' => $operation] + ); + }, $expectations), + function (string $operation, ?array $body): StreamInterface { + $template = self::$multipartUploadBodyTemplates[$operation] ?? ""; + if ($body === null) { + $body = []; + } + + foreach ($body as $key => $value) { + $template = str_replace("{{$key}}", $value, $template); + } + + return Utils::streamFor( + $template, + ); + } + ); + + $s3TransferManager = new S3TransferManager( + $s3Client, + ); + try { + $response = $s3TransferManager->upload( + new UploadRequest( + Utils::streamFor( + str_repeat('#', $contentLength), + ), + uploadRequestArgs: $requestArgs, + config: array_merge( + $config, + ['concurrency' => 1], + ), + listeners: [ + new class($totalBytesReceived, $totalPartsReceived) + extends TransferListener { + private int $totalBytesReceived; + private int $totalPartsReceived; + + public function __construct( + int &$totalBytesReceived, + int &$totalPartsReceived + ) + { + $this->totalBytesReceived =& $totalBytesReceived; + $this->totalPartsReceived =& $totalPartsReceived; + } + + /** + * @param array $context + * + * @return void + */ + public function bytesTransferred(array $context): void { + $snapshot = $context[ + TransferListener::PROGRESS_SNAPSHOT_KEY + ]; + $this->totalBytesReceived = $snapshot->getTransferredBytes(); + $this->totalPartsReceived++; + } + } + ] + ) + )->wait(); + $this->assertEquals( + "success", + $outcome['result'], + "Operation should have failed at this point" + ); + $this->assertInstanceOf( + UploadResult::class, + $response, + ); + if (isset($outcome['totalBytes'])) { + $this->assertEquals( + $outcome['totalBytes'], + $totalBytesReceived + ); + } + if (isset($outcome['totalParts'])) { + $this->assertEquals( + $outcome['totalParts'], + $totalPartsReceived + ); + } + } catch (S3TransferException | S3Exception $e) { + $this->assertEquals( + "error", + $outcome['result'], + "Operation did not expect a failure" + ); + + $this->assertTrue( + $this->assertEachWordMatchesTheErrorMessage( + $outcome['errorMessage'], + $e->getMessage() + ) + ); + } + } + + /** + * @param array $responses + * @param callable $bodyBuilder + * A callable to build the body of the response. It receives as + * parameter: + * - The operation that the response is for. + * - The body given in the expectation. + * - The headers given in the expectation. + * + * @return S3Client + */ + private function getS3ClientWithSequentialResponses( + array $responses, + callable $bodyBuilder + ): S3Client + { + $index = 0; + return new S3Client([ + 'region' => 'eu-west-1', + 'http_handler' => function () + use ($bodyBuilder, $responses, &$index) { + $response = $responses[$index++]; + if ($response['status'] < 400) { + $headers = $response['headers'] ?? []; + $body = call_user_func_array( + $bodyBuilder, + [ + $response['operation'], + $response['body'] ?? null, + &$headers + ] + ); + + $this->parseCaseHeadersToAmzHeaders($headers); + + return new Response( + $response['status'], + $headers, + $body + ); + } else { + return new RejectedPromise( + new S3TransferException( + $response['errorMessage'] ?? "" + ) + ); + } + } + ]); + } + + /** + * @param array $config + * + * @return void + */ + private function parseConfigFromCamelCaseToSnakeCase( + array &$config + ): void + { + foreach ($config as $key => $value) { + // Searches for lowercaseUPPERCASE occurrences + // Then it is replaced by using group1_group2 found. + $newKey = strtolower( + preg_replace( + "/([a-z])([A-Z])/", + "$1_$2", + $key + ) + ); + $config[$newKey] = $value; + unset($config[$key]); + } + } + + /** + * Checks if all words from the expected message appear in the actual message + * in the same order (allowing for gaps between words). + * Example that resolves to true: + * expected: "error validating config" + * actual: "error validating download config" + * reason: The words "error" -> "validating" -> "config" were found in the + * same error in the actual message. + * + * Example that resolves to false: + * expected: "error validating config" + * actual: "error in download config validation" + * reason: The words error->validating->config were not + * found in order. + * + * @param string $expectedMessage The message containing words to find + * @param string $actualMessage The message to search within + * + * @return bool True if all expected words are found in order, false otherwise + */ + private function assertEachWordMatchesTheErrorMessage( + string $expectedMessage, + string $actualMessage + ): bool + { + $expectedMessage = trim($expectedMessage); + $actualMessage = trim($actualMessage); + + // To make the validation case-insensitive + $expectedMessage = strtolower($expectedMessage); + $actualMessage = strtolower($actualMessage); + + // Split into words and filter empty elements + $expectedWords = array_filter(preg_split('/\s+/', $expectedMessage)); + $actualWords = array_filter(preg_split('/\s+/', $actualMessage)); + + if (empty($expectedWords)) { + return true; + } + + if (empty($actualWords)) { + return false; + } + + $actualIndex = 0; + $actualWordsCount = count($actualWords); + + foreach ($expectedWords as $expectedWord) { + $wordFound = false; + while ($actualIndex < $actualWordsCount) { + if ($expectedWord === $actualWords[$actualIndex]) { + $wordFound = true; + $actualIndex++; + break; + } + $actualIndex++; + } + + if (!$wordFound) { + return false; + } + } + + return true; + } + + /** + * @return Generator + */ + public function modeledDownloadCasesProvider(): Generator + { + $downloadCases = json_decode( + file_get_contents( + self::DOWNLOAD_BASE_CASES + ), + true + ); + foreach ($downloadCases as $case) { + yield $case['summary'] => [ + 'test_id' => $case['summary'], + 'config' => $case['config'], + 'download_request' => $case['downloadRequest'], + 'expectations' => $case['expectations'], + 'outcomes' => $case['outcomes'], + ]; + } + } + + /** + * @return Generator + */ + public function modeledUploadCasesProvider(): Generator + { + $downloadCases = json_decode( + file_get_contents( + self::UPLOAD_BASE_CASES + ), + true + ); + foreach ($downloadCases as $case) { + yield $case['summary'] => [ + 'test_id' => $case['summary'], + 'config' => $case['config'], + 'upload_request' => $case['uploadRequest'], + 'expectations' => $case['expectations'], + 'outcomes' => $case['outcomes'], + ]; + } + } + + /** + * @param array $requestArgs + * + * @return void + */ + private function parseRequestArgsFromCamelCaseToPascalCase( + array &$requestArgs + ): void + { + foreach ($requestArgs as $key => $value) { + $newKey = ucfirst($key); + $requestArgs[$newKey] = $value; + unset($requestArgs[$key]); + } + } + + /** + * @param array $caseHeaders + * + * @return void + */ + private function parseCaseHeadersToAmzHeaders(array &$caseHeaders): void + { + foreach ($caseHeaders as $key => $value) { + $newKey = $key; + switch ($key) { + case 'PartsCount': + $newKey = 'x-amz-mp-parts-count'; + break; + case 'ChecksumAlgorithm': + $newKey = 'x-amz-checksum-algorithm'; + break; + default: + if (preg_match('/Checksum[A-Z]+/', $key)) { + $newKey = 'x-amz-checksum-' . str_replace( + 'Checksum', + '', + $key + ); + } + } + + if ($newKey !== $key) { + $caseHeaders[$newKey] = $value; + unset($caseHeaders[$key]); + } + } + } + /** * @param array $methodsCallback If any from the callbacks below * is not provided then a default implementation will be provided. diff --git a/tests/S3/S3Transfer/test-cases/download-single-object.json b/tests/S3/S3Transfer/test-cases/download-single-object.json new file mode 100644 index 0000000000..2af9c8fc72 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/download-single-object.json @@ -0,0 +1,582 @@ +[ + { + "summary": "Test download with part GET - single object download (object size < part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "PartsCount": 1, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "single_object", + "totalBytes": 1048576, + "partCount": 1 + } + ] + }, + { + "summary": "Test download with part GET - multipart download (object size > part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 8388608-16777215/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 3, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 16777216-25165823/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 25165824, + "partCount": 3 + } + ] + }, + { + "summary": "Test download with ranged GET - single object download (object size < part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "1048576", + "Content-Range": "bytes 0-1048575/1048576", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "single_object", + "totalBytes": 1048576, + "partCount": 1 + } + ] + }, + { + "summary": "Test download with ranged GET - multipart download (object size > part size)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=8388608-16777215", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 8388608-16777215/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=16777216-25165823", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 16777216-25165823/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 25165824, + "partCount": 3 + } + ] + }, + { + "summary": "Test download with part GET - error handling when part request fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + ] + }, + { + "summary": "Test download with ranged GET - error handling when range request fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=8388608-16777215", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + ] + }, + { + "summary": "Test download with part GET - checksum validation success", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1, + "checksumMode": "ENABLED" + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/8388608", + "PartsCount": 1, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumAlgorithm": "CRC32", + "ChecksumCRC32": "abcdef12" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "single_object", + "totalBytes": 8388608, + "partCount": 1, + "checksumValidated": true, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test download with part GET - checksum validation failure", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1, + "checksumMode": "ENABLED" + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/8388608", + "PartsCount": 1, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumAlgorithm": "CRC32", + "ChecksumCRC32": "abcdef12" + }, + "body": "CORRUPTED_DATA" + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ChecksumMismatch", + "errorMessage": "Calculated checksum did not match the expected value" + } + ] + }, + { + "summary": "Test download with part GET - multipart download with uneven last part", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/10485760", + "PartsCount": 2, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2097152", + "Content-Range": "bytes 8388608-10485759/10485760", + "PartsCount": 2, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 10485760, + "partCount": 2, + "lastPartSize": 2097152 + } + ] + }, + { + "summary": "Test download with ranged GET - multipart download with uneven last part", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "RANGE", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=0-8388607" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/10485760", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "headers": { + "Range": "bytes=8388608-16777215", + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "2097152", + "Content-Range": "bytes 8388608-10485759/10485760", + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "downloadType": "multipart", + "totalBytes": 10485760, + "partCount": 2, + "lastPartSize": 2097152 + } + ] + }, + { + "summary": "Test download with part GET - validation failure when part count mismatch", + "config": { + "targetPartSizeBytes": 8388608, + "multipartDownloadType": "PART", + "responseChecksumValidation": "WHEN_SUPPORTED" + }, + "downloadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "checksumMode": "ENABLED" + }, + "expectations": [ + { + "request": { + "operation": "GetObject", + "partNumber": 1 + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 0-8388607/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + }, + { + "request": { + "operation": "GetObject", + "partNumber": 2, + "headers": { + "If-Match": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + }, + "response": { + "status": 200, + "headers": { + "Content-Length": "8388608", + "Content-Range": "bytes 8388608-16777215/25165824", + "PartsCount": 3, + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"" + } + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ValidationError", + "errorMessage": "Expected 3 parts but only received 2" + } + ] + } +] \ No newline at end of file diff --git a/tests/S3/S3Transfer/test-cases/upload-single-object.json b/tests/S3/S3Transfer/test-cases/upload-single-object.json new file mode 100644 index 0000000000..209bec9217 --- /dev/null +++ b/tests/S3/S3Transfer/test-cases/upload-single-object.json @@ -0,0 +1,746 @@ +[ + { + "summary": "Test upload with single object upload (object size < multipart threshold)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "PutObject", + "contentLength": 10485760 + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumAlgorithm": "CRC32", + "ChecksumCRC32": "abcdef12" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "single_object", + "totalBytes": 10485760 + } + ] + }, + { + "summary": "Test upload with multipart upload (object size > multipart threshold)", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 25165824, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 3, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"b54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "87654321" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + }, + { + "PartNumber": 3, + "ETag": "\"b54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "87654321" + } + ] + }, + "checksumAlgorithm": "CRC32", + "mpuObjectSize": 25165824 + }, + "response": { + "status": 200, + "body": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "Bucket": "example-bucket", + "Key": "example-object", + "Location": "https://example-bucket.s3.amazonaws.com/example-object" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "multipart", + "totalBytes": 25165824, + "partCount": 3, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test upload with multipart upload and uneven last part", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 2097152, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + ] + }, + "checksumAlgorithm": "CRC32", + "mpuObjectSize": 10485760 + }, + "response": { + "status": 200, + "body": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "multipart", + "totalBytes": 10485760, + "partCount": 2, + "lastPartSize": 2097152, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test upload with multipart upload and full object checksum", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 16777216, + "checksumAlgorithm": "CRC32", + "checksumCRC32": "fedcba98", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32", + "checksumType": "FULL_OBJECT" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + ] + }, + "checksumAlgorithm": "CRC32", + "checksumType": "FULL_OBJECT", + "checksumCRC32": "fedcba98", + "mpuObjectSize": 16777216 + }, + "response": { + "status": 200, + "body": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "Bucket": "example-bucket", + "Key": "example-object", + "Location": "https://example-bucket.s3.amazonaws.com/example-object", + "ChecksumCRC32": "fedcba98" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "multipart", + "totalBytes": 16777216, + "partCount": 2, + "checksumAlgorithm": "CRC32", + "checksumType": "FULL_OBJECT" + } + ] + }, + { + "summary": "Test upload with single object upload and checksum calculation", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "PutObject", + "contentLength": 10485760, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6\"", + "ChecksumCRC32": "abcdef12" + } + } + } + ], + "outcomes": [ + { + "result": "success", + "uploadType": "single_object", + "totalBytes": 10485760, + "checksumAlgorithm": "CRC32" + } + ] + }, + { + "summary": "Test upload with multipart upload - error handling when part upload fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 25165824, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error", + "abortedMultipartUpload": true + } + ] + }, + { + "summary": "Test upload with multipart upload - error handling when CompleteMultipartUpload fails", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 16777216, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 2, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + } + }, + { + "request": { + "operation": "CompleteMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "multipartUpload": { + "Parts": [ + { + "PartNumber": 1, + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + }, + { + "PartNumber": 2, + "ETag": "\"0c78aef83f66abc1fa1e8477f296d394\"", + "ChecksumCRC32": "12345678" + } + ] + }, + "checksumAlgorithm": "CRC32", + "mpuObjectSize": 16777216 + }, + "response": { + "status": 500, + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error" + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "InternalServerError", + "errorMessage": "Internal Server Error", + "abortedMultipartUpload": true + } + ] + }, + { + "summary": "Test upload with multipart upload - validation failure when part count mismatch", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 25165824, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 8388608, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ValidationError", + "errorMessage": "Expected 3 parts but only received 1", + "abortedMultipartUpload": true + } + ] + }, + { + "summary": "Test upload with multipart upload - validation failure when part size mismatch", + "config": { + "targetPartSizeBytes": 8388608, + "multipartUploadThresholdBytes": 16777216, + "requestChecksumCalculation": "WHEN_SUPPORTED" + }, + "uploadRequest": { + "bucket": "example-bucket", + "key": "example-object", + "contentLength": 16777216, + "checksumAlgorithm": "CRC32", + "metadata": { + "contentType": "application/octet-stream" + } + }, + "expectations": [ + { + "request": { + "operation": "CreateMultipartUpload", + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "body": { + "UploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "Bucket": "example-bucket", + "Key": "example-object" + } + } + }, + { + "request": { + "operation": "UploadPart", + "partNumber": 1, + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA", + "contentLength": 4194304, + "checksumAlgorithm": "CRC32" + }, + "response": { + "status": 200, + "headers": { + "ETag": "\"a54357aff0632cce46d942af68356b38\"", + "ChecksumCRC32": "abcdef12" + } + } + }, + { + "request": { + "operation": "AbortMultipartUpload", + "uploadId": "VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA" + }, + "response": { + "status": 204 + } + } + ], + "outcomes": [ + { + "result": "error", + "errorCode": "ValidationError", + "errorMessage": "Part size mismatch: expected 8388608 bytes but got 4194304 bytes for part 1", + "abortedMultipartUpload": true + } + ] + } +] \ No newline at end of file From fa664b1be057b258bad3bf395e51466ba6555869 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 28 Aug 2025 08:32:03 -0700 Subject: [PATCH 58/62] fix: tests with part count - Expected part count validation was added and hence in the tests where not part counts were provided a failure was happening. - Replace rmdir by the clean up directory helper method from the TestUtility implementation. --- tests/S3/S3Transfer/S3TransferManagerTest.php | 169 +++++------------- 1 file changed, 47 insertions(+), 122 deletions(-) diff --git a/tests/S3/S3Transfer/S3TransferManagerTest.php b/tests/S3/S3Transfer/S3TransferManagerTest.php index 3e508b47ce..19917aee81 100644 --- a/tests/S3/S3Transfer/S3TransferManagerTest.php +++ b/tests/S3/S3Transfer/S3TransferManagerTest.php @@ -29,7 +29,6 @@ use Closure; use Exception; use Generator; -use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Promise\Create; use GuzzleHttp\Promise\RejectedPromise; use GuzzleHttp\Psr7\Response; @@ -589,19 +588,22 @@ public function testUploadDirectoryValidatesProvidedDirectory( $this->assertTrue(true); } - $manager = new S3TransferManager( - $this->getS3ClientMock(), - ); - $manager->uploadDirectory( - new UploadDirectoryRequest( - $directory, - "Bucket", - ) - )->wait(); - // Clean up resources - if ($isDirectoryValid) { - rmdir($directory); - } + try { + $manager = new S3TransferManager( + $this->getS3ClientMock(), + ); + $manager->uploadDirectory( + new UploadDirectoryRequest( + $directory, + "Bucket", + ) + )->wait(); + } finally { + // Clean up resources + if ($isDirectoryValid) { + TestsUtility::cleanUpDir($directory); + } + } } /** @@ -664,7 +666,7 @@ public function testUploadDirectoryFailsOnInvalidFilter(): void ) )->wait(); } finally { - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -729,10 +731,7 @@ public function testUploadDirectoryFileFilter(): void )->wait(); $this->assertEquals($validFilesCount, $calledTimes); } finally { - foreach ($filesCreated as $filePathName) { - unlink($filePathName); - } - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -791,12 +790,7 @@ public function testUploadDirectoryRecursive(): void $this->assertTrue($validated); } } finally { - foreach ($files as $file) { - unlink($file); - } - - rmdir($subDirectory); - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -861,12 +855,7 @@ public function testUploadDirectoryNonRecursive(): void } } } finally { - foreach ($files as $file) { - unlink($file); - } - - rmdir($subDirectory); - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1027,10 +1016,7 @@ public function testUploadDirectoryUsesProvidedPrefix(): void $this->assertTrue($validated, "Key {$key} should have been validated"); } } finally { - foreach ($files as $file) { - unlink($file); - } - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1093,10 +1079,7 @@ public function testUploadDirectoryUsesProvidedDelimiter(): void $this->assertTrue($validated, "Key {$key} should have been validated"); } } finally { - foreach ($files as $file) { - unlink($file); - } - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1127,7 +1110,7 @@ public function testUploadDirectoryFailsOnInvalidPutObjectRequestCallback(): voi ) )->wait(); } finally { - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1183,11 +1166,7 @@ public function testUploadDirectoryPutObjectRequestCallbackWorks(): void )->wait(); $this->assertEquals(count($files), $called); } finally { - foreach ($files as $file) { - unlink($file); - } - - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1266,11 +1245,7 @@ public function testUploadDirectoryUsesFailurePolicy(): void )->wait(); $this->assertTrue($called); } finally { - foreach ($files as $file) { - unlink($file); - } - - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1301,7 +1276,7 @@ public function testUploadDirectoryFailsOnInvalidFailurePolicy(): void ) )->wait(); } finally { - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1344,10 +1319,7 @@ public function testUploadDirectoryFailsWhenFileContainsProvidedDelimiter(): voi ) )->wait(); } finally { - foreach ($files as $file) { - unlink($file); - } - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1409,10 +1381,7 @@ public function testUploadDirectoryTracksMultipleFiles(): void ); } } finally { - foreach ($files as $file) { - unlink($file); - } - rmdir($directory); + TestsUtility::cleanUpDir($directory); } } @@ -1502,6 +1471,7 @@ public function testDownloadWorksWithS3UriAsSource(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); }, @@ -1536,6 +1506,7 @@ public function testDownloadWorksWithBucketAndKeyAsSource(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); }, @@ -1588,6 +1559,7 @@ public function testDownloadAppliesChecksumMode( if ($command->getName() === MultipartDownloader::GET_OBJECT_COMMAND) { return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); } @@ -1704,6 +1676,7 @@ public function testDownloadChoosesMultipartDownloadType( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); } @@ -1885,7 +1858,7 @@ public function testDownloadDirectoryCreatesDestinationDirectory(): void )->wait(); $this->assertFileExists($destinationDirectory); } finally { - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -1961,7 +1934,7 @@ public function testDownloadDirectoryAppliesS3Prefix( $this->assertTrue($called); $this->assertTrue($listObjectsCalled); } finally { - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2076,7 +2049,7 @@ public function testDownloadDirectoryAppliesDelimiter( $this->assertTrue($called); $this->assertTrue($listObjectsCalled); } finally { - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2154,7 +2127,7 @@ public function testDownloadDirectoryFailsOnInvalidFilter(): void )->wait(); $this->assertTrue($called); } finally { - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2211,7 +2184,7 @@ public function testDownloadDirectoryFailsOnInvalidFailurePolicy(): void )->wait(); $this->assertTrue($called); } finally { - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2250,6 +2223,7 @@ public function testDownloadDirectoryUsesFailurePolicy(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); } @@ -2290,12 +2264,7 @@ public function testDownloadDirectoryUsesFailurePolicy(): void )->wait(); $this->assertTrue($called); } finally { - $file = $destinationDirectory . '/file1.txt'; - if (file_exists($file)) { - unlink($file); - } - - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2341,6 +2310,7 @@ public function testDownloadDirectoryAppliesFilter( return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); }, @@ -2384,22 +2354,7 @@ public function testDownloadDirectoryAppliesFilter( ); } } finally { - $dirs = []; - foreach ($objectList as $object) { - if (file_exists($destinationDirectory . "/" . $object['Key'])) { - unlink($destinationDirectory . "/" . $object['Key']); - } - - $dirs [dirname($destinationDirectory . "/" . $object['Key'])] = true; - } - - foreach ($dirs as $dir => $_) { - if (is_dir($dir)) { - rmdir($dir); - } - } - - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2540,7 +2495,7 @@ public function testDownloadDirectoryFailsOnInvalidGetObjectRequestCallback(): v ) )->wait(); } finally { - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2570,6 +2525,7 @@ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void return Create::promiseFor(new Result([ 'Body' => Utils::streamFor(), + 'PartsCount' => 1, '@metadata' => [] ])); }, @@ -2616,22 +2572,7 @@ public function testDownloadDirectoryGetObjectRequestCallbackWorks(): void )->wait(); $this->assertTrue($called); } finally { - $dirs = []; - foreach ($listObjectsContent as $object) { - $file = $destinationDirectory . "/" . $object['Key']; - if (file_exists($file)) { - $dirs[dirname($file)] = true; - unlink($file); - } - } - - foreach (array_keys($dirs) as $dir) { - if (is_dir($dir)) { - rmdir($dir); - } - } - - rmdir($destinationDirectory); + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2670,6 +2611,7 @@ public function testDownloadDirectoryCreateFiles( 'Body' => Utils::streamFor( "Test file " . $command['Key'] ), + 'PartsCount' => 1, '@metadata' => [] ])); }, @@ -2712,25 +2654,7 @@ public function testDownloadDirectoryCreateFiles( ); } } finally { - $dirs = []; - foreach ($expectedFileKeys as $key) { - $file = $destinationDirectory . "/" . $key; - if (file_exists($file)) { - unlink($file); - } - - $dirs [dirname($file)] = true; - } - - foreach ($dirs as $dir => $_) { - if (is_dir($dir)) { - rmdir($dir); - } - } - - if (is_dir($destinationDirectory)) { - rmdir($destinationDirectory); - } + TestsUtility::cleanUpDir($destinationDirectory); } } @@ -2818,6 +2742,7 @@ public function testResolvesOutsideTargetDirectory( 'Body' => Utils::streamFor( "Test file " . $command['Key'] ), + 'PartsCount' => 1, '@metadata' => [] ])); }, From d2d46880ae6b87646bf2f5c1f82da0e45cbf0f63 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 28 Aug 2025 08:51:37 -0700 Subject: [PATCH 59/62] fix: test invalid checksum --- tests/S3/S3Transfer/MultipartUploaderTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index 7c377cbaeb..79ae63c516 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -1107,7 +1107,7 @@ public function inputArgumentsPerOperationProvider(): Generator "SSECustomerKey" => 'test-sse-customer-key', "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', 'ChecksumType' => 'FULL_OBJECT', - 'ChecksumCRC32' => 'b71d0814', // From default algorithm used + 'ChecksumCRC32' => 'tx0IFA==', // From default algorithm used ], ], 'expects_error' => false, From e6a141f2c5f6674a2e8a95ae5471341594b6e956 Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Thu, 28 Aug 2025 08:58:36 -0700 Subject: [PATCH 60/62] chore: address PR feedback - Rename Exceptions to Exception --- src/S3/S3Transfer/AbstractMultipartUploader.php | 4 ++-- .../{Exceptions => Exception}/FileDownloadException.php | 2 +- .../{Exceptions => Exception}/ProgressTrackerException.php | 2 +- .../{Exceptions => Exception}/S3TransferException.php | 2 +- src/S3/S3Transfer/Models/DownloadRequest.php | 2 +- src/S3/S3Transfer/MultipartDownloader.php | 2 +- src/S3/S3Transfer/Progress/SingleProgressTracker.php | 2 +- src/S3/S3Transfer/S3TransferManager.php | 2 +- src/S3/S3Transfer/Utils/FileDownloadHandler.php | 2 +- tests/S3/S3Transfer/MultipartUploaderTest.php | 2 +- tests/S3/S3Transfer/RangeGetMultipartDownloaderTest.php | 2 +- tests/S3/S3Transfer/S3TransferManagerTest.php | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) rename src/S3/S3Transfer/{Exceptions => Exception}/FileDownloadException.php (61%) rename src/S3/S3Transfer/{Exceptions => Exception}/ProgressTrackerException.php (62%) rename src/S3/S3Transfer/{Exceptions => Exception}/S3TransferException.php (61%) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index d7ad802525..8cb0507c9c 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -6,7 +6,7 @@ use Aws\CommandPool; use Aws\ResultInterface; use Aws\S3\S3ClientInterface; -use Aws\S3\S3Transfer\Exceptions\S3TransferException; +use Aws\S3\S3Transfer\Exception\S3TransferException; use Aws\S3\S3Transfer\Models\S3TransferManagerConfig; use Aws\S3\S3Transfer\Progress\TransferListener; use Aws\S3\S3Transfer\Progress\TransferListenerNotifier; @@ -54,7 +54,7 @@ abstract class AbstractMultipartUploader implements PromisorInterface protected ?TransferProgressSnapshot $currentSnapshot; /** - * This will be used for custom or default checksum. + * For custom or default checksum. * * @var string|null */ diff --git a/src/S3/S3Transfer/Exceptions/FileDownloadException.php b/src/S3/S3Transfer/Exception/FileDownloadException.php similarity index 61% rename from src/S3/S3Transfer/Exceptions/FileDownloadException.php rename to src/S3/S3Transfer/Exception/FileDownloadException.php index a80cde583a..4f9dee8776 100644 --- a/src/S3/S3Transfer/Exceptions/FileDownloadException.php +++ b/src/S3/S3Transfer/Exception/FileDownloadException.php @@ -1,5 +1,5 @@ Date: Fri, 29 Aug 2025 12:12:38 -0700 Subject: [PATCH 61/62] chore: address some PR feedback - Remove full object checksum calculate since it is not recommended. - Address some styling issues. --- .../S3Transfer/AbstractMultipartUploader.php | 16 +++++++----- src/S3/S3Transfer/MultipartUploader.php | 26 ++----------------- tests/S3/S3Transfer/MultipartUploaderTest.php | 4 --- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/src/S3/S3Transfer/AbstractMultipartUploader.php b/src/S3/S3Transfer/AbstractMultipartUploader.php index 8cb0507c9c..166d9e1da9 100644 --- a/src/S3/S3Transfer/AbstractMultipartUploader.php +++ b/src/S3/S3Transfer/AbstractMultipartUploader.php @@ -18,7 +18,7 @@ use Throwable; /** - * Abstract base class for multipart operations (upload/copy). + * Abstract base class for multipart operations */ abstract class AbstractMultipartUploader implements PromisorInterface { @@ -31,16 +31,16 @@ abstract class AbstractMultipartUploader implements PromisorInterface /** @var S3ClientInterface */ protected readonly S3ClientInterface $s3Client; - /** @var array @ */ + /** @var array */ protected readonly array $requestArgs; - /** @var array @ */ + /** @var array */ protected readonly array $config; /** @var string|null */ protected string|null $uploadId; - /** @var array @ */ + /** @var array */ protected array $parts; /** @var array */ @@ -67,6 +67,9 @@ abstract class AbstractMultipartUploader implements PromisorInterface */ protected ?string $requestChecksumAlgorithm; + /** @var bool */ + private bool $isFullObjectChecksum; + /** * @param S3ClientInterface $s3Client * @param array $requestArgs @@ -96,6 +99,7 @@ public function __construct( $this->parts = $parts; $this->currentSnapshot = $currentSnapshot; $this->listenerNotifier = $listenerNotifier; + $this->isFullObjectChecksum = false; } /** @@ -181,10 +185,10 @@ protected function createMultipartOperation(): PromiseInterface if ($this->requestChecksum !== null) { $createMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; + $this->isFullObjectChecksum = true; } elseif ($this->config['request_checksum_calculation'] === 'when_supported') { $this->requestChecksumAlgorithm = $createMultipartUploadArgs['ChecksumAlgorithm'] ?? self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM; - $createMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $createMultipartUploadArgs['ChecksumAlgorithm'] = $this->requestChecksumAlgorithm; } @@ -225,7 +229,7 @@ protected function completeMultipartOperation(): PromiseInterface ]; $completeMultipartUploadArgs['MpuObjectSize'] = $this->getTotalSize(); - if ($this->requestChecksum !== null) { + if ($this->isFullObjectChecksum && $this->requestChecksum !== null) { $completeMultipartUploadArgs['ChecksumType'] = self::CHECKSUM_TYPE_FULL_OBJECT; $completeMultipartUploadArgs[ 'Checksum' . strtoupper($this->requestChecksumAlgorithm) diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 6aeb308cb8..6295525f2b 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -138,22 +138,13 @@ protected function processMultipartOperation(): PromiseInterface $partNo = count($this->parts); $uploadPartCommandArgs['UploadId'] = $this->uploadId; // Customer provided checksum - $hashBody = false; if ($this->requestChecksum !== null) { - // To avoid default calculation + // To avoid default calculation for individual parts $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); } elseif ($this->requestChecksumAlgorithm !== null) { // Normalize algorithm name - $algoName = strtolower($this->requestChecksumAlgorithm); - if ($algoName === self::DEFAULT_CHECKSUM_CALCULATION_ALGORITHM) { - $algoName = 'crc32b'; - } - - $hashBody = true; - $this->hashContext = hash_init($algoName); - // To avoid default calculation - $uploadPartCommandArgs['@context']['request_checksum_calculation'] = 'when_required'; + $this->requestChecksumAlgorithm = strtolower($this->requestChecksumAlgorithm); unset($uploadPartCommandArgs['Checksum'. strtoupper($this->requestChecksumAlgorithm)]); } @@ -166,10 +157,6 @@ protected function processMultipartOperation(): PromiseInterface break; } - if ($hashBody) { - hash_update($this->hashContext, $read); - } - $partBody = Utils::streamFor($read); $uploadPartCommandArgs['PartNumber'] = $partNo; @@ -199,15 +186,6 @@ protected function processMultipartOperation(): PromiseInterface } } - if ($hashBody) { - $this->requestChecksum = base64_encode( - hash_final( - $this->hashContext, - true - ) - ); - } - return (new CommandPool( $this->s3Client, $commands, diff --git a/tests/S3/S3Transfer/MultipartUploaderTest.php b/tests/S3/S3Transfer/MultipartUploaderTest.php index b62f570724..fc830eb487 100644 --- a/tests/S3/S3Transfer/MultipartUploaderTest.php +++ b/tests/S3/S3Transfer/MultipartUploaderTest.php @@ -1084,8 +1084,6 @@ public function inputArgumentsPerOperationProvider(): Generator "StorageClass" => 'test-storage-class', "Tagging" => 'test-tagging', "WebsiteRedirectLocation" => 'test-website-redirect-location', - 'ChecksumType' => 'FULL_OBJECT', - 'ChecksumAlgorithm' => 'crc32', ], 'UploadPart' => [ "Bucket" => 'test-bucket', @@ -1106,8 +1104,6 @@ public function inputArgumentsPerOperationProvider(): Generator "SSECustomerAlgorithm" => 'test-sse-customer-algorithm', "SSECustomerKey" => 'test-sse-customer-key', "SSECustomerKeyMD5" => 'test-sse-customer-key-md5', - 'ChecksumType' => 'FULL_OBJECT', - 'ChecksumCRC32' => 'tx0IFA==', // From default algorithm used ], ], 'expects_error' => false, From ec383d2f4877ddb960644cac80d3badb4fcb33cb Mon Sep 17 00:00:00 2001 From: Yenfry Herrera Feliz Date: Fri, 29 Aug 2025 12:35:11 -0700 Subject: [PATCH 62/62] chore: some minor refactor - Make some statements multilines. - Add comments describing functionality. --- src/S3/S3Transfer/MultipartDownloader.php | 3 ++- src/S3/S3Transfer/MultipartUploader.php | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/S3/S3Transfer/MultipartDownloader.php b/src/S3/S3Transfer/MultipartDownloader.php index 84565236de..62ace190e1 100644 --- a/src/S3/S3Transfer/MultipartDownloader.php +++ b/src/S3/S3Transfer/MultipartDownloader.php @@ -190,7 +190,8 @@ public function promise(): PromiseInterface if ($this->currentPartNo !== $this->objectPartsCount) { throw new S3TransferException( - "Expected number of parts `$this->objectPartsCount` to have been transferred but got `$this->currentPartNo`." + "Expected number of parts `$this->objectPartsCount`" + . " to have been transferred but got `$this->currentPartNo`." ); } diff --git a/src/S3/S3Transfer/MultipartUploader.php b/src/S3/S3Transfer/MultipartUploader.php index 6295525f2b..77df8bc9e2 100644 --- a/src/S3/S3Transfer/MultipartUploader.php +++ b/src/S3/S3Transfer/MultipartUploader.php @@ -13,7 +13,6 @@ use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\Psr7\LazyOpenStream; use GuzzleHttp\Psr7\Utils; -use HashContext; use Psr\Http\Message\StreamInterface; use Throwable; @@ -36,9 +35,6 @@ class MultipartUploader extends AbstractMultipartUploader /** @var StreamInterface */ private StreamInterface $body; - /** @var HashContext */ - private HashContext $hashContext; - public function __construct( S3ClientInterface $s3Client, array $requestArgs, @@ -74,6 +70,9 @@ public function upload(): UploadResult } /** + * Parses the source into an instance of + * StreamInterface to be read. + * * @param string|StreamInterface $source * * @return StreamInterface @@ -86,7 +85,8 @@ private function parseBody( // Make sure the files exists if (!is_readable($source)) { throw new \InvalidArgumentException( - "The source for this upload must be either a readable file path or a valid stream." + "The source for this upload must be either a" + . " readable file path or a valid stream." ); } $body = new LazyOpenStream($source, 'r'); @@ -106,6 +106,10 @@ private function parseBody( } /** + * Evaluates if custom checksum has been provided, + * and if so then, the values are placed in the + * respective properties. + * * @return void */ private function evaluateCustomChecksum(): void @@ -128,6 +132,11 @@ private function evaluateCustomChecksum(): void } } + /** + * Process a multipart upload operation. + * + * @return PromiseInterface + */ protected function processMultipartOperation(): PromiseInterface { $uploadPartCommandArgs = $this->requestArgs; @@ -181,7 +190,8 @@ protected function processMultipartOperation(): PromiseInterface if ($partNo > $partsCount) { return Create::rejectionFor( - "The current part `$partNo` is over the expected number of parts `$partsCount`" + "The current part `$partNo` is over " + . "the expected number of parts `$partsCount`" ); } }