diff --git a/Classes/Controller/MediaController.php b/Classes/Controller/MediaController.php index 6168ce868..3f7c3e437 100644 --- a/Classes/Controller/MediaController.php +++ b/Classes/Controller/MediaController.php @@ -14,15 +14,38 @@ * source code. */ +use Flowpack\Media\Ui\Service\ConfigurationService; +use Flowpack\Media\Ui\Tus\PartialUploadFileCacheAdapter; +use Flowpack\Media\Ui\Tus\TusEventHandler; use Neos\Flow\Annotations as Flow; +use Neos\Flow\Log\Utility\LogEnvironment; +use Neos\Flow\Utility\Environment; +use Neos\Flow\Utility\Exception; use Neos\Fusion\View\FusionView; use Neos\Neos\Controller\Module\AbstractModuleController; +use Neos\Utility\Exception\FilesException; +use Neos\Utility\Files; +use Psr\Log\LoggerInterface; +use TusPhp\Events\TusEvent; +use TusPhp\Tus\Server; /** * @Flow\Scope("singleton") */ class MediaController extends AbstractModuleController { + /** + * @Flow\Inject + * @var TusEventHandler + */ + protected $tusEventHandler; + + /** + * @Flow\Inject + * @var Environment + */ + protected $environment; + /** * @var FusionView */ @@ -33,6 +56,24 @@ class MediaController extends AbstractModuleController */ protected $defaultViewObjectName = FusionView::class; + /** + * @Flow\Inject + * @var PartialUploadFileCacheAdapter + */ + protected $partialUploadFileCacheAdapater; + + /** + * @Flow\Inject + * @var ConfigurationService + */ + protected $configurationService; + + /** + * @Flow\Inject + * @var LoggerInterface + */ + protected $logger; + /** * @var array */ @@ -46,4 +87,40 @@ class MediaController extends AbstractModuleController public function indexAction(): void { } + + /** + * @throws Exception + * @throws FilesException + * @Flow\SkipCsrfProtection + */ + public function uploadAction(): string + { + $uploadDirectory = Files::concatenatePaths([$this->environment->getPathToTemporaryDirectory(), 'TusUpload']); + if (!file_exists($uploadDirectory)) { + Files::createDirectoryRecursively($uploadDirectory); + } + + $server = new Server($this->partialUploadFileCacheAdapater); + $server->setApiPath($this->controllerContext->getRequest()->getHttpRequest()->getUri()->getPath())/** @phpstan-ignore-line */ + ->setUploadDir($uploadDirectory) + ->setMaxUploadSize($this->configurationService->getMaximumUploadFileSize()) + ->event() + ->addListener('tus-server.upload.complete', function (TusEvent $event) { + $this->tusEventHandler->processUploadedFile($event); + }); + + $server->event()->addListener('tus-server.upload.created', function (TusEvent $event) { + $this->logger->debug(sprintf('A new TUS file upload session was started for file "%s"', $event->getFile()->getName()), LogEnvironment::fromMethodName(__METHOD__)); + }); + + $server->event()->addListener('tus-server.upload.progress', function (TusEvent $event) { + $this->logger->debug(sprintf('Resumed TUS file upload for file "%s"', $event->getFile()->getName()), LogEnvironment::fromMethodName(__METHOD__)); + }); + + $response = $server->serve(); + $this->controllerContext->getResponse()->setStatusCode($response->getStatusCode()); + + $response->send(); + return ''; + } } diff --git a/Classes/GraphQL/Resolver/Type/QueryResolver.php b/Classes/GraphQL/Resolver/Type/QueryResolver.php index b9df3b8d4..1dc7acd53 100644 --- a/Classes/GraphQL/Resolver/Type/QueryResolver.php +++ b/Classes/GraphQL/Resolver/Type/QueryResolver.php @@ -17,6 +17,7 @@ use Flowpack\Media\Ui\Exception as MediaUiException; use Flowpack\Media\Ui\GraphQL\Context\AssetSourceContext; use Flowpack\Media\Ui\Service\AssetChangeLog; +use Flowpack\Media\Ui\Service\ConfigurationService; use Flowpack\Media\Ui\Service\SimilarityService; use Flowpack\Media\Ui\Service\UsageDetailsService; use Neos\Flow\Annotations as Flow; @@ -94,6 +95,12 @@ class QueryResolver implements ResolverInterface */ protected $persistenceManager; + /** + * @Flow\Inject + * @var ConfigurationService + */ + protected $configurationService; + /** * @Flow\InjectConfiguration(package="Flowpack.Media.Ui") * @var array @@ -136,7 +143,8 @@ public function assetCount($_, array $variables, AssetSourceContext $assetSource protected function createAssetProxyQuery( array $variables, AssetSourceContext $assetSourceContext - ): ?AssetProxyQueryInterface { + ): ?AssetProxyQueryInterface + { [ 'assetSourceId' => $assetSourceId, 'tagId' => $tagId, @@ -260,39 +268,13 @@ public function assetUsageCount($_, array $variables, AssetSourceContext $assetS public function config($_): array { return [ - 'uploadMaxFileSize' => $this->getMaximumFileUploadSize(), - 'uploadMaxFileUploadLimit' => $this->getMaximumFileUploadLimit(), + 'maximumUploadChunkSize' => $this->configurationService->getMaximumUploadChunkSize(), + 'maximumUploadFileSize' => $this->configurationService->getMaximumUploadFileSize(), + 'maximumUploadFileCount' => $this->configurationService->getMaximumUploadFileCount(), 'currentServerTime' => (new \DateTime())->format(DATE_W3C), ]; } - /** - * Returns the lowest configured maximum upload file size - * - * @return int - */ - protected function getMaximumFileUploadSize(): int - { - try { - return (int)min( - Files::sizeStringToBytes(ini_get('post_max_size')), - Files::sizeStringToBytes(ini_get('upload_max_filesize')) - ); - } catch (FilesException $e) { - return 0; - } - } - - /** - * Returns the maximum number of files that can be uploaded - * - * @return int - */ - protected function getMaximumFileUploadLimit(): int - { - return (int)($this->settings['maximumFileUploadLimit'] ?? 10); - } - /** * Provides a filterable list of asset proxies. These are the main entities for media management. * @@ -305,7 +287,8 @@ public function assets( $_, array $variables, AssetSourceContext $assetSourceContext - ): ?AssetProxyQueryResultInterface { + ): ?AssetProxyQueryResultInterface + { ['limit' => $limit, 'offset' => $offset] = $variables + ['limit' => 20, 'offset' => 0]; $query = $this->createAssetProxyQuery($variables, $assetSourceContext); diff --git a/Classes/Service/ConfigurationService.php b/Classes/Service/ConfigurationService.php new file mode 100644 index 000000000..fc696c034 --- /dev/null +++ b/Classes/Service/ConfigurationService.php @@ -0,0 +1,69 @@ +configuration['maximumUploadFileSize'] ?? '100MB'); + } catch (FilesException $e) { + return 0; + } + } + + /** + * Returns the maximum of server capable upload size and configured maximum chunk size + * + * @return int + */ + public function getMaximumUploadChunkSize(): int + { + try { + return min( + (int)(Files::sizeStringToBytes($this->configuration['maximumUploadChunkSize']) ?? '5MB'), + (int)Files::sizeStringToBytes(ini_get('post_max_size')), + (int)Files::sizeStringToBytes(ini_get('upload_max_filesize')) + ); + } catch (FilesException $e) { + return 5 * 1024 * 1024; + } + } + + /** + * Returns the maximum number of files that can be uploaded + * + * @return int + */ + public function getMaximumUploadFileCount(): int + { + return (int)($this->configuration['maximumUploadFileCount'] ?? 10); + } +} diff --git a/Classes/Service/UsageDetailsService.php b/Classes/Service/UsageDetailsService.php index 83fbed4aa..c57499692 100644 --- a/Classes/Service/UsageDetailsService.php +++ b/Classes/Service/UsageDetailsService.php @@ -15,6 +15,8 @@ */ use Doctrine\ORM\EntityManagerInterface; +use Doctrine\ORM\NonUniqueResultException; +use Doctrine\ORM\NoResultException; use Flowpack\Media\Ui\Domain\Model\Dto\AssetUsageDetails; use Flowpack\Media\Ui\Exception; use GuzzleHttp\Psr7\ServerRequest; @@ -353,6 +355,8 @@ protected function getAssetVariantFilterClause(string $alias): string * Returns number of assets which have no usage reference provided by `Flowpack.EntityUsage` * * @throws Exception + * @throws NoResultException + * @throws NonUniqueResultException */ public function getUnusedAssetCount(): int { diff --git a/Classes/Tus/PartialUploadFileCacheAdapter.php b/Classes/Tus/PartialUploadFileCacheAdapter.php new file mode 100644 index 000000000..4d530e690 --- /dev/null +++ b/Classes/Tus/PartialUploadFileCacheAdapter.php @@ -0,0 +1,108 @@ +partialUploadFileCache->get($key); + if (!is_string($contents)) { + return null; + } + + $contents = json_decode($contents, true, 512, JSON_THROW_ON_ERROR); + + if ($withExpired) { + return $contents; + } + + if (!$contents) { + return null; + } + + $isExpired = Carbon::parse($contents['expires_at'])->lt(Carbon::now()); + + return $isExpired ? null : $contents; + } + + public function set(string $key, $value) + { + $contents = $this->get($key) ?? []; + + if (\is_array($value)) { + $contents = $value + $contents; + } else { + $contents[] = $value; + } + + $this->partialUploadFileCache->set($this->getPrefix() . $key, json_encode($contents)); + + return true; + } + + public function delete(string $key): bool + { + return $this->partialUploadFileCache->remove($key); + } + + public function deleteAll(array $keys): bool + { + $this->partialUploadFileCache->flush(); + return true; + } + + /** + * @throws PropertyNotAccessibleException + */ + public function getTtl(): int + { + return 60*60*24; + return (int)ObjectAccess::getProperty($this->partialUploadFileCache->getBackend(), 'defaultLifetime', true); + } + + public function keys(): array + { + // @todo implement a replacement for keys() for flow cache backends + return []; + } + + public function setPrefix(string $prefix): Cacheable + { + return $this; + } + + public function getPrefix(): string + { + return ''; + } +} diff --git a/Classes/Tus/TusEventHandler.php b/Classes/Tus/TusEventHandler.php new file mode 100644 index 000000000..74fd06028 --- /dev/null +++ b/Classes/Tus/TusEventHandler.php @@ -0,0 +1,80 @@ +resourceManager->importResource($event->getFile()->getFilePath()); + $event->getFile()->deleteFiles([$event->getFile()->getFilePath()]); + + if ($this->assetRepository->findOneByResourceSha1($persistentResource->getSha1())) { + $this->logger->info(sprintf('The uploaded file "%s" (Sha1: %s) already existed in the resource management.', $persistentResource->getFilename(), $persistentResource->getSha1()), LogEnvironment::fromMethodName(__METHOD__)); + return; + } + + $targetType = $this->assetModelMappingStrategy->map($persistentResource); + $asset = new $targetType($persistentResource); + $this->assetService->getRepository($asset)->add($asset); + + $this->logger->info(sprintf('The uploaded file "%s" (Sha1: %s) was successfully imported.', $persistentResource->getFilename(), $persistentResource->getSha1()), LogEnvironment::fromMethodName(__METHOD__)); + } +} diff --git a/Configuration/Caches.yaml b/Configuration/Caches.yaml index cba1ba896..9d0938b31 100644 --- a/Configuration/Caches.yaml +++ b/Configuration/Caches.yaml @@ -1,3 +1,7 @@ Flowpack_Media_Ui_PollingCache: frontend: Neos\Cache\Frontend\StringFrontend backend: Neos\Cache\Backend\FileBackend + +Flowpack_Media_Ui_TusCache: + frontend: Neos\Cache\Frontend\StringFrontend + backend: Neos\Cache\Backend\FileBackend diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index 678767d78..8035b25e2 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -10,3 +10,13 @@ 2: # cache lifetime value: 10 + +'Flowpack\Media\Ui\Tus\PartialUploadFileCacheAdapter': + properties: + partialUploadFileCache: + object: + factoryObjectName: Neos\Flow\Cache\CacheManager + factoryMethodName: getCache + arguments: + 1: + value: Flowpack_Media_Ui_TusCache diff --git a/Configuration/Routes.yaml b/Configuration/Routes.yaml index ce99b0e77..b4aa20958 100644 --- a/Configuration/Routes.yaml +++ b/Configuration/Routes.yaml @@ -5,3 +5,12 @@ package: 't3n.GraphQL' variables: 'endpoint': 'media-assets' + +- name: 'Media Ui Upload' + uriPattern: 'neos/management/mediaui/upload(/{resourceIdentifier})' + defaults: + '@package': 'Flowpack.Media.Ui' + '@controller': 'Media' + '@format': 'json' + '@action': 'upload' + 'resourceIdentifier': '' diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 6f687b629..63d1052b6 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -1,4 +1,6 @@ Flowpack: Media: Ui: - maximumFileUploadLimit: 10 + maximumUploadFileCount: 10 + maximumUploadFileSize: 100MB + maximumUploadChunkSize: 5MB diff --git a/Resources/Private/GraphQL/schema.root.graphql b/Resources/Private/GraphQL/schema.root.graphql index dd6d9cd13..909b33970 100644 --- a/Resources/Private/GraphQL/schema.root.graphql +++ b/Resources/Private/GraphQL/schema.root.graphql @@ -136,8 +136,9 @@ type Mutation { Configuration object containing helpful parameters for API interaction """ type Config { - uploadMaxFileSize: FileSize! - uploadMaxFileUploadLimit: Int! + maximumUploadFileSize: FileSize! + maximumUploadChunkSize: FileSize! + maximumUploadFileCount: Int! currentServerTime: DateTime! } diff --git a/Resources/Private/JavaScript/core/package.json b/Resources/Private/JavaScript/core/package.json index ffc7e2070..4235e69f7 100644 --- a/Resources/Private/JavaScript/core/package.json +++ b/Resources/Private/JavaScript/core/package.json @@ -4,7 +4,8 @@ "license": "GNU GPLv3", "private": true, "dependencies": { - "pubsub-js": "^1.9.3" + "pubsub-js": "^1.9.3", + "tus-js-client": "^2.3.0" }, "devDependencies": { "@types/pubsub-js": "^1.8.2" diff --git a/Resources/Private/JavaScript/core/src/hooks/useConfigQuery.ts b/Resources/Private/JavaScript/core/src/hooks/useConfigQuery.ts index ef96a7e71..13f129c54 100644 --- a/Resources/Private/JavaScript/core/src/hooks/useConfigQuery.ts +++ b/Resources/Private/JavaScript/core/src/hooks/useConfigQuery.ts @@ -3,8 +3,9 @@ import { CONFIG } from '../queries'; interface ConfigQueryResult { config: { - uploadMaxFileSize: number; - uploadMaxFileUploadLimit: number; + maximumUploadFileSize: number; + maximumUploadChunkSize: number; + maximumUploadFileCount: number; currentServerTime: Date; }; } diff --git a/Resources/Private/JavaScript/core/src/hooks/useUploadFiles.ts b/Resources/Private/JavaScript/core/src/hooks/useUploadFiles.ts index a8bb770e9..46f3b2b5f 100644 --- a/Resources/Private/JavaScript/core/src/hooks/useUploadFiles.ts +++ b/Resources/Private/JavaScript/core/src/hooks/useUploadFiles.ts @@ -1,17 +1,91 @@ -import { useMutation } from '@apollo/client'; +import * as tus from 'tus-js-client'; +import { UploadOptions } from 'tus-js-client'; +import { useIntl, useMediaUi, useNotify } from '@media-ui/core/src'; +import { useConfigQuery } from '@media-ui/core/src/hooks'; +import { useState } from 'react'; -import { UPLOAD_FILES } from '../mutations'; -import { FileUploadResult } from '../interfaces'; +export interface FileUploadState { + fileName: string; + uploadPercentage: string; +} export default function useUploadFiles() { - const [action, { error, data, loading }] = useMutation<{ uploadFiles: FileUploadResult[] }>(UPLOAD_FILES); + const { config } = useConfigQuery(); + const { refetchAssets } = useMediaUi(); + const Notify = useNotify(); + const { translate } = useIntl(); + const [uploadState, setUploadState] = useState([] as FileUploadState[]); + const [loading, setLoading] = useState(false); + let numberOfFiles = 0; + + const options: UploadOptions = { + endpoint: `${window.location.protocol}//${window.location.host}/neos/management/mediaui/upload`, + chunkSize: config.maximumUploadChunkSize, + onError: (error) => { + numberOfFiles--; + if (numberOfFiles === 0) { + setLoading(false); + } + Notify.error(translate('fileUpload.error', 'Upload failed'), error.message); + }, + onSuccess: () => { + numberOfFiles--; + if (numberOfFiles === 0) { + Notify.ok(translate('uploadDialog.uploadFinished', 'Upload finished')); + setLoading(false); + } + refetchAssets(); + }, + removeFingerprintOnSuccess: true, + }; + + const uploadFiles = (files: File[]) => { + numberOfFiles = files.length; + files.forEach((file) => { + options.metadata = { + filename: file.name, + filetype: file.type, + }; + + options.onProgress = (bytesUploaded, bytesTotal) => { + setLoading(true); + const percentage = ((bytesUploaded / bytesTotal) * 100).toFixed(0); + setUploadState((prevState) => { + let newState: FileUploadState[] = []; + if (prevState.length > 0) { + newState = [...prevState]; + if (newState.find((uploadState) => uploadState.fileName === file.name)) { + newState.find( + (uploadState) => uploadState.fileName === file.name + ).uploadPercentage = percentage; + } else { + newState.push({ + fileName: file.name, + uploadPercentage: percentage, + }); + } + } else { + newState.push({ + fileName: file.name, + uploadPercentage: percentage, + }); + } + return newState; + }); + }; + + const upload = new tus.Upload(file, options); - const uploadFiles = (files: File[]) => - action({ - variables: { - files, - }, + upload.findPreviousUploads().then((previousUploads) => { + if (previousUploads.length) { + previousUploads.forEach((previousUpload) => { + upload.resumeFromPreviousUpload(previousUpload); + }); + } + upload.start(); + }); }); + }; - return { uploadFiles, uploadState: data?.uploadFiles || [], error, loading }; + return { uploadFiles, uploadState, loading }; } diff --git a/Resources/Private/JavaScript/core/src/queries/config.ts b/Resources/Private/JavaScript/core/src/queries/config.ts index f1b4bb412..4b7a973f4 100644 --- a/Resources/Private/JavaScript/core/src/queries/config.ts +++ b/Resources/Private/JavaScript/core/src/queries/config.ts @@ -3,8 +3,9 @@ import { gql } from '@apollo/client'; const CONFIG = gql` query CONFIG { config { - uploadMaxFileSize - uploadMaxFileUploadLimit + maximumUploadFileSize + maximumUploadChunkSize + maximumUploadFileCount currentServerTime } } diff --git a/Resources/Private/JavaScript/media-module/src/components/Dialogs/UploadDialog.tsx b/Resources/Private/JavaScript/media-module/src/components/Dialogs/UploadDialog.tsx index c29e8dac4..55c4048e7 100644 --- a/Resources/Private/JavaScript/media-module/src/components/Dialogs/UploadDialog.tsx +++ b/Resources/Private/JavaScript/media-module/src/components/Dialogs/UploadDialog.tsx @@ -71,6 +71,9 @@ const useStyles = createUseMediaUiStyles((theme: MediaUiTheme) => ({ marginLeft: theme.spacing.half, userSelect: 'none', }, + '& $progressBarInner': { + marginLeft: '0', + }, }, img: { position: 'absolute', @@ -113,9 +116,18 @@ const useStyles = createUseMediaUiStyles((theme: MediaUiTheme) => ({ backgroundColor: theme.colors.error, }, }, + progress: { + borderColor: theme.colors.warn, + }, warning: { color: theme.colors.warn, }, + progressBarInner: { + extend: 'stateOverlay', + backgroundColor: theme.colors.warn, + width: '0', + height: '100%', + }, })); interface UploadedFile extends File { @@ -130,7 +142,6 @@ const UploadDialog: React.FC = () => { const { dummyImage } = useMediaUi(); const { config } = useConfigQuery(); const { uploadFiles, uploadState, loading } = useUploadFiles(); - const { refetchAssets } = useMediaUi(); const [files, setFiles] = useState([]); const uploadPossible = !loading && files.length > 0; @@ -156,8 +167,8 @@ const UploadDialog: React.FC = () => { // TODO: Show rejection reason to user Notify.warning(translate('uploadDialog.warning.fileRejected', 'The given file cannot be uploaded.')); }, - maxSize: config?.uploadMaxFileSize || 0, - maxFiles: config?.uploadMaxFileUploadLimit || 1, + maxSize: config.maximumUploadFileSize, + maxFiles: config.maximumUploadFileCount, multiple: true, preventDropOnDocument: true, }); @@ -169,15 +180,8 @@ const UploadDialog: React.FC = () => { }, [files]); const handleUpload = useCallback(() => { - uploadFiles(files) - .then(() => { - Notify.ok(translate('uploadDialog.uploadFinished', 'Upload finished')); - refetchAssets(); - }) - .catch((error) => { - Notify.error(translate('fileUpload.error', 'Upload failed'), error); - }); - }, [uploadFiles, Notify, translate, files, refetchAssets]); + uploadFiles(files); + }, [uploadFiles, files]); const handleRequestClose = useCallback(() => { setFiles([]); @@ -216,20 +220,19 @@ const UploadDialog: React.FC = () => { "Drag 'n' drop some files here, or click to select files" )}

- {config?.uploadMaxFileSize > 0 && ( + {config.maximumUploadFileSize > 0 && (

{translate( 'uploadDialog.maxFileSize', 'Maximum file size is {size} and file limit is {limit}', { - size: humanFileSize(config.uploadMaxFileSize), - limit: config.uploadMaxFileUploadLimit, + size: humanFileSize(config.maximumUploadFileSize), + limit: config.maximumUploadFileCount, } )}

)} - {loading &&

Uploading...

} {files.length > 0 && (