From dfadac0837b057fb119195de9d135f52807fc7e1 Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Mon, 18 Dec 2023 14:25:20 +0800 Subject: [PATCH 1/6] Add job queue system to generate books Use the Symfony Messenger component to pass ebook generation requests from the web front-end to a server-processed queue. Bug: T345406 --- .env | 2 + composer.json | 1 + composer.lock | 301 +++++++++++++++++- config/packages/messenger.yaml | 11 + src/Controller/ExportController.php | 64 ++-- src/Message/CreateBookMessage.php | 35 ++ .../CreateBookMessageHandler.php | 48 +++ 7 files changed, 440 insertions(+), 22 deletions(-) create mode 100644 config/packages/messenger.yaml create mode 100644 src/Message/CreateBookMessage.php create mode 100644 src/MessageHandler/CreateBookMessageHandler.php diff --git a/.env b/.env index 1c2b94fd..4efa4d6d 100644 --- a/.env +++ b/.env @@ -28,6 +28,8 @@ APP_SECRET=2a025c1d3afd0715b3b86562f327b7a0 DATABASE_URL=mysql://db_user:db_password@127.0.0.1:3306/db_name?serverVersion=5.7 ###< doctrine/doctrine-bundle ### +MESSENGER_TRANSPORT_DSN=doctrine://wsexport + ###> symfony/mailer ### MAILER_DSN=smtp://mail.tools.wmflabs.org:25 ###< symfony/mailer ### diff --git a/composer.json b/composer.json index 5134202e..1d232a30 100644 --- a/composer.json +++ b/composer.json @@ -39,6 +39,7 @@ "symfony/dotenv": "5.4.*", "symfony/framework-bundle": "5.4.*", "symfony/mailer": "5.4.*", + "symfony/messenger": "^5.4", "symfony/monolog-bundle": "^3.6", "symfony/process": "5.4.*", "symfony/stopwatch": "5.4.*", diff --git a/composer.lock b/composer.lock index 007fee26..a36d16c0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2a350a61926e9ff3ccce9bc75cfe7c1c", + "content-hash": "ff1b9b3ac022fb31815ae509b65b365d", "packages": [ { "name": "composer/package-versions-deprecated", @@ -2826,6 +2826,75 @@ }, "time": "2019-03-08T08:55:37+00:00" }, + { + "name": "symfony/amqp-messenger", + "version": "v5.4.31", + "source": { + "type": "git", + "url": "https://github.com/symfony/amqp-messenger.git", + "reference": "8ba6a2c482d3fce9d450b702098ca033bbe42de4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/amqp-messenger/zipball/8ba6a2c482d3fce9d450b702098ca033bbe42de4", + "reference": "8ba6a2c482d3fce9d450b702098ca033bbe42de4", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/messenger": "^5.3|^6.0" + }, + "require-dev": { + "symfony/event-dispatcher": "^4.4|^5.0|^6.0", + "symfony/process": "^4.4|^5.0|^6.0", + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/serializer": "^4.4|^5.0|^6.0" + }, + "type": "symfony-messenger-bridge", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\Bridge\\Amqp\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony AMQP extension Messenger Bridge", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/amqp-messenger/tree/v5.4.31" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-03T16:16:43+00:00" + }, { "name": "symfony/cache", "version": "v5.4.32", @@ -3452,6 +3521,79 @@ ], "time": "2023-10-31T07:58:33+00:00" }, + { + "name": "symfony/doctrine-messenger", + "version": "v5.4.33", + "source": { + "type": "git", + "url": "https://github.com/symfony/doctrine-messenger.git", + "reference": "3a5ef7be3f21ba72f0c498280dd6b022cb6f33b5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/doctrine-messenger/zipball/3a5ef7be3f21ba72f0c498280dd6b022cb6f33b5", + "reference": "3a5ef7be3f21ba72f0c498280dd6b022cb6f33b5", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/messenger": "^5.1|^6.0", + "symfony/service-contracts": "^1.1|^2|^3" + }, + "conflict": { + "doctrine/dbal": "<2.13", + "doctrine/persistence": "<1.3" + }, + "require-dev": { + "doctrine/dbal": "^2.13|^3|^4", + "doctrine/persistence": "^1.3|^2|^3", + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/serializer": "^4.4|^5.0|^6.0" + }, + "type": "symfony-messenger-bridge", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\Bridge\\Doctrine\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Doctrine Messenger Bridge", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/doctrine-messenger/tree/v5.4.33" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-30T11:57:55+00:00" + }, { "name": "symfony/dotenv", "version": "v5.4.30", @@ -4468,6 +4610,96 @@ ], "time": "2023-11-03T16:16:43+00:00" }, + { + "name": "symfony/messenger", + "version": "v5.4.31", + "source": { + "type": "git", + "url": "https://github.com/symfony/messenger.git", + "reference": "8f74256d181141d83649e9bee5caf34328feb3c8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/messenger/zipball/8f74256d181141d83649e9bee5caf34328feb3c8", + "reference": "8f74256d181141d83649e9bee5caf34328feb3c8", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "psr/log": "^1|^2|^3", + "symfony/amqp-messenger": "^5.1|^6.0", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/doctrine-messenger": "^5.1|^6.0", + "symfony/polyfill-php80": "^1.16", + "symfony/redis-messenger": "^5.1|^6.0" + }, + "conflict": { + "symfony/event-dispatcher": "<4.4", + "symfony/framework-bundle": "<4.4", + "symfony/http-kernel": "<4.4", + "symfony/serializer": "<5.0" + }, + "require-dev": { + "psr/cache": "^1.0|^2.0|^3.0", + "symfony/console": "^5.4|^6.0", + "symfony/dependency-injection": "^5.3|^6.0", + "symfony/event-dispatcher": "^4.4|^5.0|^6.0", + "symfony/http-kernel": "^4.4|^5.0|^6.0", + "symfony/process": "^4.4|^5.0|^6.0", + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/routing": "^4.4|^5.0|^6.0", + "symfony/serializer": "^5.0|^6.0", + "symfony/service-contracts": "^1.1|^2|^3", + "symfony/stopwatch": "^4.4|^5.0|^6.0", + "symfony/validator": "^4.4|^5.0|^6.0" + }, + "suggest": { + "enqueue/messenger-adapter": "For using the php-enqueue library as a transport." + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Samuel Roze", + "email": "samuel.roze@gmail.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Helps applications send and receive messages to/from other applications or via message queues", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/messenger/tree/v5.4.31" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-03T16:16:43+00:00" + }, { "name": "symfony/mime", "version": "v5.4.26", @@ -5355,6 +5587,73 @@ ], "time": "2023-08-07T10:36:04+00:00" }, + { + "name": "symfony/redis-messenger", + "version": "v5.4.31", + "source": { + "type": "git", + "url": "https://github.com/symfony/redis-messenger.git", + "reference": "9735a30ac8f37b42f1714bc1634dc8066b34e72f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/redis-messenger/zipball/9735a30ac8f37b42f1714bc1634dc8066b34e72f", + "reference": "9735a30ac8f37b42f1714bc1634dc8066b34e72f", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/messenger": "^5.1|^6.0" + }, + "require-dev": { + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/serializer": "^4.4|^5.0|^6.0" + }, + "type": "symfony-messenger-bridge", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\Bridge\\Redis\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Redis extension Messenger Bridge", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/redis-messenger/tree/v5.4.31" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-03T16:16:43+00:00" + }, { "name": "symfony/routing", "version": "v5.4.33", diff --git a/config/packages/messenger.yaml b/config/packages/messenger.yaml new file mode 100644 index 00000000..d0d8c29e --- /dev/null +++ b/config/packages/messenger.yaml @@ -0,0 +1,11 @@ +framework: + messenger: + + transports: + async: + dsn: "%env(MESSENGER_TRANSPORT_DSN)%" + retry_strategy: + max_retries: 0 + + routing: + 'App\Message\CreateBookMessage': async diff --git a/src/Controller/ExportController.php b/src/Controller/ExportController.php index e7d9e30c..bb69459c 100644 --- a/src/Controller/ExportController.php +++ b/src/Controller/ExportController.php @@ -2,12 +2,14 @@ namespace App\Controller; +use App\Book; use App\BookCreator; use App\Entity\GeneratedBook; use App\Exception\WsExportException; use App\FileCache; use App\FontProvider; use App\GeneratorSelector; +use App\Message\CreateBookMessage; use App\Refresh; use App\Repository\CreditRepository; use App\Util\Api; @@ -23,6 +25,7 @@ use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\ResponseHeaderBag; +use Symfony\Component\Messenger\MessageBusInterface; // phpcs:ignore use Symfony\Component\Routing\Annotation\Route; use Symfony\Component\Stopwatch\Stopwatch; @@ -79,6 +82,7 @@ public function refresh( Request $request, Api $api, CacheItemPoolInterface $cac */ public function home( Request $request, + MessageBusInterface $bus, Api $api, FontProvider $fontProvider, GeneratorSelector $generatorSelector, @@ -102,7 +106,7 @@ public function home( $response = new Response(); if ( $request->query->get( 'page' ) ) { try { - return $this->export( $request, $api, $fontProvider, $generatorSelector, $creditRepo, $fileCache ); + return $this->export( $request, $bus, $api, $fontProvider, $generatorSelector, $creditRepo, $fileCache ); } catch ( WsExportException $ex ) { $exception = $ex; $response->setStatusCode( $ex->getResponseCode() ); @@ -131,6 +135,7 @@ public function home( private function export( Request $request, + MessageBusInterface $bus, Api $api, FontProvider $fontProvider, GeneratorSelector $generatorSelector, @@ -138,7 +143,7 @@ private function export( FileCache $fileCache ) { // Get params. - $page = $request->query->get( 'page' ); + $page = $this->getTitle( $request ); $format = $this->getFormat( $request ); $font = $this->getFont( $request, $fontProvider ); // The `credits` checkbox submits as 'false' to disable, so needs extra filtering. @@ -152,31 +157,48 @@ private function export( } // Generate ebook. - $options = [ 'images' => $images, 'fonts' => $font, 'credits' => $credits ]; + + $options = [ 'images' => $images, 'fonts' => $font, 'credits' => $credits, 'categories' => false ]; $creator = BookCreator::forApi( $api, $format, $options, $generatorSelector, $creditRepo, $fileCache ); - $creator->create( $page ); - // Send file. - $response = new BinaryFileResponse( $creator->getFilePath() ); - $response->headers->set( 'X-Robots-Tag', 'none' ); - $response->headers->set( 'Content-Description', 'File Transfer' ); - $response->headers->set( 'Content-Type', $creator->getMimeType() ); - $response->setContentDisposition( ResponseHeaderBag::DISPOSITION_ATTACHMENT, $creator->getFilename() ); - $response->deleteFileAfterSend(); + $book = new Book(); + $book->lang = $api->getLang(); + $book->title = $page; + $book->options = $options; - // Log book generation. - if ( $this->enableStats ) { - try { - $genBook = new GeneratedBook( $creator->getBook(), $format, $this->stopwatch->stop( 'generate-book' ) ); - $this->entityManager->persist( $genBook ); - $this->entityManager->flush(); - } catch ( DriverException $e ) { - // There was an error writing to tools-db. - // Silently ignore as this shouldn't prevent the book from being downloaded. + $filePathHash = md5( serialize( $book ) . $format ); + $filePath = $fileCache->getDirectory() . '/ebook_' . $filePathHash . '.' . $creator->getExtension(); + + $bus->dispatch( new CreateBookMessage( $book, $filePath, $format ) ); + + for ( $i = 0; $i < 30; $i++ ) { + if ( file_exists( $filePath ) ) { + // Send file. + $response = new BinaryFileResponse( $filePath ); + $response->headers->set( 'X-Robots-Tag', 'none' ); + $response->headers->set( 'Content-Description', 'File Transfer' ); + $response->headers->set( 'Content-Type', $creator->getMimeType() ); + $filename = str_replace( [ '/', '\\' ], '_', $page ) . '.' . $creator->getExtension(); + $response->setContentDisposition( ResponseHeaderBag::DISPOSITION_ATTACHMENT, $filename ); + $response->deleteFileAfterSend(); + + // Log book generation. + if ( $this->enableStats ) { + try { + $genBook = new GeneratedBook( $book, $format, $this->stopwatch->stop( 'generate-book' ) ); + $this->entityManager->persist( $genBook ); + $this->entityManager->flush(); + } catch ( DriverException $e ) { + // There was an error writing to tools-db. + // Silently ignore as this shouldn't prevent the book from being downloaded. + } + } + return $response; } + sleep( 1 ); } - return $response; + throw new WsExportException( 'error-no-file-generated', [], 500 ); } /** diff --git a/src/Message/CreateBookMessage.php b/src/Message/CreateBookMessage.php new file mode 100644 index 00000000..f2c6856e --- /dev/null +++ b/src/Message/CreateBookMessage.php @@ -0,0 +1,35 @@ +book = $book; + $this->filePath = $filePath; + $this->format = $format; + } + + public function getBook(): Book { + return $this->book; + } + + public function getFilePath(): string { + return $this->filePath; + } + + public function getFormat(): string { + return $this->format; + } +} diff --git a/src/MessageHandler/CreateBookMessageHandler.php b/src/MessageHandler/CreateBookMessageHandler.php new file mode 100644 index 00000000..2cf0fbf4 --- /dev/null +++ b/src/MessageHandler/CreateBookMessageHandler.php @@ -0,0 +1,48 @@ +api = $api; + $this->generatorSelector = $generatorSelector; + $this->creditRepo = $creditRepo; + $this->fileCache = $fileCache; + } + + public function __invoke( CreateBookMessage $message ) { + if ( file_exists( $message->getFilePath() ) ) { + return; + } + $this->api->setLang( $message->getBook()->lang ); + $creator = BookCreator::forApi( $this->api, $message->getFormat(), $message->getBook()->options, $this->generatorSelector, $this->creditRepo, $this->fileCache ); + $creator->create( $message->getBook()->title ); + rename( $creator->getFilePath(), $message->getFilePath() ); + } +} From 5ea4a16661eb9fdf6130c7bbd9725fd473b8967e Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Mon, 18 Dec 2023 17:50:01 +0800 Subject: [PATCH 2/6] Add setup-transports to CI --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1fd4bf5..91e58149 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: run: | composer install ./bin/console doctrine:migrations:migrate --no-interaction + ./bin/console messenger:setup-transports npm ci - name: Test From acefbf30ebeb5295d3bc644c9dba67ec2c099aad Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Tue, 19 Dec 2023 13:41:31 +0800 Subject: [PATCH 3/6] Add expiry to message, etc. --- config/packages/messenger.yaml | 3 +++ src/Controller/ExportController.php | 18 +++++++++++------- src/Message/CreateBookMessage.php | 10 +++++++++- .../CreateBookMessageHandler.php | 4 ++++ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/config/packages/messenger.yaml b/config/packages/messenger.yaml index d0d8c29e..3cfc9cb2 100644 --- a/config/packages/messenger.yaml +++ b/config/packages/messenger.yaml @@ -1,11 +1,14 @@ framework: messenger: + failure_transport: failed + transports: async: dsn: "%env(MESSENGER_TRANSPORT_DSN)%" retry_strategy: max_retries: 0 + failed: "%env(MESSENGER_TRANSPORT_DSN)%?queue_name=failed" routing: 'App\Message\CreateBookMessage': async diff --git a/src/Controller/ExportController.php b/src/Controller/ExportController.php index bb69459c..51760ec6 100644 --- a/src/Controller/ExportController.php +++ b/src/Controller/ExportController.php @@ -151,29 +151,32 @@ private function export( // The `images` checkbox submits as 'false' to disable, so needs extra filtering. $images = filter_var( $request->query->get( 'images', true ), FILTER_VALIDATE_BOOL ); + // Fetch the first page to check that it exists. If it does, it'll be cached and this won't be done again. + $api->getPageAsync( $page )->wait(); + // Start timing. if ( $this->enableStats ) { $this->stopwatch->start( 'generate-book' ); } - // Generate ebook. - + // Set up ebook details. $options = [ 'images' => $images, 'fonts' => $font, 'credits' => $credits, 'categories' => false ]; $creator = BookCreator::forApi( $api, $format, $options, $generatorSelector, $creditRepo, $fileCache ); - $book = new Book(); $book->lang = $api->getLang(); $book->title = $page; $book->options = $options; + // Dispatch a message to the job queue with the details of the book to generate and where to save the output. $filePathHash = md5( serialize( $book ) . $format ); - $filePath = $fileCache->getDirectory() . '/ebook_' . $filePathHash . '.' . $creator->getExtension(); - - $bus->dispatch( new CreateBookMessage( $book, $filePath, $format ) ); + $filePath = $fileCache->getDirectory() . '/ebook_' . $filePathHash . '_' . uniqid() . '.' . $creator->getExtension(); + // @todo Don't hardcode the expiry here. + $bus->dispatch( new CreateBookMessage( $book, $filePath, $format, time() + 30 ) ); + // Loop while checking for the existence of the generated output file. for ( $i = 0; $i < 30; $i++ ) { if ( file_exists( $filePath ) ) { - // Send file. + // The file has been generated now, so send it. $response = new BinaryFileResponse( $filePath ); $response->headers->set( 'X-Robots-Tag', 'none' ); $response->headers->set( 'Content-Description', 'File Transfer' ); @@ -198,6 +201,7 @@ private function export( sleep( 1 ); } + // If the file wasn't generated by the job queue, tell the user. throw new WsExportException( 'error-no-file-generated', [], 500 ); } diff --git a/src/Message/CreateBookMessage.php b/src/Message/CreateBookMessage.php index f2c6856e..9d889b1b 100644 --- a/src/Message/CreateBookMessage.php +++ b/src/Message/CreateBookMessage.php @@ -15,10 +15,14 @@ final class CreateBookMessage { /** @var string */ private $format; - public function __construct( Book $book, string $filePath, string $format ) { + /** @var int Timestamp that this message should expire. */ + private $expiry; + + public function __construct( Book $book, string $filePath, string $format, int $expiry ) { $this->book = $book; $this->filePath = $filePath; $this->format = $format; + $this->expiry = $expiry; } public function getBook(): Book { @@ -32,4 +36,8 @@ public function getFilePath(): string { public function getFormat(): string { return $this->format; } + + public function getExpiry(): int { + return $this->expiry; + } } diff --git a/src/MessageHandler/CreateBookMessageHandler.php b/src/MessageHandler/CreateBookMessageHandler.php index 2cf0fbf4..ac8e0251 100644 --- a/src/MessageHandler/CreateBookMessageHandler.php +++ b/src/MessageHandler/CreateBookMessageHandler.php @@ -8,6 +8,7 @@ use App\Message\CreateBookMessage; use App\Repository\CreditRepository; use App\Util\Api; +use Exception; use Symfony\Component\Messenger\Handler\MessageHandlerInterface; final class CreateBookMessageHandler implements MessageHandlerInterface { @@ -37,6 +38,9 @@ public function __construct( } public function __invoke( CreateBookMessage $message ) { + if ( $message->getExpiry() < time() ) { + throw new Exception( 'createbook-message-expired' ); + } if ( file_exists( $message->getFilePath() ) ) { return; } From b0ab9007ffa50c869872565e1a9c83dab3f677ea Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Tue, 19 Dec 2023 15:19:21 +0800 Subject: [PATCH 4/6] Fix tests --- i18n/en.json | 1 + i18n/qqq.json | 1 + src/Controller/ExportController.php | 15 +++++++++------ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 3aa89bb9..8f263aeb 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -70,6 +70,7 @@ "onwikiconfig-failure": "Unable to retrieve the on-wiki configuration from: $1", "exception-fetching-credits": "Error fetching credits information. Please try exporting again. If this error persists, check the box in 'Options' to remove credits.", "exception-book-conversion": "Error converting the format of the book.", + "exception-no-file-generated": "The book was not generated within $1 seconds.", "epub-title-page": "Title page", "epub-exported-date": "Exported from Wikisource on $1", "epub-about": "About" diff --git a/i18n/qqq.json b/i18n/qqq.json index 464c8f04..3e15172f 100644 --- a/i18n/qqq.json +++ b/i18n/qqq.json @@ -76,6 +76,7 @@ "onwikiconfig-failure": "Error message displayed if the on-wiki configuration fetching failed.\n\n* $1 - the URL of the config JSON page on Wikisource.", "exception-fetching-credits": "Error message displayed when credits information could not be retrieved.", "exception-book-conversion": "Error message displayed when a book could not be converted to another format.", + "exception-no-file-generated": "Error message displayed if the job queue takes too long to generate a book.\n\nParameters:\n\n* $1 — integer number of seconds that it took to not generate the book.", "epub-title-page": "Name of the title page in exported books (used in the table of contents).", "epub-exported-date": "Phrase used on the title page to indicate the book export date.\n\n$1 - the localized date.", "epub-about": "Name of the 'about' page in exported books (used in the table of contents)." diff --git a/src/Controller/ExportController.php b/src/Controller/ExportController.php index 51760ec6..8e74058c 100644 --- a/src/Controller/ExportController.php +++ b/src/Controller/ExportController.php @@ -27,6 +27,7 @@ use Symfony\Component\HttpFoundation\ResponseHeaderBag; use Symfony\Component\Messenger\MessageBusInterface; // phpcs:ignore +use Symfony\Component\Messenger\Stamp\HandledStamp; use Symfony\Component\Routing\Annotation\Route; use Symfony\Component\Stopwatch\Stopwatch; @@ -143,7 +144,7 @@ private function export( FileCache $fileCache ) { // Get params. - $page = $this->getTitle( $request ); + $page = $request->query->get( 'page' ); $format = $this->getFormat( $request ); $font = $this->getFont( $request, $fontProvider ); // The `credits` checkbox submits as 'false' to disable, so needs extra filtering. @@ -151,9 +152,6 @@ private function export( // The `images` checkbox submits as 'false' to disable, so needs extra filtering. $images = filter_var( $request->query->get( 'images', true ), FILTER_VALIDATE_BOOL ); - // Fetch the first page to check that it exists. If it does, it'll be cached and this won't be done again. - $api->getPageAsync( $page )->wait(); - // Start timing. if ( $this->enableStats ) { $this->stopwatch->start( 'generate-book' ); @@ -167,11 +165,16 @@ private function export( $book->title = $page; $book->options = $options; + // Fetch the first page to check that it exists. + // If it does, it'll be cached and this won't be done again; if it doesn't, this will throw an exception. + $api->getPageAsync( $page )->wait(); + // Dispatch a message to the job queue with the details of the book to generate and where to save the output. $filePathHash = md5( serialize( $book ) . $format ); $filePath = $fileCache->getDirectory() . '/ebook_' . $filePathHash . '_' . uniqid() . '.' . $creator->getExtension(); // @todo Don't hardcode the expiry here. - $bus->dispatch( new CreateBookMessage( $book, $filePath, $format, time() + 30 ) ); + $ttl = 30; + $bus->dispatch( new CreateBookMessage( $book, $filePath, $format, time() + $ttl ) ); // Loop while checking for the existence of the generated output file. for ( $i = 0; $i < 30; $i++ ) { @@ -202,7 +205,7 @@ private function export( } // If the file wasn't generated by the job queue, tell the user. - throw new WsExportException( 'error-no-file-generated', [], 500 ); + throw new WsExportException( 'no-file-generated', [ $ttl ], 500 ); } /** From 26a4f4b8be39ffb06ae2c9909336915a913776b7 Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Tue, 19 Dec 2023 15:22:45 +0800 Subject: [PATCH 5/6] phpcs --- src/Controller/ExportController.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Controller/ExportController.php b/src/Controller/ExportController.php index 8e74058c..8716a49f 100644 --- a/src/Controller/ExportController.php +++ b/src/Controller/ExportController.php @@ -27,7 +27,6 @@ use Symfony\Component\HttpFoundation\ResponseHeaderBag; use Symfony\Component\Messenger\MessageBusInterface; // phpcs:ignore -use Symfony\Component\Messenger\Stamp\HandledStamp; use Symfony\Component\Routing\Annotation\Route; use Symfony\Component\Stopwatch\Stopwatch; From c038af5feb7445c5bf058c2ec8ee38d059d94f61 Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Tue, 19 Dec 2023 16:33:18 +0800 Subject: [PATCH 6/6] Add test config for Messenger --- config/packages/test/messenger.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 config/packages/test/messenger.yaml diff --git a/config/packages/test/messenger.yaml b/config/packages/test/messenger.yaml new file mode 100644 index 00000000..3fb77873 --- /dev/null +++ b/config/packages/test/messenger.yaml @@ -0,0 +1,7 @@ +framework: + messenger: + transports: + async: "sync://" + failed: "sync://" + # routing: + # 'App\Message\CreateBookMessage': async