diff --git a/.env b/.env index 1c2b94fd..4efa4d6d 100644 --- a/.env +++ b/.env @@ -28,6 +28,8 @@ APP_SECRET=2a025c1d3afd0715b3b86562f327b7a0 DATABASE_URL=mysql://db_user:db_password@127.0.0.1:3306/db_name?serverVersion=5.7 ###< doctrine/doctrine-bundle ### +MESSENGER_TRANSPORT_DSN=doctrine://wsexport + ###> symfony/mailer ### MAILER_DSN=smtp://mail.tools.wmflabs.org:25 ###< symfony/mailer ### diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1fd4bf5..91e58149 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: run: | composer install ./bin/console doctrine:migrations:migrate --no-interaction + ./bin/console messenger:setup-transports npm ci - name: Test diff --git a/composer.json b/composer.json index 5134202e..1d232a30 100644 --- a/composer.json +++ b/composer.json @@ -39,6 +39,7 @@ "symfony/dotenv": "5.4.*", "symfony/framework-bundle": "5.4.*", "symfony/mailer": "5.4.*", + "symfony/messenger": "^5.4", "symfony/monolog-bundle": "^3.6", "symfony/process": "5.4.*", "symfony/stopwatch": "5.4.*", diff --git a/composer.lock b/composer.lock index 007fee26..a36d16c0 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2a350a61926e9ff3ccce9bc75cfe7c1c", + "content-hash": "ff1b9b3ac022fb31815ae509b65b365d", "packages": [ { "name": "composer/package-versions-deprecated", @@ -2826,6 +2826,75 @@ }, "time": "2019-03-08T08:55:37+00:00" }, + { + "name": "symfony/amqp-messenger", + "version": "v5.4.31", + "source": { + "type": "git", + "url": "https://github.com/symfony/amqp-messenger.git", + "reference": "8ba6a2c482d3fce9d450b702098ca033bbe42de4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/amqp-messenger/zipball/8ba6a2c482d3fce9d450b702098ca033bbe42de4", + "reference": "8ba6a2c482d3fce9d450b702098ca033bbe42de4", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/messenger": "^5.3|^6.0" + }, + "require-dev": { + "symfony/event-dispatcher": "^4.4|^5.0|^6.0", + "symfony/process": "^4.4|^5.0|^6.0", + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/serializer": "^4.4|^5.0|^6.0" + }, + "type": "symfony-messenger-bridge", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\Bridge\\Amqp\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony AMQP extension Messenger Bridge", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/amqp-messenger/tree/v5.4.31" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-03T16:16:43+00:00" + }, { "name": "symfony/cache", "version": "v5.4.32", @@ -3452,6 +3521,79 @@ ], "time": "2023-10-31T07:58:33+00:00" }, + { + "name": "symfony/doctrine-messenger", + "version": "v5.4.33", + "source": { + "type": "git", + "url": "https://github.com/symfony/doctrine-messenger.git", + "reference": "3a5ef7be3f21ba72f0c498280dd6b022cb6f33b5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/doctrine-messenger/zipball/3a5ef7be3f21ba72f0c498280dd6b022cb6f33b5", + "reference": "3a5ef7be3f21ba72f0c498280dd6b022cb6f33b5", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/messenger": "^5.1|^6.0", + "symfony/service-contracts": "^1.1|^2|^3" + }, + "conflict": { + "doctrine/dbal": "<2.13", + "doctrine/persistence": "<1.3" + }, + "require-dev": { + "doctrine/dbal": "^2.13|^3|^4", + "doctrine/persistence": "^1.3|^2|^3", + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/serializer": "^4.4|^5.0|^6.0" + }, + "type": "symfony-messenger-bridge", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\Bridge\\Doctrine\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Doctrine Messenger Bridge", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/doctrine-messenger/tree/v5.4.33" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-30T11:57:55+00:00" + }, { "name": "symfony/dotenv", "version": "v5.4.30", @@ -4468,6 +4610,96 @@ ], "time": "2023-11-03T16:16:43+00:00" }, + { + "name": "symfony/messenger", + "version": "v5.4.31", + "source": { + "type": "git", + "url": "https://github.com/symfony/messenger.git", + "reference": "8f74256d181141d83649e9bee5caf34328feb3c8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/messenger/zipball/8f74256d181141d83649e9bee5caf34328feb3c8", + "reference": "8f74256d181141d83649e9bee5caf34328feb3c8", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "psr/log": "^1|^2|^3", + "symfony/amqp-messenger": "^5.1|^6.0", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/doctrine-messenger": "^5.1|^6.0", + "symfony/polyfill-php80": "^1.16", + "symfony/redis-messenger": "^5.1|^6.0" + }, + "conflict": { + "symfony/event-dispatcher": "<4.4", + "symfony/framework-bundle": "<4.4", + "symfony/http-kernel": "<4.4", + "symfony/serializer": "<5.0" + }, + "require-dev": { + "psr/cache": "^1.0|^2.0|^3.0", + "symfony/console": "^5.4|^6.0", + "symfony/dependency-injection": "^5.3|^6.0", + "symfony/event-dispatcher": "^4.4|^5.0|^6.0", + "symfony/http-kernel": "^4.4|^5.0|^6.0", + "symfony/process": "^4.4|^5.0|^6.0", + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/routing": "^4.4|^5.0|^6.0", + "symfony/serializer": "^5.0|^6.0", + "symfony/service-contracts": "^1.1|^2|^3", + "symfony/stopwatch": "^4.4|^5.0|^6.0", + "symfony/validator": "^4.4|^5.0|^6.0" + }, + "suggest": { + "enqueue/messenger-adapter": "For using the php-enqueue library as a transport." + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Samuel Roze", + "email": "samuel.roze@gmail.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Helps applications send and receive messages to/from other applications or via message queues", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/messenger/tree/v5.4.31" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-03T16:16:43+00:00" + }, { "name": "symfony/mime", "version": "v5.4.26", @@ -5355,6 +5587,73 @@ ], "time": "2023-08-07T10:36:04+00:00" }, + { + "name": "symfony/redis-messenger", + "version": "v5.4.31", + "source": { + "type": "git", + "url": "https://github.com/symfony/redis-messenger.git", + "reference": "9735a30ac8f37b42f1714bc1634dc8066b34e72f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/redis-messenger/zipball/9735a30ac8f37b42f1714bc1634dc8066b34e72f", + "reference": "9735a30ac8f37b42f1714bc1634dc8066b34e72f", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/messenger": "^5.1|^6.0" + }, + "require-dev": { + "symfony/property-access": "^4.4|^5.0|^6.0", + "symfony/serializer": "^4.4|^5.0|^6.0" + }, + "type": "symfony-messenger-bridge", + "autoload": { + "psr-4": { + "Symfony\\Component\\Messenger\\Bridge\\Redis\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony Redis extension Messenger Bridge", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/redis-messenger/tree/v5.4.31" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-11-03T16:16:43+00:00" + }, { "name": "symfony/routing", "version": "v5.4.33", diff --git a/config/packages/messenger.yaml b/config/packages/messenger.yaml new file mode 100644 index 00000000..3cfc9cb2 --- /dev/null +++ b/config/packages/messenger.yaml @@ -0,0 +1,14 @@ +framework: + messenger: + + failure_transport: failed + + transports: + async: + dsn: "%env(MESSENGER_TRANSPORT_DSN)%" + retry_strategy: + max_retries: 0 + failed: "%env(MESSENGER_TRANSPORT_DSN)%?queue_name=failed" + + routing: + 'App\Message\CreateBookMessage': async diff --git a/config/packages/test/messenger.yaml b/config/packages/test/messenger.yaml new file mode 100644 index 00000000..3fb77873 --- /dev/null +++ b/config/packages/test/messenger.yaml @@ -0,0 +1,7 @@ +framework: + messenger: + transports: + async: "sync://" + failed: "sync://" + # routing: + # 'App\Message\CreateBookMessage': async diff --git a/i18n/en.json b/i18n/en.json index 3aa89bb9..8f263aeb 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -70,6 +70,7 @@ "onwikiconfig-failure": "Unable to retrieve the on-wiki configuration from: $1", "exception-fetching-credits": "Error fetching credits information. Please try exporting again. If this error persists, check the box in 'Options' to remove credits.", "exception-book-conversion": "Error converting the format of the book.", + "exception-no-file-generated": "The book was not generated within $1 seconds.", "epub-title-page": "Title page", "epub-exported-date": "Exported from Wikisource on $1", "epub-about": "About" diff --git a/i18n/qqq.json b/i18n/qqq.json index 464c8f04..3e15172f 100644 --- a/i18n/qqq.json +++ b/i18n/qqq.json @@ -76,6 +76,7 @@ "onwikiconfig-failure": "Error message displayed if the on-wiki configuration fetching failed.\n\n* $1 - the URL of the config JSON page on Wikisource.", "exception-fetching-credits": "Error message displayed when credits information could not be retrieved.", "exception-book-conversion": "Error message displayed when a book could not be converted to another format.", + "exception-no-file-generated": "Error message displayed if the job queue takes too long to generate a book.\n\nParameters:\n\n* $1 — integer number of seconds that it took to not generate the book.", "epub-title-page": "Name of the title page in exported books (used in the table of contents).", "epub-exported-date": "Phrase used on the title page to indicate the book export date.\n\n$1 - the localized date.", "epub-about": "Name of the 'about' page in exported books (used in the table of contents)." diff --git a/src/Controller/ExportController.php b/src/Controller/ExportController.php index e7d9e30c..8716a49f 100644 --- a/src/Controller/ExportController.php +++ b/src/Controller/ExportController.php @@ -2,12 +2,14 @@ namespace App\Controller; +use App\Book; use App\BookCreator; use App\Entity\GeneratedBook; use App\Exception\WsExportException; use App\FileCache; use App\FontProvider; use App\GeneratorSelector; +use App\Message\CreateBookMessage; use App\Refresh; use App\Repository\CreditRepository; use App\Util\Api; @@ -23,6 +25,7 @@ use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\ResponseHeaderBag; +use Symfony\Component\Messenger\MessageBusInterface; // phpcs:ignore use Symfony\Component\Routing\Annotation\Route; use Symfony\Component\Stopwatch\Stopwatch; @@ -79,6 +82,7 @@ public function refresh( Request $request, Api $api, CacheItemPoolInterface $cac */ public function home( Request $request, + MessageBusInterface $bus, Api $api, FontProvider $fontProvider, GeneratorSelector $generatorSelector, @@ -102,7 +106,7 @@ public function home( $response = new Response(); if ( $request->query->get( 'page' ) ) { try { - return $this->export( $request, $api, $fontProvider, $generatorSelector, $creditRepo, $fileCache ); + return $this->export( $request, $bus, $api, $fontProvider, $generatorSelector, $creditRepo, $fileCache ); } catch ( WsExportException $ex ) { $exception = $ex; $response->setStatusCode( $ex->getResponseCode() ); @@ -131,6 +135,7 @@ public function home( private function export( Request $request, + MessageBusInterface $bus, Api $api, FontProvider $fontProvider, GeneratorSelector $generatorSelector, @@ -151,32 +156,55 @@ private function export( $this->stopwatch->start( 'generate-book' ); } - // Generate ebook. - $options = [ 'images' => $images, 'fonts' => $font, 'credits' => $credits ]; + // Set up ebook details. + $options = [ 'images' => $images, 'fonts' => $font, 'credits' => $credits, 'categories' => false ]; $creator = BookCreator::forApi( $api, $format, $options, $generatorSelector, $creditRepo, $fileCache ); - $creator->create( $page ); + $book = new Book(); + $book->lang = $api->getLang(); + $book->title = $page; + $book->options = $options; - // Send file. - $response = new BinaryFileResponse( $creator->getFilePath() ); - $response->headers->set( 'X-Robots-Tag', 'none' ); - $response->headers->set( 'Content-Description', 'File Transfer' ); - $response->headers->set( 'Content-Type', $creator->getMimeType() ); - $response->setContentDisposition( ResponseHeaderBag::DISPOSITION_ATTACHMENT, $creator->getFilename() ); - $response->deleteFileAfterSend(); + // Fetch the first page to check that it exists. + // If it does, it'll be cached and this won't be done again; if it doesn't, this will throw an exception. + $api->getPageAsync( $page )->wait(); - // Log book generation. - if ( $this->enableStats ) { - try { - $genBook = new GeneratedBook( $creator->getBook(), $format, $this->stopwatch->stop( 'generate-book' ) ); - $this->entityManager->persist( $genBook ); - $this->entityManager->flush(); - } catch ( DriverException $e ) { - // There was an error writing to tools-db. - // Silently ignore as this shouldn't prevent the book from being downloaded. + // Dispatch a message to the job queue with the details of the book to generate and where to save the output. + $filePathHash = md5( serialize( $book ) . $format ); + $filePath = $fileCache->getDirectory() . '/ebook_' . $filePathHash . '_' . uniqid() . '.' . $creator->getExtension(); + // @todo Don't hardcode the expiry here. + $ttl = 30; + $bus->dispatch( new CreateBookMessage( $book, $filePath, $format, time() + $ttl ) ); + + // Loop while checking for the existence of the generated output file. + for ( $i = 0; $i < 30; $i++ ) { + if ( file_exists( $filePath ) ) { + // The file has been generated now, so send it. + $response = new BinaryFileResponse( $filePath ); + $response->headers->set( 'X-Robots-Tag', 'none' ); + $response->headers->set( 'Content-Description', 'File Transfer' ); + $response->headers->set( 'Content-Type', $creator->getMimeType() ); + $filename = str_replace( [ '/', '\\' ], '_', $page ) . '.' . $creator->getExtension(); + $response->setContentDisposition( ResponseHeaderBag::DISPOSITION_ATTACHMENT, $filename ); + $response->deleteFileAfterSend(); + + // Log book generation. + if ( $this->enableStats ) { + try { + $genBook = new GeneratedBook( $book, $format, $this->stopwatch->stop( 'generate-book' ) ); + $this->entityManager->persist( $genBook ); + $this->entityManager->flush(); + } catch ( DriverException $e ) { + // There was an error writing to tools-db. + // Silently ignore as this shouldn't prevent the book from being downloaded. + } + } + return $response; } + sleep( 1 ); } - return $response; + // If the file wasn't generated by the job queue, tell the user. + throw new WsExportException( 'no-file-generated', [ $ttl ], 500 ); } /** diff --git a/src/Message/CreateBookMessage.php b/src/Message/CreateBookMessage.php new file mode 100644 index 00000000..9d889b1b --- /dev/null +++ b/src/Message/CreateBookMessage.php @@ -0,0 +1,43 @@ +book = $book; + $this->filePath = $filePath; + $this->format = $format; + $this->expiry = $expiry; + } + + public function getBook(): Book { + return $this->book; + } + + public function getFilePath(): string { + return $this->filePath; + } + + public function getFormat(): string { + return $this->format; + } + + public function getExpiry(): int { + return $this->expiry; + } +} diff --git a/src/MessageHandler/CreateBookMessageHandler.php b/src/MessageHandler/CreateBookMessageHandler.php new file mode 100644 index 00000000..ac8e0251 --- /dev/null +++ b/src/MessageHandler/CreateBookMessageHandler.php @@ -0,0 +1,52 @@ +api = $api; + $this->generatorSelector = $generatorSelector; + $this->creditRepo = $creditRepo; + $this->fileCache = $fileCache; + } + + public function __invoke( CreateBookMessage $message ) { + if ( $message->getExpiry() < time() ) { + throw new Exception( 'createbook-message-expired' ); + } + if ( file_exists( $message->getFilePath() ) ) { + return; + } + $this->api->setLang( $message->getBook()->lang ); + $creator = BookCreator::forApi( $this->api, $message->getFormat(), $message->getBook()->options, $this->generatorSelector, $this->creditRepo, $this->fileCache ); + $creator->create( $message->getBook()->title ); + rename( $creator->getFilePath(), $message->getFilePath() ); + } +}