Skip to content

Commit

Permalink
[WIP][FEATURE] Add indexer status console command, see #216
Browse files Browse the repository at this point in the history
Additionally, clean up the indexing report
  • Loading branch information
christianbltr committed Mar 22, 2024
1 parent 620b2cd commit 4925335
Show file tree
Hide file tree
Showing 8 changed files with 186 additions and 83 deletions.
80 changes: 80 additions & 0 deletions Classes/Command/IndexerStatusCommand.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
<?php

namespace Tpwd\KeSearch\Command;

/***************************************************************
* Copyright notice
* (c) 2024 Christian Bülter
* All rights reserved
* This script is part of the TYPO3 project. The TYPO3 project is
* free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
* The GNU General Public License can be found at
* http://www.gnu.org/copyleft/gpl.html.
* This script is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* This copyright notice MUST APPEAR in all copies of the script!
***************************************************************/

use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use Tpwd\KeSearch\Service\IndexerStatusService;

/**
* Command for starting the index process of ke_search
*/
class IndexerStatusCommand extends Command implements LoggerAwareInterface
{
use LoggerAwareTrait;

protected IndexerStatusService $indexerStatusService;

public function __construct(IndexerStatusService $indexerStatusService)
{
$this->indexerStatusService = $indexerStatusService;
parent::__construct();
}

/**
* Configure command
*/
protected function configure()
{
$this->setHelp(
'Shows the current status of the indexer. If mode is "short", only the status is'
. ' shown ("running" or "idle"), otherwise a detailed report is displayed.'
);

$this->addOption(
'mode',
'm',
InputOption::VALUE_OPTIONAL,
'Mode, either "full" (default) or "short".',
'full'
);
}

/**
* Runs the index process for ke_search
*/
protected function execute(InputInterface $input, OutputInterface $output): int
{
$io = new SymfonyStyle($input, $output);
if ($input->getOption('mode') === 'short') {
$output = $this->indexerStatusService->isRunning() ? 'running' : 'idle';
} else {
$output = $this->indexerStatusService->getStatusReport(IndexerStatusService::INDEXER_STATUS_REPORT_FORMAT_PLAIN);
}
$io->writeln($output);
return self::SUCCESS;
}
}
44 changes: 19 additions & 25 deletions Classes/Controller/BackendModuleController.php
Original file line number Diff line number Diff line change
Expand Up @@ -455,13 +455,12 @@ public function getLastIndexingReport()
*/
public function printIndexerConfigurations($indexerConfigurations)
{
$content = '<div id="kesearch-startindexing-indexers"><h2>Indexer configurations</h2>';
// show indexer names
$content = '<div id="kesearch-startindexing-indexers">';
if ($indexerConfigurations) {
$content .= '<div class="row"><div class="col-md-8">';
$content .= '<div class="table-fit"><table class="table table-striped table-hover">';
$content .= '<colgroup><col><col width="100"><col width="100"><col width="100"></colgroup>';
$content .= '<tr><th></th><th>Type</th><th>UID</th><th>PID</th></tr>';
$content .= '<tr><th>Indexer configuration</th><th>Type</th><th>UID</th><th>PID</th></tr>';
foreach ($indexerConfigurations as $indexerConfiguration) {
$content .= '<tr>'
. '<td>' . $this->encode($indexerConfiguration['title']) . '</td>'
Expand Down Expand Up @@ -490,46 +489,41 @@ public function printIndexerConfigurations($indexerConfigurations)
*/
public function printNumberOfRecords()
{
$content = '<div id="kesearch-startindexing-statistics"><h2>Index statistics</h2>';
$content = '<div id="kesearch-startindexing-statistics">';
$numberOfRecords = $this->indexRepository->getTotalNumberOfRecords();

if ($numberOfRecords) {
$content .=
'<p>'
. LocalizationUtility::translate(
'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xlf:index_contains',
'KeSearch'
)
. ' <strong>'
. $numberOfRecords
. '</strong> '
$content .= '<div class="row"><div class="col-md-8">';
$content .= '<div class="alert alert-info">';
$content .= LocalizationUtility::translate(
'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xlf:index_contains',
'KeSearch'
)
. ' <strong>' . $numberOfRecords . '</strong> '
. LocalizationUtility::translate(
'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xlf:records',
'KeSearch'
)
. '</p>';
) . '.<br />' . chr(10);

$lastRun = $this->registry->get('tx_kesearch', 'lastRun');
if ($lastRun) {
$content .= '<p>'
. LocalizationUtility::translate(
'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xlf:last_indexing',
'KeSearch'
)
. ' '
. SearchHelper::formatTimestamp($lastRun['endTime'])
. '.</p>';
$content .= LocalizationUtility::translate(
'LLL:EXT:ke_search/Resources/Private/Language/locallang_mod.xlf:last_indexing',
'KeSearch'
)
. ' ' . SearchHelper::formatTimestamp($lastRun['endTime']);
}
$content .= '</div>';
$content .= '</div></div>';

$content .= '<div class="row"><div class="col-md-8">';
$content .= '<div class="table-fit"><table class="table table-striped table-hover">';
$content .= '<colgroup><col><col width="100"></colgroup>';
$content .= '<tr><th>Type</th><th>Count</th></tr>';
$content .= '<tr><th>Type of indexed content</th><th>Count</th></tr>';

/** @var IndexRepository $indexRepository */
$indexRepository = GeneralUtility::makeInstance(IndexRepository::class);
$results_per_type = $indexRepository->getNumberOfRecordsInIndexPerType();
$first = true;
foreach ($results_per_type as $type => $count) {
$content .= '<tr><td><span class="label label-primary">' . $type . '</span></td><td>' . $count . '</td></tr>';
}
Expand Down
35 changes: 5 additions & 30 deletions Classes/Controller/IndexerStatusController.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,48 +8,24 @@
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use Tpwd\KeSearch\Service\IndexerStatusService;
use Tpwd\KeSearch\Utility\TimeUtility;

class IndexerStatusController
{
private ResponseFactoryInterface $responseFactory;
private IndexerStatusService $indexerStatusService;

public function __construct(ResponseFactoryInterface $responseFactory, IndexerStatusService $indexerStatusService) {
public function __construct(ResponseFactoryInterface $responseFactory, IndexerStatusService $indexerStatusService)
{
$this->responseFactory = $responseFactory;
$this->indexerStatusService = $indexerStatusService;
}

public function getStatusAction(ServerRequestInterface $request): ResponseInterface
{
$isRunning = $this->indexerStatusService->isRunning();
$indexerStatus = [];
$html = '<div class="alert alert-notice">Indexer is idle.</div>';

if ($isRunning) {
$indexerStartTime = $this->indexerStatusService->getIndexerStartTime();
$indexerStatus = $this->indexerStatusService->getIndexerStatus();
$html = '<div class="alert alert-success">Indexer is running.</div>';
if ($indexerStatus['indexers'] ?? false) {
$html .= '<div class="table-fit"><table class="table table-striped table-hover">';
foreach ($indexerStatus['indexers'] as $singleIndexerStatus) {
$statusLine = htmlspecialchars($singleIndexerStatus['statusText'], ENT_QUOTES, 'UTF-8');
if ($singleIndexerStatus['status'] == $this->indexerStatusService::INDEXER_STATUS_RUNNING) {
$statusLine = '<tr class="table-success"><td><strong>' . $statusLine . '</strong></td></tr>';
} else {
$statusLine = '<tr><td>' . $statusLine . '</td></tr>';
}
$html .= $statusLine;
}
$html .= '</table></div>';
$html .= '<div class="alert alert-notice">Indexer is running since ' . TimeUtility::getRunningTimeHumanReadable($indexerStartTime) . '.' . '</div>';
}
}

$result = [
'running' => $isRunning,
'indexers' => $indexerStatus['indexers'] ?? [],
'html' => $html,
'running' => $this->indexerStatusService->isRunning(),
'indexers' => $this->indexerStatusService->getIndexerStatus()['indexers'] ?? [],
'html' => $this->indexerStatusService->getStatusReport(),
];

$response = $this->responseFactory->createResponse()
Expand All @@ -60,5 +36,4 @@ public function getStatusAction(ServerRequestInterface $request): ResponseInterf

return $response;
}

}
22 changes: 12 additions & 10 deletions Classes/Indexer/IndexerRunner.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
use Tpwd\KeSearch\Lib\SearchHelper;
use Tpwd\KeSearch\Service\IndexerStatusService;
use Tpwd\KeSearch\Utility\AdditionalWordCharactersUtility;
use Tpwd\KeSearch\Utility\TimeUtility;
use TYPO3\CMS\Core\Log\Logger;
use TYPO3\CMS\Core\Log\LogManager;
use TYPO3\CMS\Core\Mail\MailMessage;
Expand Down Expand Up @@ -233,7 +234,7 @@ public function startIndexing($verbose = true, $extConf = [], $mode = '', $index
}
$this->indexerStatusService->setFinishedStatus($indexerConfig);
}
$content .= '</table></div>';
$content .= '</table></div>' . chr(10);

// process index cleanup
$content .= $this->cleanUpIndex($indexingMode);
Expand All @@ -247,13 +248,13 @@ public function startIndexing($verbose = true, $extConf = [], $mode = '', $index
// log finishing
$indexingTime = $this->endTime - $this->startTime;
$content .= '<div class="alert alert-success">';
$content .= '<h3>Finished</h3>';
$content .= '<h3>Finished</h3>' . chr(10);

$message = 'Indexing finished at ' . SearchHelper::formatTimestamp($this->endTime) . ' (took ' . $this->formatTime($indexingTime) . ').';
$content .= $message;
$this->logger->info($message);

$message = '<br /><i>Index contains ' . $this->indexRepository->getTotalNumberOfRecords() . ' entries.</i>';
$message = '<br />Index contains ' . $this->indexRepository->getTotalNumberOfRecords() . ' entries.';
$content .= $message;
$this->logger->info($message);

Expand Down Expand Up @@ -344,7 +345,7 @@ public function renderIndexingReport($searchObj, $message = '')
}
}

$content .= '</td>';
$content .= '</td>' . chr(10);

// duration, show sec or ms
$content .= '<td>';
Expand All @@ -355,9 +356,9 @@ public function renderIndexingReport($searchObj, $message = '')
if ($duration > 1000) {
$duration /= 1000;
$duration = (int)$duration;
$content .= $duration . ' s.';
$content .= TimeUtility::getSecondsHumanReadable($duration);
} else {
$content .= $duration . ' ms.';
$content .= $duration . ' ms';
}
$content .= '</i>';
}
Expand All @@ -376,8 +377,9 @@ public function renderIndexingReport($searchObj, $message = '')
public function createPlaintextReport($content)
{
$content = str_ireplace(['<span class="title">', '<br />', '<br>', '<br/>', '</span>', '</p>'], chr(10), $content);
$report = preg_replace('~[ ]{2,}~', '', strip_tags($content));
return $report;
$content = preg_replace('~[ ]{2,}~', '', strip_tags($content));
$content = str_ireplace('Indexer configurationModeInfoTime', '', $content);
return $content;
}

/**
Expand Down Expand Up @@ -501,7 +503,7 @@ public function cleanUpProcessAfterIndexing()
public function cleanUpIndex(int $indexingMode)
{
$content = '<div class="alert alert-notice">';
$content .= '<h3>Cleanup</h3>';
$content .= '<h3>Cleanup</h3>' . chr(10);
if ($indexingMode == IndexerBase::INDEXING_MODE_FULL) {
$this->logger->info('Cleanup started');
$startMicrotime = microtime(true);
Expand Down Expand Up @@ -543,7 +545,7 @@ public function cleanUpIndex(int $indexingMode)

// calculate duration of indexing process
$duration = ceil((microtime(true) - $startMicrotime) * 1000);
$content .= '<i>Cleanup process took ' . $duration . ' ms.</i>' . "\n";
$content .= 'Cleanup process took ' . $duration . ' ms.' . "\n";
} else {
$message = 'Skipping cleanup in incremental mode.';
$this->logger->info($message);
Expand Down
49 changes: 44 additions & 5 deletions Classes/Service/IndexerStatusService.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace Tpwd\KeSearch\Service;

use Tpwd\KeSearch\Lib\SearchHelper;
use Tpwd\KeSearch\Utility\TimeUtility;
use TYPO3\CMS\Core\Registry;

class IndexerStatusService
Expand All @@ -14,6 +15,8 @@ class IndexerStatusService
public const INDEXER_STATUS_SCHEDULED = 0;
public const INDEXER_STATUS_RUNNING = 1;
public const INDEXER_STATUS_FINISHED = 2;
public const INDEXER_STATUS_REPORT_FORMAT_HTML = 'html';
public const INDEXER_STATUS_REPORT_FORMAT_PLAIN = 'plain';

private Registry $registry;

Expand Down Expand Up @@ -58,7 +61,7 @@ public function setScheduledStatus(array $indexerConfig)
'totalRecords' => 0,
'statusText' =>
'"' . $indexerConfig['title'] . '"'
. ' is scheduled for execution'
. ' is scheduled for execution',
];
$this->setIndexerStatus($indexerStatus);
}
Expand Down Expand Up @@ -87,7 +90,7 @@ public function setRunningStatus(array $indexerConfig, int $currentRecordCount =
'totalRecords' => $totalRecordCount,
'statusText' =>
'"' . $indexerConfig['title'] . '"'
. ' is running'
. ' is running',
];
if ($currentRecordCount >= 0 && $totalRecordCount >= 0) {
$indexerStatus['indexers'][$indexerConfig['uid']]['statusText'] .=
Expand Down Expand Up @@ -121,18 +124,54 @@ public function setLastRunTime(int $startTime, int $endTime, int $indexingTime):
[
'startTime' => $startTime,
'endTime' => $endTime,
'indexingTime' => $indexingTime
'indexingTime' => $indexingTime,
]
);
}

/**
* removes all entries from ke_search registry
*
* @return void
*/
public function clearAll(): void
{
$this->registry->removeAllByNamespace(self::INDEXER_STATUS_REGISTRY_NAMESPACE);
}

public function getStatusReport(string $format = self::INDEXER_STATUS_REPORT_FORMAT_HTML): string
{
$plain = [];
if ($this->isRunning()) {
$indexerStartTime = $this->getIndexerStartTime();
$indexerStatus = $this->getIndexerStatus();
$message = 'Indexer is running.';
$html = '<div class="alert alert-success">' . $message . '</div>';
$plain[] = $message;
if ($indexerStatus['indexers'] ?? false) {
$html .= '<div class="table-fit"><table class="table table-striped table-hover">';
foreach ($indexerStatus['indexers'] as $singleIndexerStatus) {
$statusLine = htmlspecialchars($singleIndexerStatus['statusText'], ENT_QUOTES, 'UTF-8');
if ($singleIndexerStatus['status'] == self::INDEXER_STATUS_RUNNING) {
$statusLine = '<tr class="table-success"><td><strong>' . $statusLine . '</strong></td></tr>';
} else {
$statusLine = '<tr><td>' . $statusLine . '</td></tr>';
}
$html .= $statusLine;
$plain[] = $singleIndexerStatus['statusText'];
}
$html .= '</table></div>';
$message = 'Indexer is running since ' . TimeUtility::getRunningTimeHumanReadable($indexerStartTime) . '.';
$html .= '<div class="alert alert-notice">' . $message . '</div>';
$plain[] = $message;
}
} else {
$message = 'Indexer is idle.';
$html = '<div class="alert alert-notice">' . $message . '</div>';
$plain[] = $message;
}

if ($format == self::INDEXER_STATUS_REPORT_FORMAT_PLAIN) {
return implode(chr(10), $plain);
}
return $html;
}
}
Loading

0 comments on commit 4925335

Please sign in to comment.