Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance optimizations for backups check #240

Merged
38 changes: 38 additions & 0 deletions docs/available-checks/backups.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,41 @@ Health::checks([
->atLeastSizeInMb(20),
]);
```

### Check backups on external filesystems

You can use the `onDisk` method to specify any disk you have configured in Laravel.
This is useful when the backups are stored on an external filesystem.

```php
use Spatie\Health\Facades\Health;
use Spatie\Health\Checks\Checks\BackupsCheck;

Health::checks([
BackupsCheck::new()
->onDisk('backups')
->locatedAt('backups'),
]);
```

Checking backup files on external filesystems can be slow if you have a lot of backup files.
* You can use the `parseModifiedFormat` method to get the modified date of the file from the name instead of reaching out to the file and read its metadata. This strips out the file folder and file extension and uses the remaining string to parse the date with `Carbon::createFromFormat`.
* You can also limit the size check to only the first and last backup files by using the `onlyCheckSizeOnFirstAndLast` method. Otherwise the check needs to reach out to all files and check the file sizes.

These two things can speed up the check of ~200 files on an S3 bucket from about 30 seconds to about 1 second.

```php
use Spatie\Health\Facades\Health;
use Spatie\Health\Checks\Checks\BackupsCheck;

Health::checks([
BackupsCheck::new()
->onDisk('backups')
->parseModifiedFormat('Y-m-d_H-i-s'),
->atLeastSizeInMb(20),
->onlyCheckSizeOnFirstAndLast()
]);
```

For files that contains more than just the date you can use something like parseModifiedFormat('\b\a\c\k\u\p_Ymd_His')
which would parse a file with the name similar to `backup_20240101_120000.sql.zip`.
135 changes: 87 additions & 48 deletions src/Checks/Checks/BackupsCheck.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,12 @@ class BackupsCheck extends Check

protected ?Carbon $oldestShouldHaveBeenMadeAfter = null;

protected ?string $parseModifiedUsing = null;

protected int $minimumSizeInMegabytes = 0;

protected bool $onlyCheckSizeOnFirstAndLast = false;

protected ?int $minimumNumberOfBackups = null;

protected ?int $maximumNumberOfBackups = null;
Expand All @@ -35,13 +39,20 @@ public function locatedAt(string $globPath): self
return $this;
}

public function onDisk($disk)
public function onDisk(string $disk): static
{
$this->disk = Storage::disk($disk);

return $this;
}

public function parseModifiedFormat(string $parseModifiedFormat = 'Y-m-d_H-i-s'): self
{
$this->parseModifiedUsing = $parseModifiedFormat;

return $this;
}

public function youngestBackShouldHaveBeenMadeBefore(Carbon $date): self
{
$this->youngestShouldHaveBeenMadeBefore = $date;
Expand All @@ -56,9 +67,17 @@ public function oldestBackShouldHaveBeenMadeAfter(Carbon $date): self
return $this;
}

public function atLeastSizeInMb(int $minimumSizeInMegabytes): self
public function atLeastSizeInMb(int $minimumSizeInMegabytes, bool $onlyCheckFirstAndLast = false): self
{
$this->minimumSizeInMegabytes = $minimumSizeInMegabytes;
$this->onlyCheckSizeOnFirstAndLast = $onlyCheckFirstAndLast;

return $this;
}

public function onlyCheckSizeOnFirstAndLast(bool $onlyCheckSizeOnFirstAndLast = true): self
{
$this->onlyCheckSizeOnFirstAndLast = $onlyCheckSizeOnFirstAndLast;

return $this;
}
Expand All @@ -73,80 +92,100 @@ public function numberOfBackups(?int $min = null, ?int $max = null): self

public function run(): Result
{
$files = collect($this->disk ? $files = $this->disk->files($this->locatedAt) : File::glob($this->locatedAt));
$eligibleBackups = $this->getBackupFiles();

if ($files->isEmpty()) {
return Result::make()->failed('No backups found');
}
$backupCount = $eligibleBackups->count();

$eligableBackups = $files
->map(function (string $path) {
return new BackupFile($path, $this->disk);
})
->filter(function (BackupFile $file) {
return $file->size() >= $this->minimumSizeInMegabytes * 1024 * 1024;
});
$result = Result::make()->meta([
'minimum_size' => $this->minimumSizeInMegabytes.'MB',
'backup_count' => $backupCount,
]);

if ($eligableBackups->isEmpty()) {
return Result::make()->failed('No backups found that are large enough');
if ($backupCount === 0) {
return $result->failed('No backups found');
}

if ($this->minimumNumberOfBackups) {
if ($eligableBackups->count() < $this->minimumNumberOfBackups) {
return Result::make()->failed('Not enough backups found');
}
if ($this->minimumNumberOfBackups && $backupCount < $this->minimumNumberOfBackups) {
return $result->failed('Not enough backups found');
}

if ($this->maximumNumberOfBackups) {
if ($eligableBackups->count() > $this->maximumNumberOfBackups) {
return Result::make()->failed('Too many backups found');
}
if ($this->maximumNumberOfBackups && $backupCount > $this->maximumNumberOfBackups) {
return $result->failed('Too many backups found');
}

if ($this->youngestShouldHaveBeenMadeBefore) {
if ($this->youngestBackupIsToolOld($eligableBackups)) {
return Result::make()
->failed('Youngest backup was too old');
}
$youngestBackup = $this->getYoungestBackup($eligibleBackups);
$oldestBackup = $this->getOldestBackup($eligibleBackups);

$result->appendMeta([
'youngest_backup' => $youngestBackup ? Carbon::createFromTimestamp($youngestBackup->lastModified())->toDateTimeString() : null,
'oldest_backup' => $oldestBackup ? Carbon::createFromTimestamp($oldestBackup->lastModified())->toDateTimeString() : null,
]);

if ($this->youngestBackupIsToolOld($youngestBackup)) {
return $result->failed('The youngest backup was too old');
}

if ($this->oldestShouldHaveBeenMadeAfter) {
if ($this->oldestBackupIsTooYoung($eligableBackups)) {
return Result::make()
->failed('Oldest backup was too young');
}
if ($this->oldestBackupIsTooYoung($oldestBackup)) {
return $result->failed('The oldest backup was too young');
}

return Result::make()->ok();
$backupsToCheckSizeOn = $this->onlyCheckSizeOnFirstAndLast
? collect([$youngestBackup, $oldestBackup])
: $eligibleBackups;

if ($backupsToCheckSizeOn->filter(
fn(BackupFile $file) => $file->size() >= $this->minimumSizeInMegabytes * 1024 * 1024
)->isEmpty()) {
return $result->failed('Backups are not large enough');
}

return $result->ok();
}

/**
* @param Collection<SymfonyFile> $backups
*/
protected function youngestBackupIsToolOld(Collection $backups): bool
protected function getBackupFiles(): Collection
{
/** @var SymfonyFile|null $youngestBackup */
$youngestBackup = $backups
return collect(
$this->disk
? $this->disk->files($this->locatedAt)
: File::glob($this->locatedAt ?? '')
)->map(function (string $path) {
return new BackupFile($path, $this->disk, $this->parseModifiedUsing);
});
}

protected function getYoungestBackup(Collection $backups): ?BackupFile
{
return $backups
->sortByDesc(fn (BackupFile $file) => $file->lastModified())
->first();
}

protected function youngestBackupIsToolOld(?BackupFile $youngestBackup): bool
{
if ($this->youngestShouldHaveBeenMadeBefore === null) {
return false;
}

$threshold = $this->youngestShouldHaveBeenMadeBefore->getTimestamp();

return $youngestBackup->lastModified() <= $threshold;
return !$youngestBackup || $youngestBackup->lastModified() <= $threshold;
}

/**
* @param Collection<SymfonyFile> $backups
*/
protected function oldestBackupIsTooYoung(Collection $backups): bool
protected function getOldestBackup(Collection $backups): ?BackupFile
{
/** @var SymfonyFile|null $oldestBackup */
$oldestBackup = $backups
return $backups
->sortBy(fn (BackupFile $file) => $file->lastModified())
->first();

}
protected function oldestBackupIsTooYoung(?BackupFile $oldestBackup): bool
{
if ($this->oldestShouldHaveBeenMadeAfter === null) {
return false;
}

$threshold = $this->oldestShouldHaveBeenMadeAfter->getTimestamp();

return $oldestBackup->lastModified() >= $threshold;
return !$oldestBackup || $oldestBackup->lastModified() >= $threshold;
}
}
7 changes: 7 additions & 0 deletions src/Checks/Result.php
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,13 @@ public function meta(array $meta): self
return $this;
}

public function appendMeta($meta): self
{
$this->meta = array_merge($this->meta, $meta);

return $this;
}

public function endedAt(CarbonInterface $carbon): self
{
$this->ended_at = $carbon;
Expand Down
16 changes: 15 additions & 1 deletion src/Support/BackupFile.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

namespace Spatie\Health\Support;

use Carbon\Exceptions\InvalidFormatException;
use Illuminate\Contracts\Filesystem\Filesystem;
use Illuminate\Support\Carbon;
use Illuminate\Support\Str;
use Symfony\Component\HttpFoundation\File\File as SymfonyFile;

class BackupFile
Expand All @@ -12,6 +15,7 @@ class BackupFile
public function __construct(
protected string $path,
protected ?Filesystem $disk = null,
protected ?string $parseModifiedUsing = null,
) {
if (! $disk) {
$this->file = new SymfonyFile($path);
Expand All @@ -28,8 +32,18 @@ public function size(): int
return $this->file ? $this->file->getSize() : $this->disk->size($this->path);
}

public function lastModified(): int
public function lastModified(): ?int
{
if ($this->parseModifiedUsing) {
$filename = Str::of($this->path)->afterLast('/')->before('.');

try {
return (int) Carbon::createFromFormat($this->parseModifiedUsing, $filename)->timestamp;
} catch (InvalidFormatException $e) {
return null;
}
}

return $this->file ? $this->file->getMTime() : $this->disk->lastModified($this->path);
}
}
67 changes: 67 additions & 0 deletions tests/Checks/BackupsCheckTest.php
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
<?php

use Carbon\Carbon;
use Illuminate\Support\Facades\Storage;
use Spatie\Health\Checks\Checks\BackupsCheck;
use Spatie\Health\Enums\Status;
use Spatie\Health\Facades\Health;
use Spatie\Health\Support\BackupFile;
use Spatie\TemporaryDirectory\TemporaryDirectory;

use function Spatie\PestPluginTestTime\testTime;
Expand Down Expand Up @@ -221,3 +223,68 @@
->run();
expect($result)->status->toBe(Status::failed());
});

it('can parse modified time from file name', function ($format) {
Storage::fake('backups');

$now = now();
Storage::disk('backups')->put('backups/'.$now->format($format).'.zip', 'content');

$result1 = $this->backupsCheck
->onDisk('backups')
->locatedAt('backups')
->parseModifiedFormat($format)
->oldestBackShouldHaveBeenMadeAfter($now->subMinutes(5))
->run();

testTime()->addMinutes(6);

$backupFile = new BackupFile('backups/'.$now->format($format).'.zip', Storage::disk('backups'), $format);

expect($backupFile->lastModified())->toBe($now->timestamp);

$result2 = $this->backupsCheck
->onDisk('backups')
->locatedAt('backups')
->parseModifiedFormat($format)
->oldestBackShouldHaveBeenMadeAfter(now()->subMinutes(5))
->run();

expect($result1)->status->toBe(Status::failed())
->and($result2)->status->toBe(Status::ok());

testTime()->addMinutes(2);

$result = $this->backupsCheck
->locatedAt($this->temporaryDirectory->path('*.zip'))
->oldestBackShouldHaveBeenMadeAfter(now()->subMinutes(5))
->run();
expect($result)->status->toBe(Status::failed());
})->with([
['Y-m-d_H-i-s'],
['Ymd_His'],
['YmdHis'],
['\B\a\c\k\u\p_Ymd_His'],
]);

it('can check the size of only the first and last backup files', function () {
$now = now()->startOfMinute();

addTestFile($this->temporaryDirectory->path('hey1.zip'), date: $now, sizeInMb: 5);
addTestFile($this->temporaryDirectory->path('hey2.zip'), date: $now->addMinutes(10), sizeInMb: 10);
addTestFile($this->temporaryDirectory->path('hey3.zip'), date: $now->addMinutes(20), sizeInMb: 5);

$result1 = $this->backupsCheck
->locatedAt($this->temporaryDirectory->path('*.zip'))
->atLeastSizeInMb(9)
->run();

$result2 = $this->backupsCheck
->locatedAt($this->temporaryDirectory->path('*.zip'))
->atLeastSizeInMb(9)
->onlyCheckSizeOnFirstAndLast()
->run();

expect($result1)->status->toBe(Status::ok())
->and($result2)->status->toBe(Status::failed());
});
Loading