From 0b6026c6f1d11afc8f3c875043b803ffa318531b Mon Sep 17 00:00:00 2001 From: Gangai Amaran <44192331+gangaiamaran@users.noreply.github.com> Date: Sat, 31 Aug 2024 23:34:25 +0530 Subject: [PATCH 1/5] mysql image and laravel 11 added --- .github/workflows/run-tests.yml | 46 ++++++++++++++++++++++++++++----- .gitignore | 2 ++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 4a15a91..3271c5d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,17 +1,42 @@ name: run-tests -on: [ push, pull_request ] +on: [push, pull_request] jobs: test: runs-on: ubuntu-latest + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: test_db + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping --silent" + --health-interval=10s + --health-timeout=5s + --health-retries=3 + strategy: fail-fast: true matrix: - php: [ 8.1, 8.2, 8.3 ] - laravel: [ 10.* ] - testbench: [ 8.* ] - dependency-version: [ prefer-stable ] + php: [8.1, 8.2, 8.3] + laravel: [10.*, 11.*] + dependency-version: [prefer-stable] + include: + - laravel: 10.* + testbench: 8.* + - laravel: 11.* + testbench: 9.* + exclude: + - laravel: 10.* + php: 8.0 + - laravel: 11.* + php: 8.1 + - laravel: 11.* + php: 8.0 name: P${{ matrix.php }} - L${{ matrix.laravel }} - ${{ matrix.dependency-version }} @@ -23,7 +48,7 @@ jobs: uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php }} - extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, fileinfo + extensions: dom, curl, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, fileinfo, pdo_mysql coverage: none - name: Setup problem matchers @@ -38,5 +63,14 @@ jobs: env: COMPOSER_AUTH: ${{ secrets.COMPOSER_AUTH }} + - name: Set up environment variables + run: | + echo "DB_CONNECTION=mysql" >> $GITHUB_ENV + echo "DB_HOST=127.0.0.1" >> $GITHUB_ENV + echo "DB_PORT=3306" >> $GITHUB_ENV + echo "DB_DATABASE=test_db" >> $GITHUB_ENV + echo "DB_USERNAME=root" >> $GITHUB_ENV + echo "DB_PASSWORD=root" >> $GITHUB_ENV + - name: Execute tests run: composer test diff --git a/.gitignore b/.gitignore index 321d425..8e0a583 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ auth.json phpunit.xml .phpunit.result.cache .phpunit.cache + +.idea From 671553088927d56cd9b98e3afeb7b9dc2a75523f Mon Sep 17 00:00:00 2001 From: Gangai Amaran <44192331+gangaiamaran@users.noreply.github.com> Date: Sat, 31 Aug 2024 23:49:30 +0530 Subject: [PATCH 2/5] doctrine to Schema conversion --- .../Commands/FindInvalidDatabaseValues.php | 15 ++- .../Commands/FindRiskyDatabaseColumns.php | 99 ++++++++++--------- src/DatabaseToolkitServiceProvider.php | 7 +- .../Commands/FindRiskyDatabaseColumnsTest.php | 45 +++++++++ tests/TestCase.php | 18 +++- 5 files changed, 128 insertions(+), 56 deletions(-) diff --git a/src/Console/Commands/FindInvalidDatabaseValues.php b/src/Console/Commands/FindInvalidDatabaseValues.php index 5e716c0..1afbc21 100644 --- a/src/Console/Commands/FindInvalidDatabaseValues.php +++ b/src/Console/Commands/FindInvalidDatabaseValues.php @@ -20,15 +20,21 @@ final class FindInvalidDatabaseValues extends DatabaseInspectionCommand private const CHECK_TYPE_LONG_TEXT = 'long_text'; private const CHECK_TYPE_LONG_STRING = 'long_string'; - /** @var string The name and signature of the console command. */ + /** + * @var string The name and signature of the console command. + */ protected $signature = 'database:find-invalid-values {connection=default} {--check=* : Check only specific types of issues. Available types: {null, datetime, long_text, long_string}}'; - /** @var string The console command description. */ + /** + * @var string The console command description. + */ protected $description = 'Find invalid data created in non-strict SQL mode.'; private int $valuesWithIssuesFound = 0; - /** @throws \Doctrine\DBAL\Exception */ + /** + * @throws \Doctrine\DBAL\Exception + */ public function handle(ConnectionResolverInterface $connections): int { $connection = $this->getConnection($connections); @@ -106,8 +112,7 @@ private function checkNullOnNotNullableColumn(Column $column, Connection $connec private function checkForInvalidDatetimeValues(Column $column, Connection $connection, Table $table): void { $integerProbablyUsedForTimestamp = in_array($column->getType()->getName(), [Types::INTEGER, Types::BIGINT], true) && (str_contains($column->getName(), 'timestamp') || str_ends_with($column->getName(), '_at')); - if ( - $integerProbablyUsedForTimestamp + if ($integerProbablyUsedForTimestamp || in_array($column->getType()->getName(), [Types::DATE_MUTABLE, Types::DATE_IMMUTABLE, Types::DATETIME_MUTABLE, Types::DATETIME_IMMUTABLE, Types::DATETIMETZ_MUTABLE, Types::DATETIMETZ_IMMUTABLE], true) ) { $columnName = $column->getName(); diff --git a/src/Console/Commands/FindRiskyDatabaseColumns.php b/src/Console/Commands/FindRiskyDatabaseColumns.php index 48b7cb6..1add242 100644 --- a/src/Console/Commands/FindRiskyDatabaseColumns.php +++ b/src/Console/Commands/FindRiskyDatabaseColumns.php @@ -2,13 +2,13 @@ namespace InteractionDesignFoundation\LaravelDatabaseToolkit\Console\Commands; -use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Schema\Table; use Illuminate\Database\Connection; use Illuminate\Database\ConnectionResolverInterface; use Illuminate\Database\Console\DatabaseInspectionCommand; use Illuminate\Database\MySqlConnection; +use Illuminate\Support\Arr; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Schema; use Symfony\Component\Console\Attribute\AsCommand; /** @@ -26,19 +26,25 @@ #[AsCommand('database:find-risky-columns')] final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand { - /** @var string The name and signature of the console command. */ + /** + * @var string The name and signature of the console command. + */ protected $signature = 'database:find-risky-columns {connection=default} {--threshold=70 : Percentage occupied rows number on which the command should treat it as an issue}'; - /** @var string The console command description. */ + /** + * @var string The console command description. + */ protected $description = 'Find risky auto-incremental columns on databases which values are close to max possible values.'; - /** @var array */ + /** + * @var array + */ private array $columnMinsAndMaxs = [ 'integer' => [ 'min' => -2_147_483_648, 'max' => 2_147_483_647, ], - 'unsigned integer' => [ + 'int unsigned' => [ 'min' => 0, 'max' => 4_294_967_295, ], @@ -46,15 +52,15 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand 'min' => -9_223_372_036_854_775_808, 'max' => 9_223_372_036_854_775_807, ], - 'unsigned bigint' => [ + 'bigint unsigned' => [ 'min' => 0, - 'max' => 18_446_744_073_709_551_615, + 'max' => 180_709_551_615, ], 'tinyint' => [ 'min' => -128, 'max' => 127, ], - 'unsigned tinyint' => [ + 'tinyint unsigned' => [ 'min' => 0, 'max' => 255, ], @@ -62,7 +68,7 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand 'min' => -32_768, 'max' => 32_767, ], - 'unsigned smallint' => [ + 'smallint unsigned' => [ 'min' => 0, 'max' => 65_535, ], @@ -70,7 +76,7 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand 'min' => -8_388_608, 'max' => 8_388_607, ], - 'unsigned mediumint' => [ + 'mediumint unsigned' => [ 'min' => 0, 'max' => 16_777_215, ], @@ -78,7 +84,7 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand 'min' => -99999999999999999999999999999.99999999999999999999999999999, 'max' => 99999999999999999999999999999.99999999999999999999999999999, ], - 'unsigned decimal' => [ + 'decimal unsigned' => [ 'min' => 0, 'max' => 99999999999999999999999999999.99999999999999999999999999999, ], @@ -87,19 +93,15 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand public function handle(ConnectionResolverInterface $connections): int { $thresholdAlarmPercentage = (float) $this->option('threshold'); - - $connection = $this->getConnection($connections); - $schema = $connection->getDoctrineSchemaManager(); + $connection = Schema::getConnection(); if (! $connection instanceof MySqlConnection) { throw new \InvalidArgumentException('Command supports MySQL DBs only.'); } - $this->registerTypeMappings($schema->getDatabasePlatform()); - $outputTable = []; - foreach ($schema->listTables() as $table) { - $riskyColumnsInfo = $this->processTable($table, $connection, $thresholdAlarmPercentage); + foreach (Schema::getTables() as $table) { + $riskyColumnsInfo = $this->processTable(Arr::get($table, 'name'), $connection, $thresholdAlarmPercentage); if (is_array($riskyColumnsInfo)) { $outputTable = [...$outputTable, ...$riskyColumnsInfo]; } @@ -120,41 +122,44 @@ public function handle(ConnectionResolverInterface $connections): int return self::FAILURE; } - /** @return list>|null */ - private function processTable(Table $table, Connection $connection, float $thresholdAlarmPercentage): ?array + /** + * @return list>|null + */ + private function processTable(string $tableName, Connection $connection, float $thresholdAlarmPercentage): ?array { - $this->comment("Table {$connection->getDatabaseName()}.{$table->getName()}: checking...", 'v'); + $this->comment("Table {$connection->getDatabaseName()}.{$tableName}: checking...", 'v'); + + $tableSize = $this->getTableSize($connection, $tableName); - $tableSize = $this->getTableSize($connection, $table->getName()); if ($tableSize === null) { $tableSize = -1; // not critical info, we can skip this issue } - /** @var \Illuminate\Support\Collection $columns */ - $columns = collect($table->getColumns()) - ->filter(static fn(Column $column): bool => $column->getAutoincrement()); + /** + * @var \Illuminate\Support\Collection $getColumns + */ + $columns = collect(Schema::getColumns($tableName))->filter( + static fn($column): bool => Arr::get($column, 'auto_increment') === true + ); $riskyColumnsInfo = []; foreach ($columns as $column) { - $columnName = $column->getName(); - $columnType = $column->getType()->getName(); - if ($column->getUnsigned()) { - $columnType = "unsigned {$columnType}"; - } + $columnName = Arr::get($column, 'name'); + $columnType = Arr::get($column, 'type'); $this->comment("\t{$columnName} is autoincrement.", 'vvv'); $maxValueForColumnKey = $this->getMaxValueForColumn($columnType); - $currentHighestValue = $this->getCurrentHighestValueForColumn($connection->getDatabaseName(), $table->getName(), $columnName); + $currentHighestValue = $this->getCurrentHighestValueForColumn($connection->getDatabaseName(), $tableName, $columnName); $percentageUsed = round($currentHighestValue / $maxValueForColumnKey * 100, 4); if ($percentageUsed >= $thresholdAlarmPercentage) { - $this->error("{$connection->getDatabaseName()}.{$table->getName()}.{$columnName} is full for {$percentageUsed}% (threshold for allowed usage is {$thresholdAlarmPercentage}%)", 'quiet'); + $this->error("{$connection->getDatabaseName()}.{$tableName}.{$columnName} is full for {$percentageUsed}% (threshold for allowed usage is {$thresholdAlarmPercentage}%)", 'quiet'); $riskyColumnsInfo[] = [ - 'table' => "{$connection->getDatabaseName()}.{$table->getName()}", + 'table' => "{$connection->getDatabaseName()}.{$tableName}", 'column' => $columnName, 'type' => $columnType, 'size' => $this->formatBytes($tableSize, 2), @@ -165,26 +170,13 @@ private function processTable(Table $table, Connection $connection, float $thres } } - $this->comment("Table {$connection->getDatabaseName()}.{$table->getName()}: OK", 'vv'); + $this->comment("Table {$connection->getDatabaseName()}.{$tableName}: OK", 'vv'); return count($riskyColumnsInfo) > 0 ? $riskyColumnsInfo : null; } - private function getConnection(ConnectionResolverInterface $connections): Connection - { - $connectionName = $this->argument('connection'); - if ($connectionName === 'default') { - $connectionName = config('database.default'); - } - - $connection = $connections->connection($connectionName); - assert($connection instanceof Connection); - - return $connection; - } - private function getMaxValueForColumn(string $columnType): int | float { if (array_key_exists($columnType, $this->columnMinsAndMaxs)) { @@ -218,4 +210,15 @@ private function formatBytes(int $size, int $precision): string $suffix = $suffixes[$index]; return round(1024 ** ($base - floor($base)), $precision).$suffix; } + protected function getTableSize($connection, string $table) + { + $result = $connection->selectOne( + 'SELECT (data_length + index_length) AS size FROM information_schema.TABLES WHERE table_schema = ? AND table_name = ?', [ + $connection->getDatabaseName(), + $table, + ] + ); + + return Arr::wrap((array) $result)['size']; + } } diff --git a/src/DatabaseToolkitServiceProvider.php b/src/DatabaseToolkitServiceProvider.php index 25a4b86..0dcedd0 100644 --- a/src/DatabaseToolkitServiceProvider.php +++ b/src/DatabaseToolkitServiceProvider.php @@ -10,15 +10,18 @@ final class DatabaseToolkitServiceProvider extends ServiceProvider { /** * Bootstrap any package services. + * * @see https://laravel.com/docs/master/packages#commands */ public function boot(): void { if ($this->app->runningInConsole()) { - $this->commands([ + $this->commands( + [ FindInvalidDatabaseValues::class, FindRiskyDatabaseColumns::class, - ]); + ] + ); } } } diff --git a/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php b/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php index ff46e4c..079d3ad 100644 --- a/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php +++ b/tests/Console/Commands/FindRiskyDatabaseColumnsTest.php @@ -2,7 +2,10 @@ namespace Tests\Console\Commands; +use Illuminate\Database\Schema\Blueprint; use Illuminate\Foundation\Testing\Concerns\InteractsWithConsole; +use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Schema; use Illuminate\Testing\PendingCommand; use InteractionDesignFoundation\LaravelDatabaseToolkit\Console\Commands\FindRiskyDatabaseColumns; use PHPUnit\Framework\Attributes\CoversClass; @@ -17,9 +20,51 @@ final class FindRiskyDatabaseColumnsTest extends TestCase #[Test] public function it_works_with_default_threshold(): void { + Schema::create( + 'dummy_table_1', function (Blueprint $table) { + $table->tinyIncrements('id')->startingValue(100); + $table->string('name')->nullable(); + } + ); + DB::table('dummy_table_1')->insert(['name' => 'foo']); + $pendingCommand = $this->artisan(FindRiskyDatabaseColumns::class); assert($pendingCommand instanceof PendingCommand); $pendingCommand->assertExitCode(0); } + + #[Test] + public function it_works_with_custom_threshold(): void + { + Schema::create( + 'dummy_table_2', function (Blueprint $table) { + $table->tinyIncrements('id')->startingValue(130); + $table->string('name')->nullable(); + } + ); + DB::table('dummy_table_2')->insert(['name' => 'foo']); + + $pendingCommand = $this->artisan(FindRiskyDatabaseColumns::class, ['--threshold' => 50]); + + assert($pendingCommand instanceof PendingCommand); + $pendingCommand->assertExitCode(1); + } + + #[Test] + public function it_fails_with_exceeding_threshold_tinyint(): void + { + Schema::create( + 'dummy_table_3', function (Blueprint $table) { + $table->tinyIncrements('id')->startingValue(200); + $table->string('name')->nullable(); + } + ); + DB::table('dummy_table_3')->insert(['name' => 'foo']); + + $pendingCommand = $this->artisan(FindRiskyDatabaseColumns::class); + + assert($pendingCommand instanceof PendingCommand); + $pendingCommand->assertExitCode(1); + } } diff --git a/tests/TestCase.php b/tests/TestCase.php index 912b418..58f2297 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -2,13 +2,20 @@ namespace Tests; +use Illuminate\Support\Facades\Schema; +use Illuminate\Support\Str; use InteractionDesignFoundation\LaravelDatabaseToolkit\DatabaseToolkitServiceProvider; abstract class TestCase extends \Orchestra\Testbench\TestCase { + protected function setUp(): void + { + parent::setUp(); + } /** * Load package service provider. - * @param \Illuminate\Foundation\Application $app + * + * @param \Illuminate\Foundation\Application $app * @return list */ protected function getPackageProviders($app): array @@ -17,4 +24,13 @@ protected function getPackageProviders($app): array DatabaseToolkitServiceProvider::class, ]; } + + protected function tearDown(): void + { + collect(Schema::getTableListing()) + ->filter(fn($table) => Str::startsWith($table, 'dummy')) + ->each(fn($tableName) => Schema::dropIfExists($tableName)); + + parent::tearDown(); + } } From 2671cbb21e2d25dc8605098d0a343948a94e42b6 Mon Sep 17 00:00:00 2001 From: Gangai Amaran <44192331+gangaiamaran@users.noreply.github.com> Date: Sun, 1 Sep 2024 00:45:16 +0530 Subject: [PATCH 3/5] fix typo --- src/Console/Commands/FindRiskyDatabaseColumns.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Console/Commands/FindRiskyDatabaseColumns.php b/src/Console/Commands/FindRiskyDatabaseColumns.php index 1add242..4aca4fa 100644 --- a/src/Console/Commands/FindRiskyDatabaseColumns.php +++ b/src/Console/Commands/FindRiskyDatabaseColumns.php @@ -54,7 +54,7 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand ], 'bigint unsigned' => [ 'min' => 0, - 'max' => 180_709_551_615, + 'max' => 18_446_744_073_709_551_615, ], 'tinyint' => [ 'min' => -128, From c3d0d7fd43511231c941b1e444cbf2538272070e Mon Sep 17 00:00:00 2001 From: Gangai Amaran <44192331+gangaiamaran@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:47:56 +0530 Subject: [PATCH 4/5] Removed total size calculation method --- src/Console/Commands/FindRiskyDatabaseColumns.php | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/Console/Commands/FindRiskyDatabaseColumns.php b/src/Console/Commands/FindRiskyDatabaseColumns.php index 4aca4fa..9bdbef7 100644 --- a/src/Console/Commands/FindRiskyDatabaseColumns.php +++ b/src/Console/Commands/FindRiskyDatabaseColumns.php @@ -210,15 +210,4 @@ private function formatBytes(int $size, int $precision): string $suffix = $suffixes[$index]; return round(1024 ** ($base - floor($base)), $precision).$suffix; } - protected function getTableSize($connection, string $table) - { - $result = $connection->selectOne( - 'SELECT (data_length + index_length) AS size FROM information_schema.TABLES WHERE table_schema = ? AND table_name = ?', [ - $connection->getDatabaseName(), - $table, - ] - ); - - return Arr::wrap((array) $result)['size']; - } } From 235eebff10891a344006405500e0b28a2d63006a Mon Sep 17 00:00:00 2001 From: Gangai Amaran <44192331+gangaiamaran@users.noreply.github.com> Date: Mon, 16 Sep 2024 12:04:29 +0530 Subject: [PATCH 5/5] get table size moved to schema --- src/Console/Commands/FindRiskyDatabaseColumns.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Console/Commands/FindRiskyDatabaseColumns.php b/src/Console/Commands/FindRiskyDatabaseColumns.php index 9bdbef7..c142669 100644 --- a/src/Console/Commands/FindRiskyDatabaseColumns.php +++ b/src/Console/Commands/FindRiskyDatabaseColumns.php @@ -10,6 +10,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; use Symfony\Component\Console\Attribute\AsCommand; +use function Laravel\Prompts\table; /** * Inspired by @see https://medium.com/beyn-technology/ill-never-forget-this-number-4294967295-0xffffffff-c9ad4b72f53a @@ -101,7 +102,7 @@ public function handle(ConnectionResolverInterface $connections): int $outputTable = []; foreach (Schema::getTables() as $table) { - $riskyColumnsInfo = $this->processTable(Arr::get($table, 'name'), $connection, $thresholdAlarmPercentage); + $riskyColumnsInfo = $this->processTable($table, $connection, $thresholdAlarmPercentage); if (is_array($riskyColumnsInfo)) { $outputTable = [...$outputTable, ...$riskyColumnsInfo]; } @@ -125,11 +126,12 @@ public function handle(ConnectionResolverInterface $connections): int /** * @return list>|null */ - private function processTable(string $tableName, Connection $connection, float $thresholdAlarmPercentage): ?array + private function processTable(array $table, Connection $connection, float $thresholdAlarmPercentage): ?array { + $tableName = Arr::get($table, 'name'); $this->comment("Table {$connection->getDatabaseName()}.{$tableName}: checking...", 'v'); - $tableSize = $this->getTableSize($connection, $tableName); + $tableSize = Arr::get($table, 'size'); if ($tableSize === null) { $tableSize = -1; // not critical info, we can skip this issue