Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove doctrine/dbal dependency #6

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Remove doctrine/dbal dependency #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
82e27ea
Remove doctrine/dbal dependency
alies-dev Nov 22, 2024
bd1218f
Do not run tests over L10
alies-dev Nov 22, 2024
4e48d19
Run tests over PHP 8.4
alies-dev Nov 22, 2024
3b152e3
Add testbench to the matrix
alies-dev Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions .github/workflows/run-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,21 +22,12 @@ jobs:
strategy:
fail-fast: true
matrix:
php: [8.1, 8.2, 8.3]
laravel: [10.*, 11.*]
php: [8.2, 8.3, 8.4]
laravel: [11.*]
dependency-version: [prefer-stable]
include:
- laravel: 10.*
testbench: 8.*
- laravel: 11.*
testbench: 9.*
exclude:
- laravel: 10.*
php: 8.0
- laravel: 11.*
php: 8.1
- laravel: 11.*
php: 8.0

name: P${{ matrix.php }} - L${{ matrix.laravel }} - ${{ matrix.dependency-version }}

Expand Down
9 changes: 4 additions & 5 deletions composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,13 @@
"console commands"
],
"require": {
"php": "^8.1",
"doctrine/dbal": "^3.7",
"laravel/framework": "^10.2 || ^11.0"
"php": "^8.2",
"laravel/framework": "^11.0"
},
"require-dev": {
"interaction-design-foundation/coding-standard": "0.*",
"orchestra/testbench": "^8.0",
"phpunit/phpunit": "^10.1 || ^11.0"
"orchestra/testbench": "^9.0",
"phpunit/phpunit": "^11.0"
},
"minimum-stability": "dev",
"prefer-stable": true,
Expand Down
79 changes: 36 additions & 43 deletions src/Console/Commands/FindInvalidDatabaseValues.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,12 @@

namespace InteractionDesignFoundation\LaravelDatabaseToolkit\Console\Commands;

use Doctrine\DBAL\Schema\Column;
use Doctrine\DBAL\Schema\Table;
use Doctrine\DBAL\Types\Types;
use Illuminate\Database\Connection;
use Illuminate\Database\ConnectionResolverInterface;
use Illuminate\Database\Console\DatabaseInspectionCommand;
use Illuminate\Database\MySqlConnection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Symfony\Component\Console\Attribute\AsCommand;

#[AsCommand('database:find-invalid-values')]
Expand All @@ -20,34 +18,27 @@ final class FindInvalidDatabaseValues extends DatabaseInspectionCommand
private const CHECK_TYPE_LONG_TEXT = 'long_text';
private const CHECK_TYPE_LONG_STRING = 'long_string';

/**
* @var string The name and signature of the console command.
*/
/** @var string The name and signature of the console command. */
protected $signature = 'database:find-invalid-values {connection=default} {--check=* : Check only specific types of issues. Available types: {null, datetime, long_text, long_string}}';

/**
* @var string The console command description.
*/
/** @var string The console command description. */
protected $description = 'Find invalid data created in non-strict SQL mode.';

private int $valuesWithIssuesFound = 0;

/**
* @throws \Doctrine\DBAL\Exception
*/
public function handle(ConnectionResolverInterface $connections): int
{
$connection = $this->getConnection($connections);
$schema = $connection->getDoctrineSchemaManager();
if (!$connection instanceof MySqlConnection) {
throw new \InvalidArgumentException('Command supports MySQL DBs only.');
}

$this->registerTypeMappings($schema->getDatabasePlatform());
$tables = Schema::getConnection()->getDoctrineSchemaManager()->listTableNames();

foreach ($schema->listTables() as $table) {
foreach ($table->getColumns() as $column) {
$this->processColumn($column, $table, $connection);
foreach ($tables as $tableName) {
$columns = Schema::getConnection()->getDoctrineSchemaManager()->listTableColumns($tableName);
foreach ($columns as $column) {
$this->processColumn($column, $tableName, $connection);
}
}

Expand All @@ -60,24 +51,24 @@ public function handle(ConnectionResolverInterface $connections): int
return self::FAILURE;
}

private function processColumn(Column $column, Table $table, Connection $connection): void
private function processColumn(object $column, string $tableName, Connection $connection): void
{
$this->info("{$table->getName()}.{$column->getName()}:\t{$column->getType()->getName()}", 'vvv');
$this->info("{$tableName}.{$column->getName()}:\t{$column->getType()->getName()}", 'vvv');

if ($this->shouldRunCheckType(self::CHECK_TYPE_NULL)) {
$this->checkNullOnNotNullableColumn($column, $connection, $table);
$this->checkNullOnNotNullableColumn($column, $connection, $tableName);
}

if ($this->shouldRunCheckType(self::CHECK_TYPE_DATETIME)) {
$this->checkForInvalidDatetimeValues($column, $connection, $table);
$this->checkForInvalidDatetimeValues($column, $connection, $tableName);
}

if ($this->shouldRunCheckType(self::CHECK_TYPE_LONG_TEXT)) {
$this->checkForTooLongTextTypeValues($column, $connection, $table);
$this->checkForTooLongTextTypeValues($column, $connection, $tableName);
}

if ($this->shouldRunCheckType(self::CHECK_TYPE_LONG_STRING)) {
$this->checkForTooLongStringTypeValues($column, $connection, $table);
$this->checkForTooLongStringTypeValues($column, $connection, $tableName);
}
}

Expand All @@ -94,71 +85,73 @@ private function getConnection(ConnectionResolverInterface $connections): Connec
return $connection;
}

private function checkNullOnNotNullableColumn(Column $column, Connection $connection, Table $table): void
private function checkNullOnNotNullableColumn(object $column, Connection $connection, string $tableName): void
{
if ($column->getNotnull()) {
$columnName = $column->getName();

$nullsOnNotNullableColumnCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE `{$columnName}` IS NULL")->count;
$nullsOnNotNullableColumnCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$tableName}` WHERE `{$columnName}` IS NULL")->count;
if ($nullsOnNotNullableColumnCount > 0) {
$this->error("{$table->getName()}.{$columnName} has {$nullsOnNotNullableColumnCount} NULLs but the column is not nullable.");
$this->error("{$tableName}.{$columnName} has {$nullsOnNotNullableColumnCount} NULLs but the column is not nullable.");
$this->valuesWithIssuesFound += $nullsOnNotNullableColumnCount;
} else {
$this->comment("\t".self::CHECK_TYPE_NULL.': OK', 'vvv');
}
}
}

private function checkForInvalidDatetimeValues(Column $column, Connection $connection, Table $table): void
private function checkForInvalidDatetimeValues(object $column, Connection $connection, string $tableName): void
{
$integerProbablyUsedForTimestamp = in_array($column->getType()->getName(), [Types::INTEGER, Types::BIGINT], true) && (str_contains($column->getName(), 'timestamp') || str_ends_with($column->getName(), '_at'));
if ($integerProbablyUsedForTimestamp
|| in_array($column->getType()->getName(), [Types::DATE_MUTABLE, Types::DATE_IMMUTABLE, Types::DATETIME_MUTABLE, Types::DATETIME_IMMUTABLE, Types::DATETIMETZ_MUTABLE, Types::DATETIMETZ_IMMUTABLE], true)
$columnType = $column->getType()->getName();
$columnName = $column->getName();

$integerProbablyUsedForTimestamp = in_array($columnType, ['integer', 'bigint'], true) && (str_contains($columnName, 'timestamp') || str_ends_with($columnName, '_at'));
if (
$integerProbablyUsedForTimestamp
|| in_array($columnType, ['date', 'datetime', 'timestamp'], true)
) {
$columnName = $column->getName();

$invalidDatetimeRecordsCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE `{$columnName}` <= 1")->count;
$invalidDatetimeRecordsCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$tableName}` WHERE `{$columnName}` <= 1")->count;
if ($invalidDatetimeRecordsCount > 0) {
$this->error("{$table->getName()}.{$columnName} has {$invalidDatetimeRecordsCount} invalid datetime values.");
$this->error("{$tableName}.{$columnName} has {$invalidDatetimeRecordsCount} invalid datetime values.");
$this->valuesWithIssuesFound += $invalidDatetimeRecordsCount;
} else {
$this->comment("\t".self::CHECK_TYPE_DATETIME.': OK', 'vvv');
}
}
}

private function checkForTooLongTextTypeValues(Column $column, Connection $connection, Table $table): void
private function checkForTooLongTextTypeValues(object $column, Connection $connection, string $tableName): void
{
if ($column->getType()->getName() === Types::TEXT) {
if ($column->getType()->getName() === 'text') {
$columnName = $column->getName();

$tooLongTextValuesCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE LENGTH(`{$columnName}`) > @@max_allowed_packet;")->count;
$tooLongTextValuesCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$tableName}` WHERE LENGTH(`{$columnName}`) > @@max_allowed_packet;")->count;
if ($tooLongTextValuesCount > 0) {
$this->error("{$table->getName()}.{$columnName} has {$tooLongTextValuesCount} too long text values.");
$this->error("{$tableName}.{$columnName} has {$tooLongTextValuesCount} too long text values.");
$this->valuesWithIssuesFound += $tooLongTextValuesCount;
} else {
$this->comment("\t".self::CHECK_TYPE_LONG_TEXT.': OK', 'vvv');
}
}
}

private function checkForTooLongStringTypeValues(Column $column, Connection $connection, Table $table): void
private function checkForTooLongStringTypeValues(object $column, Connection $connection, string $tableName): void
{
if (in_array($column->getType()->getName(), [Types::STRING, Types::ASCII_STRING], true)) {
if (in_array($column->getType()->getName(), ['string', 'ascii_string'], true)) {
$columnName = $column->getName();

$maxLength = $column->getLength();

if (is_int($maxLength) && $maxLength !== 0) {
$tooLongStringValuesCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$table->getName()}` WHERE LENGTH(`{$columnName}`) > {$maxLength};")->count;
$tooLongStringValuesCount = DB::selectOne("SELECT COUNT(`{$columnName}`) AS `count` FROM {$connection->getDatabaseName()}.`{$tableName}` WHERE LENGTH(`{$columnName}`) > {$maxLength};")->count;
if ($tooLongStringValuesCount > 0) {
$this->error("{$table->getName()}.{$columnName} has {$tooLongStringValuesCount} too long string values (longer than {$maxLength} chars).");
$this->error("{$tableName}.{$columnName} has {$tooLongStringValuesCount} too long string values (longer than {$maxLength} chars).");
$this->valuesWithIssuesFound += $tooLongStringValuesCount;
} else {
$this->comment("\t".self::CHECK_TYPE_LONG_STRING.': OK', 'vvv');
}
} else {
$this->warn("Could not find max length for {$table->getName()}.{$columnName} column.");
$this->warn("Could not find max length for {$tableName}.{$columnName} column.");
}
}
}
Expand Down
74 changes: 38 additions & 36 deletions src/Console/Commands/FindRiskyDatabaseColumns.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@
use Illuminate\Database\ConnectionResolverInterface;
use Illuminate\Database\Console\DatabaseInspectionCommand;
use Illuminate\Database\MySqlConnection;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Symfony\Component\Console\Attribute\AsCommand;
use function Laravel\Prompts\table;

/**
* Inspired by @see https://medium.com/beyn-technology/ill-never-forget-this-number-4294967295-0xffffffff-c9ad4b72f53a
Expand All @@ -27,65 +25,59 @@
#[AsCommand('database:find-risky-columns')]
final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand
{
/**
* @var string The name and signature of the console command.
*/
/** @var string The name and signature of the console command. */
protected $signature = 'database:find-risky-columns {connection=default} {--threshold=70 : Percentage occupied rows number on which the command should treat it as an issue}';

/**
* @var string The console command description.
*/
/** @var string The console command description. */
protected $description = 'Find risky auto-incremental columns on databases which values are close to max possible values.';

/**
* @var array<string, array{min: int|float, max: int|float}>
*/
/** @var array<string, array{min: int|float, max: int|float}> */
private array $columnMinsAndMaxs = [
'integer' => [
'min' => -2_147_483_648,
'max' => 2_147_483_647,
],
'int unsigned' => [
'unsigned integer' => [
'min' => 0,
'max' => 4_294_967_295,
],
'bigint' => [
'min' => -9_223_372_036_854_775_808,
'max' => 9_223_372_036_854_775_807,
],
'bigint unsigned' => [
'unsigned bigint' => [
'min' => 0,
'max' => 18_446_744_073_709_551_615,
],
'tinyint' => [
'min' => -128,
'max' => 127,
],
'tinyint unsigned' => [
'unsigned tinyint' => [
'min' => 0,
'max' => 255,
],
'smallint' => [
'min' => -32_768,
'max' => 32_767,
],
'smallint unsigned' => [
'unsigned smallint' => [
'min' => 0,
'max' => 65_535,
],
'mediumint' => [
'min' => -8_388_608,
'max' => 8_388_607,
],
'mediumint unsigned' => [
'unsigned mediumint' => [
'min' => 0,
'max' => 16_777_215,
],
'decimal' => [
'min' => -99999999999999999999999999999.99999999999999999999999999999,
'max' => 99999999999999999999999999999.99999999999999999999999999999,
],
'decimal unsigned' => [
'unsigned decimal' => [
'min' => 0,
'max' => 99999999999999999999999999999.99999999999999999999999999999,
],
Expand All @@ -94,15 +86,17 @@ final class FindRiskyDatabaseColumns extends DatabaseInspectionCommand
public function handle(ConnectionResolverInterface $connections): int
{
$thresholdAlarmPercentage = (float) $this->option('threshold');
$connection = Schema::getConnection();

$connection = $this->getConnection($connections);
if (! $connection instanceof MySqlConnection) {
throw new \InvalidArgumentException('Command supports MySQL DBs only.');
}

$outputTable = [];
$tables = Schema::getConnection()->getDoctrineSchemaManager()->listTableNames();

foreach (Schema::getTables() as $table) {
$riskyColumnsInfo = $this->processTable($table, $connection, $thresholdAlarmPercentage);
foreach ($tables as $tableName) {
$riskyColumnsInfo = $this->processTable($tableName, $connection, $thresholdAlarmPercentage);
if (is_array($riskyColumnsInfo)) {
$outputTable = [...$outputTable, ...$riskyColumnsInfo];
}
Expand All @@ -123,32 +117,27 @@ public function handle(ConnectionResolverInterface $connections): int
return self::FAILURE;
}

/**
* @return list<array<string, string>>|null
*/
private function processTable(array $table, Connection $connection, float $thresholdAlarmPercentage): ?array
/** @return list<array<string, string>>|null */
private function processTable(string $tableName, Connection $connection, float $thresholdAlarmPercentage): ?array
{
$tableName = Arr::get($table, 'name');
$this->comment("Table {$connection->getDatabaseName()}.{$tableName}: checking...", 'v');

$tableSize = Arr::get($table, 'size');

$tableSize = $this->getTableSize($connection, $tableName);
if ($tableSize === null) {
$tableSize = -1; // not critical info, we can skip this issue
}

/**
* @var \Illuminate\Support\Collection<int, Schema> $getColumns
*/
$columns = collect(Schema::getColumns($tableName))->filter(
static fn($column): bool => Arr::get($column, 'auto_increment') === true
);
$columns = Schema::getConnection()->getDoctrineSchemaManager()->listTableColumns($tableName);
$autoIncrementColumns = array_filter($columns, fn($column) => $column->getAutoincrement());

$riskyColumnsInfo = [];

foreach ($columns as $column) {
$columnName = Arr::get($column, 'name');
$columnType = Arr::get($column, 'type');
foreach ($autoIncrementColumns as $column) {
$columnName = $column->getName();
$columnType = $column->getType()->getName();
if ($column->getUnsigned()) {
$columnType = "unsigned {$columnType}";
}

$this->comment("\t{$columnName} is autoincrement.", 'vvv');

Expand Down Expand Up @@ -179,6 +168,19 @@ private function processTable(array $table, Connection $connection, float $thres
: null;
}

private function getConnection(ConnectionResolverInterface $connections): Connection
{
$connectionName = $this->argument('connection');
if ($connectionName === 'default') {
$connectionName = config('database.default');
}

$connection = $connections->connection($connectionName);
assert($connection instanceof Connection);

return $connection;
}

private function getMaxValueForColumn(string $columnType): int | float
{
if (array_key_exists($columnType, $this->columnMinsAndMaxs)) {
Expand Down
Loading