Skip to content

Commit

Permalink
Merge pull request #7 from dicoding-dev/backport/chunk-by-id
Browse files Browse the repository at this point in the history 
[Backport] `chunkById` Database query builder method
  • Loading branch information
@rizqyhi
rizqyhi authored Sep 22, 2023
2 parents af060a6 + 1012bf1 commit 8db410b
Show file tree
Hide file tree
Showing 2 changed files with 210 additions and 0 deletions.
94 changes: 94 additions & 0 deletions src/Illuminate/Database/Query/Builder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use Illuminate\Database\ConnectionInterface;
use Illuminate\Database\Query\Grammars\Grammar;
use Illuminate\Database\Query\Processors\Processor;
use RuntimeException;

class Builder {

Expand Down Expand Up @@ -1521,6 +1522,99 @@ public function chunk($count, callable $callback)
}
}

/**
* Chunk the results of a query by comparing IDs.
*
* @param int $count
* @param callable $callback
* @param string|null $column
* @param string|null $alias
* @return bool
*/
public function chunkById(int $count, callable $callback, string|null $column = null, string|null $alias = null): bool
{
$column ??= $this->defaultKeyName();

$alias ??= $column;

$lastId = null;

$page = 1;

do {
$clone = clone $this;

// We'll execute the query for the given page and get the results. If there are
// no results we can just break and return from here. When there are results
// we will call the callback with the current chunk of these results here.
$results = $clone->forPageAfterId($count, $lastId, $column)->get();

$countResults = count($results);

if ($countResults === 0) {
break;
}

// On each chunk result set, we will pass them to the callback and then let the
// developer take care of everything within the callback, which allows us to
// keep the memory low for spinning through large result sets for working.
if ($callback($results, $page) === false) {
return false;
}

$lastId = data_get(end($results), $alias);

if ($lastId === null) {
throw new RuntimeException("The chunkById operation was aborted because the [{$alias}] column is not present in the query result.");
}

unset($results);

$page++;
} while ($countResults === $count);

return true;
}

/**
* Constrain the query to the next "page" of results after a given ID.
*
* @param int $perPage
* @param int|null $lastId
* @param string $column
* @return $this
*/
protected function forPageAfterId(int $perPage = 15, int|null $lastId = 0, string $column = 'id'): Builder
{
$this->orders = $this->removeExistingOrdersFor($column);

if (! is_null($lastId)) {
$this->where($column, '>', $lastId);
}

return $this->orderBy($column, 'asc')
->limit($perPage);
}

/**
* Get an array with all orders with a given column removed.
*
* @param string $column
* @return array
*/
protected function removeExistingOrdersFor(string $column): array
{
return Collection::make($this->orders)
->reject(function ($order) use ($column) {
return isset($order['column']) && $order['column'] === $column;
})->values()->all();
}

private function defaultKeyName(): string
{
return 'id';
}

/**
* Get an array with the values of a given column.
*
Expand Down
116 changes: 116 additions & 0 deletions tests/Database/DatabaseQueryBuilderTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
use Illuminate\Database\Query\Grammars\Grammar;
use Illuminate\Database\Query\Processors\Processor;
use Illuminate\Pagination\Factory;
use Illuminate\Support\Collection;
use L4\Tests\BackwardCompatibleTestCase;
use Mockery as m;
use Mockery\MockInterface;

class DatabaseQueryBuilderTest extends BackwardCompatibleTestCase
{
Expand Down Expand Up @@ -1363,6 +1365,108 @@ public function testMergeBuildersBindingOrder(): void
$this->assertEquals(['foo', 'bar', 'baz'], $builder->getBindings());
}

public function testChunkByIdOnArrays(): void
{
$builder = $this->getMockQueryBuilder();
$builder->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = [['someIdField' => 1], ['someIdField' => 2]];
$chunk2 = [['someIdField' => 10], ['someIdField' => 11]];
$chunk3 = [];
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->times(3)->andReturn($chunk1, $chunk2, $chunk3);

$callbackAssertor = m::mock(stdClass::class);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk2);
$callbackAssertor->shouldReceive('doSomething')->never()->with($chunk3);

$builder->chunkById(2, function ($results) use ($callbackAssertor) {
$callbackAssertor->doSomething($results);
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithLastChunkComplete(): void
{
$builder = $this->getMockQueryBuilder();
$builder->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = [(object) ['someIdField' => 1], (object) ['someIdField' => 2]];
$chunk2 = [(object) ['someIdField' => 10], (object) ['someIdField' => 11]];
$chunk3 = [];
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->times(3)->andReturn($chunk1, $chunk2, $chunk3);

$callbackAssertor = m::mock(stdClass::class);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk2);
$callbackAssertor->shouldReceive('doSomething')->never()->with($chunk3);

$builder->chunkById(2, function ($results) use ($callbackAssertor) {
$callbackAssertor->doSomething($results);
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithLastChunkPartial(): void
{
$builder = $this->getMockQueryBuilder();
$builder->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = [(object) ['someIdField' => 1], (object) ['someIdField' => 2]];
$chunk2 = [(object) ['someIdField' => 10]];
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->times(2)->andReturn($chunk1, $chunk2);

$callbackAssertor = m::mock(stdClass::class);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk2);

$builder->chunkById(2, function ($results) use ($callbackAssertor) {
$callbackAssertor->doSomething($results);
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithCountZero(): void
{
$builder = $this->getMockQueryBuilder();
$builder->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk = [];
$builder->shouldReceive('forPageAfterId')->once()->with(0, 0, 'someIdField')->andReturnSelf();
$builder->shouldReceive('get')->times(1)->andReturn($chunk);

$callbackAssertor = m::mock(stdClass::class);
$callbackAssertor->shouldReceive('doSomething')->never();

$builder->chunkById(0, function ($results) use ($callbackAssertor) {
$callbackAssertor->doSomething($results);
}, 'someIdField');
}

public function testChunkPaginatesUsingIdWithAlias(): void
{
$builder = $this->getMockQueryBuilder();
$builder->orders[] = ['column' => 'foobar', 'direction' => 'asc'];

$chunk1 = [(object) ['table_id' => 1], (object) ['table_id' => 10]];
$chunk2 = [];
$builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'table.id')->andReturnSelf();
$builder->shouldReceive('forPageAfterId')->once()->with(2, 10, 'table.id')->andReturnSelf();
$builder->shouldReceive('get')->times(2)->andReturn($chunk1, $chunk2);

$callbackAssertor = m::mock(stdClass::class);
$callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1);
$callbackAssertor->shouldReceive('doSomething')->never()->with($chunk2);

$builder->chunkById(2, function ($results) use ($callbackAssertor) {
$callbackAssertor->doSomething($results);
}, 'table.id', 'table_id');
}

protected function getBuilder(): Builder
{
Expand Down Expand Up @@ -1411,4 +1515,16 @@ protected function getMySqlBuilderWithProcessor(): Builder
return new Builder(m::mock(ConnectionInterface::class), $grammar, $processor);
}

/**
* @return MockInterface|\Illuminate\Database\Query\Builder
*/
protected function getMockQueryBuilder(): MockInterface|Builder
{
return m::mock(Builder::class, [
m::mock(ConnectionInterface::class),
new Grammar,
m::mock(Processor::class),
])->makePartial()->shouldAllowMockingProtectedMethods();
}

}

0 comments on commit 8db410b

Please sign in to comment.