Skip to content

Commit 4f6a452

Browse files
k2idevmechelon
k2idev
authored and
mechelon
committed
Added TableDefinition::outputInChunksOf(int $chunkSize)
to allow for generation of chunked INSERT statements Added test for outputInChunksOf() Added snapshot Modified LaravelMaskedDump::dumpTableData() to check for chunked output generation
1 parent b253c6f commit 4f6a452

File tree

4 files changed

+148
-15
lines changed

4 files changed

+148
-15
lines changed

‎src/LaravelMaskedDump.php

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -111,30 +111,80 @@ protected function dumpTableData(TableDefinition $table)
111111
{
112112
$query = '';
113113

114-
$queryBuilder = $this->definition->getConnection()
115-
->table($table->getDoctrineTable()->getName());
114+
$queryBuilder = $this->definition->getConnection()->table($table->getDoctrineTable()->getName());
116115

117116
$table->modifyQuery($queryBuilder);
118117

119-
$queryBuilder->get()
120-
->each(function ($row, $index) use ($table, &$query) {
121-
$row = $this->transformResultForInsert((array)$row, $table);
122-
$tableName = $table->getDoctrineTable()->getName();
118+
119+
if($table->getChunkSize() > 0) {
120+
121+
$data = $queryBuilder->get();
122+
123+
if($data->isEmpty()) {
124+
return "";
125+
}
126+
127+
$tableName = $table->getDoctrineTable()->getName();
128+
$columns = array_keys((array)$data->first());
129+
$column_names = "(`" . join('`, `', $columns) . "`)";
130+
131+
// When tables have 1000+ rows we must split them in reasonably sized chunks of e.g. 100
132+
// otherwise the INSERT statement will fail
133+
// this returns a collection of value tuples
134+
135+
$valuesChunks = $data
136+
->chunk($table->getChunkSize())
137+
->map(function($chunk) use($table) {
138+
// for each chunk we generate a list of VALUES for the INSERT statement
139+
// (1, 'some 1', 'data A'),
140+
// (2, 'some 2', 'data B'),
141+
// (3, 'some 3', 'data C'),
142+
// ... etc
143+
144+
$values = $chunk->map(function($row) use($table) {
145+
$row = $this->transformResultForInsert((array)$row, $table);
146+
$query = '(' . join(', ', $row) . ')';
147+
return $query;
148+
})->join(', ');
149+
150+
return $values;
151+
});
152+
153+
// Now we generate the INSERT statements for each chunk of values
154+
// INSERT INTO table <list of columns> VALUES (1, 'some 1', 'data A'), (2, 'some 2', 'data B'), (3, 'some 3', 'data C')...
155+
$insert_statement = $valuesChunks->map(
156+
157+
function($values) use($table, $tableName, $column_names) {
158+
159+
return "INSERT INTO `${tableName}` $column_names VALUES " . $values .';';
160+
161+
})
162+
->join(PHP_EOL);
163+
164+
return $insert_statement . PHP_EOL;
165+
166+
} else {
167+
168+
// orig
169+
$queryBuilder->get()
170+
->each(function ($row, $index) use ($table, &$query) {
171+
$row = $this->transformResultForInsert((array)$row, $table);
172+
$tableName = $table->getDoctrineTable()->getName();
123173

124174
$query .= "INSERT INTO `$tableName` (`" . implode('`, `', array_keys($row)) . '`) VALUES ';
125175
$query .= "(";
126176

127-
$firstColumn = true;
128-
foreach ($row as $value) {
129-
if (!$firstColumn) {
130-
$query .= ", ";
177+
$firstColumn = true;
178+
foreach ($row as $value) {
179+
if (!$firstColumn) {
180+
$query .= ", ";
181+
}
182+
$query .= $value;
183+
$firstColumn = false;
131184
}
132-
$query .= $value;
133-
$firstColumn = false;
134-
}
135185

136-
$query .= ");" . PHP_EOL;
137-
});
186+
$query .= ");" . PHP_EOL;
187+
});
138188

139189
return $query;
140190
}

‎src/TableDefinitions/TableDefinition.php

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ class TableDefinition
1515
protected $dumpType;
1616
protected $query;
1717
protected $columns = [];
18+
protected $chunkSize = 0;
1819

1920
public function __construct(Table $table)
2021
{
@@ -36,6 +37,13 @@ public function fullDump()
3637
return $this;
3738
}
3839

40+
public function outputInChunksOf(int $chunkSize)
41+
{
42+
$this->chunkSize = $chunkSize;
43+
44+
return $this;
45+
}
46+
3947
public function query(callable $callable)
4048
{
4149
$this->query = $callable;
@@ -68,6 +76,11 @@ public function findColumn(string $column)
6876
return false;
6977
}
7078

79+
public function getChunkSize()
80+
{
81+
return $this->chunkSize;
82+
}
83+
7184
public function getDoctrineTable()
7285
{
7386
return $this->table;

‎tests/DumperTest.php

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,4 +198,45 @@ public function it_does_remove_excluded_tables_from_allTables()
198198

199199
$this->assertMatchesTextSnapshot(file_get_contents($outputFile));
200200
}
201+
202+
/** @test */
203+
public function it_creates_chunked_insert_statements_for_a_table()
204+
{
205+
$this->loadLaravelMigrations();
206+
207+
DB::table('users')
208+
->insert(['name' => 'Marcel1', 'email' => 'marcel1@beyondco.de', 'password' => 'test',
209+
'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
210+
]);
211+
DB::table('users')
212+
->insert(['name' => 'Marcel2', 'email' => 'marcel2@beyondco.de', 'password' => 'test',
213+
'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
214+
]);
215+
DB::table('users')
216+
->insert(['name' => 'Marcel3', 'email' => 'marcel3@beyondco.de', 'password' => 'test',
217+
'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
218+
]);
219+
DB::table('users')
220+
->insert(['name' => 'Marcel4', 'email' => 'marcel4@beyondco.de', 'password' => 'test',
221+
'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
222+
]);
223+
DB::table('users')
224+
->insert(['name' => 'Marcel5', 'email' => 'marcel5@beyondco.de', 'password' => 'test',
225+
'created_at' => '2021-01-01 00:00:00', 'updated_at' => '2021-01-01 00:00:00',
226+
]);
227+
228+
$outputFile = base_path('test.sql');
229+
230+
$this->app['config']['masked-dump.default'] = DumpSchema::define()
231+
->allTables()
232+
->table('users', function($table) {
233+
return $table->outputInChunksOf(3);
234+
});
235+
236+
$this->artisan('db:masked-dump', [
237+
'output' => $outputFile
238+
]);
239+
240+
$this->assertMatchesTextSnapshot(file_get_contents($outputFile));
241+
}
201242
}

‎tests/__snapshots__/DumperTest__it_creates_chunked_insert_statements_for_a_table__1.txt

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
DROP TABLE IF EXISTS `failed_jobs`;
2+
CREATE TABLE failed_jobs (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, uuid VARCHAR(255) NOT NULL COLLATE BINARY, connection CLOB NOT NULL COLLATE BINARY, queue CLOB NOT NULL COLLATE BINARY, payload CLOB NOT NULL COLLATE BINARY, exception CLOB NOT NULL COLLATE BINARY, failed_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL);CREATE UNIQUE INDEX failed_jobs_uuid_unique ON failed_jobs (uuid);
3+
LOCK TABLES `failed_jobs` WRITE;
4+
ALTER TABLE `failed_jobs` DISABLE KEYS;
5+
ALTER TABLE `failed_jobs` ENABLE KEYS;
6+
UNLOCK TABLES;
7+
DROP TABLE IF EXISTS `migrations`;
8+
CREATE TABLE migrations (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, migration VARCHAR(255) NOT NULL COLLATE BINARY, batch INTEGER NOT NULL);
9+
LOCK TABLES `migrations` WRITE;
10+
ALTER TABLE `migrations` DISABLE KEYS;
11+
INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '2014_10_12_000000_testbench_create_users_table', '1');
12+
INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '2014_10_12_100000_testbench_create_password_resets_table', '1');
13+
INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '2019_08_19_000000_testbench_create_failed_jobs_table', '1');
14+
ALTER TABLE `migrations` ENABLE KEYS;
15+
UNLOCK TABLES;
16+
DROP TABLE IF EXISTS `password_resets`;
17+
CREATE TABLE password_resets (email VARCHAR(255) NOT NULL COLLATE BINARY, token VARCHAR(255) NOT NULL COLLATE BINARY, created_at DATETIME DEFAULT NULL);CREATE INDEX password_resets_email_index ON password_resets (email);
18+
LOCK TABLES `password_resets` WRITE;
19+
ALTER TABLE `password_resets` DISABLE KEYS;
20+
ALTER TABLE `password_resets` ENABLE KEYS;
21+
UNLOCK TABLES;
22+
DROP TABLE IF EXISTS `users`;
23+
CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL COLLATE BINARY, email VARCHAR(255) NOT NULL COLLATE BINARY, email_verified_at DATETIME DEFAULT NULL, password VARCHAR(255) NOT NULL COLLATE BINARY, remember_token VARCHAR(255) DEFAULT NULL COLLATE BINARY, created_at DATETIME DEFAULT NULL, updated_at DATETIME DEFAULT NULL);CREATE UNIQUE INDEX users_email_unique ON users (email);
24+
LOCK TABLES `users` WRITE;
25+
ALTER TABLE `users` DISABLE KEYS;
26+
INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel1', 'marcel1@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'), ('2', 'Marcel2', 'marcel2@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'), ('3', 'Marcel3', 'marcel3@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00');
27+
INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('4', 'Marcel4', 'marcel4@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'), ('5', 'Marcel5', 'marcel5@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00');
28+
ALTER TABLE `users` ENABLE KEYS;
29+
UNLOCK TABLES;

0 commit comments

Comments
 (0)