Skip to content

Commit d6a2966

Browse files
mechelonmechelon
committed
Remove schema from dump output; support for SQLite and PostgreSQL; outputInChunksOf (by @k2idev, #9)
1 parent 4f6a452 commit d6a2966

12 files changed

+51
-474
lines changed

‎config/masked-dump.php

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,4 @@
1919
});
2020
$table->mask('password');
2121
})
22-
->schemaOnly('failed_jobs')
23-
->schemaOnly('password_reset_tokens'),
2422
];

‎src/DumpSchema.php

Lines changed: 5 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,6 @@ public static function define($connectionName = null)
2828
return new static($connectionName);
2929
}
3030

31-
public function schemaOnly(string $tableName)
32-
{
33-
return $this->table($tableName, function (TableDefinition $table) {
34-
$table->schemaOnly();
35-
});
36-
}
37-
3831
public function table(string $tableName, callable $tableDefinition)
3932
{
4033
$this->customizedTables[$tableName] = $tableDefinition;
@@ -109,58 +102,21 @@ protected function createDoctrineTables(array $tables): array
109102
foreach ($tables as $table) {
110103
$columns = $this->getBuilder()->getColumns($table['name']);
111104

112-
$table = new Table($table['name']);
113-
105+
$doctrineTable = new Table($table['name']);
114106
foreach ($columns as $column) {
115-
$type = $this->mapType($column['type_name']);
116-
$table->addColumn(
107+
108+
$doctrineTable->addColumn(
117109
$column['name'],
118-
$type
110+
Types::STRING, // doesn't matter, but is required
119111
);
120112
}
121113

122-
$doctrineTables[] = $table;
114+
$doctrineTables[] = $doctrineTable;
123115
}
124116

125117
return $doctrineTables;
126118
}
127119

128-
protected function mapType(string $typeName): string
129-
{
130-
switch ($typeName) {
131-
case 'char':
132-
case 'varchar':
133-
return Types::STRING;
134-
case 'int':
135-
case 'integer':
136-
return Types::INTEGER;
137-
case 'text':
138-
case 'longtext':
139-
case 'mediumtext':
140-
return Types::TEXT;
141-
case 'date':
142-
return Types::DATE_MUTABLE;
143-
case 'datetime':
144-
case 'timestamp':
145-
return Types::DATETIME_MUTABLE;
146-
case 'bigint':
147-
case 'mediumint':
148-
return Types::BIGINT;
149-
case 'tinyint':
150-
case 'smallint':
151-
return Types::SMALLINT;
152-
case 'binary':
153-
return Types::BINARY;
154-
case 'json':
155-
return Types::JSON;
156-
case 'decimal':
157-
return Types::DECIMAL;
158-
default:
159-
return Types::TEXT;
160-
}
161-
}
162-
163-
164120
public function load()
165121
{
166122
$this->loadAvailableTables();

‎src/LaravelMaskedDump.php

Lines changed: 36 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@
22

33
namespace BeyondCode\LaravelMaskedDumper;
44

5-
use Doctrine\DBAL\Schema\Schema;
65
use Illuminate\Console\OutputStyle;
76
use BeyondCode\LaravelMaskedDumper\TableDefinitions\TableDefinition;
87
use Doctrine\DBAL\Platforms\MariaDBPlatform;
98
use Doctrine\DBAL\Platforms\MySQLPlatform;
109
use Doctrine\DBAL\Platforms\SqlitePlatform;
1110
use Illuminate\Database\Connection as DatabaseConnection;
1211
use Doctrine\DBAL\Platforms\AbstractPlatform;
12+
use Doctrine\DBAL\Platforms\PostgreSQLPlatform;
1313

1414
class LaravelMaskedDump
1515
{
@@ -22,11 +22,18 @@ class LaravelMaskedDump
2222
/** @var AbstractPlatform */
2323
protected $platform;
2424

25+
/** @var string */
26+
protected $escapeString = "`";
27+
2528
public function __construct(DumpSchema $definition, OutputStyle $output)
2629
{
2730
$this->definition = $definition;
2831
$this->output = $output;
2932
$this->platform = $this->getPlatform($this->definition->getConnection());
33+
34+
if($this->platform instanceof PostgreSQLPlatform) {
35+
$this->escapeString = '"';
36+
}
3037
}
3138

3239
public function dump()
@@ -38,15 +45,8 @@ public function dump()
3845
$overallTableProgress = $this->output->createProgressBar(count($tables));
3946

4047
foreach ($tables as $tableName => $table) {
41-
$query .= "DROP TABLE IF EXISTS `$tableName`;" . PHP_EOL;
42-
$query .= $this->dumpSchema($table);
43-
4448
if ($table->shouldDumpData()) {
45-
$query .= $this->lockTable($tableName);
46-
4749
$query .= $this->dumpTableData($table);
48-
49-
$query .= $this->unlockTable($tableName);
5050
}
5151

5252
$overallTableProgress->advance();
@@ -73,40 +73,22 @@ protected function transformResultForInsert($row, TableDefinition $table)
7373
})->toArray();
7474
}
7575

76-
protected function dumpSchema(TableDefinition $table)
77-
{
78-
$schema = new Schema([$table->getDoctrineTable()]);
79-
80-
return implode(";", $schema->toSql($this->platform)) . ";" . PHP_EOL;
81-
}
82-
8376
protected function getPlatform(DatabaseConnection $connection)
8477
{
8578
switch ($connection->getDriverName()) {
8679
case 'mysql':
8780
return new MySQLPlatform;
81+
case 'pgsql':
82+
return new PostgreSQLPlatform;
83+
case 'sqlite':
84+
return new SqlitePlatform;
8885
case 'mariadb':
8986
return new MariaDBPlatform;
9087
default:
91-
if ($connection->getDriverName() === 'sqlite' && $this->isTesting()) {
92-
return new SqlitePlatform;
93-
}
9488
throw new \RuntimeException("Unsupported platform: {$connection->getDriverName()}. Please check the documentation for more information.");
9589
}
9690
}
9791

98-
protected function lockTable(string $tableName)
99-
{
100-
return "LOCK TABLES `$tableName` WRITE;" . PHP_EOL .
101-
"ALTER TABLE `$tableName` DISABLE KEYS;" . PHP_EOL;
102-
}
103-
104-
protected function unlockTable(string $tableName)
105-
{
106-
return "ALTER TABLE `$tableName` ENABLE KEYS;" . PHP_EOL .
107-
"UNLOCK TABLES;" . PHP_EOL;
108-
}
109-
11092
protected function dumpTableData(TableDefinition $table)
11193
{
11294
$query = '';
@@ -115,64 +97,47 @@ protected function dumpTableData(TableDefinition $table)
11597

11698
$table->modifyQuery($queryBuilder);
11799

100+
$tableName = $table->getDoctrineTable()->getName();
101+
$tableName = "$this->escapeString$tableName$this->escapeString";
118102

119-
if($table->getChunkSize() > 0) {
103+
if ($table->getChunkSize() > 0) {
120104

121105
$data = $queryBuilder->get();
122106

123-
if($data->isEmpty()) {
107+
if ($data->isEmpty()) {
124108
return "";
125109
}
126110

127111
$tableName = $table->getDoctrineTable()->getName();
128112
$columns = array_keys((array)$data->first());
129-
$column_names = "(`" . join('`, `', $columns) . "`)";
130-
131-
// When tables have 1000+ rows we must split them in reasonably sized chunks of e.g. 100
132-
// otherwise the INSERT statement will fail
133-
// this returns a collection of value tuples
113+
$column_names = "($this->escapeString" . join("$this->escapeString, $this->escapeString", $columns) . "$this->escapeString)";
134114

135115
$valuesChunks = $data
136-
->chunk($table->getChunkSize())
137-
->map(function($chunk) use($table) {
138-
// for each chunk we generate a list of VALUES for the INSERT statement
139-
// (1, 'some 1', 'data A'),
140-
// (2, 'some 2', 'data B'),
141-
// (3, 'some 3', 'data C'),
142-
// ... etc
143-
144-
$values = $chunk->map(function($row) use($table) {
145-
$row = $this->transformResultForInsert((array)$row, $table);
146-
$query = '(' . join(', ', $row) . ')';
147-
return $query;
148-
})->join(', ');
149-
150-
return $values;
151-
});
152-
153-
// Now we generate the INSERT statements for each chunk of values
154-
// INSERT INTO table <list of columns> VALUES (1, 'some 1', 'data A'), (2, 'some 2', 'data B'), (3, 'some 3', 'data C')...
155-
$insert_statement = $valuesChunks->map(
156-
157-
function($values) use($table, $tableName, $column_names) {
158-
159-
return "INSERT INTO `${tableName}` $column_names VALUES " . $values .';';
116+
->chunk($table->getChunkSize())
117+
->map(function ($chunk) use ($table) {
118+
$values = $chunk->map(function ($row) use ($table) {
119+
$row = $this->transformResultForInsert((array)$row, $table);
120+
$query = '(' . join(', ', $row) . ')';
121+
return $query;
122+
})->join(', ');
123+
124+
return $values;
125+
});
160126

161-
})
162-
->join(PHP_EOL);
127+
$insert_statement = $valuesChunks->map(function ($values) use ($table, $tableName, $column_names) {
128+
return "INSERT INTO $tableName $column_names VALUES " . $values . ';';
129+
})
130+
->join(PHP_EOL);
163131

164132
return $insert_statement . PHP_EOL;
165-
166133
} else {
167-
168-
// orig
169134
$queryBuilder->get()
170-
->each(function ($row, $index) use ($table, &$query) {
135+
->each(function ($row, $index) use ($table, &$query, $tableName) {
171136
$row = $this->transformResultForInsert((array)$row, $table);
172-
$tableName = $table->getDoctrineTable()->getName();
173137

174-
$query .= "INSERT INTO `$tableName` (`" . implode('`, `', array_keys($row)) . '`) VALUES ';
175-
$query .= "(";
138+
$query .= "INSERT INTO $tableName ($this->escapeString" . implode("$this->escapeString, $this->escapeString", array_keys($row)) . "$this->escapeString) VALUES ";
139+
140+
$query .= "(";
176141

177142
$firstColumn = true;
178143
foreach ($row as $value) {
@@ -185,11 +150,8 @@ function($values) use($table, $tableName, $column_names) {
185150

186151
$query .= ");" . PHP_EOL;
187152
});
153+
}
188154

189155
return $query;
190156
}
191-
192-
protected function isTesting(): bool {
193-
return config('app.env') === 'workbench' || config('app.env') === 'ci';
194-
}
195157
}

‎src/TableDefinitions/TableDefinition.php

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,6 @@ public function __construct(Table $table)
2323
$this->dumpType = static::DUMP_FULL;
2424
}
2525

26-
public function schemaOnly()
27-
{
28-
$this->dumpType = static::DUMP_SCHEMA;
29-
30-
return $this;
31-
}
32-
3326
public function fullDump()
3427
{
3528
$this->dumpType = static::DUMP_FULL;

‎tests/DumperTest.php

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,7 @@ public function it_can_dump_certain_tables_as_schema_only()
161161
$outputFile = base_path('test.sql');
162162

163163
$this->app['config']['masked-dump.default'] = DumpSchema::define()
164-
->allTables()
165-
->schemaOnly('migrations')
166-
->schemaOnly('users');
164+
->allTables();
167165

168166
$this->artisan('db:masked-dump', [
169167
'output' => $outputFile

‎tests/__snapshots__/DumperTest__it_can_dump_all_tables_without_modifications__1.txt

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,4 @@
1-
DROP TABLE IF EXISTS `cache`;
2-
CREATE TABLE cache ("key" VARCHAR(255) NOT NULL, value CLOB NOT NULL, expiration INTEGER NOT NULL);
3-
LOCK TABLES `cache` WRITE;
4-
ALTER TABLE `cache` DISABLE KEYS;
5-
ALTER TABLE `cache` ENABLE KEYS;
6-
UNLOCK TABLES;
7-
DROP TABLE IF EXISTS `cache_locks`;
8-
CREATE TABLE cache_locks ("key" VARCHAR(255) NOT NULL, owner VARCHAR(255) NOT NULL, expiration INTEGER NOT NULL);
9-
LOCK TABLES `cache_locks` WRITE;
10-
ALTER TABLE `cache_locks` DISABLE KEYS;
11-
ALTER TABLE `cache_locks` ENABLE KEYS;
12-
UNLOCK TABLES;
13-
DROP TABLE IF EXISTS `failed_jobs`;
14-
CREATE TABLE failed_jobs (id INTEGER NOT NULL, uuid VARCHAR(255) NOT NULL, connection CLOB NOT NULL, queue CLOB NOT NULL, payload CLOB NOT NULL, exception CLOB NOT NULL, failed_at DATETIME NOT NULL);
15-
LOCK TABLES `failed_jobs` WRITE;
16-
ALTER TABLE `failed_jobs` DISABLE KEYS;
17-
ALTER TABLE `failed_jobs` ENABLE KEYS;
18-
UNLOCK TABLES;
19-
DROP TABLE IF EXISTS `job_batches`;
20-
CREATE TABLE job_batches (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL, total_jobs INTEGER NOT NULL, pending_jobs INTEGER NOT NULL, failed_jobs INTEGER NOT NULL, failed_job_ids CLOB NOT NULL, options CLOB NOT NULL, cancelled_at INTEGER NOT NULL, created_at INTEGER NOT NULL, finished_at INTEGER NOT NULL);
21-
LOCK TABLES `job_batches` WRITE;
22-
ALTER TABLE `job_batches` DISABLE KEYS;
23-
ALTER TABLE `job_batches` ENABLE KEYS;
24-
UNLOCK TABLES;
25-
DROP TABLE IF EXISTS `jobs`;
26-
CREATE TABLE jobs (id INTEGER NOT NULL, queue VARCHAR(255) NOT NULL, payload CLOB NOT NULL, attempts INTEGER NOT NULL, reserved_at INTEGER NOT NULL, available_at INTEGER NOT NULL, created_at INTEGER NOT NULL);
27-
LOCK TABLES `jobs` WRITE;
28-
ALTER TABLE `jobs` DISABLE KEYS;
29-
ALTER TABLE `jobs` ENABLE KEYS;
30-
UNLOCK TABLES;
31-
DROP TABLE IF EXISTS `migrations`;
32-
CREATE TABLE migrations (id INTEGER NOT NULL, migration VARCHAR(255) NOT NULL, batch INTEGER NOT NULL);
33-
LOCK TABLES `migrations` WRITE;
34-
ALTER TABLE `migrations` DISABLE KEYS;
351
INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1');
362
INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1');
373
INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1');
38-
ALTER TABLE `migrations` ENABLE KEYS;
39-
UNLOCK TABLES;
40-
DROP TABLE IF EXISTS `password_reset_tokens`;
41-
CREATE TABLE password_reset_tokens (email VARCHAR(255) NOT NULL, token VARCHAR(255) NOT NULL, created_at DATETIME NOT NULL);
42-
LOCK TABLES `password_reset_tokens` WRITE;
43-
ALTER TABLE `password_reset_tokens` DISABLE KEYS;
44-
ALTER TABLE `password_reset_tokens` ENABLE KEYS;
45-
UNLOCK TABLES;
46-
DROP TABLE IF EXISTS `sessions`;
47-
CREATE TABLE sessions (id VARCHAR(255) NOT NULL, user_id INTEGER NOT NULL, ip_address VARCHAR(255) NOT NULL, user_agent CLOB NOT NULL, payload CLOB NOT NULL, last_activity INTEGER NOT NULL);
48-
LOCK TABLES `sessions` WRITE;
49-
ALTER TABLE `sessions` DISABLE KEYS;
50-
ALTER TABLE `sessions` ENABLE KEYS;
51-
UNLOCK TABLES;
52-
DROP TABLE IF EXISTS `users`;
53-
CREATE TABLE users (id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, email VARCHAR(255) NOT NULL, email_verified_at DATETIME NOT NULL, password VARCHAR(255) NOT NULL, remember_token VARCHAR(255) NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL);
54-
LOCK TABLES `users` WRITE;
55-
ALTER TABLE `users` DISABLE KEYS;
564
INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00');
57-
ALTER TABLE `users` ENABLE KEYS;
58-
UNLOCK TABLES;

0 commit comments

Comments
 (0)