Skip to content

Commit 671b3f8

Browse files
committed
Merge branch '352-logicaldump-exclude-tables' into 'master'
feat: add excludeTables to logical dump (#348) Closes #352 and #348 See merge request postgres-ai/database-lab!536
2 parents ff916d4 + bac3ab1 commit 671b3f8

File tree

4 files changed

+109
-32
lines changed

4 files changed

+109
-32
lines changed

engine/configs/config.example.logical_generic.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,15 @@ retrieval:
219219
# Do not specify the databases section to take all databases.
220220
# databases:
221221
# database1:
222-
# # Option for a partial dump. Do not specify the tables section to dump all available tables.
222+
# Options for a partial dump.
223+
# Do not specify the tables section to dump all available tables.
224+
# Corresponds to the --table option of pg_dump.
223225
# tables:
224226
# - table1
227+
# Do not dump data for any of the tables matching pattern.
228+
# Corresponds to the --exclude-table option of pg_dump.
229+
# excludeTables:
230+
# - table2
225231
# database2:
226232
# databaseN:
227233

engine/configs/config.example.logical_rds_iam.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,9 +281,15 @@ retrieval:
281281
# format: directory
282282
# # Compression (only for plain-text dumps): "gzip", "bzip2", or "no". Default: "no".
283283
# compression: no
284-
# # Option for a partial restore. Do not specify the tables section to restore all available tables.
284+
# Options for a partial dump.
285+
# Do not specify the tables section to dump all available tables.
286+
# Corresponds to the --table option of pg_dump.
285287
# tables:
286288
# - table1
289+
# Do not dump data for any of the tables matching pattern.
290+
# Corresponds to the --exclude-table option of pg_dump.
291+
# excludeTables:
292+
# - table2
287293
# database2:
288294
# databaseN:
289295

engine/internal/retrieval/engine/postgres/logical/dump.go

Lines changed: 51 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,11 @@ type Source struct {
9898

9999
// DumpDefinition describes a database for dumping.
100100
type DumpDefinition struct {
101-
Tables []string `yaml:"tables"`
102-
Format string `yaml:"format"`
103-
Compression compressionType `yaml:"compression"`
104-
dbName string
101+
Tables []string `yaml:"tables"`
102+
ExcludeTables []string `yaml:"excludeTables"`
103+
Format string `yaml:"format"`
104+
Compression compressionType `yaml:"compression"`
105+
dbName string
105106
}
106107

107108
type dumpJobConfig struct {
@@ -435,13 +436,23 @@ func (d *DumpJob) cleanupDumpLocation(ctx context.Context, dumpContID string, db
435436
}
436437

437438
func (d *DumpJob) dumpDatabase(ctx context.Context, dumpContID, dbName string, dumpDefinition DumpDefinition) error {
438-
dumpCommand := d.buildLogicalDumpCommand(dbName, dumpDefinition.Tables)
439-
log.Msg("Running dump command: ", dumpCommand)
439+
dumpCommand := d.buildLogicalDumpCommand(dbName, dumpDefinition)
440+
441+
if len(dumpDefinition.Tables) > 0 ||
442+
len(dumpDefinition.ExcludeTables) > 0 {
443+
log.Msg("Partial dump")
444+
445+
if len(dumpDefinition.Tables) > 0 {
446+
log.Msg("Including tables: ", strings.Join(dumpDefinition.Tables, ", "))
447+
}
440448

441-
if len(dumpDefinition.Tables) > 0 {
442-
log.Msg("Partial dump will be run. Tables for dumping: ", strings.Join(dumpDefinition.Tables, ", "))
449+
if len(dumpDefinition.ExcludeTables) > 0 {
450+
log.Msg("Excluding tables: ", strings.Join(dumpDefinition.ExcludeTables, ", "))
451+
}
443452
}
444453

454+
log.Msg("Running dump command: ", dumpCommand)
455+
445456
if output, err := d.performDumpCommand(ctx, dumpContID, types.ExecConfig{
446457
Tty: true,
447458
Cmd: dumpCommand,
@@ -488,7 +499,12 @@ func setupPGData(ctx context.Context, dockerClient *client.Client, dataDir strin
488499
return nil
489500
}
490501

491-
func updateConfigs(ctx context.Context, dockerClient *client.Client, dataDir, contID string, configs map[string]string) error {
502+
func updateConfigs(
503+
ctx context.Context,
504+
dockerClient *client.Client,
505+
dataDir, contID string,
506+
configs map[string]string,
507+
) error {
492508
log.Dbg("Stopping container to update configuration")
493509

494510
tools.StopContainer(ctx, dockerClient, contID, cont.StopTimeout)
@@ -605,21 +621,38 @@ func (d *DumpJob) getExecEnvironmentVariables() []string {
605621
return execEnvs
606622
}
607623

608-
func (d *DumpJob) buildLogicalDumpCommand(dbName string, tables []string) []string {
609-
optionalArgs := map[string]string{
610-
"--host": d.config.db.Host,
611-
"--port": strconv.Itoa(d.config.db.Port),
612-
"--username": d.config.db.Username,
613-
"--dbname": dbName,
614-
"--jobs": strconv.Itoa(d.DumpOptions.ParallelJobs),
624+
func (d *DumpJob) buildLogicalDumpCommand(dbName string, dump DumpDefinition) []string {
625+
// don't use map here, it creates inconsistency in the order of arguments
626+
dumpCmd := []string{"pg_dump", "--create"}
627+
628+
if d.config.db.Host != "" {
629+
dumpCmd = append(dumpCmd, "--host", d.config.db.Host)
630+
}
631+
632+
if d.config.db.Port > 0 {
633+
dumpCmd = append(dumpCmd, "--port", strconv.Itoa(d.config.db.Port))
615634
}
616635

617-
dumpCmd := append([]string{"pg_dump", "--create"}, prepareCmdOptions(optionalArgs)...)
636+
if d.config.db.Username != "" {
637+
dumpCmd = append(dumpCmd, "--username", d.config.db.Username)
638+
}
618639

619-
for _, table := range tables {
640+
if dbName != "" {
641+
dumpCmd = append(dumpCmd, "--dbname", dbName)
642+
}
643+
644+
if d.DumpOptions.ParallelJobs > 0 {
645+
dumpCmd = append(dumpCmd, "--jobs", strconv.Itoa(d.DumpOptions.ParallelJobs))
646+
}
647+
648+
for _, table := range dump.Tables {
620649
dumpCmd = append(dumpCmd, "--table", table)
621650
}
622651

652+
for _, table := range dump.ExcludeTables {
653+
dumpCmd = append(dumpCmd, "--exclude-table", table)
654+
}
655+
623656
// Define if restore directly or export to dump location.
624657
if d.DumpOptions.Restore.Enabled {
625658
dumpCmd = append(dumpCmd, "--format", customFormat)
@@ -652,18 +685,6 @@ func (d *DumpJob) buildLogicalRestoreCommand(dbName string) []string {
652685
return restoreCmd
653686
}
654687

655-
func prepareCmdOptions(options map[string]string) []string {
656-
cmdOptions := []string{}
657-
658-
for optionKey, optionValue := range options {
659-
if optionValue != "" {
660-
cmdOptions = append(cmdOptions, optionKey, optionValue)
661-
}
662-
}
663-
664-
return cmdOptions
665-
}
666-
667688
func (d *DumpJob) markDatabaseData() error {
668689
if err := d.dbMarker.CreateConfig(); err != nil {
669690
return errors.Wrap(err, "failed to create a DBMarker config of the database")

engine/internal/retrieval/engine/postgres/logical/restore_test.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,50 @@ func TestRestoreCommandBuilding(t *testing.T) {
122122
}
123123
}
124124

125+
func TestDumpCommandBuilding(t *testing.T) {
126+
logicalJob := &DumpJob{
127+
config: dumpJobConfig{
128+
db: Connection{
129+
Host: "localhost",
130+
Port: 5432,
131+
DBName: "postgres",
132+
Username: "john",
133+
Password: "secret",
134+
},
135+
},
136+
}
137+
138+
testCases := []struct {
139+
copyOptions DumpOptions
140+
command []string
141+
}{
142+
{
143+
copyOptions: DumpOptions{
144+
ParallelJobs: 1,
145+
DumpLocation: "/tmp/db.dump",
146+
Databases: map[string]DumpDefinition{
147+
"testDB": {
148+
Tables: []string{"test", "users"},
149+
ExcludeTables: []string{
150+
"test2",
151+
"users2",
152+
},
153+
},
154+
},
155+
},
156+
command: []string{"pg_dump", "--create", "--host", "localhost", "--port", "5432", "--username", "john", "--dbname", "testDB", "--jobs", "1", "--table", "test", "--table", "users", "--exclude-table", "test2", "--exclude-table", "users2", "--format", "directory", "--file", "/tmp/db.dump/testDB"},
157+
},
158+
}
159+
160+
for _, tc := range testCases {
161+
logicalJob.DumpOptions = tc.copyOptions
162+
for dbName, definition := range tc.copyOptions.Databases {
163+
dumpCommand := logicalJob.buildLogicalDumpCommand(dbName, definition)
164+
assert.Equal(t, tc.command, dumpCommand)
165+
}
166+
}
167+
}
168+
125169
func TestDiscoverDumpDirectories(t *testing.T) {
126170
t.Skip("docker client is required")
127171

0 commit comments

Comments
 (0)