From 337423e137039c3bd03f4f81ba1de1446af20288 Mon Sep 17 00:00:00 2001 From: doorknob88 Date: Tue, 6 May 2025 15:00:44 -0700 Subject: [PATCH] Added LibSQL support --- database/libsql/README.md | 19 ++ .../migrations/33_create_table.down.sql | 1 + .../migrations/33_create_table.up.sql | 3 + .../migrations/44_alter_table.down.sql | 1 + .../examples/migrations/44_alter_table.up.sql | 1 + database/libsql/libsql.go | 297 ++++++++++++++++++ database/libsql/libsql_test.go | 133 ++++++++ go.mod | 5 +- go.sum | 8 + 9 files changed, 467 insertions(+), 1 deletion(-) create mode 100644 database/libsql/README.md create mode 100644 database/libsql/examples/migrations/33_create_table.down.sql create mode 100644 database/libsql/examples/migrations/33_create_table.up.sql create mode 100644 database/libsql/examples/migrations/44_alter_table.down.sql create mode 100644 database/libsql/examples/migrations/44_alter_table.up.sql create mode 100644 database/libsql/libsql.go create mode 100644 database/libsql/libsql_test.go diff --git a/database/libsql/README.md b/database/libsql/README.md new file mode 100644 index 000000000..ddae9c4a2 --- /dev/null +++ b/database/libsql/README.md @@ -0,0 +1,19 @@ +# libsql + +`libsql://file:path/to/database?query` +`libsql://your-turso-db.turso.io?authToken=your-auth-token` + +Unlike other migrate database drivers, the libsql driver will automatically wrap each migration in an implicit transaction by default. Migrations must not contain explicit `BEGIN` or `COMMIT` statements. This behavior may change in a future major release. (See below for a workaround.) + +Refer to [upstream documentation](https://github.com/tursodatabase/go-libsql#usage) for a complete list of query parameters supported by the libsql database driver. The auxiliary query parameters listed below may be supplied to tailor migrate behavior. All auxiliary query parameters are optional. + +| URL Query | WithInstance Config | Description | +|------------|---------------------|-------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table. Defaults to `schema_migrations`. | +| `x-no-tx-wrap` | `NoTxWrap` | Disable implicit transactions when `true`. Migrations may, and should, contain explicit `BEGIN` and `COMMIT` statements. | + +## Notes + +* Uses the `github.com/tursodatabase/go-libsql` libsql db driver. +* Supports local file databases, in-memory databases (`libsql://file::memory:`) and remote Turso databases. +``` diff --git a/database/libsql/examples/migrations/33_create_table.down.sql b/database/libsql/examples/migrations/33_create_table.down.sql new file mode 100644 index 000000000..72d18c554 --- /dev/null +++ b/database/libsql/examples/migrations/33_create_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/libsql/examples/migrations/33_create_table.up.sql b/database/libsql/examples/migrations/33_create_table.up.sql new file mode 100644 index 000000000..5ad3404d1 --- /dev/null +++ b/database/libsql/examples/migrations/33_create_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE pets ( + name string +); \ No newline at end of file diff --git a/database/libsql/examples/migrations/44_alter_table.down.sql b/database/libsql/examples/migrations/44_alter_table.down.sql new file mode 100644 index 000000000..72d18c554 --- /dev/null +++ b/database/libsql/examples/migrations/44_alter_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/libsql/examples/migrations/44_alter_table.up.sql b/database/libsql/examples/migrations/44_alter_table.up.sql new file mode 100644 index 000000000..f0682fcca --- /dev/null +++ b/database/libsql/examples/migrations/44_alter_table.up.sql @@ -0,0 +1 @@ +ALTER TABLE pets ADD predator bool; diff --git a/database/libsql/libsql.go b/database/libsql/libsql.go new file mode 100644 index 000000000..f2e4553ae --- /dev/null +++ b/database/libsql/libsql.go @@ -0,0 +1,297 @@ +package libsql + +import ( + "database/sql" + "fmt" + "io" + nurl "net/url" + "strconv" + "strings" + + "go.uber.org/atomic" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/tursodatabase/go-libsql" +) + +func init() { + database.Register("libsql", &Sqlite{}) +} + +var DefaultMigrationsTable = "schema_migrations" +var ( + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string + NoTxWrap bool +} + +type Sqlite struct { + db *sql.DB + isLocked atomic.Bool + + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + mx := &Sqlite{ + db: instance, + config: config, + } + if err := mx.ensureVersionTable(); err != nil { + return nil, err + } + return mx, nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Sqlite type. +func (m *Sqlite) ensureVersionTable() (err error) { + if err = m.Lock(); err != nil { + return err + } + + defer func() { + if e := m.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, m.config.MigrationsTable, m.config.MigrationsTable) + + if _, err := m.db.Exec(query); err != nil { + return err + } + return nil +} + +func (m *Sqlite) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "libsql://", "", 1) + db, err := sql.Open("libsql", dbfile) + if err != nil { + return nil, err + } + + qv := purl.Query() + + migrationsTable := qv.Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + + noTxWrap := false + if v := qv.Get("x-no-tx-wrap"); v != "" { + noTxWrap, err = strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("x-no-tx-wrap: %s", err) + } + } + + mx, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + NoTxWrap: noTxWrap, + }) + if err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) Close() error { + return m.db.Close() +} + +func (m *Sqlite) Drop() (err error) { + query := `SELECT name FROM sqlite_master WHERE type = 'table';` + tables, err := m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if err := tables.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if len(tableNames) > 0 { + for _, t := range tableNames { + query := "DROP TABLE " + t + if m.config.NoTxWrap { + _, err = m.db.Exec(query) + } else { + err = m.executeQuery(query) + } + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if m.config.NoTxWrap { + _, _ = m.db.Exec("ROLLBACK;") + } + + query := "VACUUM" + // VACUUM cannot be run inside a transaction + _, err = m.db.Exec(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +func (m *Sqlite) Lock() error { + if !m.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (m *Sqlite) Unlock() error { + if !m.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (m *Sqlite) Run(migration io.Reader) error { + migr, err := io.ReadAll(migration) + if err != nil { + return err + } + query := string(migr[:]) + + if m.config.NoTxWrap { + return m.executeQueryNoTx(query) + } + return m.executeQuery(query) +} + +func (m *Sqlite) executeQuery(query string) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +func (m *Sqlite) executeQueryNoTx(query string) error { + if _, err := m.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + return nil +} + +func (m *Sqlite) SetVersion(version int, dirty bool) error { + query := "DELETE FROM " + m.config.MigrationsTable + + if m.config.NoTxWrap { + if _, err := m.db.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if version >= 0 || (version == database.NilVersion && dirty) { + insertQuery := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) + if _, err := m.db.Exec(insertQuery, version, dirty); err != nil { + return &database.Error{OrigErr: err, Query: []byte(insertQuery)} + } + } + return nil + } + + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *Sqlite) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} diff --git a/database/libsql/libsql_test.go b/database/libsql/libsql_test.go new file mode 100644 index 000000000..de64ec3cb --- /dev/null +++ b/database/libsql/libsql_test.go @@ -0,0 +1,133 @@ +package libsql + +import ( + "database/sql" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/tursodatabase/go-libsql" +) + +func Test(t *testing.T) { + dir := t.TempDir() + t.Logf("DB path : %s\n", filepath.Join(dir, "libsql.db")) + p := &Sqlite{} + addr := fmt.Sprintf("libsql://file:%s", filepath.Join(dir, "libsql.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) +} + +func TestMigrate(t *testing.T) { + dir := t.TempDir() + t.Logf("DB path : %s\n", filepath.Join(dir, "libsql.db")) + + db, err := sql.Open("libsql", filepath.Join(dir, "libsql.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + driver, err := WithInstance(db, &Config{}) + if err != nil { + t.Fatal(err) + } + + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { + t.Fatal(err) + } + dt.TestMigrate(t, m) +} + +func TestMigrationTable(t *testing.T) { + dir := t.TempDir() + + t.Logf("DB path : %s\n", filepath.Join(dir, "libsql.db")) + + db, err := sql.Open("libsql", filepath.Join(dir, "libsql.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + + config := &Config{ + MigrationsTable: "my_migration_table", + } + driver, err := WithInstance(db, config) + if err != nil { + t.Fatal(err) + } + m, err := migrate.NewWithDatabaseInstance( + "file://./examples/migrations", + "ql", driver) + if err != nil { + t.Fatal(err) + } + t.Log("UP") + err = m.Up() + if err != nil { + t.Fatal(err) + } + + _, err = db.Query(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable)) + if err != nil { + t.Fatal(err) + } +} + +func TestNoTxWrap(t *testing.T) { + dir := t.TempDir() + t.Logf("DB path : %s\n", filepath.Join(dir, "libsql.db")) + p := &Sqlite{} + addr := fmt.Sprintf("libsql://file:%s?x-no-tx-wrap=true", filepath.Join(dir, "libsql.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + // An explicit BEGIN statement would ordinarily fail without x-no-tx-wrap. + // (Transactions in sqlite may not be nested.) + dt.Test(t, d, []byte("BEGIN; CREATE TABLE t (Qty int, Name string); COMMIT;")) +} + +func TestNoTxWrapInvalidValue(t *testing.T) { + dir := t.TempDir() + t.Logf("DB path : %s\n", filepath.Join(dir, "libsql.db")) + p := &Sqlite{} + addr := fmt.Sprintf("libsql://file:%s?x-no-tx-wrap=yeppers", filepath.Join(dir, "libsql.db")) + _, err := p.Open(addr) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "x-no-tx-wrap") + assert.Contains(t, err.Error(), "invalid syntax") + } +} + +func TestMigrateWithDirectoryNameContainsWhitespaces(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "libsql.db") + t.Logf("DB path : %s\n", dbPath) + p := &Sqlite{} + addr := fmt.Sprintf("libsql://file:%s", dbPath) + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) +} diff --git a/go.mod b/go.mod index 3c20151f2..9b83d2ca1 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( ) require ( + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -51,11 +52,13 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kr/text v0.2.0 // indirect + github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/tursodatabase/go-libsql v0.0.0-20250416102726-983f7e9acb0e // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/otel v1.29.0 // indirect @@ -175,7 +178,7 @@ require ( gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.36.0 // indirect - golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.12.0 // indirect diff --git a/go.sum b/go.sum index e30a51a2a..2cf1f11be 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= @@ -447,6 +449,8 @@ github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06 h1:JLvn7D+wXjH9g4Jsjo+VqmzTUpl/LX7vfr6VOfSWTdM= +github.com/libsql/sqlite-antlr4-parser v0.0.0-20240327125255-dbf53b6cbf06/go.mod h1:FUkZ5OHjlGPjnM2UyGJz9TypXQFgYqw6AFNO1UiROTM= github.com/markbates/pkger v0.15.1 h1:3MPelV53RnGSW07izx5xGxl4e/sdRD6zqseIk0rMASY= github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -569,6 +573,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tursodatabase/go-libsql v0.0.0-20250416102726-983f7e9acb0e h1:DUEcD8ukLWxIlcRWWJSuAX6IbEQln2bc7t9HOT45FFk= +github.com/tursodatabase/go-libsql v0.0.0-20250416102726-983f7e9acb0e/go.mod h1:TjsB2miB8RW2Sse8sdxzVTdeGlx74GloD5zJYUC38d8= github.com/xanzy/go-gitlab v0.15.0 h1:rWtwKTgEnXyNUGrOArN7yyc3THRkpYcKXIXia9abywQ= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -660,6 +666,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=