From f7835b8b0a5642c4bd71b2c6467944027bd8ef4f Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 21 Aug 2023 17:58:18 -0700 Subject: [PATCH 01/12] spike: update db address and credentials --- example/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/example/main.go b/example/main.go index 8601754..1667e6f 100644 --- a/example/main.go +++ b/example/main.go @@ -27,9 +27,11 @@ func main() { flag.Parse() db := pg.Connect(&pg.Options{ - User: "postgres", - Database: "pg_migrations_example", - }) + Addr: "10.164.0.189:26257", + User: "pvos_dia_db_rw", + Password: "c0f47960-9c78-4039-9760-c0ff390b3259_pvos_dia_db_rw", + Database: "pvos_dia_db", + }) oldVersion, newVersion, err := migrations.Run(db, flag.Args()...) if err != nil { From 5c7b1e2fef5143f14a8408a7fdb940e454a6f4b5 Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 21 Aug 2023 18:17:07 -0700 Subject: [PATCH 02/12] debug: print oldVersion and newVersion --- example/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/example/main.go b/example/main.go index 1667e6f..542d2b8 100644 --- a/example/main.go +++ b/example/main.go @@ -34,6 +34,7 @@ func main() { }) oldVersion, newVersion, err := migrations.Run(db, flag.Args()...) + fmt.Println("example/main.go: --------------- oldVersion: ", oldVersion, ", newVersion: ", newVersion) if err != nil { exitf(err.Error()) } From a83c6fa4c5c543ccff2186ce09abbd384091866c Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 21 Aug 2023 19:15:14 -0700 Subject: [PATCH 03/12] spike: skip TLS verification --- example/main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/example/main.go b/example/main.go index 542d2b8..8f209ea 100644 --- a/example/main.go +++ b/example/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "crypto/tls" "github.com/go-pg/migrations/v8" "github.com/go-pg/pg/v10" @@ -27,10 +28,11 @@ func main() { flag.Parse() db := pg.Connect(&pg.Options{ - Addr: "10.164.0.189:26257", - User: "pvos_dia_db_rw", - Password: "c0f47960-9c78-4039-9760-c0ff390b3259_pvos_dia_db_rw", - Database: "pvos_dia_db", + Addr: "10.164.0.189:26257", + User: "pvos_dia_db_rw", + Password: "c0f47960-9c78-4039-9760-c0ff390b3259_pvos_dia_db_rw", + Database: "pvos_dia_db", + TLSConfig: &tls.Config{InsecureSkipVerify: true}, }) oldVersion, newVersion, err := migrations.Run(db, flag.Args()...) From 13e96410859152872d3f7874133ef8c5b762791d Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Tue, 22 Aug 2023 15:01:26 -0700 Subject: [PATCH 04/12] debug: add more debug logs --- collection.go | 2 ++ example/main.go | 1 + 2 files changed, 3 insertions(+) diff --git a/collection.go b/collection.go index 29c8d99..2b9bd67 100644 --- a/collection.go +++ b/collection.go @@ -385,6 +385,7 @@ func (c *Collection) Run(db DB, a ...string) (oldVersion, newVersion int64, err return } + fmt.Println("c.Run: ----------- a:", a) cmd := "up" if len(a) > 0 { cmd = a[0] @@ -409,6 +410,7 @@ func (c *Collection) Run(db DB, a ...string) (oldVersion, newVersion int64, err } filename := fmtMigrationFilename(version+1, strings.Join(a[1:], "_")) + fmt.Println("c.Run: ----------------- filename: ", filename) err = createMigrationFile(filename) if err != nil { return diff --git a/example/main.go b/example/main.go index 8f209ea..0328e58 100644 --- a/example/main.go +++ b/example/main.go @@ -35,6 +35,7 @@ func main() { TLSConfig: &tls.Config{InsecureSkipVerify: true}, }) + fmt.Println("main.go: flag:Args(): ", flag.Args()) oldVersion, newVersion, err := migrations.Run(db, flag.Args()...) fmt.Println("example/main.go: --------------- oldVersion: ", oldVersion, ", newVersion: ", newVersion) if err != nil { From 6c90136690d093441ac67c01f3df73bcb09638bb Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Tue, 22 Aug 2023 16:54:04 -0700 Subject: [PATCH 05/12] debug: add more debug logs --- collection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/collection.go b/collection.go index 2b9bd67..ab602f4 100644 --- a/collection.go +++ b/collection.go @@ -385,6 +385,8 @@ func (c *Collection) Run(db DB, a ...string) (oldVersion, newVersion int64, err return } + fmt.Println("c.Run: -------------- migrations: ", migrations) + fmt.Println("c.Run: ----------- a:", a) cmd := "up" if len(a) > 0 { From 5c43ad2e7ca39a3afa2284ac0724e2415cd5c388 Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Thu, 31 Aug 2023 18:09:13 -0700 Subject: [PATCH 06/12] debug: add more logs --- collection.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/collection.go b/collection.go index ab602f4..2303844 100644 --- a/collection.go +++ b/collection.go @@ -144,6 +144,7 @@ func migrationFile() string { // DiscoverSQLMigrations scan the dir for files with .sql extension // and adds discovered SQL migrations to the collection. func (c *Collection) DiscoverSQLMigrations(dir string) error { + fmt.Println("c.DiscoverSQlMigrations: ---------------------- dir: ", dir) dir, err := filepath.Abs(dir) if err != nil { return err @@ -155,6 +156,7 @@ func (c *Collection) DiscoverSQLMigrations(dir string) error { // DiscoverSQLMigrations scan the dir from the given filesystem for files with .sql extension // and adds discovered SQL migrations to the collection. func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir string) error { + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem function is called!!!!!!!!") if c.isVisitedDir(dir) { return nil } @@ -195,16 +197,19 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir // Sort files to have consistent errors. sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() }) - for _, f := range files { + for i, f := range files { + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ---------------------- f: ", f, ", i: ", i) if f.IsDir() { continue } fileName := f.Name() if !strings.HasSuffix(fileName, ".sql") { + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: -------------------- fileName", fileName, " does not have suffix .sql") continue } + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ------------------ file", fileName, " running strings.IndeByte") idx := strings.IndexByte(fileName, '_') if idx == -1 { err := fmt.Errorf( @@ -214,10 +219,12 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir } version, err := strconv.ParseInt(fileName[:idx], 10, 64) + if err != nil { return err } + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ------------- version", version) m := newMigration(version) filePath := filepath.Join(dir, fileName) @@ -243,7 +250,8 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir "file=%q must have extension .up.sql or .down.sql", fileName) } - for _, m := range ms { + for i, m := range ms { + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: i:", i, ", m:", m) c.addMigration(m) } @@ -254,6 +262,7 @@ func (c *Collection) isVisitedDir(dir string) bool { c.mu.Lock() defer c.mu.Unlock() + fmt.Println("c.isVisiteddir: ---------------- dir: ", dir, ", visitedDirs: ", c.visitedDirs) if _, ok := c.visitedDirs[dir]; ok { return true } From a84b75bfe158dccf2afeabb3c9aa43a45d2c945d Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Thu, 31 Aug 2023 19:11:56 -0700 Subject: [PATCH 07/12] debug: add more debug logs --- collection.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/collection.go b/collection.go index 2303844..24cf22f 100644 --- a/collection.go +++ b/collection.go @@ -35,6 +35,10 @@ func (m *Migration) String() string { return strconv.FormatInt(m.Version, 10) } +func (m *Migration) PrintAsString() { + fmt.Println("Migration: Version: ", m.Version, ", UpTx: ", m.UpTx, ", func(DB) up: ", m.Up, ", DownTx: ", m.DownTx, ", func(DB) down", m.Down) +} + type Collection struct { tableName string sqlAutodiscoverDisabled bool @@ -176,6 +180,7 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir var ms []*Migration newMigration := func(version int64) *Migration { + fmt.Println("newMigration func is called, input version: ", version) for i := range ms { m := ms[i] if m.Version == version { @@ -198,14 +203,14 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() }) for i, f := range files { - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ---------------------- f: ", f, ", i: ", i) + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ---------------------- f: ", f, ", i: ", i, ", so continue to loop another file") if f.IsDir() { continue } fileName := f.Name() if !strings.HasSuffix(fileName, ".sql") { - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: -------------------- fileName", fileName, " does not have suffix .sql") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: -------------------- fileName", fileName, " does not have suffix .sql, so continue to loop another file") continue } @@ -368,6 +373,7 @@ func (c *Collection) MustRegisterTx(fns ...func(DB) error) { } func (c *Collection) Migrations() []*Migration { + fmt.Println("c.Migrations: ------------------ c.Migrations() is called.") if !c.sqlAutodiscoverDisabled { _ = c.DiscoverSQLMigrations(filepath.Dir(migrationFile())) From fb685ffdb757281dbf3eeaec4561d1be0813f776 Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 11 Sep 2023 19:53:01 -0700 Subject: [PATCH 08/12] debug: update debug log to make them neat --- collection.go | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/collection.go b/collection.go index 24cf22f..3dff004 100644 --- a/collection.go +++ b/collection.go @@ -39,6 +39,10 @@ func (m *Migration) PrintAsString() { fmt.Println("Migration: Version: ", m.Version, ", UpTx: ", m.UpTx, ", func(DB) up: ", m.Up, ", DownTx: ", m.DownTx, ", func(DB) down", m.Down) } +func (m *Migration) PrintAsStringWithIndex(i int) { + fmt.Println("Migration: index: ", i, " Version: ", m.Version, ", UpTx: ", m.UpTx, ", func(DB) up: ", m.Up, ", DownTx: ", m.DownTx, ", func(DB) down", m.Down) +} + type Collection struct { tableName string sqlAutodiscoverDisabled bool @@ -148,7 +152,7 @@ func migrationFile() string { // DiscoverSQLMigrations scan the dir for files with .sql extension // and adds discovered SQL migrations to the collection. func (c *Collection) DiscoverSQLMigrations(dir string) error { - fmt.Println("c.DiscoverSQlMigrations: ---------------------- dir: ", dir) + fmt.Println("c.DiscoverSQlMigrations: dir: ", dir) dir, err := filepath.Abs(dir) if err != nil { return err @@ -203,18 +207,26 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() }) for i, f := range files { - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ---------------------- f: ", f, ", i: ", i, ", so continue to loop another file") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: f: ", f, ", i: ", i, ", so continue to loop another file") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: current m []*Migrations:") + for j, m := range ms { + + } + if f.IsDir() { continue } fileName := f.Name() if !strings.HasSuffix(fileName, ".sql") { - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: -------------------- fileName", fileName, " does not have suffix .sql, so continue to loop another file") + for i, m := range ms { + fmt("c.DiscoverSQLMigrationsFromFilesystem:") + } + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: fileName", fileName, " does not have suffix .sql, so continue to loop another file") continue } - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ------------------ file", fileName, " running strings.IndeByte") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: file", fileName, " running strings.IndeByte") idx := strings.IndexByte(fileName, '_') if idx == -1 { err := fmt.Errorf( @@ -229,7 +241,7 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir return err } - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: ------------- version", version) + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: - version", version) m := newMigration(version) filePath := filepath.Join(dir, fileName) @@ -267,7 +279,7 @@ func (c *Collection) isVisitedDir(dir string) bool { c.mu.Lock() defer c.mu.Unlock() - fmt.Println("c.isVisiteddir: ---------------- dir: ", dir, ", visitedDirs: ", c.visitedDirs) + fmt.Println("c.isVisiteddir: dir: ", dir, ", visitedDirs: ", c.visitedDirs) if _, ok := c.visitedDirs[dir]; ok { return true } @@ -295,7 +307,7 @@ func newSQLMigration(fs http.FileSystem, filePath string) func(DB) error { for scanner.Scan() { b := scanner.Bytes() - const prefix = "--gopg:" + const prefix = "gopg:" if bytes.HasPrefix(b, []byte(prefix)) { b = b[len(prefix):] if bytes.Equal(b, []byte("split")) { @@ -373,7 +385,7 @@ func (c *Collection) MustRegisterTx(fns ...func(DB) error) { } func (c *Collection) Migrations() []*Migration { - fmt.Println("c.Migrations: ------------------ c.Migrations() is called.") + fmt.Println("c.Migrations: c.Migrations() is called.") if !c.sqlAutodiscoverDisabled { _ = c.DiscoverSQLMigrations(filepath.Dir(migrationFile())) @@ -400,9 +412,9 @@ func (c *Collection) Run(db DB, a ...string) (oldVersion, newVersion int64, err return } - fmt.Println("c.Run: -------------- migrations: ", migrations) + fmt.Println("c.Run: migrations: ", migrations) - fmt.Println("c.Run: ----------- a:", a) + fmt.Println("c.Run: - a:", a) cmd := "up" if len(a) > 0 { cmd = a[0] @@ -427,7 +439,7 @@ func (c *Collection) Run(db DB, a ...string) (oldVersion, newVersion int64, err } filename := fmtMigrationFilename(version+1, strings.Join(a[1:], "_")) - fmt.Println("c.Run: ----------------- filename: ", filename) + fmt.Println("c.Run: - filename: ", filename) err = createMigrationFile(filename) if err != nil { return From 9075ccf184486de2b91b58a9570f190557a8eb1f Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 11 Sep 2023 20:07:19 -0700 Subject: [PATCH 09/12] debug: add more debug logs --- collection.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/collection.go b/collection.go index 3dff004..64c6674 100644 --- a/collection.go +++ b/collection.go @@ -56,7 +56,9 @@ func NewCollection(migrations ...*Migration) *Collection { c := &Collection{ tableName: "gopg_migrations", } - for _, m := range migrations { + fmt.Println("NewCollection(): len(migrations): ", len(migrations)) + for i, m := range migrations { + m.PrintAsStringWithIndex(i) c.addMigration(m) } return c @@ -209,9 +211,6 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir for i, f := range files { fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: f: ", f, ", i: ", i, ", so continue to loop another file") fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: current m []*Migrations:") - for j, m := range ms { - - } if f.IsDir() { continue @@ -219,14 +218,15 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir fileName := f.Name() if !strings.HasSuffix(fileName, ".sql") { + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: fileName", fileName, " does not have suffix .sql, so continue to loop another file") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: len(ms): ", len(ms)) for i, m := range ms { - fmt("c.DiscoverSQLMigrationsFromFilesystem:") + m.PrintAsStringWithIndex(i) } - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: fileName", fileName, " does not have suffix .sql, so continue to loop another file") continue } - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: file", fileName, " running strings.IndeByte") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: file", fileName, " running strings.IndexByte") idx := strings.IndexByte(fileName, '_') if idx == -1 { err := fmt.Errorf( @@ -241,7 +241,7 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir return err } - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: - version", version) + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: version", version) m := newMigration(version) filePath := filepath.Join(dir, fileName) From 5e4d3fca2a48de63915dc2c6e48901f93427beb5 Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 11 Sep 2023 20:14:56 -0700 Subject: [PATCH 10/12] debug: add more debug logs --- collection.go | 1 + 1 file changed, 1 insertion(+) diff --git a/collection.go b/collection.go index 64c6674..2ece7d1 100644 --- a/collection.go +++ b/collection.go @@ -339,6 +339,7 @@ func newSQLMigration(fs http.FileSystem, filePath string) func(DB) error { } for _, q := range queries { + fmt.Println("newSQLMigration: q: ", queries) _, err = db.Exec(q) if err != nil { return err From 5d20d84e6e8dfd28acb6458e7dd6326e32335f9b Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 11 Sep 2023 20:30:31 -0700 Subject: [PATCH 11/12] debug: correct some debug logs --- collection.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collection.go b/collection.go index 2ece7d1..27d0a3c 100644 --- a/collection.go +++ b/collection.go @@ -209,7 +209,7 @@ func (c *Collection) DiscoverSQLMigrationsFromFilesystem(fs http.FileSystem, dir sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() }) for i, f := range files { - fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: f: ", f, ", i: ", i, ", so continue to loop another file") + fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: f: ", f, ", i: ", i) fmt.Println("c.DiscoverSQLMigrationsFromFilesystem: current m []*Migrations:") if f.IsDir() { From 75eba4cb67c43041182291c111c686a38b804e8f Mon Sep 17 00:00:00 2001 From: Jeff Liang Date: Mon, 11 Sep 2023 21:06:35 -0700 Subject: [PATCH 12/12] debug: add more debug logs --- collection.go | 3 +++ default.go | 2 ++ example/1_initial.go | 1 + example/2_add_id.go | 1 + example/3_seed_data.go | 1 + 5 files changed, 8 insertions(+) diff --git a/collection.go b/collection.go index 27d0a3c..0d3a9e5 100644 --- a/collection.go +++ b/collection.go @@ -84,11 +84,13 @@ func (c *Collection) DisableSQLAutodiscover(flag bool) *Collection { // Register registers new database migration. Must be called // from a file with name like "1_initialize_db.go". func (c *Collection) Register(fns ...func(DB) error) error { + fmt.Println("c.Register is called!!!!!!") return c.register(false, fns...) } // RegisterTx is like Register, but migration will be run in a transaction. func (c *Collection) RegisterTx(fns ...func(DB) error) error { + fmt.Println("c.RegisterTx is called!!!!!!") return c.register(true, fns...) } @@ -745,6 +747,7 @@ func (c *Collection) begin(db DB) (*pg.Tx, int64, error) { } func extractVersionGo(name string) (int64, error) { + fmt.Println("extractVersionGo: name: ", name) base := filepath.Base(name) if !strings.HasSuffix(name, ".go") { return 0, fmt.Errorf("file=%q must have extension .go", base) diff --git a/default.go b/default.go index 1ac0ae3..9710e79 100644 --- a/default.go +++ b/default.go @@ -1,4 +1,5 @@ package migrations +import "fmt" var DefaultCollection = NewCollection() @@ -32,6 +33,7 @@ func MustRegister(fns ...func(DB) error) { } func MustRegisterTx(fns ...func(DB) error) { + fmt.Println("MustRegisterTx function is called") DefaultCollection.MustRegisterTx(fns...) } diff --git a/example/1_initial.go b/example/1_initial.go index 72d281e..40a8e2c 100644 --- a/example/1_initial.go +++ b/example/1_initial.go @@ -7,6 +7,7 @@ import ( ) func init() { + fmt.Println("1_initial.go: init function is called") migrations.MustRegisterTx(func(db migrations.DB) error { fmt.Println("creating table my_table...") _, err := db.Exec(`CREATE TABLE my_table()`) diff --git a/example/2_add_id.go b/example/2_add_id.go index 4341cad..071fe90 100644 --- a/example/2_add_id.go +++ b/example/2_add_id.go @@ -7,6 +7,7 @@ import ( ) func init() { + fmt.Println("2_add_id.go: init function is called!!!!") migrations.MustRegisterTx(func(db migrations.DB) error { fmt.Println("adding id column...") _, err := db.Exec(`ALTER TABLE my_table ADD id serial`) diff --git a/example/3_seed_data.go b/example/3_seed_data.go index 804e7de..80cf719 100644 --- a/example/3_seed_data.go +++ b/example/3_seed_data.go @@ -7,6 +7,7 @@ import ( ) func init() { + fmt.Println("3_seed_data.go: init() function is called") migrations.MustRegisterTx(func(db migrations.DB) error { fmt.Println("seeding my_table...") _, err := db.Exec(`INSERT INTO my_table VALUES (1)`)