-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Bits for copying source to target dbs * Use transaction for destination DB * Function comment clarified * extract new method MakeDbInstance for use by import; do not migrate the input db * Add some collision detection * switch to streaming approach for data transfer * Tiny formatting change * Remove blank lines that lint dislikes * Fix lint errors plus double-registration error for sqlite driver when the import args are both sqlite:// * Start / WIP for import int test * validate the source and target db using rownumbers * undo baseFixtures rename * Trust primary key / unique constraints to prevent collisions * Wrap import destination in Tx per table * Additional entitites for db import to verify * WIP more explicit handling, less generics * Revert "WIP more explicit handling, less generics" Also switched to "string" tablenames to avoid generics/polymophism issues * Fix lint * Importer working for experiments, sqlite -> pg * Comments and additional test * Fix lint * Another lint complaint * remove comments and dry-run support * Fix experiment_id association * Add some additional testing fields * Fix lint complaint * disable import and testing of apps, dashboards * Fixture rename * Restore missing ArchiveRuns() function * Add slices module and use to detect sqlite driver registration * Add struct to hold import functions as methods * Missed this chunk for adding Importer struct * remove unneeded err check * Add table-contents test and rename helpers * remove separate declaration of variable * Refactor db instance management to factory pattern * Fix postgres factory * comments and gofumpt * Test both dbs in the integration-tests container * Use makefile to pass var from github action to docker-compose * Lint complaints * Lower-case private methods * moar lowercase/private structs * FML_DATABASE_URI is set via Env -> docker-compose.yaml, so don't need these -e args * db.reset() move to shared code path so PG also uses * PR requests * PR change requests * PR refactors * Fumpt and change comments only * Rename request from PR * Rename Db() accessor to GormDB() * Rename Db() accessor to GormDB() in tests * PR requests error catch and changed error message --------- Co-authored-by: Geoff Wilson <[email protected]>
- Loading branch information
Showing
21 changed files
with
514 additions
and
329 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package database | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "github.com/rotisserie/eris" | ||
| ) | ||
|
|
||
| // MakeDBProvider will create a DbProvider of the correct type from the parameters. | ||
| func MakeDBProvider( | ||
| dsn string, slowThreshold time.Duration, poolMax int, reset bool, migrate bool, artifactRoot string, | ||
| ) (db DBProvider, err error) { | ||
| dsnURL, err := url.Parse(dsn) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid database URL: %w", err) | ||
| } | ||
| switch dsnURL.Scheme { | ||
| case "sqlite": | ||
| db, err = NewSqliteDBInstance( | ||
| *dsnURL, | ||
| slowThreshold, | ||
| poolMax, | ||
| reset, | ||
| ) | ||
| if err != nil { | ||
| return nil, eris.Wrap(err, "error creating sqlite provider") | ||
| } | ||
| case "postgres", "postgresql": | ||
| db, err = NewPostgresDBInstance( | ||
| *dsnURL, | ||
| slowThreshold, | ||
| poolMax, | ||
| reset, | ||
| ) | ||
| if err != nil { | ||
| return nil, eris.Wrap(err, "error creating postgres provider") | ||
| } | ||
| default: | ||
| { | ||
| return nil, eris.New("unsupported database type") | ||
| } | ||
| } | ||
|
|
||
| if reset { | ||
| if err := db.Reset(); err != nil { | ||
| db.Close() | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| if err := checkAndMigrate(migrate, db); err != nil { | ||
| db.Close() | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := createDefaultExperiment(artifactRoot, db); err != nil { | ||
| db.Close() | ||
| return nil, err | ||
| } | ||
|
|
||
| return db, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package database | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestMakeDBProvider(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| dsn string | ||
| expectedDialector string | ||
| }{ | ||
| { | ||
| name: "WithSqliteURI", | ||
| dsn: "sqlite:///tmp/fasttrack.db", | ||
| expectedDialector: "sqlite", | ||
| }, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| DB = nil | ||
| db, err := MakeDBProvider( | ||
| tt.dsn, | ||
| time.Second*2, | ||
| 2, | ||
| false, | ||
| false, | ||
| "s3://somewhere", | ||
| ) | ||
| assert.Nil(t, err) | ||
| assert.NotNil(t, db) | ||
| assert.Equal(t, tt.expectedDialector, db.GormDB().Dialector.Name()) | ||
|
|
||
| // expecting the global 'DB' not to be set | ||
| assert.Nil(t, DB) | ||
|
|
||
| err = db.Close() | ||
| assert.Nil(t, err) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package database | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "strings" | ||
| "time" | ||
|
|
||
| log "github.com/sirupsen/logrus" | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| // DBProvider is the interface to access the DB. | ||
| type DBProvider interface { | ||
| GormDB() *gorm.DB | ||
| Dsn() string | ||
| Close() error | ||
| Reset() error | ||
| } | ||
|
|
||
| // DB is a global gorm.DB reference | ||
| var DB *gorm.DB | ||
|
|
||
| // DBInstance is the base concrete type for DbProvider. | ||
| type DBInstance struct { | ||
| *gorm.DB | ||
| dsn string | ||
| closers []io.Closer | ||
| } | ||
|
|
||
| // Close will invoke the closers. | ||
| func (db *DBInstance) Close() error { | ||
| for _, c := range db.closers { | ||
| err := c.Close() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // Dsn will return the dsn string. | ||
| func (db *DBInstance) Dsn() string { | ||
| return db.dsn | ||
| } | ||
|
|
||
| // Db will return the gorm DB. | ||
| func (db *DBInstance) GormDB() *gorm.DB { | ||
| return db.DB | ||
| } | ||
|
|
||
| // createDefaultExperiment will create the default experiment if needed. | ||
| func createDefaultExperiment(artifactRoot string, db DBProvider) error { | ||
| if tx := db.GormDB().First(&Experiment{}, 0); tx.Error != nil { | ||
| if errors.Is(tx.Error, gorm.ErrRecordNotFound) { | ||
| log.Info("Creating default experiment") | ||
| var id int32 = 0 | ||
| ts := time.Now().UTC().UnixMilli() | ||
| exp := Experiment{ | ||
| ID: &id, | ||
| Name: "Default", | ||
| LifecycleStage: LifecycleStageActive, | ||
| CreationTime: sql.NullInt64{ | ||
| Int64: ts, | ||
| Valid: true, | ||
| }, | ||
| LastUpdateTime: sql.NullInt64{ | ||
| Int64: ts, | ||
| Valid: true, | ||
| }, | ||
| } | ||
| if tx := db.GormDB().Create(&exp); tx.Error != nil { | ||
| return fmt.Errorf("error creating default experiment: %s", tx.Error) | ||
| } | ||
|
|
||
| exp.ArtifactLocation = fmt.Sprintf("%s/%d", strings.TrimRight(artifactRoot, "/"), *exp.ID) | ||
| if tx := db.GormDB().Model(&exp).Update("ArtifactLocation", exp.ArtifactLocation); tx.Error != nil { | ||
| return fmt.Errorf("error updating artifact_location for experiment '%s': %s", exp.Name, tx.Error) | ||
| } | ||
| } else { | ||
| return fmt.Errorf("unable to find default experiment: %s", tx.Error) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.