feat(sqlmigration): consolidate all sqlmigrations into one package (#…

…7018)

* feat(sqlmigration): add sqlmigrations

* feat(sqlmigration): test sqlmigrations

* feat(sqlmigration): add remaining factories

* feat(sqlmigration): consolidate into single package

* fix(telemetrystore): remove existing env variables

* fix(telemetrystore): fix DSN

ee/query-service/app/server.go

@@ -11,7 +11,6 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof" // http profiler
-	"os"
 	"regexp"
 	"time"
 
@@ -149,25 +148,20 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	}
 
 	var reader interfaces.DataConnector
-	storage := os.Getenv("STORAGE")
-	if storage == "clickhouse" {
-		zap.L().Info("Using ClickHouse as datastore ...")
-		qb := db.NewDataConnector(
-			serverOptions.SigNoz.SQLStore.SQLxDB(),
-			serverOptions.SigNoz.TelemetryStore.ClickHouseDB(),
-			serverOptions.PromConfigPath,
-			lm,
-			serverOptions.Cluster,
-			serverOptions.UseLogsNewSchema,
-			serverOptions.UseTraceNewSchema,
-			fluxIntervalForTraceDetail,
-			serverOptions.SigNoz.Cache,
-		)
-		go qb.Start(readerReady)
-		reader = qb
-	} else {
-		return nil, fmt.Errorf("storage type: %s is not supported in query service", storage)
-	}
+	qb := db.NewDataConnector(
+		serverOptions.SigNoz.SQLStore.SQLxDB(),
+		serverOptions.SigNoz.TelemetryStore.ClickHouseDB(),
+		serverOptions.PromConfigPath,
+		lm,
+		serverOptions.Cluster,
+		serverOptions.UseLogsNewSchema,
+		serverOptions.UseTraceNewSchema,
+		fluxIntervalForTraceDetail,
+		serverOptions.SigNoz.Cache,
+	)
+	go qb.Start(readerReady)
+	reader = qb
+
 	skipConfig := &basemodel.SkipConfig{}
 	if serverOptions.SkipTopLvlOpsPath != "" {
 		// read skip config

ee/query-service/dao/sqlite/modelDao.go

@@ -7,7 +7,6 @@ import (
 	basedao "go.signoz.io/signoz/pkg/query-service/dao"
 	basedsql "go.signoz.io/signoz/pkg/query-service/dao/sqlite"
 	baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
-	"go.uber.org/zap"
 )
 
 type modelDao struct {
@@ -29,41 +28,6 @@ func (m *modelDao) checkFeature(key string) error {
 	return m.flags.CheckFeature(key)
 }
 
-func columnExists(db *sqlx.DB, tableName, columnName string) bool {
-	query := fmt.Sprintf("PRAGMA table_info(%s);", tableName)
-	rows, err := db.Query(query)
-	if err != nil {
-		zap.L().Error("Failed to query table info", zap.Error(err))
-		return false
-	}
-	defer rows.Close()
-
-	var (
-		cid        int
-		name       string
-		ctype      string
-		notnull    int
-		dflt_value *string
-		pk         int
-	)
-	for rows.Next() {
-		err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt_value, &pk)
-		if err != nil {
-			zap.L().Error("Failed to scan table info", zap.Error(err))
-			return false
-		}
-		if name == columnName {
-			return true
-		}
-	}
-	err = rows.Err()
-	if err != nil {
-		zap.L().Error("Failed to scan table info", zap.Error(err))
-		return false
-	}
-	return false
-}
-
 // InitDB creates and extends base model DB repository
 func InitDB(inputDB *sqlx.DB) (*modelDao, error) {
 	dao, err := basedsql.InitDB(inputDB)
@@ -73,69 +37,6 @@ func InitDB(inputDB *sqlx.DB) (*modelDao, error) {
 	// set package variable so dependent base methods (e.g. AuthCache)  will work
 	basedao.SetDB(dao)
 	m := &modelDao{ModelDaoSqlite: dao}
-
-	table_schema := `
-	PRAGMA foreign_keys = ON;
-	CREATE TABLE IF NOT EXISTS org_domains(
-		id TEXT PRIMARY KEY,
-		org_id TEXT NOT NULL,
-		name VARCHAR(50) NOT NULL UNIQUE,
-		created_at INTEGER NOT NULL,
-		updated_at INTEGER,
-		data TEXT  NOT NULL,
-		FOREIGN KEY(org_id) REFERENCES organizations(id)
-	);
-	CREATE TABLE IF NOT EXISTS personal_access_tokens (
-		id INTEGER PRIMARY KEY AUTOINCREMENT,
-		role TEXT NOT NULL,
-		user_id TEXT NOT NULL,
-		token TEXT NOT NULL UNIQUE,
-		name TEXT NOT NULL,
-		created_at INTEGER NOT NULL,
-		expires_at INTEGER NOT NULL,
-		updated_at INTEGER NOT NULL,
-		last_used INTEGER NOT NULL,
-		revoked BOOLEAN NOT NULL,
-		updated_by_user_id TEXT NOT NULL,
-		FOREIGN KEY(user_id) REFERENCES users(id)
-	);
-	`
-
-	_, err = m.DB().Exec(table_schema)
-	if err != nil {
-		return nil, fmt.Errorf("error in creating tables: %v", err.Error())
-	}
-
-	if !columnExists(m.DB(), "personal_access_tokens", "role") {
-		_, err = m.DB().Exec("ALTER TABLE personal_access_tokens ADD COLUMN role TEXT NOT NULL DEFAULT 'ADMIN';")
-		if err != nil {
-			return nil, fmt.Errorf("error in adding column: %v", err.Error())
-		}
-	}
-	if !columnExists(m.DB(), "personal_access_tokens", "updated_at") {
-		_, err = m.DB().Exec("ALTER TABLE personal_access_tokens ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0;")
-		if err != nil {
-			return nil, fmt.Errorf("error in adding column: %v", err.Error())
-		}
-	}
-	if !columnExists(m.DB(), "personal_access_tokens", "last_used") {
-		_, err = m.DB().Exec("ALTER TABLE personal_access_tokens ADD COLUMN last_used INTEGER NOT NULL DEFAULT 0;")
-		if err != nil {
-			return nil, fmt.Errorf("error in adding column: %v", err.Error())
-		}
-	}
-	if !columnExists(m.DB(), "personal_access_tokens", "revoked") {
-		_, err = m.DB().Exec("ALTER TABLE personal_access_tokens ADD COLUMN revoked BOOLEAN NOT NULL DEFAULT FALSE;")
-		if err != nil {
-			return nil, fmt.Errorf("error in adding column: %v", err.Error())
-		}
-	}
-	if !columnExists(m.DB(), "personal_access_tokens", "updated_by_user_id") {
-		_, err = m.DB().Exec("ALTER TABLE personal_access_tokens ADD COLUMN updated_by_user_id TEXT NOT NULL DEFAULT '';")
-		if err != nil {
-			return nil, fmt.Errorf("error in adding column: %v", err.Error())
-		}
-	}
 	return m, nil
 }
 

ee/query-service/license/db.go

@@ -10,7 +10,6 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/mattn/go-sqlite3"
 
-	"go.signoz.io/signoz/ee/query-service/license/sqlite"
 	"go.signoz.io/signoz/ee/query-service/model"
 	basemodel "go.signoz.io/signoz/pkg/query-service/model"
 	"go.uber.org/zap"
@@ -28,10 +27,6 @@ func NewLicenseRepo(db *sqlx.DB) Repo {
 	}
 }
 
-func (r *Repo) InitDB(inputDB *sqlx.DB) error {
-	return sqlite.InitDB(inputDB)
-}
-
 func (r *Repo) GetLicensesV3(ctx context.Context) ([]*model.LicenseV3, error) {
 	licensesData := []model.LicenseDB{}
 	licenseV3Data := []*model.LicenseV3{}

ee/query-service/license/manager.go

@@ -2,7 +2,6 @@ package license
 
 import (
 	"context"
-	"fmt"
 	"sync/atomic"
 	"time"
 
@@ -50,11 +49,6 @@ func StartManager(db *sqlx.DB, features ...basemodel.Feature) (*Manager, error)
 	}
 
 	repo := NewLicenseRepo(db)
-	err := repo.InitDB(db)
-	if err != nil {
-		return nil, fmt.Errorf("failed to initiate license repo: %v", err)
-	}
-
 	m := &Manager{
 		repo: &repo,
 	}

ee/query-service/main.go

@@ -18,7 +18,6 @@ import (
 	"go.signoz.io/signoz/pkg/config/fileprovider"
 	"go.signoz.io/signoz/pkg/query-service/auth"
 	baseconst "go.signoz.io/signoz/pkg/query-service/constants"
-	"go.signoz.io/signoz/pkg/query-service/migrate"
 	"go.signoz.io/signoz/pkg/query-service/version"
 	"go.signoz.io/signoz/pkg/signoz"
 	"google.golang.org/grpc"
@@ -183,12 +182,6 @@ func main() {
 		zap.L().Info("JWT secret key set successfully.")
 	}
 
-	if err := migrate.Migrate(signoz.SQLStore.SQLxDB()); err != nil {
-		zap.L().Error("Failed to migrate", zap.Error(err))
-	} else {
-		zap.L().Info("Migration successful")
-	}
-
 	server, err := app.NewServer(serverOptions)
 	if err != nil {
 		zap.L().Fatal("Failed to create server", zap.Error(err))

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/knadh/koanf v1.5.0
 	github.com/knadh/koanf/v2 v2.1.1
 	github.com/mailru/easyjson v0.7.7
-	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/mattn/go-sqlite3 v1.14.24
 	github.com/oklog/oklog v0.3.2
 	github.com/open-telemetry/opamp-go v0.5.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.111.0

go.sum

@@ -521,8 +521,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
-github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
+github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=

pkg/config/configtest/config.go

@@ -0,0 +1,13 @@
+package configtest
+
+import (
+	"go.signoz.io/signoz/pkg/config"
+	"go.signoz.io/signoz/pkg/config/envprovider"
+)
+
+func NewResolverConfig() config.ResolverConfig {
+	return config.ResolverConfig{
+		Uris:              []string{"env:"},
+		ProviderFactories: []config.ProviderFactory{envprovider.NewFactory()},
+	}
+}

pkg/instrumentation/logger.go

@@ -10,7 +10,20 @@ import (
 func NewLogger(config Config, wrappers ...loghandler.Wrapper) *slog.Logger {
 	logger := slog.New(
 		loghandler.New(
-			slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: config.Logs.Level, AddSource: true}),
+			slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: config.Logs.Level, AddSource: true, ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
+				// This is more in line with OpenTelemetry semantic conventions
+				if a.Key == slog.SourceKey {
+					a.Key = "code"
+					return a
+				}
+
+				if a.Key == slog.TimeKey {
+					a.Key = "timestamp"
+					return a
+				}
+
+				return a
+			}}),
 			wrappers...,
 		),
 	)

pkg/query-service/agentConf/db.go

@@ -8,7 +8,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
-	"go.signoz.io/signoz/pkg/query-service/agentConf/sqlite"
 	"go.signoz.io/signoz/pkg/query-service/model"
 	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
@@ -19,10 +18,6 @@ type Repo struct {
 	db *sqlx.DB
 }
 
-func (r *Repo) initDB(inputDB *sqlx.DB) error {
-	return sqlite.InitDB(inputDB)
-}
-
 func (r *Repo) GetConfigHistory(
 	ctx context.Context, typ ElementTypeDef, limit int,
 ) ([]ConfigVersion, *model.ApiError) {

pkg/query-service/agentConf/manager.go

@@ -65,10 +65,6 @@ func Initiate(options *ManagerOptions) (*Manager, error) {
 		configSubscribers: map[string]func(){},
 	}
 
-	err := m.initDB(options.DB)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not init agentConf db")
-	}
 	return m, nil
 }