feat(reconciliation): different time for payments and ledger (#986)

formancehq · Dec 11, 2023 · f9bf5e1 · f9bf5e1
1 parent bea0e76
commit f9bf5e1
Show file tree

Hide file tree

Showing 17 changed files with 10,844 additions and 208 deletions.
diff --git a/.gitignore b/.gitignore
@@ -32,4 +32,5 @@ worktrees
 
 # Ignore SDKS
 sdks
-.DS_Store
+.DS_Store
+openapi/node_modules
diff --git a/components/ledger/libs/otlp/http_client.go b/components/ledger/libs/otlp/http_client.go
@@ -21,7 +21,7 @@ func (t WithBodiesTracingHTTPTransport) RoundTrip(req *http.Request) (*http.Resp
 	}
 
 	rsp, err := t.underlying.RoundTrip(req)
-	if t.debug || rsp.StatusCode >= 400 {
+	if t.debug || err != nil || rsp.StatusCode >= 400 {
 		span := trace.SpanFromContext(req.Context())
 		span.SetAttributes(attribute.String("raw-request", string(rawRequest)))
 		if err != nil {

diff --git a/components/operator/internal/modules/module.go b/components/operator/internal/modules/module.go
@@ -5,13 +5,11 @@ import (
 	"fmt"
 	"sort"
 
-	"golang.org/x/mod/semver"
-	appsv1 "k8s.io/api/apps/v1"
-
-	"github.com/pkg/errors"
-
 	"github.com/formancehq/operator/apis/stack/v1beta3"
 	"github.com/formancehq/stack/libs/go-libs/collectionutils"
+	"github.com/pkg/errors"
+	"golang.org/x/mod/semver"
+	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -27,7 +25,6 @@ type Cron struct {
 
 type DatabaseMigration struct {
 	// optional, will be the commit hash if empty in the kube object
-	Name          string
 	Shutdown      bool
 	Command       []string
 	AdditionalEnv func(config ReconciliationConfig) []EnvVar
@@ -359,10 +356,13 @@ func (r *moduleReconciler) runPreUpgradeMigration(ctx context.Context, module Mo
 
 func (r *moduleReconciler) runDatabaseMigration(ctx context.Context, version string, migration DatabaseMigration, postgresConfig v1beta3.PostgresConfig) (bool, error) {
 	logger := log.FromContext(ctx)
-	name := fmt.Sprintf("%s-%s-db-migration", r.module.Name(), version)
-	if migration.Name != "" {
-		name = fmt.Sprintf("%s-%s-db-migration", r.module.Name(), migration.Name)
+	if !semver.IsValid(version) {
+		// Commit hash
+		if len(version) > 7 {
+			version = version[:7]
+		}
 	}
+	name := fmt.Sprintf("%s-%s-db-migration", r.module.Name(), version)
 	return r.RunJob(ctx, name,
 		func() error {
 			if migration.Shutdown {

diff --git a/components/operator/internal/modules/reconciliation/handler.go b/components/operator/internal/modules/reconciliation/handler.go
@@ -24,7 +24,6 @@ func (o module) Versions() map[string]modules.Version {
 	return map[string]modules.Version{
 		"v0.0.0": {
 			DatabaseMigration: &modules.DatabaseMigration{
-				Name:     "v0.0.0",
 				Shutdown: false,
 				Command:  []string{"migrate", "up"},
 				AdditionalEnv: func(config modules.ReconciliationConfig) []modules.EnvVar {

diff --git a/ee/reconciliation/internal/api/reconciliation.go b/ee/reconciliation/internal/api/reconciliation.go
@@ -17,14 +17,15 @@ import (
 )
 
 type reconciliationResponse struct {
-	ID               string              `json:"id"`
-	PolicyID         string              `json:"policyID"`
-	CreatedAt        time.Time           `json:"createdAt"`
-	ReconciledAt     time.Time           `json:"reconciledAt"`
-	Status           string              `json:"status"`
-	PaymentsBalances map[string]*big.Int `json:"paymentsBalances"`
-	LedgerBalances   map[string]*big.Int `json:"ledgerBalances"`
-	Error            string              `json:"error"`
+	ID                   string              `json:"id"`
+	PolicyID             string              `json:"policyID"`
+	CreatedAt            time.Time           `json:"createdAt"`
+	ReconciledAtLedger   time.Time           `json:"reconciledAtLedger"`
+	ReconciledAtPayments time.Time           `json:"reconciledAtPayments"`
+	Status               string              `json:"status"`
+	PaymentsBalances     map[string]*big.Int `json:"paymentsBalances"`
+	LedgerBalances       map[string]*big.Int `json:"ledgerBalances"`
+	Error                string              `json:"error"`
 }
 
 func reconciliationHandler(b backend.Backend) http.HandlerFunc {
@@ -54,14 +55,15 @@ func reconciliationHandler(b backend.Backend) http.HandlerFunc {
 		}
 
 		data := &reconciliationResponse{
-			ID:               res.ID.String(),
-			PolicyID:         policyID,
-			CreatedAt:        res.CreatedAt,
-			ReconciledAt:     res.ReconciledAt,
-			Status:           res.Status.String(),
-			PaymentsBalances: res.PaymentsBalances,
-			LedgerBalances:   res.LedgerBalances,
-			Error:            res.Error,
+			ID:                   res.ID.String(),
+			PolicyID:             policyID,
+			CreatedAt:            res.CreatedAt,
+			ReconciledAtLedger:   res.ReconciledAtLedger,
+			ReconciledAtPayments: res.ReconciledAtPayments,
+			Status:               res.Status.String(),
+			PaymentsBalances:     res.PaymentsBalances,
+			LedgerBalances:       res.LedgerBalances,
+			Error:                res.Error,
 		}
 
 		api.Ok(w, data)
@@ -79,14 +81,15 @@ func getReconciliationHandler(b backend.Backend) http.HandlerFunc {
 		}
 
 		data := &reconciliationResponse{
-			ID:               res.ID.String(),
-			PolicyID:         res.PolicyID.String(),
-			CreatedAt:        res.CreatedAt,
-			ReconciledAt:     res.ReconciledAt,
-			Status:           res.Status.String(),
-			PaymentsBalances: res.PaymentsBalances,
-			LedgerBalances:   res.LedgerBalances,
-			Error:            res.Error,
+			ID:                   res.ID.String(),
+			PolicyID:             res.PolicyID.String(),
+			CreatedAt:            res.CreatedAt,
+			ReconciledAtLedger:   res.ReconciledAtLedger,
+			ReconciledAtPayments: res.ReconciledAtPayments,
+			Status:               res.Status.String(),
+			PaymentsBalances:     res.PaymentsBalances,
+			LedgerBalances:       res.LedgerBalances,
+			Error:                res.Error,
 		}
 
 		api.Ok(w, data)