Improve indexer rest api error code and supports to query file info i…

…n batch (#79)

* Extract common code for rest api

* refactor local gateway

* refine code for api framework

* simplify indexer gateway server code

* rename file

* Add error code for rest api

* refactor errors

* refine code

* Supports to get file info from indexer in batch

* update readme

* limit batch size to get file info

README.md

@@ -160,3 +160,38 @@ There are two options for uploading:
     ```
 
 > **Note:** The `proof` field should contain a [`merkle.Proof`](https://github.com/0glabs/0g-storage-client/blob/8780c5020928a79fb60ed7dea26a42d9876ecfae/core/merkle/proof.go#L20) object, which is used to verify the integrity of each segment.
+
+### Query File Info
+
+Users could query file information by `cid` (transaction sequence number or file merkle root):
+
+```
+GET /file/info/{cid}
+```
+
+or query multiple files in batch:
+
+```
+GET /files/info?cid={cid1}&cid={cid2}&cid={cid3}
+```
+
+Note, the batch API will return `null` if specified `cid` not found.
+
+### HTTP Response
+
+Basically, the REST APIs return 2 kinds of HTTP status code:
+
+- `200`: success or business error.
+- `600`: server internal error.
+
+The HTTP response body is in JSON format, including `code`, `message` and `data`:
+
+```go
+type BusinessError struct {
+	Code    int         `json:"code"`
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+}
+```
+
+There are several pre-defined [common errors](/common/api/errors.go) and [business errors](/indexer/gateway/errors.go), and those errors may contain different `data` for detailed error context.

common/api/controller.go

@@ -0,0 +1,41 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/go-playground/validator/v10"
+)
+
+const httpStatusCodeInternalError = 600
+
+var ErrHandled = new(BusinessError)
+
+func Wrap(controller func(c *gin.Context) (interface{}, error)) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		result, err := controller(c)
+		if err == ErrHandled {
+			return
+		}
+
+		if err != nil {
+			switch e := err.(type) {
+			case *BusinessError:
+				// custom business error
+				if e != ErrHandled {
+					c.JSON(http.StatusOK, e)
+				}
+			case validator.ValidationErrors:
+				// binding error
+				c.JSON(http.StatusOK, ErrValidation.WithData(e))
+			default:
+				// internal server error
+				c.JSON(httpStatusCodeInternalError, ErrInternal.WithData(e))
+			}
+		} else if result == nil {
+			c.JSON(http.StatusOK, ErrNil)
+		} else {
+			c.JSON(http.StatusOK, ErrNil.WithData(result))
+		}
+	}
+}

common/api/errors.go

@@ -0,0 +1,30 @@
+package api
+
+// General errors
+var (
+	ErrNil        = NewBusinessError(0, "Success")
+	ErrValidation = NewBusinessError(1, "Invalid parameter")
+	ErrInternal   = NewBusinessError(2, "Internal server error")
+)
+
+type BusinessError struct {
+	Code    int         `json:"code"`
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+}
+
+func NewBusinessError(code int, message string) *BusinessError {
+	return &BusinessError{code, message, nil}
+}
+
+func NewBusinessErrorWithData(code int, message string, data interface{}) *BusinessError {
+	return &BusinessError{code, message, data}
+}
+
+func (err *BusinessError) Error() string {
+	return err.Message
+}
+
+func (be *BusinessError) WithData(data interface{}) *BusinessError {
+	return NewBusinessErrorWithData(be.Code, be.Message, data)
+}

common/api/server.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/gin-contrib/cors"
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+)
+
+type RouteFactory func(router *gin.Engine)
+
+type RouterOption struct {
+	RecoveryDisabled bool
+	LoggerForced     bool
+	OriginsAllowed   []string
+}
+
+func MustServe(endpoint string, factory RouteFactory, option ...RouterOption) {
+	if err := Serve(endpoint, factory, option...); err != http.ErrServerClosed {
+		logrus.WithError(err).Fatal("Failed to serve API")
+	}
+}
+
+func Serve(endpoint string, factory RouteFactory, option ...RouterOption) error {
+	router := newRouter(factory, option...)
+
+	server := http.Server{
+		Addr:    endpoint,
+		Handler: router,
+	}
+
+	return server.ListenAndServe()
+}
+
+func newRouter(factory RouteFactory, option ...RouterOption) *gin.Engine {
+	var opt RouterOption
+	if len(option) > 0 {
+		opt = option[0]
+	}
+
+	router := gin.New()
+
+	if !opt.RecoveryDisabled {
+		router.Use(gin.Recovery())
+	}
+
+	router.Use(newCorsMiddleware(opt.OriginsAllowed))
+
+	if opt.LoggerForced || logrus.IsLevelEnabled(logrus.DebugLevel) {
+		router.Use(gin.Logger())
+	}
+
+	factory(router)
+
+	return router
+}
+
+func newCorsMiddleware(origins []string) gin.HandlerFunc {
+	conf := cors.DefaultConfig()
+	conf.AllowMethods = append(conf.AllowMethods, "OPTIONS")
+	conf.AllowHeaders = append(conf.AllowHeaders, "*")
+
+	if len(origins) == 0 {
+		conf.AllowAllOrigins = true
+	} else {
+		conf.AllowOrigins = origins
+	}
+
+	return cors.New(conf)
+}

gateway/local_apis.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	zg_common "github.com/0glabs/0g-storage-client/common"
+	"github.com/0glabs/0g-storage-client/common/api"
 	"github.com/0glabs/0g-storage-client/core"
 	"github.com/0glabs/0g-storage-client/node"
 	"github.com/0glabs/0g-storage-client/transfer"
@@ -118,20 +119,20 @@ func uploadLocalFile(c *gin.Context) (interface{}, error) {
 	}
 
 	if input.Node < 0 || input.Node >= len(allClients) {
-		return nil, ErrValidation.WithData("node index out of bound")
+		return nil, api.ErrValidation.WithData("node index out of bound")
 	}
 
 	uploader, err := transfer.NewUploader(context.Background(), nil, []*node.ZgsClient{allClients[input.Node]}, zg_common.LogOption{Logger: logrus.StandardLogger()})
 	if err != nil {
-		return nil, ErrValidation.WithData(err)
+		return nil, api.ErrValidation.WithData(err)
 	}
 
 	filename := getFilePath(input.Path, false)
 
 	// Open file to upload
 	file, err := core.Open(filename)
 	if err != nil {
-		return nil, ErrValidation.WithData(err)
+		return nil, api.ErrValidation.WithData(err)
 	}
 	defer file.Close()
 
@@ -154,7 +155,7 @@ func downloadFileLocal(c *gin.Context) (interface{}, error) {
 	}
 
 	if input.Node < 0 || input.Node >= len(allClients) {
-		return nil, ErrValidation.WithData("node index out of bound")
+		return nil, api.ErrValidation.WithData("node index out of bound")
 	}
 
 	downloader, err := transfer.NewDownloader([]*node.ZgsClient{allClients[input.Node]}, zg_common.LogOption{Logger: logrus.StandardLogger()})

gateway/server.go

@@ -1,17 +1,12 @@
 package gateway
 
 import (
-	"net/http"
-
+	"github.com/0glabs/0g-storage-client/common/api"
 	"github.com/0glabs/0g-storage-client/node"
-	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
-	"github.com/go-playground/validator/v10"
 	"github.com/sirupsen/logrus"
 )
 
-const httpStatusInternalError = 600
-
 var allClients []*node.ZgsClient
 
 func MustServeLocal(nodes []*node.ZgsClient) {
@@ -21,59 +16,12 @@ func MustServeLocal(nodes []*node.ZgsClient) {
 
 	allClients = nodes
 
-	server := http.Server{
-		Addr:    "127.0.0.1:6789",
-		Handler: newLocalRouter(),
-	}
-
-	if err := server.ListenAndServe(); err != http.ErrServerClosed {
-		logrus.WithError(err).Fatal("Failed to serve API")
-	}
-}
-
-func newLocalRouter() *gin.Engine {
-	router := gin.New()
-	router.Use(gin.Recovery())
-	if logrus.IsLevelEnabled(logrus.DebugLevel) {
-		router.Use(gin.Logger())
-	}
-	router.Use(middlewareCors())
-
-	localApi := router.Group("/local")
-	localApi.GET("/nodes", wrap(listNodes))
-	localApi.GET("/file", wrap(getLocalFileInfo))
-	localApi.GET("/status", wrap(getFileStatus))
-	localApi.POST("/upload", wrap(uploadLocalFile))
-	localApi.POST("/download", wrap(downloadFileLocal))
-
-	return router
-}
-
-func wrap(controller func(*gin.Context) (interface{}, error)) func(*gin.Context) {
-	return func(c *gin.Context) {
-		result, err := controller(c)
-		if err != nil {
-			switch e := err.(type) {
-			case *BusinessError:
-				c.JSON(http.StatusOK, e)
-			case validator.ValidationErrors: // binding error
-				c.JSON(http.StatusOK, ErrValidation.WithData(e.Error()))
-			default: // internal server error
-				c.JSON(httpStatusInternalError, ErrInternalServer.WithData(err.Error()))
-			}
-		} else if result == nil {
-			c.JSON(http.StatusOK, ErrNil)
-		} else {
-			c.JSON(http.StatusOK, ErrNil.WithData(result))
-		}
-	}
-}
-
-func middlewareCors() gin.HandlerFunc {
-	conf := cors.DefaultConfig()
-	conf.AllowMethods = append(conf.AllowMethods, "OPTIONS")
-	conf.AllowHeaders = append(conf.AllowHeaders, "*")
-	conf.AllowAllOrigins = true
-
-	return cors.New(conf)
+	api.MustServe("127.0.0.1:6789", func(router *gin.Engine) {
+		localApi := router.Group("/local")
+		localApi.GET("/nodes", api.Wrap(listNodes))
+		localApi.GET("/file", api.Wrap(getLocalFileInfo))
+		localApi.GET("/status", api.Wrap(getFileStatus))
+		localApi.POST("/upload", api.Wrap(uploadLocalFile))
+		localApi.POST("/download", api.Wrap(downloadFileLocal))
+	})
 }

indexer/gateway/controller.go

@@ -2,7 +2,6 @@ package gateway
 
 import (
 	"context"
-	"fmt"
 	"strconv"
 
 	"github.com/0glabs/0g-storage-client/common/shard"
@@ -85,12 +84,12 @@ func (ctrl *RestController) getAvailableStorageNodes(ctx context.Context, cid Ci
 func (ctrl *RestController) fetchFileInfo(ctx context.Context, cid Cid) (*node.FileInfo, error) {
 	clients, err := ctrl.getAvailableStorageNodes(ctx, cid)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get available storage nodes: %v", err)
+		return nil, errors.WithMessage(err, "Failed to get available storage nodes")
 	}
 
 	fileInfo, err := getOverallFileInfo(ctx, clients, cid)
 	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve file info from storage nodes: %v", err)
+		return nil, errors.WithMessage(err, "Failed to retrieve file info from storage nodes")
 	}
 
 	if fileInfo != nil {
@@ -100,7 +99,7 @@ func (ctrl *RestController) fetchFileInfo(ctx context.Context, cid Cid) (*node.F
 	// Attempt retrieval from trusted clients as a fallback
 	fileInfo, err = getOverallFileInfo(ctx, ctrl.nodeManager.TrustedClients(), cid)
 	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve file info from trusted clients: %v", err)
+		return nil, errors.WithMessage(err, "Failed to retrieve file info from trusted clients")
 	}
 
 	return fileInfo, nil