Exploration for MSC4097+Fastly (authenticated media redirects)

CHANGELOG.md

@@ -7,17 +7,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
-*Nothing yet.*
+### Added
+
+* Add *unstable* support for [MSC3916: Authentication for media](https://github.com/matrix-org/matrix-spec-proposals/pull/3916).
+  * **Note**: MMR will *not* attempt to use authentication to download media over federation in this version. 
+    * ***Subject to change during development.*** 
 
 ## [1.3.4] - February 9, 2024
 
 ### Added
 
 * Dendrite homeservers can now have their media imported safely, and `adminApiKind` may be set to `dendrite`.
-* Exporting MMR's data to Synapse is now possible with `import_to_synapse`. To use it, first run `gdpr_export` or similar.
-* Errors encountered during a background task, such as an API-induced export, are exposed as `error_message` in the admin API.
-* MMR will follow redirects on federated downloads up to 5 hops.
-* S3-backed datastores can have download requests redirected to a public-facing CDN rather than being proxied through MMR. See `publicBaseUrl` under the S3 datastore config.
 
 ### Changed
 

api/_apimeta/auth.go

@@ -16,6 +16,10 @@ type UserInfo struct {
 	IsShared    bool
 }
 
+type ServerInfo struct {
+	ServerName string
+}
+
 func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) {
 	isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
 	isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr)

api/_auth_cache/auth_cache.go

@@ -12,7 +12,7 @@ import (
 	"github.com/t2bot/matrix-media-repo/matrix"
 )
 
-var tokenCache = cache.New(0*time.Second, 30*time.Second)
+var tokenCache = cache.New(cache.NoExpiration, 30*time.Second)
 var rwLock = &sync.RWMutex{}
 var regexCache = make(map[string]*regexp.Regexp)
 

api/_routers/97-require-server-auth.go

@@ -0,0 +1,30 @@
+package _routers
+
+import (
+	"net/http"
+
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/common"
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+	"github.com/t2bot/matrix-media-repo/matrix"
+)
+
+type GeneratorWithServerFn = func(r *http.Request, ctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{}
+
+func RequireServerAuth(generator GeneratorWithServerFn) GeneratorFn {
+	return func(r *http.Request, ctx rcontext.RequestContext) interface{} {
+		serverName, err := matrix.ValidateXMatrixAuth(r, true)
+		if err != nil {
+			ctx.Log.Debug("Error with X-Matrix auth: ", err)
+			return &_responses.ErrorResponse{
+				Code:         common.ErrCodeForbidden,
+				Message:      "no auth provided (required)",
+				InternalCode: common.ErrCodeMissingToken,
+			}
+		}
+		return generator(r, ctx, _apimeta.ServerInfo{
+			ServerName: serverName,
+		})
+	}
+}

api/_routers/98-use-rcontext.go

@@ -95,20 +95,24 @@ func (c *RContextRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 beforeParseDownload:
 	log.Infof("Replying with result: %T %+v", res, res)
 	if downloadRes, isDownload := res.(*_responses.DownloadResponse); isDownload {
-		ranges, err := http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes)
-		if errors.Is(err, http_range.ErrInvalid) {
-			proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
-			res = _responses.BadRequest("invalid range header")
-			goto beforeParseDownload // reprocess `res`
-		} else if errors.Is(err, http_range.ErrNoOverlap) {
-			proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
-			res = _responses.BadRequest("out of range")
-			goto beforeParseDownload // reprocess `res`
-		}
-		if len(ranges) > 1 {
-			proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
-			res = _responses.BadRequest("only 1 range is supported")
-			goto beforeParseDownload // reprocess `res`
+		var ranges []http_range.Range
+		var err error
+		if downloadRes.SizeBytes > 0 {
+			ranges, err = http_range.ParseRange(r.Header.Get("Range"), downloadRes.SizeBytes, rctx.Config.Downloads.DefaultRangeChunkSizeBytes)
+			if errors.Is(err, http_range.ErrInvalid) {
+				proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+				res = _responses.BadRequest("invalid range header")
+				goto beforeParseDownload // reprocess `res`
+			} else if errors.Is(err, http_range.ErrNoOverlap) {
+				proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+				res = _responses.BadRequest("out of range")
+				goto beforeParseDownload // reprocess `res`
+			}
+			if len(ranges) > 1 {
+				proposedStatusCode = http.StatusRequestedRangeNotSatisfiable
+				res = _responses.BadRequest("only 1 range is supported")
+				goto beforeParseDownload // reprocess `res`
+			}
 		}
 
 		contentType = downloadRes.ContentType

api/custom/federation.go

@@ -34,6 +34,9 @@ func GetFederationInfo(r *http.Request, rctx rcontext.RequestContext, user _apim
 
 	versionUrl := url + "/_matrix/federation/v1/version"
 	versionResponse, err := matrix.FederatedGet(versionUrl, hostname, rctx)
+	if versionResponse != nil {
+		defer versionResponse.Body.Close()
+	}
 	if err != nil {
 		rctx.Log.Error(err)
 		sentry.CaptureException(err)

api/routes.go

@@ -18,6 +18,7 @@ import (
 
 const PrefixMedia = "/_matrix/media"
 const PrefixClient = "/_matrix/client"
+const PrefixFederation = "/_matrix/federation"
 
 func buildRoutes() http.Handler {
 	counter := &_routers.RequestCounter{}
@@ -36,13 +37,25 @@ func buildRoutes() http.Handler {
 	register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute)
 	register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute)
 	register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter))
-	register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter))
+	previewUrlRoute := makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)
+	register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, previewUrlRoute)
 	register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", counter))
-	register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter))
+	configRoute := makeRoute(_routers.RequireAccessToken(r0.PublicConfig), "config", counter)
+	register([]string{"GET"}, PrefixMedia, "config", mxSpecV3TransitionCS, router, configRoute)
 	register([]string{"POST"}, PrefixClient, "logout", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.Logout), "logout", counter))
 	register([]string{"POST"}, PrefixClient, "logout/all", mxSpecV3TransitionCS, router, makeRoute(_routers.RequireAccessToken(r0.LogoutAll), "logout_all", counter))
 	register([]string{"POST"}, PrefixMedia, "create", mxV1, router, makeRoute(_routers.RequireAccessToken(v1.CreateMedia), "create", counter))
 
+	// MSC3916 - Authentication & endpoint API separation
+	register([]string{"GET"}, PrefixClient, "media/preview_url", msc3916, router, previewUrlRoute)
+	register([]string{"GET"}, PrefixClient, "media/config", msc3916, router, configRoute)
+	authedDownloadRoute := makeRoute(_routers.RequireAccessToken(unstable.ClientDownloadMedia), "download", counter)
+	register([]string{"GET"}, PrefixClient, "media/download/:server/:mediaId/:filename", msc3916, router, authedDownloadRoute)
+	register([]string{"GET"}, PrefixClient, "media/download/:server/:mediaId", msc3916, router, authedDownloadRoute)
+	register([]string{"GET"}, PrefixClient, "media/thumbnail/:server/:mediaId", msc3916, router, makeRoute(_routers.RequireAccessToken(r0.ThumbnailMedia), "thumbnail", counter))
+	register([]string{"GET"}, PrefixFederation, "media/download/:server/:mediaId", msc3916, router, makeRoute(_routers.RequireServerAuth(unstable.FederationDownloadMedia), "download", counter))
+	register([]string{"GET"}, PrefixFederation, "media/thumbnail/:server/:mediaId", msc3916, router, makeRoute(_routers.RequireServerAuth(unstable.FederationThumbnailMedia), "thumbnail", counter))
+
 	// Custom features
 	register([]string{"GET"}, PrefixMedia, "local_copy/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.LocalCopy), "local_copy", counter))
 	register([]string{"GET"}, PrefixMedia, "info/:server/:mediaId", mxUnstable, router, makeRoute(_routers.RequireAccessToken(unstable.MediaInfo), "info", counter))
@@ -129,6 +142,7 @@ var (
 	//mxAllSpec            matrixVersions = []string{"r0", "v1", "v3", "unstable", "unstable/io.t2bot.media" /* and MSC routes */}
 	mxUnstable           matrixVersions = []string{"unstable", "unstable/io.t2bot.media"}
 	msc4034              matrixVersions = []string{"unstable/org.matrix.msc4034"}
+	msc3916              matrixVersions = []string{"unstable/org.matrix.msc3916"}
 	mxSpecV3Transition   matrixVersions = []string{"r0", "v1", "v3"}
 	mxSpecV3TransitionCS matrixVersions = []string{"r0", "v3"}
 	mxR0                 matrixVersions = []string{"r0"}

api/unstable/msc3916_download.go

@@ -0,0 +1,37 @@
+package unstable
+
+import (
+	"bytes"
+	"net/http"
+
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/api/r0"
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+	"github.com/t2bot/matrix-media-repo/util/readers"
+)
+
+func ClientDownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+	r.URL.Query().Set("allow_remote", "true")
+	return r0.DownloadMedia(r, rctx, user)
+}
+
+func FederationDownloadMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} {
+	r.URL.Query().Set("allow_remote", "false")
+
+	res := r0.DownloadMedia(r, rctx, _apimeta.UserInfo{})
+	if dl, ok := res.(*_responses.DownloadResponse); ok {
+		return &_responses.DownloadResponse{
+			ContentType: "multipart/mixed",
+			Filename:    "",
+			SizeBytes:   0,
+			Data: readers.NewMultipartReader(
+				&readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))},
+				&readers.MultipartPart{ContentType: dl.ContentType, FileName: dl.Filename, Reader: dl.Data},
+			),
+			TargetDisposition: "attachment",
+		}
+	} else {
+		return res
+	}
+}

api/unstable/msc3916_thumbnail.go

@@ -0,0 +1,32 @@
+package unstable
+
+import (
+	"bytes"
+	"net/http"
+
+	"github.com/t2bot/matrix-media-repo/api/_apimeta"
+	"github.com/t2bot/matrix-media-repo/api/_responses"
+	"github.com/t2bot/matrix-media-repo/api/r0"
+	"github.com/t2bot/matrix-media-repo/common/rcontext"
+	"github.com/t2bot/matrix-media-repo/util/readers"
+)
+
+func FederationThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} {
+	r.URL.Query().Set("allow_remote", "false")
+
+	res := r0.ThumbnailMedia(r, rctx, _apimeta.UserInfo{})
+	if dl, ok := res.(*_responses.DownloadResponse); ok {
+		return &_responses.DownloadResponse{
+			ContentType: "multipart/mixed",
+			Filename:    "",
+			SizeBytes:   0,
+			Data: readers.NewMultipartReader(
+				&readers.MultipartPart{ContentType: "application/json", Reader: readers.MakeCloser(bytes.NewReader([]byte("{}")))},
+				&readers.MultipartPart{ContentType: dl.ContentType, FileName: dl.Filename, Reader: dl.Data},
+			),
+			TargetDisposition: "attachment",
+		}
+	} else {
+		return res
+	}
+}

cdn/fastly.go

@@ -0,0 +1,57 @@
+package cdn
+
+import (
+	"context"
+	"errors"
+
+	"github.com/fastly/fastly-go/fastly"
+)
+
+type FastlyCdn struct {
+	cli            *fastly.APIClient
+	ctx            context.Context
+	serviceId      string
+	dictionaryName string
+}
+
+func NewFastlyCdn(apiKey string, serviceId string, dictionaryName string) *FastlyCdn {
+	if apiKey == "" || serviceId == "" || dictionaryName == "" {
+		return nil
+	}
+
+	client := fastly.NewAPIClient(fastly.NewConfiguration())
+	ctx := fastly.NewAPIKeyContext(apiKey)
+	return &FastlyCdn{
+		cli:            client,
+		ctx:            ctx,
+		serviceId:      serviceId,
+		dictionaryName: dictionaryName,
+	}
+}
+
+func (f *FastlyCdn) SetDictionaryItem(key string, value string) error {
+	// Find the latest service version
+	versions, _, err := f.cli.VersionAPI.ListServiceVersions(f.ctx, f.serviceId).Execute()
+	if err != nil {
+		return err
+	}
+	var latestVersion *fastly.VersionResponse
+	for _, v := range versions {
+		if v.GetActive() && (latestVersion == nil || latestVersion.GetNumber() < v.GetNumber()) {
+			latestVersion = &v
+		}
+	}
+	if latestVersion == nil {
+		return errors.New("no active service versions to configure")
+	}
+
+	// Find and update the dictionary
+	d, _, err := f.cli.DictionaryAPI.GetDictionary(f.ctx, f.serviceId, latestVersion.GetNumber(), f.dictionaryName).Execute()
+	if err != nil {
+		return err
+	}
+	req := f.cli.DictionaryItemAPI.UpsertDictionaryItem(f.ctx, f.serviceId, *d.ID, key)
+	req.ItemValue(value)
+	_, _, err = req.Execute()
+	return err
+}

common/config/access.go

@@ -222,6 +222,15 @@ func GetDomain(domain string) *DomainRepoConfig {
 	return domains[domain]
 }
 
+func AddDomainForTesting(domain string, config *DomainRepoConfig) {
+	Get() // Ensure the "main" config was loaded first
+	if config == nil {
+		c := NewDefaultDomainConfig()
+		config = &c
+	}
+	domains[domain] = config
+}
+
 func DomainConfigFrom(c MainRepoConfig) DomainRepoConfig {
 	// HACK: We should be better at this kind of inheritance
 	dc := NewDefaultDomainConfig()

config.sample.yaml

@@ -208,6 +208,10 @@ datastores:
       # capability. MMR may still be responsible for bandwidth charges incurred from going to
       # the bucket directly.
       #publicBaseUrl: "https://mycdn.example.org/"
+      cdn: "fastly"
+      fastlyApiToken: "FASTLY_API_TOKEN"
+      fastlyServiceId: "some_string"
+      fastlyDictionaryName: "mmr_encrypt_keys"
 
 # Options for controlling archives. Archives are exports of a particular user's content for
 # the purpose of GDPR or moving media to a different server.

datastores/download.go

@@ -48,6 +48,12 @@ func DownloadOrRedirect(ctx rcontext.RequestContext, ds config.DatastoreConfig,
 	}
 
 	if s3c.publicBaseUrl != "" {
+		if s3c.fastlyApi != nil {
+			err = s3c.fastlyApi.SetDictionaryItem("test", "hello world")
+			if err != nil {
+				ctx.Log.Warn("Error updating Fastly API:", err)
+			}
+		}
 		metrics.S3Operations.With(prometheus.Labels{"operation": "RedirectGetObject"}).Inc()
 		return nil, redirect(fmt.Sprintf("%s%s", s3c.publicBaseUrl, dsFileName))
 	}

datastores/s3.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/t2bot/matrix-media-repo/cdn"
 	"github.com/t2bot/matrix-media-repo/common/config"
 	"github.com/t2bot/matrix-media-repo/common/rcontext"
 )
@@ -20,6 +21,7 @@ type s3 struct {
 	storageClass  string
 	bucket        string
 	publicBaseUrl string
+	fastlyApi     *cdn.FastlyCdn
 }
 
 func ResetS3Clients() {
@@ -39,6 +41,11 @@ func getS3(ds config.DatastoreConfig) (*s3, error) {
 	storageClass, hasStorageClass := ds.Options["storageClass"]
 	useSslStr, hasSsl := ds.Options["ssl"]
 	publicBaseUrl := ds.Options["publicBaseUrl"]
+	cdnType := ds.Options["cdn"]
+
+	fastlyApiToken := ds.Options["fastlyApiToken"]
+	fastlyDictionaryName := ds.Options["fastlyDictionaryName"]
+	fastlyServiceId := ds.Options["fastlyServiceId"]
 
 	if !hasStorageClass {
 		storageClass = "STANDARD"
@@ -66,6 +73,11 @@ func getS3(ds config.DatastoreConfig) (*s3, error) {
 		bucket:        bucket,
 		publicBaseUrl: publicBaseUrl,
 	}
+
+	if cdnType == "fastly" {
+		s3c.fastlyApi = cdn.NewFastlyCdn(fastlyApiToken, fastlyServiceId, fastlyDictionaryName)
+	}
+
 	s3clients.Store(ds.Id, s3c)
 	return s3c, nil
 }