commit proxy images same way as normally uploaded ones

backend/app/cmd/server.go

@@ -586,6 +586,7 @@ func (s *ServerCommand) makeAvatarStore() (avatar.Store, error) {
 func (s *ServerCommand) makePicturesStore() (*image.Service, error) {
 	imageServiceParams := image.ServiceParams{
 		ImageAPI:     s.RemarkURL + "/api/v1/picture/",
+		ProxyAPI:     s.RemarkURL + "/api/v1/img",
 		EditDuration: s.EditDuration,
 		MaxSize:      s.Image.MaxSize,
 		MaxHeight:    s.Image.ResizeHeight,

backend/app/rest/proxy/image.go

@@ -131,7 +131,6 @@ func (p Image) cacheImage(r io.Reader, imgID string) {
 	if err != nil {
 		log.Printf("[WARN] unable to save image to the storage: %+v", err)
 	}
-	p.ImageService.Submit(func() []string { return []string{imgID} })
 }
 
 // download an image.

backend/app/rest/proxy/image_test.go

@@ -180,7 +180,6 @@ func TestImage_RoutesCachingImage(t *testing.T) {
 
 	imageStore.On("Load", mock.Anything).Once().Return(nil, nil)
 	imageStore.On("Save", mock.Anything, mock.Anything).Once().Return(nil)
-	imageStore.On("Commit", mock.Anything).Once().Return(nil)
 
 	resp, err := http.Get(ts.URL + "/?src=" + encodedImgURL)
 	require.Nil(t, err)
@@ -191,7 +190,6 @@ func TestImage_RoutesCachingImage(t *testing.T) {
 
 	imageStore.AssertCalled(t, "Load", mock.Anything)
 	imageStore.AssertCalled(t, "Save", "cached_images/4b84b15bff6ee5796152495a230e45e3d7e947d9-"+image.Sha1Str(imgURL), gopherPNGBytes())
-	imageStore.AssertCalled(t, "Commit", mock.Anything)
 }
 
 func TestImage_RoutesUsingCachedImage(t *testing.T) {

backend/app/store/image/image.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha1" //nolint:gosec // not used for cryptography
+	"encoding/base64"
 	"fmt"
 	"image"
 	// support gif and jpeg images decoding
@@ -50,6 +51,7 @@ type Service struct {
 type ServiceParams struct {
 	EditDuration time.Duration // edit period for comments
 	ImageAPI     string        // image api matching path
+	ProxyAPI     string        // proxy api matching path
 	MaxSize      int
 	MaxHeight    int
 	MaxWidth     int
@@ -146,6 +148,21 @@ func (s *Service) ExtractPictures(commentHTML string) (ids []string, err error)
 					ids = append(ids, id)
 				}
 			}
+			if strings.Contains(im, s.ProxyAPI) {
+				proxiedURL, err := url.Parse(im)
+				if err != nil {
+					return
+				}
+				imgURL, err := base64.URLEncoding.DecodeString(proxiedURL.Query().Get("src"))
+				if err != nil {
+					return
+				}
+				imgID, err := CachedImgID(string(imgURL))
+				if err != nil {
+					return
+				}
+				ids = append(ids, imgID)
+			}
 		}
 	})
 

backend/app/store/image/image_test.go

@@ -3,6 +3,8 @@ package image
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
+	"fmt"
 	"image"
 	"io"
 	"io/ioutil"
@@ -72,24 +74,52 @@ func TestService_WrongFormat(t *testing.T) {
 }
 
 func TestService_ExtractPictures(t *testing.T) {
-	svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/"}}
-	html := `blah <img src="/blah/user1/pic1.png"/> foo 
+	svc := Service{ServiceParams: ServiceParams{ImageAPI: "/blah/", ProxyAPI: "/non_existent"}}
+	html := `blah <img src="/blah/user1/pic1.png"/> foo
 <img src="/blah/user2/pic3.png"/> xyz <p>123</p> <img src="/pic3.png"/> <img src="https://i.ibb.co/0cqqqnD/ezgif-5-3b07b6b97610.png" alt="">`
 	ids, err := svc.ExtractPictures(html)
 	require.NoError(t, err)
 	require.Equal(t, 2, len(ids), "two images")
 	assert.Equal(t, "user1/pic1.png", ids[0])
 	assert.Equal(t, "user2/pic3.png", ids[1])
-}
 
-func TestService_ExtractPictures2(t *testing.T) {
-	svc := Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/"}}
-	html := "<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде." +
-		"</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля). Например одна из моих вот так выглядит на libraries.io:</p>\n\n<p><img src=\"https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png\" alt=\"bjtr0-201906-08110846-i324c.png\"/></p>\n\n<p>По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е. <code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n"
-	ids, err := svc.ExtractPictures(html)
+	svc = Service{ServiceParams: ServiceParams{ImageAPI: "https://remark42.radio-t.com/api/v1/picture/", ProxyAPI: "https://remark42.radio-t.com/api/v1/img"}}
+	html = `<p>TLDR: такое в go пока правильно посчитать трудно. То, что они считают это общее количество go packages в коде.
+</p>\n\n<p>Пакеты в го это средство организации кода, они могут быть связанны друг с другом в рамках одной библиотеки (модуля).
+Например одна из моих вот так выглядит на libraries.io:</p>\n\n
+<p><img src="https://remark42.radio-t.com/api/v1/picture/github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png" alt="bjtr0-201906-08110846-i324c.png"/></p>\n\n<p>
+По форме все верно, это все packages, но по сути это все одна библиотека организованная таким образом. При ее импорте, например посредством go mod, она выглядит как один модуль, т.е.
+<code>github.com/go-pkgz/auth v0.5.2</code>.</p>\n`
+	ids, err = svc.ExtractPictures(html)
 	require.NoError(t, err)
 	require.Equal(t, 1, len(ids), "one image in")
 	assert.Equal(t, "github_ef0f706a79cc24b17bbbb374cd234a691d034128/bjttt8ahajfmrhsula10.png", ids[0])
+
+	// proxied image
+	html = `<img src="https://remark42.radio-t.com/api/v1/img?src=aHR0cHM6Ly9ob21lcGFnZXMuY2FlLndpc2MuZWR1L35lY2U1MzMvaW1hZ2VzL2JvYXQucG5n" alt="cat.png">`
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(ids), "one image in")
+	assert.Equal(t, "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e", ids[0])
+
+	// bad url
+	html = `<img src=" https://remark42.radio-t.com/api/v1/img">`
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Empty(t, ids)
+
+	// bad src
+	html = `<img src="https://remark42.radio-t.com/api/v1/img?src=bad">`
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Empty(t, ids)
+
+	// good src with bad content
+	badURL := base64.URLEncoding.EncodeToString([]byte(" http://foo.bar"))
+	html = fmt.Sprintf(`<img src="https://remark42.radio-t.com/api/v1/img?src=%s">`, badURL)
+	ids, err = svc.ExtractPictures(html)
+	require.NoError(t, err)
+	require.Empty(t, ids)
 }
 
 func TestService_Cleanup(t *testing.T) {

backend/app/store/service/service_test.go

@@ -19,7 +19,6 @@ import (
 	"github.com/go-pkgz/lgr"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	bolt "go.etcd.io/bbolt"
 
@@ -100,7 +99,6 @@ func TestService_CreateFromPartial(t *testing.T) {
 }
 
 func TestService_CreateFromPartialWithTitle(t *testing.T) {
-
 	ks := admin.NewStaticKeyStore("secret 123")
 	eng, teardown := prepStoreEngine(t)
 	defer teardown()
@@ -1279,8 +1277,14 @@ func TestService_submitImages(t *testing.T) {
 	lgr.Setup(lgr.Debug, lgr.CallerFile, lgr.CallerFunc)
 
 	mockStore := image.MockStore{}
-	mockStore.On("Commit", mock.Anything).Times(2).Return(nil)
-	imgSvc := image.NewService(&mockStore, image.ServiceParams{EditDuration: 50 * time.Millisecond})
+	mockStore.On("Commit", "dev/pic1.png").Once().Return(nil)
+	mockStore.On("Commit", "dev/pic2.png").Once().Return(nil)
+	imgSvc := image.NewService(&mockStore,
+		image.ServiceParams{
+			EditDuration: 50 * time.Millisecond,
+			ImageAPI:     "/",
+			ProxyAPI:     "/non_existent",
+		})
 	defer imgSvc.Close(context.TODO())
 
 	// two comments for https://radio-t.com
@@ -1301,6 +1305,7 @@ func TestService_submitImages(t *testing.T) {
 
 	b.submitImages(c.Locator, c.ID)
 	time.Sleep(250 * time.Millisecond)
+	mockStore.AssertNumberOfCalls(t, "Commit", 2)
 }
 
 func TestService_ResubmitStagingImages(t *testing.T) {
@@ -1309,6 +1314,7 @@ func TestService_ResubmitStagingImages(t *testing.T) {
 		image.ServiceParams{
 			EditDuration: 10 * time.Millisecond,
 			ImageAPI:     "http://127.0.0.1:8080/api/v1/picture/",
+			ProxyAPI:     "http://127.0.0.1:8080/api/v1/img",
 		})
 	defer imgSvc.Close(context.TODO())
 
@@ -1321,6 +1327,7 @@ func TestService_ResubmitStagingImages(t *testing.T) {
 		ID: "id-0",
 		Text: `<img src="http://127.0.0.1:8080/api/v1/picture/dev_user/bqf122eq9r8ad657n3ng" alt="startrails_01.jpg"><br/>
                <img src="http://127.0.0.1:8080/api/v1/picture/dev_user/bqf321eq9r8ad657n3ng" alt="cat.png"><br/>
+               <img src="http://127.0.0.1:8080/api/v1/img?src=aHR0cHM6Ly9ob21lcGFnZXMuY2FlLndpc2MuZWR1L35lY2U1MzMvaW1hZ2VzL2JvYXQucG5n" alt="cat.png"><br/>
                <img src="https://homepages.cae.wisc.edu/~ece533/images/boat.png" alt="boat.png">`,
 		Timestamp: time.Date(2017, 12, 20, 15, 18, 22, 0, time.Local),
 		Locator:   store.Locator{URL: "https://radio-t.com", SiteID: "radio-t"},
@@ -1337,10 +1344,11 @@ func TestService_ResubmitStagingImages(t *testing.T) {
 	// wait for Submit goroutine to commit image
 	mockStore.On("Commit", "dev_user/bqf122eq9r8ad657n3ng").Once().Return(nil)
 	mockStore.On("Commit", "dev_user/bqf321eq9r8ad657n3ng").Once().Return(nil)
+	mockStore.On("Commit", "cached_images/12318fbd4c55e9d177b8b5ae197bc89c5afd8e07-a41fcb00643f28d700504256ec81cbf2e1aac53e").Once().Return(nil)
 	time.Sleep(time.Millisecond * 100)
 
 	mockStore.AssertNumberOfCalls(t, "Info", 1)
-	mockStore.AssertNumberOfCalls(t, "Commit", 2)
+	mockStore.AssertNumberOfCalls(t, "Commit", 3)
 
 	// empty answer
 	mockStoreEmpty := image.MockStore{}