Merge pull request #3 from darkweak/feat/chore/stream-response

feat(chore): stream response

Makefile

@@ -1,7 +1,13 @@
-.PHONY: lint run-caddy run-roadrunner
+.PHONY: lint run-caddy run-server run-traefik
 
 lint: ## Run golangci-lint to ensure the code quality
 	docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint golangci-lint run
 
-run-caddy: ## Build caddy binary
+run-caddy: ## Build and run caddy binary
 	cd middleware/caddy && $(MAKE) build && $(MAKE) run
+
+run-server: ## Run server main.go
+	go run middleware/server/main.go
+
+run-traefik: ## Run server main.go
+	cd middleware/traefik && $(MAKE) build && $(MAKE) run

README.md

@@ -51,18 +51,18 @@ func functionToParseESITags(b []byte, r *http.Request) []byte {
 ```bash
 xcaddy build --with github.com/darkweak/go-esi/middleware/caddy
 ```
-Refer to the [sample Caddyfile](https://github.com/darkweak/go-esi/blob/master/middleware/caddy/Caddyfile) to know how to configure that.
+Refer to the [sample Caddyfile](https://github.com/darkweak/go-esi/blob/master/middleware/caddy/Caddyfile) to know how to use that.
 
 ### Træfik middleware
-```bash
+```yaml
 # anywhere/traefik.yml
 experimental:
   plugins:
     souin:
       moduleName: github.com/darkweak/go-esi
-      version: v0.0.4
+      version: v0.0.5
 ```
-```bash
+```yaml
 # anywhere/dynamic-configuration
 http:
   routers:
@@ -74,12 +74,9 @@ http:
   middlewares:
     esi:
       plugin:
-        esi:
-          # We don't care about the configuration but we have ot declare that block 
-          # due to shitty træfik empty configuration handle.
-          disable: false
+        esi: {}
 ```
-Refer to the [sample Caddyfile](https://github.com/darkweak/go-esi/blob/master/middleware/caddy/Caddyfile) to know how to configure that.
+Refer to the [sample traefik file](https://github.com/darkweak/go-esi/blob/master/middleware/traefik/esi-configuration.yml) to know how to use that.
 
 ## TODO
 - [x] choose tag

esi/choose.go

@@ -30,7 +30,7 @@ type chooseTag struct {
 //   </esi:otherwise>
 // </esi:choose>
 // ).
-func (c *chooseTag) process(b []byte, req *http.Request) ([]byte, int) {
+func (c *chooseTag) Process(b []byte, req *http.Request) ([]byte, int) {
 	found := closeChoose.FindIndex(b)
 	if found == nil {
 		return nil, len(b)
@@ -56,3 +56,14 @@ func (c *chooseTag) process(b []byte, req *http.Request) ([]byte, int) {
 
 	return res, c.length
 }
+
+func (*chooseTag) HasClose(b []byte) bool {
+	return closeChoose.FindIndex(b) != nil
+}
+
+func (*chooseTag) GetClosePosition(b []byte) int {
+	if idx := closeChoose.FindIndex(b); idx != nil {
+		return idx[1]
+	}
+	return 0
+}

esi/comment.go

@@ -14,11 +14,22 @@ type commentTag struct {
 }
 
 // Input (e.g. comment text="This is a comment." />).
-func (c *commentTag) process(b []byte, req *http.Request) ([]byte, int) {
+func (c *commentTag) Process(b []byte, req *http.Request) ([]byte, int) {
 	found := closeComment.FindIndex(b)
 	if found == nil {
 		return nil, len(b)
 	}
 
 	return []byte{}, found[1]
 }
+
+func (*commentTag) HasClose(b []byte) bool {
+	return closeComment.FindIndex(b) != nil
+}
+
+func (*commentTag) GetClosePosition(b []byte) int {
+	if idx := closeComment.FindIndex(b); idx != nil {
+		return idx[1]
+	}
+	return 0
+}

esi/escape.go

@@ -16,7 +16,7 @@ type escapeTag struct {
 	*baseTag
 }
 
-func (e *escapeTag) process(b []byte, req *http.Request) ([]byte, int) {
+func (e *escapeTag) Process(b []byte, req *http.Request) ([]byte, int) {
 	closeIdx := closeEscape.FindIndex(b)
 
 	if closeIdx == nil {
@@ -28,3 +28,14 @@ func (e *escapeTag) process(b []byte, req *http.Request) ([]byte, int) {
 
 	return b, e.length
 }
+
+func (*escapeTag) HasClose(b []byte) bool {
+	return closeEscape.FindIndex(b) != nil
+}
+
+func (*escapeTag) GetClosePosition(b []byte) int {
+	if idx := closeEscape.FindIndex(b); idx != nil {
+		return idx[1]
+	}
+	return 0
+}

esi/esi.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 )
 
-func findTagName(b []byte) tag {
+func findTagName(b []byte) Tag {
 	name := tagname.FindSubmatch(b)
 	if name == nil {
 		return nil
@@ -43,6 +43,43 @@ func findTagName(b []byte) tag {
 	return nil
 }
 
+func HasOpenedTags(b []byte) bool {
+	return esi.FindIndex(b) != nil || escapeRg.FindIndex(b) != nil
+}
+
+func CanProcess(b []byte) bool {
+	if tag := findTagName(b); tag != nil {
+		return tag.HasClose(b)
+	}
+
+	return false
+}
+
+func ReadToTag(next []byte, pointer int) (startTagPosition, esiPointer int, t Tag) {
+	tagIdx := esi.FindIndex(next)
+	var isEscapeTag bool
+
+	if escIdx := escapeRg.FindIndex(next); escIdx != nil && (tagIdx == nil || escIdx[0] < tagIdx[0]) {
+		tagIdx = escIdx
+		tagIdx[1] = escIdx[0]
+		isEscapeTag = true
+	}
+
+	if tagIdx == nil {
+		return len(next), 0, nil
+	}
+
+	esiPointer = tagIdx[1]
+	startTagPosition = tagIdx[0]
+	t = findTagName(next[esiPointer:])
+
+	if isEscapeTag {
+		esiPointer += 7
+	}
+
+	return
+}
+
 func Parse(b []byte, req *http.Request) []byte {
 	pointer := 0
 
@@ -69,7 +106,7 @@ func Parse(b []byte, req *http.Request) []byte {
 			esiPointer += 7
 		}
 
-		res, p := t.process(next[esiPointer:], req)
+		res, p := t.Process(next[esiPointer:], req)
 		esiPointer += p
 
 		b = append(b[:pointer], append(next[:tagIdx[0]], append(res, next[esiPointer:]...)...)...)

esi/include.go

@@ -40,7 +40,7 @@ func (i *includeTag) loadAttributes(b []byte) error {
 // With or without the alt
 // With or without a space separator before the closing
 // With or without the quotes around the src/alt value.
-func (i *includeTag) process(b []byte, req *http.Request) ([]byte, int) {
+func (i *includeTag) Process(b []byte, req *http.Request) ([]byte, int) {
 	closeIdx := closeInclude.FindIndex(b)
 
 	if closeIdx == nil {
@@ -71,3 +71,14 @@ func (i *includeTag) process(b []byte, req *http.Request) ([]byte, int) {
 
 	return b, i.length
 }
+
+func (*includeTag) HasClose(b []byte) bool {
+	return closeInclude.FindIndex(b) != nil
+}
+
+func (*includeTag) GetClosePosition(b []byte) int {
+	if idx := closeInclude.FindIndex(b); idx != nil {
+		return idx[1]
+	}
+	return 0
+}

esi/remove.go

@@ -13,7 +13,7 @@ type removeTag struct {
 	*baseTag
 }
 
-func (r *removeTag) process(b []byte, req *http.Request) ([]byte, int) {
+func (r *removeTag) Process(b []byte, req *http.Request) ([]byte, int) {
 	closeIdx := closeRemove.FindIndex(b)
 	if closeIdx == nil {
 		return []byte{}, len(b)
@@ -23,3 +23,14 @@ func (r *removeTag) process(b []byte, req *http.Request) ([]byte, int) {
 
 	return []byte{}, r.length
 }
+
+func (*removeTag) HasClose(b []byte) bool {
+	return closeRemove.FindIndex(b) != nil
+}
+
+func (*removeTag) GetClosePosition(b []byte) int {
+	if idx := closeRemove.FindIndex(b); idx != nil {
+		return idx[1]
+	}
+	return 0
+}

esi/type.go

@@ -5,8 +5,10 @@ import (
 )
 
 type (
-	tag interface {
-		process([]byte, *http.Request) ([]byte, int)
+	Tag interface {
+		Process([]byte, *http.Request) ([]byte, int)
+		HasClose([]byte) bool
+		GetClosePosition([]byte) int
 	}
 
 	baseTag struct {
@@ -18,6 +20,6 @@ func newBaseTag() *baseTag {
 	return &baseTag{length: 0}
 }
 
-func (b *baseTag) process(content []byte, _ *http.Request) ([]byte, int) {
+func (b *baseTag) Process(content []byte, _ *http.Request) ([]byte, int) {
 	return []byte{}, len(content)
 }

esi/vars.go

@@ -75,7 +75,7 @@ type varsTag struct {
 }
 
 // Input (e.g. comment text="This is a comment." />).
-func (c *varsTag) process(b []byte, req *http.Request) ([]byte, int) {
+func (c *varsTag) Process(b []byte, req *http.Request) ([]byte, int) {
 	found := closeVars.FindIndex(b)
 	if found == nil {
 		return nil, len(b)
@@ -87,3 +87,14 @@ func (c *varsTag) process(b []byte, req *http.Request) ([]byte, int) {
 		return []byte(parseVariables(b, req))
 	}), c.length
 }
+
+func (*varsTag) HasClose(b []byte) bool {
+	return closeVars.FindIndex(b) != nil
+}
+
+func (*varsTag) GetClosePosition(b []byte) int {
+	if idx := closeVars.FindIndex(b); idx != nil {
+		return idx[1]
+	}
+	return 0
+}

middleware/caddy/Caddyfile

@@ -5,7 +5,7 @@
     esi
 }
 
-:80 {
+localhost:443 {
     route /chained-esi-include-1 {
         header Content-Type text/html
         respond `<esi:include src="http://domain.com/chained-esi-include-2"/>`
@@ -28,8 +28,8 @@
 
     route /* {
         esi
-
-        root * ../../fixtures
-        file_server
+        reverse_proxy 127.0.0.1:81
+        # root * ../../fixtures
+        # file_server
     }
 }

middleware/caddy/esi.go

@@ -2,15 +2,14 @@ package caddy_esi
 
 import (
 	"bytes"
-	"fmt"
 	"net/http"
 	"sync"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/darkweak/go-esi/esi"
+	"github.com/darkweak/go-esi/writer"
 )
 
 var bufPool *sync.Pool = &sync.Pool{
@@ -45,14 +44,33 @@ func (e *ESI) ServeHTTP(rw http.ResponseWriter, r *http.Request, next caddyhttp.
 	buf := bufPool.Get().(*bytes.Buffer)
 	buf.Reset()
 	defer bufPool.Put(buf)
-	cw := newWriter(buf, rw)
+	cw := writer.NewWriter(buf, rw, r)
+	go func(w *writer.Writer) {
+		w.Header().Del("Content-Length")
+		if w.Rq.ProtoMajor == 1 {
+			w.Header().Set("Content-Encoding", "chunked")
+		}
+		var i = 0
+		for {
+			if len(cw.AsyncBuf) <= i {
+				continue
+			}
+			rs := <-cw.AsyncBuf[i]
+			if rs == nil {
+				cw.Done <- true
+				break
+			}
+			_, _ = rw.Write(rs)
+			i++
+		}
+	}(cw)
 	next.ServeHTTP(cw, r)
+	cw.AsyncBuf = append(cw.AsyncBuf, make(chan []byte))
+	go func(w *writer.Writer, iteration int) {
+		w.AsyncBuf[iteration] <- nil
+	}(cw, cw.Iteration)
 
-	b := esi.Parse(cw.buf.Bytes(), r)
-
-	rw.Header().Set("Content-Length", fmt.Sprintf("%d", len(b)))
-	rw.WriteHeader(cw.status)
-	_, _ = rw.Write(b)
+	<-cw.Done
 
 	return nil
 }

middleware/caddy/go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/caddyserver/caddy/v2 v2.5.2
-	github.com/darkweak/go-esi v0.0.4
+	github.com/darkweak/go-esi v0.0.5
 )
 
 require (
@@ -111,4 +111,4 @@ require (
 	howett.net/plist v1.0.0 // indirect
 )
 
-replace github.com/darkweak/go-esi v0.0.4 => ../..
+replace github.com/darkweak/go-esi v0.0.5 => ../..