Merge pull request #2368 from thedadams/no-nginx-cache-ips

Ensure that nginx does DNS lookups

Dockerfile

@@ -38,6 +38,7 @@ COPY --from=registry /bin/registry /usr/local/bin
 COPY --from=klipper-lb /usr/bin/entry /usr/local/bin/klipper-lb
 COPY ./scripts/ds-containerd-config-path-entry /usr/local/bin
 COPY ./scripts/setup-binfmt /usr/local/bin
+COPY ./scripts/40-copy-resolv-nameserver.sh /docker-entrypoint.d/
 COPY --from=helper /usr/local/bin/acorn-helper /usr/local/bin/
 COPY --from=loglevel /usr/local/bin/loglevel /usr/local/bin/
 VOLUME /var/lib/buildkit

pkg/controller/appdefinition/deploy.go

@@ -98,7 +98,7 @@ func DeploySpec(req router.Request, resp router.Response) (err error) {
 	} else if len(objs) > 0 {
 		result = append(result, objs...)
 	}
-	if objs, err := toRouters(appInstance); err != nil {
+	if objs, err := toRouters(req.Ctx, req.Client, appInstance); err != nil {
 		return err
 	} else {
 		result = append(result, objs...)

pkg/controller/appdefinition/router.go

@@ -1,13 +1,15 @@
 package appdefinition
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"strconv"
 	"strings"
 
 	"github.com/acorn-io/baaah/pkg/typed"
 	v1 "github.com/acorn-io/runtime/pkg/apis/internal.acorn.io/v1"
+	"github.com/acorn-io/runtime/pkg/config"
 	"github.com/acorn-io/runtime/pkg/labels"
 	"github.com/acorn-io/runtime/pkg/pdb"
 	"github.com/acorn-io/runtime/pkg/ports"
@@ -23,9 +25,14 @@ import (
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func toRouters(appInstance *v1.AppInstance) (result []kclient.Object, _ error) {
+func toRouters(ctx context.Context, c kclient.Client, appInstance *v1.AppInstance) (result []kclient.Object, _ error) {
+	cfg, err := config.Get(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
 	for _, entry := range typed.Sorted(appInstance.Status.AppSpec.Routers) {
-		routerObjects, err := toRouter(appInstance, entry.Key, entry.Value)
+		routerObjects, err := toRouter(appInstance, entry.Key, entry.Value, cfg.InternalClusterDomain)
 		if err != nil {
 			return nil, err
 		}
@@ -34,12 +41,12 @@ func toRouters(appInstance *v1.AppInstance) (result []kclient.Object, _ error) {
 	return result, nil
 }
 
-func toRouter(appInstance *v1.AppInstance, routerName string, router v1.Router) (result []kclient.Object, _ error) {
+func toRouter(appInstance *v1.AppInstance, routerName string, router v1.Router, internalClusterDomain string) (result []kclient.Object, _ error) {
 	if ports.IsLinked(appInstance, routerName) || len(router.Routes) == 0 {
 		return nil, nil
 	}
 
-	conf, confName := toNginxConf(routerName, router)
+	conf, confName := toNginxConf(internalClusterDomain, appInstance.Status.Namespace, routerName, router)
 
 	podLabels := routerLabels(appInstance, router, routerName, labels.AcornAppPublicName, publicname.Get(appInstance))
 	deploymentLabels := routerLabels(appInstance, router, routerName)
@@ -165,7 +172,7 @@ func toRouter(appInstance *v1.AppInstance, routerName string, router v1.Router)
 	}, nil
 }
 
-func toNginxConf(routerName string, router v1.Router) (string, string) {
+func toNginxConf(internalClusterDomain, namespace, routerName string, router v1.Router) (string, string) {
 	buf := &strings.Builder{}
 	buf.WriteString("server {\nlisten 8080;\n")
 	for _, route := range router.Routes {
@@ -179,32 +186,38 @@ func toNginxConf(routerName string, router v1.Router) (string, string) {
 		buf.WriteString("location ")
 		buf.WriteString("= ")
 		buf.WriteString(route.Path)
-		buf.WriteString(" {\n  proxy_pass ")
-		buf.WriteString("http://")
+		buf.WriteString(" {\n  set $backend_servers ")
 		buf.WriteString(route.TargetServiceName)
-		buf.WriteString(":")
+		buf.WriteString(".")
+		buf.WriteString(namespace)
+		buf.WriteString(".")
+		buf.WriteString(internalClusterDomain)
+		buf.WriteString(";\n  proxy_pass http://$backend_servers:")
 		buf.WriteString(strconv.Itoa(port))
 		buf.WriteString(";\n  proxy_set_header X-Forwarded-Host $http_host;")
 		buf.WriteString("\n}\n")
 		if route.PathType == v1.PathTypePrefix && !strings.HasSuffix(route.Path, "/") {
 			buf.WriteString("location ")
 			buf.WriteString(route.Path)
-			buf.WriteString("/")
-			buf.WriteString(" {\n  proxy_pass ")
-			buf.WriteString("http://")
+			buf.WriteString("/ {\n  set $backend_servers ")
 			buf.WriteString(route.TargetServiceName)
-			buf.WriteString(":")
+			buf.WriteString(".")
+			buf.WriteString(namespace)
+			buf.WriteString(".")
+			buf.WriteString(internalClusterDomain)
+			buf.WriteString(";\n  proxy_pass http://$backend_servers:")
 			buf.WriteString(strconv.Itoa(port))
 			buf.WriteString(";\n  proxy_set_header X-Forwarded-Host $http_host;")
 			buf.WriteString("\n}\n")
 		}
 		if route.PathType == v1.PathTypePrefix && route.Path == "/" {
-			buf.WriteString("location ")
-			buf.WriteString("/")
-			buf.WriteString(" {\n  proxy_pass ")
-			buf.WriteString("http://")
+			buf.WriteString("location / {\n  set $backend_servers ")
 			buf.WriteString(route.TargetServiceName)
-			buf.WriteString(":")
+			buf.WriteString(".")
+			buf.WriteString(namespace)
+			buf.WriteString(".")
+			buf.WriteString(internalClusterDomain)
+			buf.WriteString(";\n  proxy_pass http://$backend_servers:")
 			buf.WriteString(strconv.Itoa(port))
 			buf.WriteString(";\n  proxy_set_header X-Forwarded-Host $http_host;")
 			buf.WriteString("\n}\n")

pkg/controller/appdefinition/testdata/router/expected.golden

@@ -66,7 +66,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: router-name-e9cec3df
+          name: router-name-30019cec
         name: conf
 status: {}
 
@@ -77,22 +77,25 @@ data:
     server {
     listen 8080;
     location = /foo {
-      proxy_pass http://foo-target:80;
+      set $backend_servers foo-target.app-created-namespace.svc.cluster.local;
+      proxy_pass http://$backend_servers:80;
       proxy_set_header X-Forwarded-Host $http_host;
     }
     location = /zzzz {
-      proxy_pass http://zzz-target:80;
+      set $backend_servers zzz-target.app-created-namespace.svc.cluster.local;
+      proxy_pass http://$backend_servers:80;
       proxy_set_header X-Forwarded-Host $http_host;
     }
     location /zzzz/ {
-      proxy_pass http://zzz-target:80;
+      set $backend_servers zzz-target.app-created-namespace.svc.cluster.local;
+      proxy_pass http://$backend_servers:80;
       proxy_set_header X-Forwarded-Host $http_host;
     }
     }
 kind: ConfigMap
 metadata:
   creationTimestamp: null
-  name: router-name-e9cec3df
+  name: router-name-30019cec
   namespace: app-created-namespace
 
 ---

scripts/40-copy-resolv-nameserver.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -eu
+
+ip=$(cat /etc/resolv.conf | grep nameserver | awk '{print $2}')
+echo "resolver $ip valid=15s;" > /etc/nginx/conf.d/resolver.conf