Optimization for redirects renderer

envs/base.env

@@ -2,5 +2,5 @@
 DEBUG=False
 TEMPLATE_DEBUG=False
 DJANGO_SETTINGS_MODULE=vaas.settings.docker
-PROMETHEUS_ENABLE=True
+PROMETHEUS_ENABLE=False
 LOG_LEVEL=INFO

envs/dev.env

@@ -2,5 +2,5 @@
 DEBUG=True
 TEMPLATE_DEBUG=True
 DJANGO_SETTINGS_MODULE=vaas.settings.docker
-PROMETHEUS_ENABLE=True
+PROMETHEUS_ENABLE=False
 LOG_LEVEL=DEBUG

vaas/vaas/router/models.py

@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 import uuid
-from typing import Dict, Tuple, List
+from typing import Dict
 from django.db import models
 from django.conf import settings
 from django.core.validators import MinValueValidator, MaxValueValidator
-from urllib.parse import urlsplit
 
 from vaas.cluster.models import DomainMapping, LogicalCluster
 from vaas.manager.models import Director
@@ -37,16 +36,6 @@ class ResponseStatusCode(models.IntegerChoices):
     def get_hashed_assertions_pks(self) -> Dict[int, int]:
         return {hash((a.given_url, a.expected_location)): a.pk for a in self.assertions.all()}
 
-    def fetch_all_destinations_mappings(self, cluster: LogicalCluster) -> Tuple[str, List[str]]:
-        """
-        Fetch tuple containing domain parsed from destination url and all found mappings for input cluster
-        """
-        all_mappings = set()
-        destination_domain = urlsplit(self.destination).netloc
-        for domain_mapping in DomainMapping.objects.filter(domain=destination_domain):
-            all_mappings = all_mappings.union(set(domain_mapping.mapped_domains(cluster)))
-        return destination_domain, list(all_mappings)
-
     @property
     def final_condition(self):
         if self.required_custom_header:

vaas/vaas/vcl/renderer.py

@@ -5,7 +5,7 @@
 import hashlib
 import time
 import functools
-from typing import Dict, List
+from typing import Dict, List, Tuple
 
 from django.conf import settings
 from django.db.models import Prefetch
@@ -186,23 +186,37 @@ def __init__(self, varnish, input_data):
             'mesh_routing': varnish.cluster.service_mesh_routing
         }
 
+    def fetch_all_destinations_mappings(self, cluster: LogicalCluster, redirect: str,
+                                        domain_mappings: List[DomainMapping]) -> Tuple[str, List[str]]:
+        """
+        Fetch tuple containing domain parsed from destination url and all found mappings for input cluster
+        """
+        all_mappings = set()
+        from urllib.parse import urlsplit
+        destination_domain = urlsplit(redirect).netloc
+        for domain_mapping in domain_mappings:
+            all_mappings = all_mappings.union(set(domain_mapping.mapped_domains(cluster)))
+        return destination_domain, list(all_mappings)
+
     @collect_processing
     def prepare_redirects(self) -> Dict[str, List[VclRedirect]]:
         redirects = {}
-        related_domains = MappingProvider(DomainMapping.objects.all()).provide_related_domains(self.varnish.cluster)
-        for redirect in self.input.redirects:
-            destination_domain, destination_mappings = redirect.fetch_all_destinations_mappings(self.varnish.cluster)
-            if str(redirect.src_domain) in related_domains:
-                for mapped_domain in redirect.src_domain.mapped_domains(self.varnish.cluster):
-                    destination = str(redirect.destination)
-                    if destination_domain == redirect.src_domain.domain:
-                        destination = destination.replace(destination_domain, mapped_domain)
-                    elif all((destination_domain, len(destination_mappings) == 1)):
-                        destination = destination.replace(destination_domain, destination_mappings[0])
-                    if entries := redirects.get(mapped_domain, []):
-                        entries.append(VclRedirect(redirect, mapped_domain, destination))
-                    else:
-                        redirects[mapped_domain] = [VclRedirect(redirect, mapped_domain, destination)]
+        related_domains = self.input.mapping_provider.provide_related_domains(self.varnish.cluster)
+        for related_domain in related_domains:
+            if related_domain in self.input.redirects.keys():
+                for redirect in self.input.redirects.get(related_domain):
+                    destination_domain, destination_mappings = self.fetch_all_destinations_mappings(
+                        self.varnish.cluster, redirect.destination, self.input.domainmappings)
+                    for mapped_domain in redirect.src_domain.mapped_domains(self.varnish.cluster):
+                        destination = str(redirect.destination)
+                        if destination_domain == redirect.src_domain.domain:
+                            destination = destination.replace(destination_domain, mapped_domain)
+                        elif all((destination_domain, len(destination_mappings) == 1)):
+                            destination = destination.replace(destination_domain, destination_mappings[0])
+                        if entries := redirects.get(mapped_domain, []):
+                            entries.append(VclRedirect(redirect, mapped_domain, destination))
+                        else:
+                            redirects[mapped_domain] = [VclRedirect(redirect, mapped_domain, destination)]
         return redirects
 
     @collect_processing
@@ -382,7 +396,7 @@ def fetch_render_data(self):
             Prefetch('clusters', queryset=LogicalCluster.objects.only('pk'), to_attr='cluster_ids'),
         ))
         self.routes.sort(key=lambda route: "{:03d}-{}".format(route.priority, route.director.name))
-        self.redirects = list(Redirect.objects.all().order_by('src_domain', 'priority'))
+        self.redirects = self.assemble_redirects()
         self.dcs = list(Dc.objects.all())
         self.template_blocks = list(VclTemplateBlock.objects.all().prefetch_related('template'))
         self.vcl_variables = list(VclVariable.objects.all())
@@ -392,6 +406,17 @@ def fetch_render_data(self):
         )
         self.distributed_backends = self.distribute_backends(backends)
         self.distributed_canary_backends = self.prepare_canary_backends(canary_backend_ids, backends)
+        self.domainmappings = list(DomainMapping.objects.all())
+        self.mapping_provider = MappingProvider(self.domainmappings)
+
+    @collect_processing
+    def assemble_redirects(self) -> Dict[str, List[Redirect]]:
+        redirects = {}
+        for redirect in Redirect.objects.all().order_by('src_domain', 'priority'):
+            if redirect.src_domain.domain not in redirects.keys():
+                redirects[redirect.src_domain.domain] = []
+            redirects[redirect.src_domain.domain].append(redirect)
+        return redirects
 
     @collect_processing
     def distribute_backends(self, backends):

vaas/vaas/vcl/tests/expected-vcl-4.0-canary.vcl

@@ -339,7 +339,7 @@ sub vcl_synth {
     if (resp.status == 989) {
         set resp.status = 200;
         set resp.http.Content-Type = "application/json";
-        synthetic ( {"{ "vcl_version" : "0bbef", "varnish_status": "disabled" }"} );
+        synthetic ( {"{ "vcl_version" : "ce716", "varnish_status": "disabled" }"} );
         return (deliver);
     }
 }
@@ -424,12 +424,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
     if (req.http.host == "example.prod.org") {
     if (req.url ~ "/source") {
@@ -444,12 +438,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
 
 # Test ROUTER

vaas/vaas/vcl/tests/expected-vcl-4.0.vcl

@@ -366,7 +366,7 @@ sub vcl_synth {
     if (resp.status == 989) {
         set resp.status = 200;
         set resp.http.Content-Type = "application/json";
-        synthetic ( {"{ "vcl_version" : "d584a", "varnish_status": "disabled" }"} );
+        synthetic ( {"{ "vcl_version" : "721d3", "varnish_status": "disabled" }"} );
         return (deliver);
     }
 }
@@ -451,12 +451,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
     if (req.http.host == "example.prod.org") {
     if (req.url ~ "/source") {
@@ -471,12 +465,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
 
 # Test ROUTER

vaas/vaas/vcl/tests/test_renderer.py

@@ -280,7 +280,7 @@ def setUp(self):
         )
 
         Redirect.objects.create(
-            src_domain=self.example_domain_mapping,
+            src_domain=self.external_domain_mapping,
             condition='req.url ~ "/external"',
             destination='http://external.com/external_destination',
             action=301,
@@ -436,23 +436,26 @@ def test_should_decorate_set_backend_tag_with_fallback_service_in_dc1(self):
     def test_should_decorate_flexible_router_tag_with_properly_mapped_destination_domain(self):
         vcl_tag_builder = VclTagBuilder(self.varnish, VclRendererInput())
         tag = vcl_tag_builder.get_expanded_tags('FLEXIBLE_ROUTER').pop()
-        assert_set_equal({'example.prod.com', 'example.prod.org'}, set(tag.parameters['redirects'].keys()))
+        assert_set_equal({'example.prod.com', 'example-external.com', 'example.prod.org'},
+                         set(tag.parameters['redirects'].keys()))
         assert_equals('example.com', tag.parameters['redirects']['example.prod.com'][1].src_domain.domain)
         assert_equals('example.com', tag.parameters['redirects']['example.prod.org'][1].src_domain.domain)
         assert_equals('http://example.prod.com/destination',
                       tag.parameters['redirects']['example.prod.com'][1].destination)
         assert_equals('http://example.prod.org/destination',
                       tag.parameters['redirects']['example.prod.org'][1].destination)
         assert_equals('http://example-external.com/external_destination',
-                      tag.parameters['redirects']['example.prod.com'][2].destination)
+                      tag.parameters['redirects']['example-external.com'][0].destination)
 
     def test_should_sort_redirects_by_priority(self):
         vcl_tag_builder = VclTagBuilder(self.varnish, VclRendererInput())
         tag = vcl_tag_builder.get_expanded_tags('FLEXIBLE_ROUTER').pop()
-        assert_set_equal({'example.prod.com', 'example.prod.org'}, set(tag.parameters['redirects'].keys()))
+        assert_set_equal({'example.prod.com', 'example-external.com', 'example.prod.org'},
+                         set(tag.parameters['redirects'].keys()))
         assert_equals('2/example.prod.com', tag.parameters['redirects']['example.prod.com'][0].id)
         assert_equals('1/example.prod.com', tag.parameters['redirects']['example.prod.com'][1].id)
-        assert_equals('3/example.prod.com', tag.parameters['redirects']['example.prod.com'][2].id)
+        assert_equals('2/example.prod.org', tag.parameters['redirects']['example.prod.org'][0].id)
+        assert_equals('3/example-external.com', tag.parameters['redirects']['example-external.com'][0].id)
 
 
 class VclRendererInputTest(TestCase):