draft-ietf-tsvwg-L4S-deployment.xml

<?xml version="1.0" encoding="US-ASCII"?>
	<!-- 
	     draft-rfcxml-general-template-annotated-00
	  
	     This template includes examples of most of the features of RFCXML with comments explaining 
	     how to customise them, and examples of how to achieve specific formatting.
	     
	     Documentation is at https://authors.ietf.org/en/templates-and-schemas
	-->
	<?xml-model href="rfc7991bis.rnc"?>  <!-- Required for schema validation and schema-aware editing -->
	<!-- <?xml-stylesheet type="text/xsl" href="rfc2629.xslt" ?> --> 
	<!-- This third-party XSLT can be enabled for direct transformations in XML processors, including most browsers -->
	
	<!DOCTYPE rfc [
	  <!ENTITY nbsp    "&#160;">
	  <!ENTITY zwsp   "&#8203;">
	  <!ENTITY nbhy   "&#8209;">
	  <!ENTITY wj     "&#8288;">
					]>
	
	<!-- Extra statement used by XSLT processors to control the output style. -->
	<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>
	<!-- Processing Instructions- PIs (for a complete list and description,
	     see file http://xml.resource.org/authoring/README.html.
	     You may find that some sphisticated editors are not able to edit PIs when palced here.
	     An alternative position is just inside the rfc elelment as noted below. -->
	<!-- Some of the more generally applicable PIs that most I-Ds might want to use -->
	<!-- Try to enforce the ID-nits conventions and DTD validity -->
	<?rfc strict="yes" ?>
	<!-- Items used when reviewing the document -->
	<!-- Controls display of <cref> elements -->
	<?rfc comments="yes" ?>
	<!-- When no, put comments at end in comments section,
	     otherwise, put inline -->
	<?rfc inline="yes" ?>
	<!-- When yes, insert editing marks: editing marks consist of a 
	     string such as <29> printed in the blank line at the 
	     beginning of each paragraph of text. -->
	<?rfc editing="no" ?>
	<!-- Create Table of Contents (ToC) and set some options for it.  
	     Note the ToC may be omitted for very short documents,but idnits insists on a ToC 
	     if the document has more than 15 pages. -->
	<?rfc toc="yes"?>
	<?rfc tocompact="yes"?>
	<!-- If "yes" eliminates blank lines before main section entries. -->
	<?rfc tocdepth="3"?>
	<!-- Sets the number of levels of sections/subsections... in ToC.
	   Can be overridden by 'toc="include"/"exclude"' on the section
	   element-->
	<!-- Choose the options for the references. 
	     Some like symbolic tags in the references (and citations) and others prefer 
	     numbers. The RFC Editor always uses symbolic tags.
	     The tags used are the anchor attributes of the references. -->
	<?rfc symrefs="yes"?>
	<?rfc sortrefs="yes" ?>
	<!-- If "yes", causes the references to be sorted in order of tags.
				 This doesn't have any effect unless symrefs is "yes"
	          also. --> 
	<!-- These two save paper: Just setting compact to "yes" makes savings by not starting each 
	     main section on a new page but does not omit the blank lines between list items. 
	     If subcompact is also "yes" the blank lines between list items are also omitted. -->
	<?rfc compact="yes" ?>
	<?rfc subcompact="no" ?>
	<!-- end of list of popular I-D processing instructions -->
	
	<!-- Information about the document.
	     category values: std, bcp, info, exp, and historic
	     For Internet-Drafts, specify attribute "ipr".
	         original ipr values are: full3978, noModification3978, noDerivatives3978), 
	         2008 IETF Trust versions: trust200811, noModificationTrust200811, noDerivativeTrust200811
	         2009/Current: trust200902, noModificationTrust200902, noDerivativesTrust200902, pre5378Trust200902
	     Also for Internet-Drafts, you must specify a value for attributes "docName" which is 
	        typically the file name under which it is filed but need not be.
	     If relevant, "iprExtract" may be specified to denote the anchor attribute value of a
	        section that can be extracted for separate publication, it is only useful when the value
	        of "ipr" does not give the Trust full rights.
	     "updates" and "obsoletes" attributes can also be specified here, their arguments are
	        comma-separated lists of RFC numbers (just the numbers) -->
	<rfc
	  xmlns:xi="http://www.w3.org/2001/XInclude"
	  category="info"
	  docName="draft-livingood-low-latency-deployment-07"
	  ipr="trust200902"
	  obsoletes=""
	  updates=""
	  submissionType="independent"
	  xml:lang="en"
	  version="3">
	
	
	  <!-- ***** FRONT MATTER ***** -->
	
	  <front>
	
	    <title abbrev="ISP Dual Queue Networking Deployment Recommendations">ISP Dual Queue Networking Deployment Recommendations</title>
	    <seriesInfo name="Internet-Draft" value="draft-livingood-low-latency-deployment-07"/>
	
	    <!-- add 'role="editor"' below for the editors if appropriate -->
	    <author fullname="Jason Livingood" initials="J" surname="Livingood">
		   <organization>
	           Comcast
	       </organization>
	      <address>
	        <postal>
	          <city>Philadelphia</city> <region>PA</region>
	          <country>USA</country>
	        </postal>
	        <email>jason_livingood@comcast.com</email>
	      </address>
	    </author>
	
	    <date month="October" day="17" year="2024"/>
	
	    <!-- Meta-data Declarations -->
	    <!-- WG name at the upper left corner of the doc,
	         IETF fine for individual submissions.  You can also
	         omit this element in which case it defaults to "Network Working Group" -
	         a hangover from the ancient history of the IETF! -->

	    <workgroup>Independent Stream</workgroup>
	
		<!-- You can add <keyword/> elements here.  They will be incorporated into HTML output
	         files in a meta tag but they have no effect on text or nroff output. -->
		<!-- <keyword>Text</keyword> (as many of those elements as needed -->
	

	    <abstract>
	      <t>The IETF's Transport Area Working Group (TSVWG) has finalized experimental RFCs for Low Latency, 
	      Low Loss, Scalable Throughput (L4S) and new Non-Queue-Building (NQB) per hop behavior. 
	      These documents describe a new architecture and protocol for deploying low latency networking. 
          Since deployment decisions are left to implementers, this document explores the potential 
	      implications of those decisions and makes recommendations that can help drive adoption and acceptance of L4S and NQB.</t>
	    </abstract>
	
		
	  </front>
	
	  <middle>
        <!--  <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
	      "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
	      document are to be interpreted as described in
		  <xref target="RFC2119">RFC 2119</xref>.</t> -->

	    <section title="Introduction">
			    
		<t>The IETF's Transport Area Working Group (TSVWG) has finalized RFCs for Low Latency, 
		Low Loss, Scalable Throughput (L4S) and Non-Queue-Building (NQB) per hop behavior <xref target="RFC9330"/> 
		<xref target="RFC9331"/> <xref target="RFC9332"/> 
		<xref target="I-D.ietf-tsvwg-l4sops"/> <xref target="I-D.ietf-tsvwg-nqb"/> 
		<xref target="I-D.ietf-tsvwg-dscp-considerations"/>. These documents do a good job of describing 
		a new architecture and protocol for deploying low latency networking. 
		But as is normal for many such standards, especially those in 
		experimental status, certain deployment decisions are ultimately left to implementers.</t>
		
		<t>This document explores the potential implications of key deployment decisions and makes 
		recommendations for those decisions that may help drive adoption. In particular, there are best practices based 
        on prior experience 
		as a network operator that should be considered and there are network neutrality types of considerations 
		as well. These technologies are benign on their own, but the way they are operationally implemented can 
		determine whether they are ultimately perceived positively and adopted by the broader Internet ecosystem. 
		That is a key issue for low latency networking, because the more applications developers and edge platforms that adopt 
		new packet marking for low latency traffic, then the greater the value to end users, so ensuring it is received well 
		is key to driving strong initial adoption.</t>
		
		<t>It is worth stating though that these decisions are not embedded in or inherent to L4S and NQB per se, but are decisions 
		that can change depending upon differing technical, regulatory, business or other requirements. Even two network operators with the 
		same type of access technology and in the same market area may choose to implement in different ways. Nevertheless, this document 
		suggests that certain specific deployment decisions can help maximize the value of low latency networking to both users and 
		network operators.</t> 
		
		<t>The IETF documents on L4S and NQB also made clear that nearly all modern application types - from video conferencing to web 
        browsing - can benefit from low latency networking. In addition, the design of the protocols also make clear 
        that applications developers are best positioned to understand the needs of their applications and to, 
        by extension, express any such low latency needs via appropriate L4S or NQB packet marking. Furthermore, 
        unlike with bandwidth priority on a highly/fully utilized link, low latency networking can better balance 
        the needs of different types of best effort flows (with some caveats - see <xref target="NewThinking"/>).</t>
		
		<t>For additional background on latency and why latency matters so much to the Internet, please 
        read <xref target="BITAG"/>.</t>
	
	    
	    <section title="A Different Understanding of Application Needs">
		    <t>In the course of working to improve the responsiveness of network protocols, the IETF 
			    concluded with their L4S and NQB work that there were fundamentally two types of Internet traffic 
			    and that these two major traffic types could benefit from having separate network processing queues 
			    in order to improve the way the Internet works for all applications, and especially for 
                interactive applications.</t>
			    
			    <t>One of the two major traffic types is Queue Building (QB) - things like file downloads and 
				    backups that are designed utilize as much network capacity as possible but with which users 
				    are usually not interacting with in 
			    real-time. The other was Non-Queue-Building (NQB) - such as DNS lookups, voice interaction 
			    with artificial intelligence (AI) assistants, video conferencing, gaming, and so on. NQB 
			    flows tend to be ones where the end user is sensitive to any delays.</t>
			    
				<t>Thus, the IETF created specifications for how to use two different network processing 
					queues. The performance value of dual queue networking (simply "low latency 
					networking" hereafter) has proven out in Comcast's 
					dual queue networking field trial <xref target="Comcast"/>. That field trial lasted for over a year and was 
					regularly reported on over time at several IETF TSVWG meetings <xref target="IETF-TSVWG-117"/> <xref target="IETF-TSVWG-118"/> 
					<xref target="IETF-TSVWG-119"/> <xref target="IETF-TSVWG-120"/> and demonstrated that 
					L4S and NQB can work deliver excellent responsiveness for a variety of applications - from video 
					conferencing to cloud gaming, DNS and other applications. It seems likely that 
					this new capability will enable entirely new classes of applications to become possible, 
					driving a wave of new Internet innovation, while also improving the applications people use 
					today.</t>

			
        <table>
            <name>Comparison of Traffic Types</name>
            <thead>
                <tr>
                    <td>May Benefit from L4S/NQB</td>
                    <td>Classic Queue Building</td>
                </tr>
            </thead>
            <tbody>
                <tr>
                    <td>User interacting with screen</td>
                    <td>Unattended background process</td>
                </tr>
                <tr>
                    <td>Video conference, audio conference, live event video stream, cloud document editing</td>
                    <td>Cloud file backup, cloud file restoration, game platform update, operating system update</td>
                </tr>
            </tbody>
        </table>

		</section>
		
		<section title="New Thinking on Low Latency Packet Processing" anchor="NewThinking">
			<t>The Introduction says that unlike with bandwidth priority on a highly/fully utilized link, low latency networking can better 
				balance the needs of different types of best effort flows.
			But this bears a bit of further discussion to understand more fully.</t>
			
			<t>L4S does *not* provide low latency in the same way as previous technologies like DiffServ Quality of Service (QoS). That prior 
			QoS approach used packet 
prioritization, where it was possible to assign a higher relative priority to certain application traffic, such as Voice over IP (VoIP) telephony. 
This approach could provide consistent and relatively low latency by assigning high priority to a partition of the capacity of a link, and then policing the 
rate of packets using that partition. This traditional approach to QoS is hierarchical in nature.</t>
			
			<t>That QoS approach 
			is to some extent predicated on an idea that network capacity is 
			very limited and that links are often highly utilized. But in today's Internet, it is increasingly the case that there is an 
			abundance of capacity to end users (e.g., symmetric 1 Gbps), which makes such traditional QoS approaches ineffective in 
			delivering ever-lower latency. This new low latency networking approach is not based on hierarchical QoS prioritization. 
			Rather, it is built upon conditional 
			priority scheduling between its two queues that operate at best effort QoS priority.</t>
			
		</section>
        </section>

        <!--  END MAJOR SECTION --> 
        <!--  START NEW MAJOR SECTION --> 
	    
	    <section title="Key Low Latency Networking Concepts">

        <t>Before proceeding into deployment recommendations, it is important to first explore a few key concepts 
        about low latency networking. This is critical because in the past many networks have thought of only 
        bandwidth and/or priority at the sole network attributes that could be adjusted in order to improve 
        application or network quality. The advent of low latency networking forces a re-examination of those 
        old assumptions, as we take into account the profound impact that latency and jitter have on the quality of 
        the end user experience.</t>
	    
			<section title="Prioritization: Same Best Effort Priority for All Traffic">
		    	<t>Low latency traffic to is not prioritized over other 
				(best effort priority) "classic" Internet traffic. That is the case over the ISP network and the 
				broader internet, though it may not not necessarily be the case for a user's in-home Wi-Fi network 
				due to the particulars of how the IEEE 802.11 wireless protocol <xref target="IEEE"/> functions 
				at the current time - see <xref target="RFC8325"/>). In addition, some user access points 
				may prioritize certain traffic (such as gaming) and some traffic such as NQB may use the 
				AC_VI Wi-Fi link layer queue <xref target="I-D.ietf-tsvwg-nqb"/>. This best effort approach stands 
				in contrast to prior differential quality of service (QoS) approaches or to what has been discussed 
				for 5G network slicing <xref target="CDT-NN"/> <xref target="van-Schewick-1A"/> 
				<xref target="van-Schewick-1B"/> <xref target="van-Schewick-2"/> <xref target="van-Schewick-3"/>.</t>
	    	</section>
	    
			<section title="Throughput: Shared Across All Traffic">
		    	<t>Low latency networking flows do not get access to greater throughput than "classic" flows. 
				Thus, a user's total provisioned or permitted throughput on an ISP access network link is 
				shared between both classic and low latency queues.</t>
	    	</section>
	    	
	    	<section title="Access-Agnostic: Can Work on All Types of Network Technology">
		    	<t>Ultimately, the emergence of low latency networking represents a fundamental new network 
			    	capability that applications can choose to utilize as their needs dictate. It reflects a new ground truth 
			    	about two fundamentally different types of application traffic and demonstrates that networks 
			    	continue to evolve in exciting ways. This new network capability can be implemented in a 
					variety of network technologies. For example in access network technologies this could be 
					implemented in DOCSIS <xref target="LLD"/>, 5G <xref target="Ericsson"/>, 
					PON <xref target="CTI"/>, and many other types of networks. Anywhere that a network bottleneck 
					could occur may benefit from this technology.</t>
	    	</section>
	    </section>
	    
        <!--  END MAJOR SECTION --> 
        <!--  START NEW MAJOR SECTION --> 

        <section title="Recommendations for Application Developers">
        <t>Application developers need to add L4S or NQB packet marking to their application, which will often depend 
        upon the capabilities of a device's operation system (OS) or a software development kit (SDK) 
        <xref target="Apple"/> that the OS developer makes available. In addition, the application server will 
        also need to support the appropriate marking and, when L4S is used, to implement a responsive congestion 
        controller.</t>

        <section title="Delivery Infrastructure Needs for L4S">
        <t>Since L4S uses the Explicit Congestion Notification (ECN) field of the packet header, to ensure 
        ECN works end-to-end, application developers need to be certain that their servers, datacenter routers, 
        and any transit, cloud provider, or content delivery network (CDN) server involved in their application 
        IS NOT altering or bleaching the ECN field. For an application to use the L4S queue, they must mark 
        their packets with the ECT(1) code point to signal L4S-capability or with the Congestion Experienced (CE) 
        code point when appropriate. Coupled with client marking, if an application client or server detects CE marks, 
        it should respond accordingly (e.g., by reducing the send rate), which typically means that the server 
        must be running a "responsive" congestion controller (i.e., is able to adjust rate based the presence or 
        absence of CE marks for L4S traffic - such as DCTCP, TCP Prague, SCReAM, and BBRv2). See Section 4.3 
        of <xref target="RFC9330"/> and Section 4.3 of <xref target="RFC9331"/> for more information about this.</t>
        </section>

        <section title="Delivery Infrastructure Needs for NQB">
        <t>Since NQB uses the DSCP-45 code point in the DiffServ part of the packet header, to ensure 
        NQB works end-to-end, application developers need to be certain that their servers, datacenter routers, 
        and any transit, cloud provider, or content delivery network (CDN) server involved in their application 
        IS NOT altering or bleaching a DSCP-45 mark. One relative advantage of NQB is that the server does not need to run 
        a special responsive congestion controller. On the other hand, NQB is geared toward bandwidth-limited sparse 
        flows rather than capacity-seeking flows, so it will depend on your application's needs. In addition, it is 
        common for networks to bleach or modify DSCP marks on ingress, which means that networks will need to change that 
        packet handling policy for NQB to function on an end-to-end basis. In contrast, it is far less common for the 
        ECN field of the packet header to be modified.</t>
        </section>

        <section title="Don't Mark Non-Delay-Sensitive Traffic for L4S or NQB">
        <t>It may seem tempting to mark all traffic for L4S or NQB handling, but this will likely harm the experience 
        of queue-building apps like file downloads. Also, remember that the network priority for L4S and NQB is still 
        best efforts and so there is not more bandwidth for L4S or NQB compared to classic traffic; 
        they share the same bandwidth and best-efforts priority. It is also possible that some flows from an 
        application can benefit, while others will not. For example, a video gaming service may benefit from using L4S or 
        NQB for real-time controller inputs and gameplay, while major game software updates would best be left in 
        the classic queue.</t>    
        </section>

       <section title="Consider Application Needs in Choosing L4S vs. NQB"> 

        <t>Determine whether your application needs "sparse" flows or "congestion-controlled" (higher capacity) 
        flows. Sparse flows that are latency senstive should be marked as NQB (thus DSCP-45). This may be things 
        like DNS queries or VoIP media flows, where maximizing the bandwidth of the flow is not necesary.</t>
        
        <t>Latency-sensitive flows that need more bandwidth are congestion controlled, and identified via ECN marking. 
        These types of applications are less limited by the application protocol itself (i.e., a small DNS query), 
        which means the application quality can improve as more bandwidth is available - such as shifting a 
        video stream or a video conference session from Standard Definition (SD) to 4K quality.</t>
        </section>
        </section>

        <!--  END MAJOR SECTION --> 
        <!--  START NEW MAJOR SECTION --> 

	    <section title="Recommended Deployment Decisions for ISPs">
		    
		    <t>Like any network or system, a good deployment design and configuration matters and 
		    can be the difference between a well-functioning and accepted system and one that experiences 
		    problems and faces user and/or developer opposition. In the context of deploying low latency networking in an ISP network, 
		    this document describes some recommendations that should help 
		    to ensure a deployment is resilient, well-accepted, and creates the environment for generating 
		    strong network effects. Following these recommendations should help avoid barriers to adoption and help provide a strong foundation 
			for growth.</t>

            <t>The first sections below will look at the end-to-end network path from when packets come into an 
            ISP's network (provider edge), then what happens within the ISP's network core, how packets are delivered 
            to customner premise equiment (CPE), and finally what may happen with the Local Area Network (LAN) in a 
            home.</t>

            <t>This section will then conclude with key deployment decisions, which have both operational as well as 
            technology policy implications.</t> 

        <section title="Application-Originated Marking vs. Middleboxes">
        <t>As noted in <xref target="Tussle"/> there has always been a tension in the end-to-end model between how 
        much intelligence and processing takes place along the end-to-end path inside of a network and how much 
        takes place at the end of the network in servers and/or end user client devices and software. In this new 
        approach to low latency networking, entry into a low latency queue depends upon marks in the packet header 
        of a particular application flow. In practice, this marking is best left to the application edge of the network, 
        rather than it being a funcyion of a so-called middlebox. As explored below, this is the most efficient, 
        least prone to error or mis-classification, and is most acceptable from the standpoint of network neutrality 
        regulation.</t>
    
	    <section title="Applications Mark Traffic, Not Middleboxes">
		    
		    <t>The best approach is for applications to mark traffic to indicate their preference for the low latency queue, 
			not the network making such a decision on its own. This is for several reasons:</t>
		    <ul>
		    <li>According to the end-to-end principle, this function is best delegated to the edge of 
		    the network as an architectural best practice (the edge being the application in this case).</li>
		    <li>Application marking maintains the loose coupling between the application and network layers, 
		    eliminating the need for close coordination between networks and application developers.</li>
		    <li>Application developers know best whether their application is compatible with low 
			    latency networking and which aspects of their traffic flows will or will not benefit.</li>
			<li>Only the application (not the network) knows whether a scalable congestion control algorithm congestion control is being 
			used on the application server. Thus, only the developer and server administrator know if they are correctly responding to 
			Congestion Experienced (CE) markings for L4S (see Section 4.1 of <xref target="RFC9331"/>). </li>
		    <li>Application traffic is almost entirely encrypted, which makes it very difficult for networks 
		    to accurately determine application protocols and to further infer which flows will benefit from low latency 
		    and which flows may be harmed because they need to build a queue. It is likely that false positives <xref target="Lotus"/> and false negatives 
			will occur if network-based 
			inference is used; all of which can be avoided simply by relying solely on application marking.</li>
		    <li>The pace of innovation and iteration is necessarily faster-moving in the application edge at layer 7, 
		    rather than in the network at layer 3 (and below) - where there is greater standards stability and a lower rate of 
		    major changes. As a result, the application layer is best suited to rapid experimentation and iteration. Network 
		    operators and equipment vendors trying to infer application needs will in comparison always be in a reactive 
		    mode, one step behind changes made in applications.</li>

<!-- ADD SOMETHING ON ERRORS - A LA LOTUS -->


		    </ul>
		    
	    </section>
	    
	    
	    <section title="Any Application Provider Can Mark Traffic - No Advance Permission"> 
		    
		    <t>Any application provider should be able to mark their traffic for the low latency queue, with no restrictions 
		    other than standards compliance or other reasonable and openly documented technical guidelines. This 
		    maintains the loose cross-layer coupling that is a key tenet of the Internet's architecture by eliminating 
		   the need for application providers and networks to coordinate and creates an environment of so-called "permissionless innovation".</t> 
		               
	    </section>

            </section>

        <section title="Interconnection Points (Provider Edge)">
        
        <section title="Allow ECN Across Domain (Peer) Boundaries">
        <t>Traffic sent TO a peer network marked with ECT(1) or CE in the ECN header MUST pass to that peer 
        without altering or removing the ECT(1) or CE marking (see exception below). Traffic FROM peers marked 
        with ECT(1) or CE in the ECN header MUST be allowed to enter the network without altering or removing 
        the ECT(1) or CE marking (see exception below). The only exception would be when a network element is 
        CE-aware and able to add a CE mark to signal that it is experiencing congestion at that hop.</t>
        
        <t>This part - allowing unmodified ECN across the network - is likely to be easier than DSCP-45 for NQB (see next section), since it 
        appears rare that networks modify the ECN header of packet flows.</t>
        </section>
        

        <section title="Allow DSCP-45 Across Domain (Peer) Boundaries">
        <t>Traffic sent TO a peer network marked with DSCP value 45 MUST pass to that peer without altering or removing the DSCP 45 marking (see exception below).
Traffic FROM peers marked with DSCP value 45 MUST be allowed to enter the network without altering or removing the DSCP 45 marking (see exception below).
Peer marking exception: Some peer networks may use DSCP 45 for purposes other than NQB LL within their network. In these cases, the peer using DSCP 45 for other purposes is responsible for remarking on ingress and egress from their network.
In all cases, traffic marked DSCP 45 must still be handled as best efforts - not a higher priority than other Internet traffic.</t>
        </section>

        </section>

        <section title="Inside the ISP Network (Core Network)">
        
        <t>Within an ISP network, packets will typically traverse edge routers, backbone routers, and regional/local routers before 
        being send into a last-mile access network to which end users are connected. In the case of ECN, there are unlikely to be 
        any issues in these hops, as ECN appears to pass end-to-end across most networks. Nevertheless, it is important to check 
        network policies and router configurations to confirm this and to validate it via packet captures at an end user CPE.</t>

        <t>Ensuring support for NQB via DSCP-45 may be more challenging, as network operators typically have unique uses of various 
        DSCP marks within their network, such as to differentiate residential internet traffic from commercial internet, transit, 
        voice services, management traffic, and so on. If DSCP-45 is already used, then a network policy will need to be created and 
        deployed that will transform ingress packets at the peer edge marked as DSCP-45 to some internal-use value, then back again to DSCP-45 
        when packets hit the end user CPE. Then the reverse must be done for egress packets, changing DSCP-45 marks from CPE to the 
        internal-use value and then at the peering edge back to DSCP-45. There will also need to be a policy to transform DSCP 
        marks from the internal-use value back to DSCP-45 at the CPE for cases when a flow is peer-to-peer, meaning directly 
        between two end users on a given ISP network.</t>

            <section title="Avoid Internal Remarking of DSCP Values if Possible">
				<t>If possible, for operational simplicity, a network should try to maintain the use of DSCP 45 on an end-to-end basis 
				without remarking in their interior network hops. This may not be possible in all networks, because some may already use DSCP 45 for some private 
				internal reason. In such cases, packets must obviously be remarked to and from DSCP 45 at the customer edge (CPE) and network ingress/egress to other 
				networks. But if DSCP 45 is not used internally, it is far simpler for network operations and troubleshooting to preserve that mark on an end 
				to end basis.</t>
			</section>	

        </section>

        <section title="Last Mile Network (Access Network)">
        
        <t>There are two hops of interest in the last mile access network. One will be a point of user aggregation, such as a Cable 
        Modem Termination System (CMTS) or Optical Line Terminal (OLT). The second is at the user location, such as a Cable Modem (CM) 
        or Optical Network Unit (ONU), both of which are example of CPE.</t>
        
        <section title="Consider Queue Protection">
				<t>The specifications in <xref target="I-D.ietf-tsvwg-nqb"/> describe a concept of Traffic Protection, also known as a 
                Queue Protection Function <xref target="I-D.briscoe-docsis-q-protection"/> or simply Queue Protection. The document says that Traffic Protection 
                is optional and may not be needed in certain networks. In the case of an ISP deploying low latency networking with 
                two queues, an ISP should consider deploying such a network function to at least detect mismarking (if not necessarily 
                to correct mismarking). This may be implemented, for example, in end user CPE, last mile network equipment, 
                and/or elsewhere in the ISP network - or closely monitors network statistics and user feedback for any indication of 
                widespread low latency packet mismarking by applications.</t>

                <t>Queue Protection can be implemented any place that there are two queues for low latency networking. For example, for 
                downstream traffic, this would be in the aggregation device (i.e., CMTS, OLT) and in the upstream direction in 
                the CPE (i.e., CM or ONU).</t>
			</section>	
        </section>

        <section title="Customer Premise Equipment (Customer Edge)">

        <t>In most residential internet services, there are typically two equipment modes. One is very simple CPE that hands off from 
        the ISP's access network (i.e., DSL, 5G, DOCSIS, PON) and provides the customer with an Ethernet interface and IP address(es). 
        The customer then connects their own router and wireless access point (often integrated into the router, typically referred 
        to as a "wireless gateway" or "wireless router"). The other model is more typical, which is that the CPE integrates a link 
        layer termination function (i.e., Cable Modem, 5G radio, or Optical Network Unit) as well as a wireless gateway.</t>

        <t>Not all ISP networks support both of these models; sometimes only a wireless gateway is available. Even in this case, some 
        users "double NAT" and install their own router and wireless access point(s) to get whatever functionality and control over their 
        home network that they desire. The cases explored below are commonplace but may not apply to all networks.</t>
        
            <section title="Support a Variety of End User Equipment"> 
		    
		   <t>To create the best foundation for growth, both customer-owned and ISP-administered Customer Premises Equipment (CPE) should be supported 
		   (assuming that is typical and technically feasible in a particular type of access network), when applicable. In mobile networks, where a user 
		   device connects directly to the network, this may be easier as it simply depends upon operating system software in that end user device. In 
		   fixed networks, there is usually CPE that demarcates between the ISP network and the user's home LAN. The software running on those CPE devices 
		   will also need to support low latency networking, as well as software in other ISP network devices where CPE connections are aggregated. The more 
		   CPE devices that are enabled, the greater the pool of potential users, and thus the broader adoption would be, positively 
           driving network effects.</t>
		   
	    </section>

            <section title="Pass Markings to Customer-Administered CPE">
				<t>In some cases, dual queue networking and associated packet marking is supported up to the ISP's demarcation point - 
                such as in a cable modem. It is recommended that packet markings should pass from such a demarcation point to 
                any attached customer-administered CPE, such as a router or wireless access point. That enables a 
				customer-administered router to implement dual queue networking, rather that it only being possible with 
				ISP-administered CPE.</t>
			</section>	

        </section>
        
        <section title="Inside the Home (Customer Local Area Network)">

        <t>As noted above with the mention of an integrated wireless gateway, and is most common, the CPE and router/wireless network 
        gear may be integrated into a single CPE device. Even though these are functionally in one piece of hardware, we can 
        think of the wide area network interface and local area network as functionally separate for purposes of this analysis.</t>

        <section title="In Home Wi-Fi LAN Configuration">
				<t>As noted above with respect to prioritization of packets in the ISP network, all packets should be 
				handled with the same best effort priority in the ISP access network and on the internet. However, in a user's home 
				Wi-Fi (wireless) local area network (WLAN), this is more complicated, 
                because there is not a precise mapping between IETF packet marking and IEEE 802.11 marking,
				 explored in 
				<xref target="RFC8325"/>. In short, today's 802.11 specifications enable a Wi-Fi network to have multiple queues, 
				using different "User Priority" and "Access 
				Category" values. At the current time, these queues are AC_BK (Background), AC_BE (Best Effort), 
				AC_VI (Video), and AC_VO (Voice).</t>

				<t>As explored in <xref target="I-D.ietf-tsvwg-nqb"/>, packets in the low latency queue 
				may be expected to be marked for the best effort (AC_BE) or video (AC_VI) wireless queue. For additional context, 
				please refer to Section 8.1 of  <xref target="I-D.ietf-tsvwg-nqb"/>. In some situations, such as a 
				user-owned wireless access point or CPE, it may not be possible for the user to select which wireless 
				queue is used. In cases where the CPE is ISP-administered, selecting a specific wireless queue may be 
				possible - though it is not yet clear what the best practice may be for this selection until ISPs and 
				application developers have more experience with low latency networking. As of the writing of this document, 
                it appears that the AC_VI queue may be used for the low latency queue in some networks - and that many 
                latency-sensitive applications are already marking their WLAN traffic for AC_VI and AC_VO.</t>
			</section>	

			<section title="Use Permissive Upstream NQB Queue Admission">
				<t>Since the IETF's NQB specification is only recently completed, many applications that have 
				been using other DSCP marks for their latency-sensitive flows have not yet shifted to adopt DSCP-45. 
				One example is the Microsoft Xbox platform <xref target="Microsoft"/>, which is using DSCP-46. 
				So in the relatively short-term, ISPs may find it beneficial to their customers to 
				use a more permissive upstream NQB admission policy, allowing DSCP-40, 45, 46, and 56 admission 
				into the low latency queue. It may take a year or more after the NQB DSCP assignment is made by IANA for  
				developers to shift to DSCP-45, given other items in their development backlog and their 
				software release schedule.</t>

			</section>	

        </section>

        <!--  END MAJOR SECTION --> 
        <!--  START NEW MAJOR SECTION --> 
	  
	 </section>
	 
	    <section title="Acknowledgements">
			<t>Thanks to Bob Briscoe, Mat Ford, Vidhi Goel, Eliot Lear, Sebastian Moeller, Sebnem Ozer, Jim Rampley, Dan Rice, Greg Skinner, 
            Greg White, and Yiannis Yiakoumis for their review and feedback on this document.</t>
		</section>
	
	    <section title="IANA Considerations">
		  <t>RFC Editor: Please remove this section before
			 publication.</t>
	      <t>This memo includes no requests to or actions for IANA.</t>
	    </section>
	
	    <section title="Security Considerations">
		  <t>The key security consideration pertains to Queue Protection. As the current time, it is recommended that 
		  implementers utilize Queue Protection, to ensure that any traffic that is incorrectly marked for low latency 
		  can be detected and remarked for the classic queue. The necessity of Queue Protection remains something of 
		  a debate, with some firmly believing it is necessary but others believing that it is not needed. The latter 
		  view is that application developers have a natural incentive to correctly mark their traffic, because to 
		  do otherwise would worsen the quality of experience (QoE) for their users. In that line of thinking, if a 
		  developer mismarks, they and/or their users will notice and they will fix that error. However, it is 
		  also conceivable that malicious software could be operating on a user's device or home network and that 
		  malicious software could try to send some much traffic to the low latency queue that the queue or both 
		  queues become unusable. This is quite similar to other "traditional" denial of servce (DoS) attacks, so it 
		  does not necessarily seems unique to low latency networking. But due to the possibility of this occuring, and 
		  low latency networking being such a new approach, it seems prudent to implement Queue Protection.</t>
	    </section>

		<section title="Regulatory Considerations">
		    <t>Network Neutrality (a.k.a. Net Neutrality) can mean a variety 
		    of things within a country, as well as between different countries, based on 
		    different opinions, market structures, business practices, laws, and regulations. Generally speaking, 
		    In the context of the United States' market, it has come to mean that Internet Service Providers (ISPs) 
			should not block, throttle, or deprioritize lawful application traffic, and should not engage in paid 
			prioritization, among other things. Net Neutrality concerns can sometimes affect the deployment 
			of new technologies by ISPs, so they should carefully consider regulatory issues when 
			making deployment decisions.</t>

			<t>As it is envisioned in the design of the IETF's new low latency networking protocols, the addition 
			of a low latency packet processing queue at a network link is merely a second 
		    packet queue and does not mean that this queue is hierarchically prioritized or that it has more 
		    capacity. As a result, low latency networking appears to pose no new Net Neutrality issues. However, 
			it is important for ISPs to keep these risks in mind as they make deployment design decisions.</t>

			<t>One key aspect of low latencty networking is that it operates, from the perspective of an ISP's 
			deployment, is application-agnostic. The ISP creates a second network queue on key network links, but 
			does not decide on their own what applications can use this queue. Rather, they add the queue and 
			packet flows are sent to that queue based on packet marking set by application developers. This 
			approach is far superior to older approaches, which caused significant Net Neutrality risks
			<xref target="Lotus"/>, 
			that used middleboxes to attempt to infer applications based on observing packet flows on ISP 
			network links.</t>

			</section>
	    
	    <section title="Revision History">
		  <t>RFC Editor: Please remove this section before
			 publication.</t>
	      <t>v00: First draft</t>
	      <t>v01: Incorporate comments from 1st version after IETF-115</t>
		  <t>v02: Incorporate feedback from the TSVWG mailing list</t>
		  <t>v03: Final feedback from TSVWG and prep for sending to ISE</t>
		  <t>v04: Refresh expiration before major revision</t>
          <t>v05: Changes from Greg Skinner and Eliot Lear</t>
          <t>v06: More changes from Eliot Lear</t>
          <t>v07: More changes from Eliot Lear</t>

	    </section>
	    
	    <section title="Open Issues">
		  <t>RFC Editor: Please remove this section before
			 publication.</t>
	      <t>- Open issues are being tracked in a GitHub repository for this document 
		      at https://github.com/jlivingood/IETF-L4S-Deployment/issues</t>
	    </section>
	    
	    
	  </middle>
	
	  <!--  *****BACK MATTER ***** -->
	
	  <back>
	    <!-- References split to informative and normative -->
	
	   <references title="Informative References">
			<xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.8325.xml"/>
			<xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.9330.xml"/>
			<xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.9331.xml"/>
			<xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.9332.xml"/>
		   <xi:include href="https://bib.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-tsvwg-l4sops.xml"/>
		   <xi:include href="https://bib.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-tsvwg-nqb.xml"/>
		   <xi:include href="https://bib.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-tsvwg-dscp-considerations.xml"/>
		   <xi:include href="https://bib.ietf.org/public/rfc/bibxml3/reference.I-D.briscoe-docsis-q-protection.xml"/>
		   
		   <reference anchor="BITAG" target="https://bitag.org/documents/BITAG_latency_explained.pdf">
			   <front>
	            <title>Latency Explained</title>
	            <author>
	              <organization>Broadband Internet Technical Advisory Group</organization>
	            </author>
	            <date year="2022" month="January" day="10"/>
			   </front>
		   </reference>
		   
		   <reference anchor="Lotus" target="https://www.eff.org/wp/packet-forgery-isps-report-comcast-affair">
			   <front>
	            <title>Packet Forgery By ISPs: A Report on the Comcast Affair</title>
	            <author fullname="Peter Eckersley" initials="P" surname="Eckerseley">
	              <organization>Electronic Frontier Foundation</organization>
	            </author>
                <author fullname="Fred von Lohmann" initials="F" surname="von Lohmann">
	              <organization>Electronic Frontier Foundation</organization>
	            </author>
                <author fullname="Seth Schoen" initials="S" surname="Schoen">
	              <organization>Electronic Frontier Foundation</organization>
	            </author>
	            <date year="2007" month="November" day="28"/>
			   </front>
		   </reference>
		   
		   <reference anchor="IETF-114-Slides" target="https://datatracker.ietf.org/meeting/114/materials/slides-114-tsvwg-update-on-l4s-work-in-ietf-114-hackathon-00.pdf">
			   <front>
	            <title>First L4S Interop Event @ IETF Hackathon</title>
	            <author fullname="Greg White" initials="G" surname="White">
	              <organization>CableLabs</organization>
	            </author>
	            <date year="2022" month="July" day="25"/>
			   </front>
			   
			</reference>
			
			<reference anchor="LLD" target="https://cablela.bs/low-latency-docsis-technology-overview-february-2019">
				<front>
					<title>Low Latency DOCSIS: Technology Overview</title>
					<author fullname="Greg White" initials="G" surname="White">
						<organization>CableLabs</organization>
						</author>
					<author fullname="Karthik Sundaresan" initials="K" surname="Sundaresan">
						<organization>CableLabs</organization>
						</author>
							<author fullname="Bob Briscoe" initials="B" surname="Briscoe">
					</author>
					<date year="2019" month="February"/>
				</front>
				
				</reference>

				<reference anchor="Ericsson" target="https://www.ericsson.com/49bc82/assets/local/reports-papers/white-papers/26052021-enabling-time-critical-applications-over-5g-with-rate-adaptation-whitepaper.pdf">
							<front>
							<title>Enabling time-critical applications over 5G with rate adaptation</title>
					<author fullname="Per Willars" initials="P" surname="Willars"><organization>Ericsson</organization></author>
					<author fullname="Emma Wittenmark" initials="E" surname="Wittenmark"><organization>Ericsson</organization></author>
					<author fullname="Henrik Ronkainen" initials="H" surname="Ronkainen"><organization>Business Area Networks</organization></author>
					<author fullname="Ingemar Johansson" initials="I" surname="Johansson"><organization>Ericsson</organization></author>
					<author fullname="Johan Strand" initials="J" surname="Strand"><organization>Business Area Networks</organization></author>
					<author fullname="Petr Ledl" initials="D" surname="Ledl"><organization>Deutsche Telekom</organization></author>
					<author fullname="Dominik Schnieders" initials="D" surname="Schnieders"><organization>Deutsche Telekom</organization></author>
	
	
							<date year="2021" month="May"/>
						</front>
							
						</reference>

			<reference anchor="CTI" target="https://www.itu.int/rec/T-REC-G.Sup71-202104-I">
							<front>
							<title>Optical line termination capabilities for supporting cooperative dynamic bandwidth assignment</title>
							<author><organization>International Telecommunications Union - Telecommunication Standardization Sector (ITU-T)</organization></author>
			
			        <date year="2021" month="April"/>
		           </front>
			           <seriesInfo name="Series G: Transmission Systems and Media, Digital Systems and Networks" value="Supplement 71" />
						</reference>

			<reference anchor="IEEE" target="https://ieeexplore.ieee.org/document/9363693">
							<front>
							<title>Part 11: Wireless LAN Medium Access Control (MAC) and Physical Layer (PHY) Specifications</title>
							<author><organization>IEEE Computer Society (IEEE)</organization></author>
			        <date year="2021" month="February" day="26"/>
		           </front>
				   		<seriesInfo name="DOI" value="10.1109/IEEESTD.2021.9363693"/>
			           <seriesInfo name="IEEE Standard for Information Technology--Telecommunications and Information Exchange between Systems - Local and Metropolitan Area Networks--Specific Requirements" value="802.11-2020" />
						</reference>

			<reference anchor="Microsoft" target="https://learn.microsoft.com/en-us/gaming/gdk/_content/gc/networking/overviews/qos-packet-tagging">
			   <front>
	            <title>Quality of service (QoS) packet tagging on Xbox consoles</title>
	            <author>
	              <organization>Microsoft</organization>
	            </author>
	            <date year="2022" month="August" day="19"/>
			   </front>
			</reference>

			<reference anchor="Comcast" target="https://corporate.comcast.com/stories/comcast-kicks-off-industrys-first-low-latency-docsis-field-trials">
			   <front>
	            <title>Comcast Kicks Off Industry's First Low Latency DOCSIS Field Trials</title>
	            <author>
	              <organization>Comcast</organization>
	            </author>
	            <date year="2023" month="June" day="16"/>
			   </front>
			</reference>


			<reference anchor="IETF-TSVWG-120" target="https://datatracker.ietf.org/doc/slides-120-tsvwg-52-comcasts-l4s-nqb-field-trials/">
			   <front>
	            <title>TSVWG Meeting at IETF-120</title>
				  <author fullname="Jason Livingood" initials="J" surname="Livingood"><organization>Comcast</organization>
				  </author>
	            <date year="2024" month="July" day="26"/>
			   </front>
			</reference>

			<reference anchor="IETF-TSVWG-119" target="https://datatracker.ietf.org/doc/slides-119-tsvwg-sessa-41-comcasts-l4s-nqb-field-trials/">
			   <front>
	            <title>TSVWG Meeting at IETF-119</title>
				  <author fullname="Jason Livingood" initials="J" surname="Livingood"><organization>Comcast</organization>
				  </author>
	            <date year="2024" month="March" day="18"/>
			   </front>
			</reference>

			<reference anchor="IETF-TSVWG-118" target="https://datatracker.ietf.org/doc/slides-118-tsvwg-sessa-61-l4s-experience/">
			   <front>
	            <title>TSVWG Meeting at IETF-118</title>
				  <author fullname="Jason Livingood" initials="J" surname="Livingood"><organization>Comcast</organization>
				  </author>
	            <date year="2023" month="November" day="18"/>
			   </front>
			</reference>

			<reference anchor="IETF-TSVWG-117" target="https://datatracker.ietf.org/doc/slides-118-tsvwg-sessa-61-l4s-experience/">
			   <front>
	            <title>TSVWG Meeting at IETF-117</title>
				  <author fullname="Jason Livingood" initials="J" surname="Livingood"><organization>Comcast</organization>
				  </author>
	            <date year="2023" month="July" day="27"/>
			   </front>
			</reference>

			<reference anchor="CDT-NN" target="https://arxiv.org/pdf/2308.05829">
			   <front>
	            <title>Slicing the Network: Maintaining Neutrality, Protecting Privacy, and Promoting Competition. 
				A technical and policy overview with recommendations for operators and regulators.</title>
				  <author fullname="Nick Doty" initials="N" surname="Doty"><organization>Center for Democracy and Technology</organization>
				  </author>
				   <author fullname="Mallory Knodel" initials="M" surname="Knodel"><organization>Center for Democracy and Technology</organization>
				  </author>
	            <date year="2023" month="April"/>
			   </front>
			</reference>

			<reference anchor="van-Schewick-1A" target="https://www.fcc.gov/ecfs/document/103120890811342/1">
			   <front>
	            <title>FCC Ex Parte In the matter of Safeguarding and Securing the Open Internet, 
				WC Docket No. 23-320</title>
				  <author fullname="Barbara van Schewick" initials="B" surname="van Schewick">
				  <organization>Center for Internet and Society, Stanford Law School</organization>
				  </author>
				  <author fullname="Scott Jordan" initials="S" surname="Jordan">
				  <organization>University of California at Irvine</organization>
				  </author>
				  <author>
				  <organization>Open Technology Institute at New America</organization>
				  </author>
				  <author>
				  <organization>Public Knowledge</organization>
				  </author>
	            <date year="2024" month="March" day="20"/>
			   </front>
			</reference>
			
			<reference anchor="van-Schewick-1B" target="https://www.fcc.gov/ecfs/document/10323701322790/2">
			   <front>
	            <title>Net Neutrality &amp; Non-BIAS Data Services</title>
				  <author fullname="Barbara van Schewick" initials="B" surname="van Schewick">
				  <organization>Center for Internet and Society, Stanford Law School</organization>
				  </author>
				  <author fullname="Scott Jordan" initials="S" surname="Jordan">
				  <organization>University of California at Irvine</organization>
				  </author>
				  <author>
				  <organization>Open Technology Institute at New America</organization>
				  </author>
				  <author>
				  <organization>Public Knowledge</organization>
				  </author>
	            <date year="2024" month="March" day="20"/>
			   </front>
			</reference>


			<reference anchor="van-Schewick-2" target="https://law.stanford.edu/wp-content/uploads/2024/08/van-Schewick-2024-5G-Network-Slicing-and-Net-Neutrality-Shetler-Steffen1.pdf">
			   <front>
	            <title>Net Neutrality &amp; 5G Network Slicing</title>
				  <author fullname="Barbara van Schewick" initials="B" surname="van Schewick">
				  <organization>Center for Internet and Society, Stanford Law School</organization>
				  </author>
	            <date year="2024" month="April" day="3"/>
			   </front>
			</reference>
			
			<reference anchor="van-Schewick-3" target="https://law.stanford.edu/wp-content/uploads/2024/08/van-Schewick-2024-5G-Network-Slicing-and-No-Throttling-Rule-20240418.pdf">
			   <front>
	            <title>Network Slicing and Net Neutrality: No Throttling Rule</title>
				  <author fullname="Barbara van Schewick" initials="B" surname="van Schewick">
				  <organization>Center for Internet and Society, Stanford Law School</organization>
				  </author>
	            <date year="2024" month="April" day="18"/>
			   </front>
			</reference>

            <reference anchor="Apple" target="https://developer.apple.com/documentation/network/testing-and-debugging-l4s-in-your-app">
			   <front>
	            <title>Testing and Debugging L4S in Your App</title>
				  <author>
				  <organization>Apple</organization>
				  </author>
			   </front>
			</reference>

			
			<reference anchor="Tussle" target="https://dl.acm.org/doi/10.1145/633025.633059">
			   <front>
	            <title>Tussle in Cyberspace: Defining Tomorrow's Internets</title>
				  <author fullname="David D. Clark" initials="D" surname="Clark">
				  <organization>MIT Lab for Computer Science</organization>
				  </author>
				  <author fullname="John Wroclawski" initials="J" surname="Wroclawski">
				  <organization>MIT Lab for Computer Science</organization>
				  </author>
                  <author fullname="Karen R. Sollins" initials="K" surname="Sollins">
				  <organization>MIT Lab for Computer Science</organization>
				  </author>
                  <author fullname="Robert Braden" initials="R" surname="Braden">
				  <organization>USC Information Sciences Institute</organization>
				  </author>
	            <date year="2002" month="August" day="19"/>
			   </front>
			</reference>
			
		   
	   </references>
	
	  </back>
	</rfc>