<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE rfcSYSTEM "rfc2629.dtd"[<!-- A set of on-line citation libraries are maintained on the xml2rfc web site. The next line defines an entity named RFC2629, which contains the necessary XML for the reference element, and is used much later in the file. This XML contains an anchor (also RFC2629) which can be used to cross-reference this item in the text. You can also use local file names instead of a URI. The environment variable XML_LIBRARY provides a search path of directories to look at to locate a relative path name for the file. There has to be one entity for each item to be referenced. --><!ENTITYRFC2234 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2234.xml"> <!ENTITY RFC2629 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.2629.xml">nbsp " "> <!ENTITYRFC4234 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.4234.xml">zwsp "​"> <!ENTITYRFC5575 SYSTEM "http://xml.resource.org/public/rfc/bibxml/reference.RFC.5575.xml"> <!-- There is also a library of current Internet Draft citations. It isn't a good idea to actually use one for the template because it might have disappeared when you come to test this template. This is the form of the entity definition <!ENTITY I-D.mrose-writing-rfcs SYSTEM "http://xml.resource.org/public/rfc/bibxml3/reference.I-D.mrose-writing-rfcs.xml"> corresponding to a draft filename draft-mrose-writing-rfcs-nn.txt. The citation will be to the most recent draft in the sequence, and is updated roughly hourly on the web site. For working group drafts, the same principle applies: file name starts draft-ietf-wgname-.. and entity file is reference.I-D.ietf-wgname-... The corresponding entity name is I-D.ietf-wgname-... (I-D.mrose-writing-rfcs for the other example). Of course this doesn't change when the draft version changes. --> <!-- Fudge for XMLmind which doesn't have this built in -->nbhy "‑"> <!ENTITYnbsp " ">wj "⁠"> ]><!-- Extra statement used by XSLT processors to control the output style. --> <?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?> <!-- Processing Instructions can be placed here but if you are editing with XMLmind (and maybe other XML editors) they are better placed after the rfc element start tag as shown below. --> <!-- Information about the document. category values: std, bcp, info, exp, and historic For Internet-Drafts, specify attribute "ipr". (ipr values are: full3667, noModification3667, noDerivatives3667), Also for Internet-Drafts, can specify values for attributes "docName" and, if relevant, "iprExtract". Note that the value for iprExtract is the anchor attribute value of a section (such as a MIB specification) that can be extracted for separate publication, and is only useful whenhe value of "ipr" is not "full3667". --> <!-- TODO: verify which attributes are specified only by the RFC editor. It appears that attributes "number", "obsoletes", "updates", and "seriesNo" are specified by the RFC editor (and not by the document author). --><rfc xmlns:xi="http://www.w3.org/2001/XInclude" submissionType="IETF" category="std"ipr="trust200902"consensus="true" docName="draft-ietf-ippm-rfc8889bis-04" number="9342" ipr="trust200902" obsoletes="8889"> <!-- Processing Instructions- PIs (for a complete list and description, see file http://xml.resource.org/authoring/README.html and below... --> <!-- Some of the more generally applicable PIs that most I-Ds might want to use --> <!-- Try to enforce the ID-nits conventions and DTD validity --> <?rfc strict="yes" ?> <!-- Items used when reviewing the document --> <?rfc comments="no" ?> <!-- Controls display of <cref> elements --> <?rfc inline="no" ?> <!-- When no, put comments at end in comments section, otherwise, put inline --> <?rfc editing="no" ?> <!-- When yes, insert editing marks: editing marks consist of a string such as <29> printed in the blank line at the beginning of each paragraph of text. --> <!-- Create Table of Contents (ToC) and set some options for it. Note the ToC may be omitted for very short documents,but idnits insists on a ToC if the document has more than 15 pages. --> <?rfc toc="yes"?> <?rfc tocompact="yes"?> <!-- If "yes" eliminates blank lines before main section entries. --> <?rfc tocdepth="3"?> <!-- Sets the number of levels of sections/subsections... in ToC --> <!-- Choose the options for the references. Some like symbolic tags in the references (and citations) and others prefer numbers. The RFC Editor always uses symbolic tags. The tags used are the anchor attributes of the references. --> <?rfc symrefs="yes"?> <?rfc sortrefs="yes" ?> <!-- If "yes", causes the references to be sorted in order of tags. This doesn't have any effect unless symrefs is "yes" also. --> <!-- These two save paper: Just setting compact to "yes" makes savings by not starting each main section on a new page but does not omit the blank lines between list items. If subcompact is also "yes" the blank lines between list items are also omitted. --> <?rfc compact="yes" ?> <?rfc subcompact="no" ?> <!-- end of list of popular I-D processing instructions --> <!-- ***** FRONT MATTER ***** -->updates="" xml:lang="en" tocInclude="true" tocDepth="3" symRefs="true" sortRefs="true" version="3"> <front><!-- The abbreviated title is used in the page header - it is only necessary if the full title is longer than 42 characters --><titleabbrev="ClustAltMark">Clusteredabbrev="Clustered Alternate-Marking Method">Clustered Alternate-Marking Method</title><!-- add 'role="editor"' below for the editors if appropriate --><seriesInfo name="RFC" value="9342"/> <author role="editor" fullname="Giuseppe Fioccola" initials="G." surname="Fioccola"> <organization>Huawei Technologies</organization> <address> <postal> <street>Riesstrasse, 25</street> <city>Munich</city> <code>80992</code> <country>Germany</country> </postal> <email>giuseppe.fioccola@huawei.com</email> </address> </author> <author fullname="Mauro Cociglio" initials="M." surname="Cociglio"> <organization>Telecom Italia</organization> <address> <postal><street></street> <city></city> <code></code> <country></country><street/> <city/> <code/> <country/> </postal> <email>mauro.cociglio@outlook.com</email> </address> </author> <author fullname="Amedeo Sapio" initials="A." surname="Sapio"> <organization>Intel Corporation</organization> <address> <postal> <street>4750 Patrick Henry Dr.</street> <city>Santa Clara</city> <region>CA</region> <code>95054</code><country>USA</country><country>United States of America</country> </postal> <email>amedeo.sapio@intel.com</email> </address> </author> <author fullname="Riccardo Sisto" initials="R." surname="Sisto"> <organization>Politecnico di Torino</organization> <address> <postal> <street>Corso Duca degli Abruzzi, 24</street> <city>Torino</city> <code>10129</code> <country>Italy</country> </postal> <email>riccardo.sisto@polito.it</email> </address> </author> <author fullname="Tianran Zhou" initials="T." surname="Zhou"> <organization>Huawei Technologies</organization> <address> <postal> <street>156 Beiqing Rd.</street> <city>Beijing</city> <code>100095</code> <region/> <country>China</country> </postal> <email>zhoutianran@huawei.com</email> </address> </author> <dateyear="2022"/> <!-- month="March" is no longer necessary note also, day="30" is optional --> <!-- WARNING: If the month and year are the current ones, xml2rfc will fill in the day for you. If only the year is specified, xml2rfc will fill in the current day and month irrespective of the day. This silliness should be fixed in v1.31. --> <!-- Meta-data Declarations --> <!-- Notice the use of & as an escape for & which would otherwise start an entity declaration, whereas we want a literal &. --> <area></area> <!-- WG name at the upperleft corner of the doc, IETF fine for individual submissions. You can also omit this element in which case in defaults to "Network Working Group" - a hangover from the ancient history of the IETF! --> <workgroup></workgroup> <!-- The DTD allows multiple area and workgroup elements but only the first one has any effect on output. --> <!-- You can add <keyword/> elements here. They will be incorporated into HTML output files in a meta tag but they have no effect on text or nroff output. -->year="2022" month="December" /> <area>tsv</area> <workgroup>ippm</workgroup> <keyword>Multipoint</keyword> <keyword>Cluster</keyword> <keyword>Performance</keyword> <keyword>Measurement</keyword> <keyword>Monitoring</keyword> <keyword>Passive</keyword> <keyword>Hybrid</keyword> <keyword>Loss</keyword> <keyword>Delay</keyword> <keyword>Delay Variation</keyword> <keyword>Hashing</keyword> <keyword>Closed-Loop</keyword> <abstract> <t>This document generalizes and expands the Alternate-Marking methodology to measure any kind of unicast flow whose packets can follow several different paths in thenetwork thatnetwork; this can result in a multipoint-to-multipoint network. The network clustering approach is presented and, for this reason, the techniqueheredescribed here is called "ClusteredAlternate-Marking".Alternate Marking". This document obsoletes RFC 8889.</t> </abstract> </front> <middle> <sectiontitle="Introduction">numbered="true" toc="default"> <name>Introduction</name> <t>The Alternate-Marking Method, as described in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, is applicable to a point-to-point path. The extension proposed in this document applies to the most general case of a multipoint-to-multipoint path and enables flexible and adaptive performance measurements in a managed network.</t> <t>The Alternate-Marking methodology consistsinof splitting the packet flow into markingblocksblocks, and the monitoring parameters are the packet counters and the timestamps for each marking period. In some applications of the Alternate-Markingmethod,Method, a lot of flows and nodes are to be monitored. MultipointAlternate-MarkingAlternate Marking aims to reduce these values and makes the performance monitoring more flexible in case a detailed analysis is not needed. For instance, by considering n measurement points and m monitored flows, the order of magnitude of the packet counters for each time interval is n*m*2 (1 per color). The number of measurement points and monitored flows may vary and depends on the portion of the network we are monitoring (core network, metro network, accessnetwork)network, etc.) and the granularity (for each service, eachcustomer).customer, etc.). So if both n and m are high values, the packet counters increase a lot, and MultipointAlternate-MarkingAlternate Marking offers a tool to control these parameters.</t> <t>The approach presented in this document is applied only to unicast flows and not to multicast. Broadcast, Unknown Unicast, and Multicast (BUM) traffic is not considered here, because traffic replication is not covered by the Multipoint Alternate-Markingmethod.Method. Furthermore, it can be applicable to anycast flows, and Equal-Cost Multipath (ECMP) paths can also be easily monitored with this technique.</t> <t><xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/> applies to point-to-point unicast flows and BUM traffic. For BUM traffic, the basic method of <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/> caneasilybe easily applied link bylink and therefore splitlink; therefore, the multicast flow tree distribution can be split into separate unicast point-to-pointlinks. While, thislinks.</t> <t>This document and its Clustered Alternate-Markingmethod applyMethod applies to multipoint-to-multipoint unicast flows, anycast, and ECMPflows.</t> <t>Therefore,flows. Therefore, the Alternate-MarkingmethodMethod can be extended to any kind of multipoint-to-multipoint paths, and the network-clustering approach presented in this document is the formalization of how to implement this property and allow a flexible and optimized performance measurement support for network management in every situation.</t> <t>Without network clustering, it is possible to applyAlternate-MarkingAlternate Marking only for all the network or per single flow. Instead, with network clustering, it is possible touse thepartitionofthe network into clusters at different levels in order to provide the needed degree of detail. In some circumstances, it is possible to monitor a multipoint network by monitoring the network clusters, without examining in depth. In case of problems (packet loss ismeasured,measured or the delay is too high), the filtering criteria could be enhanced in order to perform a detailed analysis by using a different combination of clusters up to a per-flow measurement as described in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>.</t> <t>This approach fits very well with the Closed-Loop Network and Software-Defined Network (SDN) paradigm, where the SDN orchestrator and the SDN controllers are the brains of the network and can manage flow control to the switches and routers and, in the same way, can calibrate the performance measurements depending on the desired accuracy. An SDN controller application can orchestrate how accurately the network performance monitoring is set up by applying the MultipointAlternate-MarkingAlternate Marking as described in this document.</t> <t>It is important to underline that, as an extension of <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, this is a methodology document, so the mechanism that can be used to transmit the counters and the timestamps is out of scope here.</t> <t>This document assumes that the blocks are created according to a fixed timer as per <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>. Switching after a fixed number of packets ispossiblepossible, but it is out of scope here.</t> <t>Note that the fragmented packets' case can be managed with the Alternate-Markingmethodologymethodology, and the same guidance provided insection 6 of<xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" sectionFormat="of" section="6" format="default"/>applyalso applies in the case of MultipointAlternate-Marking.</t>Alternate Marking.</t> <sectiontitle="Summarynumbered="true" toc="default"> <name>Summary of Changes from RFC8889">8889</name> <t>This document defines the Multipoint Alternate-Marking Method, addressing ambiguities and overtaking its experimental phase in the original specification <xreftarget="RFC8889"></xref>.</t>target="RFC8889" format="default"/>.</t> <t>The relevant changesare:<list style="symbols"> <t>Addedare:</t> <ul spacing="normal"> <li>Added the recommendations about the different deployments in case one or two flag bits are available for marking (<xreftarget="finding"></xref>).</t> <t>Changedtarget="finding" format="default"/>).</li> <li>Changed the structure to improvethe readability.</t> <t>Removedreadability.</li> <li>Removed the wording about the experimentation of the method and considerations that no longerapply.</t> <t>Revisedapply.</li> <li>Revised the description of detailed aspects of the methodology,e.g.e.g., synchronization andtiming.</t> </list></t>timing.</li> </ul> <t>It is important to note that all the changes are totally backward compatible with <xreftarget="RFC8889"></xref>target="RFC8889" format="default"/>, and no new additional technique has been introduced in this document compared to <xreftarget="RFC8889"></xref>.</t>target="RFC8889" format="default"/>.</t> </section> <sectiontitle="Requirements Language"> <t>Thenumbered="true" toc="default"> <name>Requirements Language</name> <t> The key words"MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY","<bcp14>MUST</bcp14>", "<bcp14>MUST NOT</bcp14>", "<bcp14>REQUIRED</bcp14>", "<bcp14>SHALL</bcp14>", "<bcp14>SHALL NOT</bcp14>", "<bcp14>SHOULD</bcp14>", "<bcp14>SHOULD NOT</bcp14>", "<bcp14>RECOMMENDED</bcp14>", "<bcp14>NOT RECOMMENDED</bcp14>", "<bcp14>MAY</bcp14>", and"OPTIONAL""<bcp14>OPTIONAL</bcp14>" in this document are to be interpreted as described inBCP 14BCP 14 <xreftarget="RFC2119"></xref>target="RFC2119"/> <xreftarget="RFC8174"></xref>target="RFC8174"/> when, and only when, they appear in all capitals, as shownhere.</t>here. </t> </section> </section> <section anchor="terms"title="Terminology">numbered="true" toc="default"> <name>Terminology</name> <t>Thedefinitionsuse of the basic terms are identical to those found inAlternate-MarkingAlternate Marking <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>. It is to be remembered that <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/> is valid for point-to-point unicast flows and BUM traffic.</t> <t>The important new termsthat need to be explainedarelisted below:<list> <t>Multipoint Alternate-Marking: Extensionexplained below:</t> <dl> <dt>Multipoint Alternate Marking: </dt> <dd>Extension to <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, valid for multipoint-to-multipoint unicast flows, anycast, and ECMP flows. It can also be referred to asClustered Alternate-Marking.</t> <t>Flow"Clustered Alternate Marking". </dd> <dt>Flow definition:The</dt> <dd>The concept of flow is generalized in this document. The identification fields are selected without any constraints and, in general, the flow can be a multipoint-to-multipoint flow, as a result of aggregate point-to-pointflows.</t> <t>Monitoring Network: Identifiedflows. </dd> <dt>Monitoring network: </dt> <dd>Identified with the nodes of the network that are the measurement points (MPs) and the links that are the connections between MPs. The monitoring network graph depends on the flow definition, so it can represent a specific flow or the entire network topology as aggregate of all the flows. Each node of the monitoring network cannot be both a source and a destination of theflow.</t> <t>Cluster: Smallestflow. </dd> <dt>Cluster: </dt> <dd>Smallest identifiable non-trivial subnetwork of the entire monitoring network graph that still satisfies the condition that the number of packets that go in is the same as the number that go out. A cluster partition algorithm, such as that found in <xreftarget="nclustering_algo"></xref>,target="nclustering_algo" format="default"/>, can be applied to split the monitoring network intoclusters.</t> <t>Multipointclusters. </dd> <dt>Multipoint metrics:Packet</dt> <dd>Packet loss,delaydelay, and delay variation are extended to the case of multipoint flows. It is possible to compute these metrics on the basis of multipoint paths in order to associate the measurements to a cluster, a combination of clusters, or the entire monitored network. For delay and delay variation, it is also possible to define the metrics on a single-packet basis, and it means that the multipoint path is used to easily couple packets between input and output nodes of a multipointpath.</t> </list></t>path. </dd> </dl> <t>The next section highlights the correlation with the terms used in <xreftarget="RFC5644">RFC 5644</xref>.</t>target="RFC5644" format="default"/>.</t> <sectiontitle="Correlationnumbered="true" toc="default"> <name>Correlation with RFC5644">5644</name> <t><xref target="RFC5644"format="default">RFC 5644</xref>format="default"/> is limited to active measurements using a single source packet or stream. Its scope is also limited to observations of corresponding packets along the path (spatial metric) and at one or more destinations (one-to-group) along the path.</t> <t>Instead, the scope of this memo is to define multiparty metrics for passive and hybrid measurements in a group-to-group topology with multiple sources and destinations.</t> <t><xref target="RFC5644"format="default">RFC 5644</xref>format="default"/> introduces metric names that can be reused here but have to be extended and rephrased to be applied to the Alternate-Marking schema:</t><t><list style="letters"> <t>the<ol spacing="normal" type="a"> <li>the multiparty metrics are not only one-to-group metrics but canbealso be group-to-groupmetrics;</t> <t>themetrics;</li> <li>the spatial metrics, used for measuring the performance of segments of asource to destinationsource-to-destination path, are applied here toclusters.</t> </list></t>clusters.</li> </ol> </section> </section> <sectiontitle="Flow Classification">numbered="true" toc="default"> <name>Flow Classification</name> <t>A unicast flow is identified by all the packets having a set of common characteristics. This definition is inspired by <xref target="RFC7011"format="default">RFC 7011</xref>.</t>format="default"/>.</t> <t>As an example, by considering a flow as all the packets sharing the same source IP address or the same destination IP address, it is easy to understand that the resulting pattern will not be a point-to-pointconnection,connection but a point-to-multipoint or multipoint-to-point connection.</t> <t>In general, a flow can be defined by a set of selection rules used to match a subset of the packets processed by the network device. These rules specify a set of Layer 3 and Layer 4 header fields (identification fields) and the relative values that must be found in matching packets.</t> <t>The choice of the identification fields directly affects the type of paths that the flow would follow in the network. In fact, it is possible to relate a set of identification fields with the pattern of the resulting graphs, as listed in <xref target="Flows"/>.</t>format="default"/>.</t> <t>A TCP 5-tuple usually identifies flows following either a single path or a point-to-point multipath (in the case of load balancing). On the contrary, a single source address selects aggregate flows following apoint-to-multipoint,point-to-multipoint path, while a multipoint-to-point path can be the result of a matching on a single destination address. In the case where a selection rule and its reverse are used for bidirectional measurements, they can correspond to a point-to-multipoint path in one direction and a multipoint-to-point path in the opposite direction.</t> <t>So the flows to be monitored are selected into the monitoring points using packet selection rules, which can also change the pattern of the monitored network.</t> <t>Note that, more generally, the flow can be defined at different levels based on the potential encapsulation, and additional conditions that are not in the packet header can also be included as part of matching criteria.</t> <t>The Alternate-MarkingmethodMethod is applicable only to a single path (and partially to a one-to-one multipath), so the extension proposed in this document is suitable also for the most general case of multipoint-to-multipoint, which embraces all the other patternsofin <xref target="Flows"/>.</t>format="default"/>.</t> <figureanchor="Flows" title="Flow Classification">anchor="Flows"> <name>Flow Classification</name> <artwork name="" type="" align="left" alt=""><![CDATA[ point-to-point single path +------+ +------+ +------+ ---<> R1 <>----<> R2 <>----<> R3 <>--- +------+ +------+ +------+ point-to-point multipath +------+ <> R2 <> / +------+ \ / \ +------+ / \ +------+ ---<> R1 <> <> R4 <>--- +------+ \ / +------+ \ / \ +------+ / <> R3 <> +------+ point-to-multipoint +------+ <> R4 <>--- / +------+ +------+ / <> R2 <> / +------+ \ +------+ / \ +------+ ---<> R1 <> <> R5 <>--- +------+ \ +------+ \ +------+ <> R3 <> +------+ \ \ +------+ <> R6 <>--- +------+ multipoint-to-point +------+ ---<> R1 <> +------+ \ \ +------+ <> R4 <> / +------+ \ +------+ / \ +------+ ---<> R2 <> <> R6 <>--- +------+ / +------+ +------+ / <> R5 <> / +------+ +------+ / ---<> R3 <> +------+ multipoint-to-multipoint +------+ +------+ ---<> R1 <> <> R6 <>--- +------+ \ / +------+ \ +------+ / <> R4 <> +------+ \ +------+ \ +------+ ---<> R2 <> <> R7 <>--- +------+ \ / +------+ \ +------+ / <> R5 <> / +------+ \ +------+ / \ +------+ ---<> R3 <> <> R8 <>--- +------+ +------+ ]]></artwork> </figure> <t>The case of unicast flow is considered in <xreftarget="Flows"/>.target="Flows" format="default"/>. The anycast flow is also covered, since it is only a special case of a unicast flow if routing is stable throughout the measurement period. Furthermore, an ECMP flow is in scope by definition, since it is a point-to-multipoint unicast flow.</t> </section> <sectiontitle="Extensionnumbered="true" toc="default"> <name>Extension of the Method to MultipointFlows">Flows</name> <t>By using the Alternate-Markingmethod,Method, only point-to-point paths can be monitored. To have an IP (TCP/UDP) flow that follows a point-to-point path, in general we have to define, with a specific value, 5 identification fields (IP Source, IP Destination, Transport Protocol, Source Port, and Destination Port).</t> <t>MultipointAlternate-MarkingAlternate Marking enables the performance measurement for multipoint flows selected by identification fields without any constraints (even the entire network production traffic). It is also possible to use multiple marking points for the same monitored flow.</t> <sectiontitle="Monitoring Network">numbered="true" toc="default"> <name>Monitoring Network</name> <t>The monitoring network is deduced from the production network by identifying the nodes of the graph that are the measurementpoints,points and the links that are the connections between measurement points. It can be modeled as a set of nodes and a set of directed arcswhichthat connect pairs of nodes.</t> <t>There are some techniques that can help with the building of the monitoring network (as an example, see <xreftarget="I-D.ietf-ippm-route"target="RFC9198" format="default"/>). In general, there are different options: the monitoring network can be obtained by considering all the possible paths for the traffic or periodically checking the traffic(e.g.(e.g., daily, weekly, and monthly) and updating the graph as appropriate, but this is up to the Network Management System (NMS) configuration.</t> <t>So a graph model of the monitoring network can be built according to the Alternate-Markingmethod:Method, where the monitored interfaces and links are identified. Only the measurement points and links where the traffic has flowed have to be represented in the graph.</t> <t>A simple example of a monitoring network graph isshowedshown in <xreftarget="Appendix"/>.</t>target="Appendix" format="default"/>.</t> <t>Each monitoring point is characterized by the packet counter that refers only to a marking period of the monitored flow. Also, it is assumed that therebeis a monitoring point at all possible egress points of the multipoint monitored network.</t> <t>The same is also applicable for the delay, but it will be described in the following sections.</t> <t>The rest of the document assumes that the traffic is going from left to right in order to simplify the explanation. But the analysis done for one direction applies equally to all directions.</t> </section> <section anchor="Nploss"title="Networknumbered="true" toc="default"> <name>Network PacketLoss">Loss</name> <t>Since all the packets of the considered flow leaving the network have previously entered the network, the number of packets counted by all the input nodes is always greater than, or equal to, the number of packets counted by all the output nodes. It is assumed that routing is stable during the measurement period while packet fragmentation must be handled as described in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>.</t> <t>In the case of no packet loss occurring in the marking period, if all the input and output points of the network domain to be monitored are measurement points, the sum of the number of packets on all the ingress interfaces equals the number on egress interfaces for the monitored flow. In this circumstance, if no packet loss occurs, the intermediate measurement points only have the task of splitting the measurement.</t> <t>It is possible to define theNetwork Packet Lossnetwork packet loss of one monitored flow for a single period. In a packet network, the number of lost packets is the number of packets counted by the input nodes minus the number of packets counted by the output nodes. This is true for every packet flow in each marking period.</t> <t>The monitored network packet loss with n input nodes and m output nodes is given by:</t><t>PL<artwork><![CDATA[ PL = (PI1 + PI2 +...+ PIn) - (PO1 + PO2 +...+POm)</t>POm) ]]></artwork> <t>where:</t><t>PL<ul> <li>PL is the network packet loss (number of lostpackets)</t> <t>PIipackets); </li> <li>PIi is the number of packets flowed through the i-th input node in thisperiod</t> <t>POjperiod; and </li> <li>POj is the number of packets flowed through the j-th output node in thisperiod</t>period. </li> </ul> <t>The equation is applied on a per-time-interval basis and a per-flowbasis:<list> <t>Thebasis:</t> <ul empty="false" spacing="normal"> <li>The reference interval is the Alternate-Marking period, as defined in <xreftarget="I-D.ietf-ippm-rfc8321bis" format="default"/>.</t> <t>Thetarget="RFC9341" format="default"/>.</li> <li>The flow definition is generalized here. Indeed, as described before, a multipoint packet flow is considered, and the identification fields can be selected without anyconstraints.</t> </list></t>constraints.</li> </ul> </section> </section> <section anchor="nclustering"title="Network Clustering">numbered="true" toc="default"> <name>Network Clustering</name> <t>The previous equation of <xreftarget="Nploss"></xref>target="Nploss" format="default"/> can determine the number of packets lost globally in the monitored network, exploiting only the data provided by the counters in the input and output nodes.</t> <t>In addition, it is possible to leverage the data provided by the other counters in the network to converge on the smallest identifiable subnetworks where the losses occur.</t> <t>As defined in <xreftarget="terms"></xref>,target="terms" format="default"/>, a cluster is a non-trivial subnetwork of the entire monitoring network graph that still satisfies the condition that the number of packets that go in is the same as the number that go out, if no packet loss occurs. According to this definition, a cluster should contain all the arcs emanating from its input nodes and all the arcs terminating at its output nodes. This ensures that we can count all the packets (and only those) exiting an input node again at the output node, whatever path they follow.</t> <t>As for the entire monitoring network graph, the cluster is defined on a per-flow basis. In a completely monitored network (a network where every network interface is monitored), each network device corresponds to a cluster, and each physical link corresponds to two clusters (one for each device).</t> <t>Clusters can have different sizes depending on the flow-filtering criteria adopted.</t> <t>Moreover, sometimes clusters can be optionally simplified. For example, when two monitored interfaces are divided by a single router (one is the input interface, the other is the output interface, and the router has only these two interfaces), instead of counting exactly twice, upon entering and leaving, it is possible to consider a single measurement point. In this case, we do not care about the internal packet loss of the router.</t> <t>It is worth highlighting that it might also be convenient to define clusters based on the topological information so that they are applicable to all the possible flows in the monitored network.</t> <t>Note that, in case of translation or encapsulation, the cluster properties must also be invariant.</t> <section anchor="nclustering_algo"title="Algorithmnumbered="true" toc="default"> <name>Algorithm for ClustersPartition">Partition</name> <t>A simple algorithm can be applied in order to split the monitoring network into clusters. This can be done for each directionseparately, indeedseparately; indeed, a node cannot be both a source and a destination. The clusters partition is based on the monitoring network graph, which can be valid for a specific flow or can also be general and valid for the entire network topology.</t> <t>It is a two-stepalgorithm:<list style="symbols"> <t>Groupalgorithm:</t> <ul spacing="normal"> <li>Group the links where there is the same startingnode;</t> <t>Joinnode;</li> <li>Join the grouped links with at least one ending node incommon.</t> </list></t>common.</li> </ul> <t>Considering that the links are unidirectional, the first step implies listing all the links as connections between two nodes and grouping the different links if they have the same starting node. Note that it is possible to start from any link, and the procedure will work. Following this classification, the second step implies eventually joining the groups classified in the first step by looking at the ending nodes. If different groups have at least one common ending node, they are put together and belong to the same set. After the application of the two steps of the algorithm, each one of the composed sets of links, together with the endpoint nodes, constitutes a cluster.</t> <t>A simple application of the clusters partition isshowedshown in <xreftarget="Appendix"/>.</t>target="Appendix" format="default"/>.</t> <t>The algorithm, as applied in the example of a point-to-multipoint network, works for the more general case of a multipoint-to-multipoint network in the same way. It should be highlighted that for a multipoint-to-multipointnetworknetwork, the multiple sourcesMUST<bcp14>MUST</bcp14> markcoherentlythe traffic coherently andMUST<bcp14>MUST</bcp14> be synchronized with all the other nodes according to the timing requirements detailed in <xreftarget="sync-timing"></xref>.</t>target="sync-timing" format="default"/>.</t> <t>When the clusters partition is done, the calculation of packet loss,delaydelay, and delay variation can be made on a cluster basis. Note that the packet counters for each marking period permit calculating the packet rate on a cluster basis, so Committed Information Rate (CIR) and Excess Information Rate (EIR) could also be deduced on a cluster basis.</t> <t>Obviously, by combining some clusters in a new connectedsubnetworksubnetwork, the packet-loss rule is still true. So it is also possible to consider combinations of clusters if and where it suits.</t> <t>In this way, in a very large network, there is no need to configure detailed filter criteria to inspect the traffic. It is possible to check a multipoint network and, in case of problems, go deep with a step-by-step cluster analysis, but only for the cluster or combination of clusters where the problem happens.</t> <t>In summary, once a flow is defined, the algorithm to build the clusters partition is based on topological information; therefore, it considers all the possible links and nodes that could potentially be crossed by the given flow, even if there is no traffic.So,So if the flow does not enter or traverse all the nodes, the counters have a non-zero value for the involved nodes and a zero value for the other nodes without traffic; but in the end, all the formulas are still valid.</t> <t>The algorithm described above is an iterative clustering algorithm since it executes steps in iterations, but it is also possible to apply a recursive clustering algorithm as detailed in <xreftarget="IEEE-ACM-ToN-MPNPM"target="IEEE-ACM-TON-MPNPM" format="default"/>.</t> <t>The complete and mathematical analysis of the possible algorithms for the clusters partition, including the considerations in terms of efficiency and a comparison between the different methods, is in the paper <xreftarget="IEEE-ACM-ToN-MPNPM"target="IEEE-ACM-TON-MPNPM" format="default"/>.</t> </section> </section> <section anchor="Mploss"title="Multipoint Packet Loss Measurement">numbered="true" toc="default"> <name>Multipoint Packet-Loss Measurement</name> <t>TheNetwork Packet Loss,network packet loss, defined in <xreftarget="Nploss"></xref>,target="Nploss" format="default"/>, valid for the entire monitored flow, can easily be extended to each multipoint path (e.g., the whole multipoint network, a cluster, or a combination of clusters). In thiswayway, it is possible to calculate Multipoint Packet Loss that is representative of a multipoint path.</t> <t>The same equation of <xreftarget="Nploss"></xref>target="Nploss" format="default"/> can be applied to a generic multipoint path like a cluster or a combination of clusters, where the number of packets are those entering and leaving the multipoint path.</t> <t>By applying the algorithm described in <xreftarget="nclustering_algo"></xref>,target="nclustering_algo" format="default"/>, it is possible to split the monitoring network into clusters. Then, packet loss can be measured on a cluster basis for each single period by considering the counters of the input and output nodes that belong to the specific cluster. This can be done for every packet flow in each marking period.</t> </section> <sectiontitle="Multipointnumbered="true" toc="default"> <name>Multipoint Delay and DelayVariation">Variation</name> <t>The same line of reasoning can be applied to delay and delay variation. The delay measurement methods defined in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/> can be extended to the case of multipoint flows. It is important to highlight that both delay and delay-variation measurements make sense in a multipoint path. The delay variation is calculated by considering the same packets selected for measuring the delay.</t> <t>In general, it is possible to perform delay and delay-variation measurements on the basis of multipoint paths or singlepackets:<list style="symbols"> <t>Delaypackets:</t> <ul spacing="normal"> <li>Delay measurements on the basis of multipoint paths mean that the delay value is representative of an entire multipoint path (e.g., the whole multipoint network, a cluster, or a combination ofclusters).</t> <t>Delayclusters).</li> <li>Delay measurements on a single-packet basis mean that it is possible to use a multipoint path just to easily couple packets between input and output nodes of a multipoint path, as described in the followingsections.</t> </list></t>sections.</li> </ul> <section anchor="Mdelay"title="Delaynumbered="true" toc="default"> <name>Delay Measurements on a Multipoint-PathsBasis">Basis</name> <section anchor="M_single-marking"title="Single-Marking Measurement">numbered="true" toc="default"> <name>Single-Marking Measurement</name> <t>Mean delay and mean delay-variation measurements can also be generalized to the case of multipoint flows. It is possible to compute the average one-way delay of packets in one block, a cluster, or the entire monitored network.</t> <t>The average latency can be measured as the difference between the weighted averages of the mean timestamps of the sets of output and input nodes. This means that, in the calculation, it is possible to weigh the timestamps with the number of packets for each endpoint.</t> <t>Note that, since the one-way delay value is representative of a multipoint path, it is possible to calculate the two-way delay of a multipoint path by summing the one-way delays of the two directions, similarly to <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>.</t> </section> </section> <sectiontitle="Delaynumbered="true" toc="default"> <name>Delay Measurements on a Single-PacketBasis">Basis</name> <section anchor="p_sd-marking"title="Single-numbered="true" toc="default"> <name>Single- and Double-MarkingMeasurement">Measurement</name> <t>Delay and delay-variation measurements associated with only one picked packet per period, both single and double marked, cannot be easily performed in a multipoint scenario since there are somelimitations:<list style="hanging"> <t>Single markinglimitations:</t> <ul> <li>Single Marking based on the first/last packet of the interval does not work properly. Indeed, by considering a point-to-multipoint scenario, it is not possible to recognize which path the first packet of each block takes over the multipoint flow in order to correlate it. This is also true for the general case of the multipoint-to-multipointscenario.</t> <t>Double markingscenario. </li> <li>Double Marking or multiplexed marking works but only through statistical means. In a point-to-multipoint scenario, by selecting only a single packet with the second marking for each block, it is possible to follow and calculate the delay for that picked packet. But the measurement can only be done for a single path in each marking period. To traverse all the paths of the multipoint flow, it can theoretically be done by continuing the measurement for the following marking periods and expect to span all the paths. In the general case of a multipoint-to-multipoint path, it is also needed to take into account the multiple source nodeswhichthat complicate the correlation of the samples. In this case, it can be possible to select the second marked packet only for a source node at a time for each block and cover the remaining source nodes one by one in the next markingperiods.</t> </list></t>periods. </li> </ul> <t>Note that, since the one-way delay measurement is done on a single-packet basis, it is always possible to calculate the two-waydelaydelay, but it is not immediate since it is necessary to couple the measurement on each single path with the opposite direction. In thiscasecase, the NMS can do the calculation.</t> <t>If a delay measurement is performed for more than one picked packet and for all the paths of the multipoint flow in the same marking period, neither thesingle-Single- nor thedouble-marking methodDouble-Marking Method are applicable in the multipoint scenario. The packets follow differentpathspaths, and it becomes very difficult to correlate marked packets in a multipoint-to-multipoint path if there are more than one per period.</t> <t>A desirable option is to monitor simultaneously all the paths of a multipoint path in the same marking period. For this purpose, hashing can be used, as reported in the next section.</t> </section> <section anchor="p_hashing"title="Hashingnumbered="true" toc="default"> <name>Hashing SelectionMethod"> <t>RFCs <xref target="RFC5474">5474</xref> and <xref target="RFC5475">5475</xref> introduce samplingMethod</name> <t>Sampling and filtering techniques for IP packetselection.</t>selection are introduced in <xref target="RFC5474"/> and <xref target="RFC5475" format="default"/>.</t> <t>The hash-based selection methodologies for delay measurement can work in a multipoint-to-multipoint path and can be used either coupled to mean delay orstand-alone.</t>standalone.</t> <t><xreftarget="IEEE-Network-PNPM"/>target="IEEE-NETWORK-PNPM" format="default"/> introduces how to use the hash method(<xref target="RFC5474">RFC 5474</xref>(see <xref target="RFC5474" format="default"/> and <xreftarget="RFC5475">RFC 5475</xref>)target="RFC5475" format="default"/>) combined with the Alternate-MarkingmethodMethod for point-to-point flows. It is also calledMixed"Mixed HashedMarkingMarking" because it refers to the conjunction of the marking method and the hashing technique. It involves only thesingle marking, indeedSingle Marking; indeed, it is supposed thatdouble markingDouble Marking is not used with hashing. The coupling of thesingle markingSingle Marking with the hashing selection allows choosing a simplified hash function since the alternation of blocks gives temporal boundaries for the hashing samples. The marking batches anchor the samples selected withhashinghashing, and this eases the correlation of the hashing packets along the path. For example, in case a hashed sample is lost, it is confined to the considered block without affecting the identification of the samples for the following blocks.</t> <t>Using the hash-based sampling, the number of samples in each block may vary a lot because it depends on the packet rate that is variable. A dynamic approach can help to have an almost fixed number of samples for each marking period, and this is a better option for making regular measurements over time. In the hash-based sampling,Alternate-MarkingAlternate Marking is used to create periods, so that hash-based samples are divided into batches, which allows anchoring the selected samples to their period. Moreover, in a dynamic hash-based sampling, it can be possible to dynamically adapt the length of the hash value to meet the current packet rate, so that the number of samples is bounded in each marking period.</t> <t>In a multipoint environment, the hashing selection may be the solution for performing delay measurements on specific packets and overcoming thesingle-Single- anddouble-markingDouble-Marking limitations.</t> </section> </section> </section> <section anchor="sync-timing"title="Synchronizationnumbered="true" toc="default"> <name>Synchronization andTiming">Timing</name> <t>It is important to consider the timing aspects, since out-of-order packets happen and have to be handled as well, as described in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>.</t> <t>However, in a multisource situation, an additional issue has to be considered. With multipoint path, the egress nodes will receive alternate marked packets in random order from different ingress nodes, and this must not affect the measurement.</t> <t>So, if we analyze a multipoint-to-multipoint path with more than one marking node, it is important to recognize the reference measurement interval. In general, the measurement interval for describing the results is the interval of the marking node that is more aligned with the start of the measurement, as reported in <xref target="measint"/>.</t>format="default"/>.</t> <t>Note that the mark switching approach based on a fixed timer is considered in this document.</t> <figureanchor="measint" title="Measurement Interval">anchor="measint"> <name>Measurement Interval</name> <artwork name="" type="" align="left" alt=""><![CDATA[ time -> start stop T(R1) |-------------| T(R2) |-------------| T(R3) |------------| ]]></artwork> </figure> <t>In <xreftarget="measint"/>,target="measint" format="default"/>, it is assumed that the node with the earliest clock (R1) identifies the right starting and ending times of the measurement, but it is just anassumption,assumption and other possibilities could occur.So,So in this case, T(R1) is the measurement interval, and its recognition is essential in order to make comparisons with other active/passive/hybridPacket Losspacket-loss metrics.</t> <t>Regarding the timing constraints of the methodology, <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/> already describes two contributions that are taken into account: the clock error between network devices and the network delay between the measurement points.</t> <t>When we expand to a multipoint environment, we have to consider that there are more marking nodes that mark the traffic based on synchronized clock time. But, due to different synchronization issues that may happen, the marking batches can be of different lengths and with different offsets when they get mixed in a multipoint flow. According to <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, the maximum clock skew between the network devices is A. Therefore, the additional gap that results between the multiple sources can be incorporated into A.</t> <figureanchor="timing" title="Timing Aspects">anchor="timing"> <name>Timing Aspects</name> <artwork name="" type="" align="left" alt=""><![CDATA[ ...BBBBBBBBB | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA | BBBBBBBBB... |<======================================>| | L | ...=========>|<==================><==================>|<==========... | L/2 L/2 | |<====>| |<====>| d | | d |<========================>| available counting interval ]]></artwork> </figure> <t>Moreover, it is assumed that the multipoint path can be modeled with a normaldistribution, otherwisedistribution; otherwise, it is necessary to reformulate based on the type of distribution. Under this assumption, the definition of the guard band d is still applicable as defined in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/> and is given by:</t><t>d<artwork><![CDATA[ d = A + D_avg +3*D_stddev,</t>3*D_stddev, ]]></artwork> <t>where A is the clock accuracy, D_avg is the average value of the network delay, and D_stddev is the standard deviation of the delay.</t> <t>As shown in <xreftarget="timing"/>target="timing" format="default"/> and according to <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, the condition that must be satisfied to enable the method to function properly is that the available counting interval must be > 0, and that means:</t><t>L<artwork><![CDATA[ L - 2d> 0.</t>> 0. ]]></artwork> <t>This formula needs to be verified for each measurement point on the multipoint path.</t> <t>Note that the timing considerations are valid for both packet loss and delay measurements.</t> </section> <section anchor="finding"title="Recommendationsnumbered="true" toc="default"> <name>Recommendations forDeployment">Deployment</name> <t>The methodology described in the previous sections can be applied to various performance measurement problems, as also explained in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>. <xreftarget="RFC8889"></xref>target="RFC8889" format="default"/> reports experimental examples and <xreftarget="IEEE-Network-PNPM"/>target="IEEE-NETWORK-PNPM" format="default"/> also includes some information about the deployment experience.</t> <t>Different deployments are possible using one flag bit, two flagbitsbits, or the hashingselection:<list> <t>Oneselection:</t> <dl> <dt>One flag:packet loss</dt> <dd>packet-loss measurementMUST<bcp14>MUST</bcp14> be done as described in <xreftarget="Mploss"></xref>target="Mploss" format="default"/> by applying the network clustering partition described in <xreftarget="nclustering"></xref>.target="nclustering" format="default"/>. Delay measurementMUST<bcp14>MUST</bcp14> be done according to theMeanmean delay calculation representative of the multipoint path, as described in <xreftarget="M_single-marking"></xref>. Single-marking methodtarget="M_single-marking" format="default"/>. A Single-Marking Method based on the first/last packet of the interval cannot be applied, as mentioned in <xreftarget="p_sd-marking"></xref>.</t> <t>Twotarget="p_sd-marking" format="default"/>. </dd> <dt>Two flags:packet loss</dt> <dd>packet-loss measurementMUST<bcp14>MUST</bcp14> be done as described in <xreftarget="Mploss"></xref>target="Mploss" format="default"/> by applying the network clustering partition described in <xreftarget="nclustering"></xref>.target="nclustering" format="default"/>. Delay measurementSHOULD<bcp14>SHOULD</bcp14> be done on asingle packetsingle-packet basis according todouble-marking method <xref target="p_sd-marking"></xref>.the Double-Marking Method (<xref target="p_sd-marking" format="default"/>). In thiscasecase, theMeanmean delay calculation (<xreftarget="M_single-marking"></xref>) MAYtarget="M_single-marking" format="default"/>) <bcp14>MAY</bcp14> also be used as a representative value of a multipoint path. The choice depends on the kind of information that is needed, as further detailedbelow.</t> <t>Onebelow. </dd> <dt>One flag with hash-based selection:packet loss</dt> <dd>packet-loss measurementMUST<bcp14>MUST</bcp14> be done as described in <xreftarget="Mploss"></xref>target="Mploss" format="default"/> by applying the network clustering partition described in <xreftarget="nclustering"></xref>.target="nclustering" format="default"/>. Hash-based selection methodologies, introduced in <xreftarget="p_hashing"></xref>, MUSTtarget="p_hashing" format="default"/>, <bcp14>MUST</bcp14> be used for delaymeasurement.</t> </list></t>measurement. </dd> </dl> <t>Similarly to <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, there are some operational guidelines to considerfor the purpose ofwhen deciding which recommendation tofollow the recommendations above anduse (i.e., one flag or two flags or one flag with hash-basedselection.<list> <t>Theselection.</t> <ul empty="false" spacing="normal"> <li>The Multipoint Alternate-MarkingmethodMethod utilizes specific flags in the packet header, so an important factor is the number of flags available for the implementation. Indeed, if there is only one flagavailableavailable, there is no other way, while if two flags areavailableavailable, the option with two flags can be considered in comparison with the option of one flag with hash-basedselection.</t> <t>Theselection.</li> <li>The duration of the Alternate-Marking period affects the frequency of themeasurementmeasurement, and this is a parameter that can be decided on the basis of the required temporal sampling. But it cannot be freely chosen, as explained in <xreftarget="sync-timing"/>.</t> <t>Thetarget="sync-timing" format="default"/>.</li> <li>The Multipoint Alternate-Marking methodologies enable packet loss,delaydelay, and delay variation calculation, but in accordance with the method used(e.g. single-marking or double-marking(e.g., Single Marking, Double Marking, or hashing selection), there is a different kind of information that can be derived. For example, to get measurements on a multipoint-paths basis, one flag can be used. To get measurements on a single-packet basis, two flags are preferred. For this reason, the type of data needed in the specific scenario is an additional element to take intoaccount.</t> <t>Theaccount.</li> <li>The Multipoint Alternate-MarkingmethodsMethods imply different computational load depending on the method employed. Therefore, the available computational resources on the measurement points can also influence the choice. As an example, mean delay calculation may require moreprocessingprocessing, and it may not be the best option to minimize the computationalload.</t> </list></t>load.</li> </ul> <t>The experiment with Multipoint Alternate-Marking methodologies confirmed the benefits of the Alternate-Marking methodology(<xref target="I-D.ietf-ippm-rfc8321bis" format="default"/>),<xref target="RFC9341" format="default"/> as its extension to the general case of multipoint-to-multipoint scenarios.</t> <t>The Multipoint Alternate-Marking MethodMUST<bcp14>MUST</bcp14> only be applied to controlled domains, as per <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>.</t> </section> <sectiontitle="Anumbered="true" toc="default"> <name>A Closed-Loop Performance-ManagementApproach">Approach</name> <t>The Multipoint Alternate-Marking framework that is introduced in this document adds flexibility to Performance Management (PM), because it can reduce the order of magnitude of the packet counters. This allows an SDN orchestrator to supervise, control, and manage PM in large networks.</t> <t>The monitoring network can be considered as a whole or split into clusters that are the smallest subnetworks (group-to-group segments), maintaining the packet-loss property for each subnetwork. The clusters can also be combined in new, connected subnetworks at different levels, depending on the detail we want to achieve.</t> <t>An SDN controller ora Network Management System (NMS)an NMS can calibrate performance measurements, since they are aware of the network topology. They can start without examining in depth. In case of necessity (packet loss ismeasured,measured or the delay is too high), the filtering criteria could be immediately reconfigured in order to perform a partition of the network by using clusters and/or different combinations of clusters. In this way, the problem can be localized in a specific cluster or a single combination of clusters, and a more detailed analysis can be performed step by step by successive approximation up to a point-to-point flow detailed analysis. This is the so-called "closed loop".</t> <t>This approach can be called "network zooming" and can be performed in two different ways:</t><t>1) change<ol> <li>change the traffic filter and select more detailedflows;</t> <t>2) activateflows; </li> <li>activate new measurement points by defining more specifiedclusters.</t>clusters. </li> </ol> <t>The network-zooming approach implies that some filters or rules arechanged and that thereforechanged; therefore, there is a transient time to wait once the new network configuration takes effect. This time can be determined by theNetwork Orchestrator/Controller,network orchestrator/controller, based on the network conditions.</t> <t>For example, if the network zooming identifies the performance problem for the traffic coming from a specific source, we need to recognize the marked signal from this specific source node and its relative path. For this purpose, we can activate all the available measurement points and better specify the flow filter criteria (i.e., 5-tuple). As an alternative, it can be enough to select packets from the specific source for delay measurements; in this case, it is possible to apply the hashing technique, as mentioned in the previous sections.</t> <t><xref target="I-D.song-opsawg-ifit-framework" format="default"/> defines an architecture where the centralizedData Collectordata collector andNetwork Managementnetwork management can apply the intelligent and flexible Alternate-Marking algorithm as previously described.</t> <t>As for <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>, it is possible to classify the traffic and mark a portion of the total traffic. For each period, the packet rate and bandwidth are calculated from the number of packets. In this way, the network orchestrator becomes aware if the traffic rate surpasses limits. In addition, more precision can be obtained by reducing the marking period; indeed, some implementations use a marking period of 1 sec or less.</t> <t>In addition, an SDN controller could also collect the measurement history.</t> <t>It is important to mention that the Multipoint Alternate-Marking framework also helps Traffic Visualization. Indeed, this methodology is very useful for identifying which path or cluster is crossed by the flow.</t> </section> <sectiontitle="Security Considerations">numbered="true" toc="default"> <name>Security Considerations</name> <t>This document specifies a method of performing measurements that does not directly affect Internet security or applications that run on the Internet. However, implementation of this method must be mindful of security and privacy concerns, as explained in <xreftarget="I-D.ietf-ippm-rfc8321bis"target="RFC9341" format="default"/>.</t> </section> <section anchor="IANA"title="IANA Considerations">numbered="true" toc="default"> <name>IANA Considerations</name> <t>This document has no IANA actions.</t> </section><section anchor="Contributors" title="Contributors"> <t>Greg Mirsky<vspace blankLines="0" /> Ericsson<vspace blankLines="0" /> Email: gregimirsky@gmail.com</t> <t>Tal Mizrahi<vspace blankLines="0" /> Huawei Technologies<vspace blankLines="0" /> Email: tal.mizrahi.phd@gmail.com</t> <t>Xiao Min<vspace blankLines="0" /> ZTE Corp.<vspace blankLines="0" /> Email: xiao.min2@zte.com.cn</t> </section> <section anchor="Acknowledgements" title="Acknowledgements"> <t>The authors would like to thank Martin Duke and Tommy Pauly for their assistance and their detailed and precious reviews.</t> </section> <!-- Possibly a 'Contributors' section ... --></middle><!-- *****BACK MATTER ***** --><back> <displayreference target="I-D.song-opsawg-ifit-framework" to="OPSAWG-IFIT-FRAMEWORK"/> <references> <name>References</name> <references> <name>Normative References</name> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2119.xml"/> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8174.xml"/> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.5475.xml"/> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.5644.xml"/> <!--References split to informative and normative[I-D.ietf-ippm-rfc8321bis] in AUTH48-DONE as of 12/13/22; companion document RFC YYY1 --><references title="Normative References"> <?rfc include='reference.RFC.2119'?> <?rfc include='reference.RFC.8174'?> <?rfc include='reference.RFC.5475'?> <?rfc include='reference.RFC.5644'?> <?rfc include='reference.I-D.ietf-ippm-rfc8321bis'?><reference anchor='RFC9341' target='https://www.rfc-editor.org/info/rfc9341'> <front> <title>Alternate-Marking Method</title> <author initials="G." surname="Fioccola" fullname="Giuseppe Fioccola" role="editor"> <organization>Huawei Technologies</organization> </author> <author initials="M." surname="Cociglio" fullname="Mauro Cociglio"> <organization>Telecom Italia</organization> </author> <author initials="G." surname="Mirsky" fullname="Greg Mirsky"> <organization>Ericsson</organization> </author> <author initials="T." surname="Mizrahi" fullname="Tal Mizrahi"> <organization>Huawei Technologies</organization> </author> <author initials="T." surname="Zhou" fullname="Tianran Zhou"> <organization>Huawei Technologies</organization> </author> <date month="December" year="2022"/> </front> <seriesInfo name="RFC" value="9341"/> <seriesInfo name="DOI" value="10.17487/RFC9341"/> </reference> </references><references title="Informative References"> <!-- A reference written by by an organization not a persoN. --> <?rfc include='reference.RFC.8889'?> <?rfc include='reference.RFC.5474'?><references> <name>Informative References</name> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8889.xml"/> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.5474.xml"/> <referenceanchor='IEEE-Network-PNPM'>anchor="IEEE-NETWORK-PNPM"> <front> <title>AM-PM: Efficient Network Telemetry using Alternate Marking</title><author> <organization>IEEE Network</organization><author surname="Mizrahi" initials="T"> <organization/> </author> <author surname="Navon" initials="G"> <organization/> </author> <author surname="Fioccola" initials="G"> <organization/> </author> <author surname="Cociglio" initials="M"> <organization/> </author> <author surname="Chen" initials="M"> <organization/> </author> <author surname="Mirsky" initials="G"> <organization/> </author> <dateyear='2019' />month="July" year="2019"/> </front> <refcontent>IEEE Network, Vol. 33, Issue 4</refcontent> <seriesInfoname='DOI' value='10.1109/MNET.2019.1800152'/>name="DOI" value="10.1109/MNET.2019.1800152"/> </reference> <referenceanchor='IEEE-ACM-ToN-MPNPM'>anchor="IEEE-ACM-TON-MPNPM"> <front> <title>Multipoint Passive Monitoring in Packet Networks</title><author> <organization>IEEE/ACM TRANSACTION ON NETWORKING</organization><author surname="Cociglio" initials="M"> <organization showOnFrontPage="true"/> </author> <author surname="Fioccola" initials="G"> <organization showOnFrontPage="true"/> </author> <author surname="Marchetto" initials="G"> <organization showOnFrontPage="true"/> </author> <author surname="Sapio" initials="A"> <organization showOnFrontPage="true"/> </author> <author surname="Sisto" initials="R"> <organization showOnFrontPage="true"/> </author> <dateyear='2019' />month="December" year="2019"/> </front> <refcontent>IEEE/ACM Transactions on Networking, Vol. 27, Issue 6</refcontent> <seriesInfoname='DOI' value='10.1109/TNET.2019.2950157'/>name="DOI" value="10.1109/TNET.2019.2950157"/> </reference><?rfc include='reference.I-D.song-opsawg-ifit-framework'?> <?rfc include='reference.I-D.ietf-ippm-route'?> <?rfc include='reference.RFC.7011'?><!-- [I-D.song-opsawg-ifit-framework] IESG state I-D Exists as of 12/13/22 --> <xi:include href="https://datatracker.ietf.org/doc/bibxml3/reference.I-D.song-opsawg-ifit-framework.xml"/> <!-- [I-D.ietf-ippm-route] Published as RFC 9198 --> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9198.xml"/> <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7011.xml"/> </references> </references> <sectiontitle="Exampleanchor="Appendix" numbered="true" toc="default"> <name>Example of Monitoring Network and ClustersPartition" anchor="Appendix">Partition</name> <t><xreftarget="monitored-graph"/>target="monitored-graph" format="default"/> shows a simple example of a monitoring network graph:</t> <figureanchor="monitored-graph" title="Monitoringanchor="monitored-graph"> <name>Monitoring NetworkGraph">Graph</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +------+ <> R6 <>--- / +------+ +------+ +------+ / <> R2 <>---<> R4 <> / +------+ \ +------+ \ / \ \ +------+ +------+ / +------+ \ +------+ <> R7 <>--- ---<> R1 <>---<> R3 <>---<> R5 <> +------+ +------+ \ +------+ \ +------+ \ \ \ \ +------+ \ \ <> R8 <>--- \ \ +------+ \ \ \ \ +------+ \ <> R9 <>--- \ +------+ \ \ +------+ <> R10 <>--- +------+ ]]></artwork> </figure> <t>In the monitoring network graph example, it is possible to identify the clusters partition by applying this two-step algorithm described in <xreftarget="nclustering_algo"/>.</t>target="nclustering_algo" format="default"/>.</t> <t>The first step identifies the followinggroups:<list style="numbers"> <t>Groupgroups:</t> <ul spacing="normal" empty="true"><li>Group 1: (R1-R2), (R1-R3),(R1-R10)</t> <t>Group(R1-R10)</li> <li>Group 2: (R2-R4),(R2-R5)</t> <t>Group(R2-R5)</li> <li>Group 3: (R3-R5),(R3-R9)</t> <t>Group(R3-R9)</li> <li>Group 4: (R4-R6),(R4-R7)</t> <t>Group(R4-R7)</li> <li>Group 5:(R5-R8)</t> </list></t> <t>And then,(R5-R8)</li> </ul> <t>Then, the second step builds the clusters partition (in particular, we can underline that Groups 2 and 3 connect together, since R5 is incommon):<list style="numbers"> <t>Clustercommon):</t> <ul spacing="normal" empty="true"><li>Cluster 1: (R1-R2), (R1-R3),(R1-R10)</t> <t>Cluster(R1-R10)</li> <li>Cluster 2: (R2-R4), (R2-R5), (R3-R5),(R3-R9)</t> <t>Cluster(R3-R9)</li> <li>Cluster 3: (R4-R6),(R4-R7)</t> <t>Cluster(R4-R7)</li> <li>Cluster 4:(R5-R8)</t> </list></t>(R5-R8)</li> </ul> <t>The flow directionhereconsidered here is from left to right. For the opposite direction, the same reasoning can be applied, and in this example, you get the same clusters partition.</t> <t>In the end, the following 4 clusters are obtained:</t> <figureanchor="clusters" title="Clusters Example">anchor="clusters"> <name>Clusters Example</name> <artwork name="" type="" align="left" alt=""><![CDATA[ Cluster 1 +------+ <> R2 <>--- / +------+ / +------+ / +------+ ---<> R1 <>---<> R3 <>--- +------+ \ +------+ \ \ \ \ \ \ \ \ \ +------+ <> R10 <>--- +------+ Cluster 2 +------+ +------+ ---<> R2 <>---<> R4 <>--- +------+ \ +------+ \ +------+ \ +------+ ---<> R3 <>---<> R5 <>--- +------+ \ +------+ \ \ \ \ \ +------+ <> R9 <>--- +------+ Cluster 3 +------+ <> R6 <>--- / +------+ +------+ / ---<> R4 <> +------+ \ \ +------+ <> R7 <>--- +------+ Cluster 4 +------+ ---<> R5 <> +------+ \ \ +------+ <> R8 <>--- +------+ ]]></artwork> </figure> <t>There are clusters with more than two nodes as well as two-node clusters. In the two-node clusters, the loss is on the link (Cluster 4). In more-than-two-node clusters, the loss is on the cluster, but we cannot know in which link (Cluster 1, 2, or 3).</t> </section> <sectiontitle="Changes Log"> <t>Changes from RFC 8889 in draft-fioccola-rfc8889bis-00 include:<list style="symbols"> <t>Minor editorial changes</t> <t>Removed section on "Examples of application"</t> </list></t> <t>Changes in draft-fioccola-rfc8889bis-01 include:<list style="symbols"> <t>Considerations on BUM traffic</t> <t>Reference to RFC8321bis for the fragmentation part</t> <t>Revised section on "Delay Measurements on a Single-Packet Basis"</t> <t>Revised section on "Timing Aspects"</t> </list></t> <t>Changes in draft-fioccola-rfc8889bis-02 include:<list style="symbols"> <t>Clarified the formula in the section on "Timing Aspects" to be aligned with RFC 8321</t> <t>Considerations on two-way delay measurements in both sections 8.1 and 8.2 on delay measurements</t> <t>Clarified in section 4.1 on "Monitoring Network" that the description is done for one direction but it can easily be extended to all direction</t> <t>New section on "Results of the Multipoint Alternate Marking Experiment"</t> </list></t> <t>Changes in draft-fioccola-rfc8889bis-03 include:<list style="symbols"> <t>Moved and renamed section on "Timing Aspects" as "Synchronization and Timing"</t> <t>Renamed old section on "Multipoint Packet Loss" as "Network Packet Loss"</t> <t>New section on "Multipoint Packet Loss Measurement"</t> <t>Renamed section on "Multipoint Performance Measurement" as "Extension of the Methodanchor="Acknowledgements" numbered="false" toc="default"> <name>Acknowledgements</name> <t>The authors would like toMultipoint Flows"</t> </list></t> <t>Changes in draft-fioccola-rfc8889bis-04/draft-ietf-ippm-rfc8889bis-00 include:<list style="symbols"> <t>Revised section 5.1 on "Algorithmthank <contact fullname="Martin Duke"/> and <contact fullname="Tommy Pauly"/> forClusters Partition"</t> </list></t> <t>Changes in draft-ietf-ippm-rfc8889bis-01 include:<list style="symbols"> <t>New section on "Summary of Changes from RFC 8889"</t> </list></t> <t>Changes in draft-ietf-ippm-rfc8889bis-02 include:<list style="symbols"> <t>Revised sections on "Single- and Double-Marking Measurement", "Hashing Selection Method"their assistance and"Synchronizationtheir detailed andTiming"</t> <t>Revised references</t> </list></t> <t>Changes in draft-ietf-ippm-rfc8889bis-03 include:<list style="symbols"> <t>Comments addressed from Last Call review</t> <t>Renamed section 9 as "Recommendations for Deployment"</t> </list></t> <t>Changes in draft-ietf-ippm-rfc8889bis-04 include:<list style="symbols"> <t>Comments addressed from Last Call review</t> </list></t>valuable reviews.</t> </section> <section anchor="Contributors" numbered="false" toc="default"> <name>Contributors</name> <author fullname="Greg Mirsky" initials="G" surname="Mirsky"> <organization>Ericsson</organization> <address> <postal> </postal> <email>gregimirsky@gmail.com</email> </address> </author> <author fullname="Tal Mizrahi" initials="T" surname="Mizrahi"> <organization>Huawei Technologies</organization> <address> <postal> </postal> <email>tal.mizrahi.phd@gmail.com</email> </address> </author> <author fullname="Xiao Min" initials="X" surname="Min"> <organization>ZTE Corp.</organization> <address> <postal> </postal> <email>xiao.min2@zte.com.cn</email> </address> </author> </section> </back> </rfc>