From f988cfd256ee483d1774c3f8dfb9bdc6f5aa66bb Mon Sep 17 00:00:00 2001 From: Andy Fingerhut Date: Sat, 3 Apr 2021 11:38:54 -0400 Subject: [PATCH 1/4] Some changes to enable proto/ptf test to pass on system using Python3 and avoiding the use of Python2. --- proto/ptf/base_test.py | 48 +++++++++++++---------- proto/ptf/bmv2/gen_bmv2_config.py | 2 +- proto/ptf/l3_host_fwd/test/l3_host_fwd.py | 48 +++++++++++++---------- proto/ptf/ptf_runner.py | 8 ++-- 4 files changed, 61 insertions(+), 45 deletions(-) diff --git a/proto/ptf/base_test.py b/proto/ptf/base_test.py index 99934c8c..58431645 100644 --- a/proto/ptf/base_test.py +++ b/proto/ptf/base_test.py @@ -63,9 +63,14 @@ def stringify(n, length): 'length' bytes, it is represented in the fewest number of bytes it does fit into without loss of precision. It always returns a string at least one byte long, even if value=width=0.""" - h = '%x' % n - s = ('0'*(len(h) % 2) + h).zfill(length*2).decode('hex') - return s + if length == 0 and n == 0: + length = 1 + while True: + try: + s = n.to_bytes(length, byteorder='big') + return s + except OverflowError: + length = length + 1 def ipv4_to_binary(addr): """Take an argument 'addr' containing an IPv4 address written as a @@ -74,10 +79,10 @@ def ipv4_to_binary(addr): client operations.""" bytes_ = [int(b, 10) for b in addr.split('.')] assert len(bytes_) == 4 - # Note: The chr(b) call below will throw exception if any b is - # outside of the range [0, 255]], so no need to add a separate - # check for that here. - return "".join(chr(b) for b in bytes_) + # Note: The bytes() call below will throw exception if any + # elements of bytes_ is outside of the range [0, 255]], so no need + # to add a separate check for that here. + return bytes(bytes_) def mac_to_binary(addr): """Take an argument 'addr' containing an Ethernet MAC address written @@ -87,10 +92,10 @@ def mac_to_binary(addr): operations.""" bytes_ = [int(b, 16) for b in addr.split(':')] assert len(bytes_) == 6 - # Note: The chr(b) call below will throw exception if any b is - # outside of the range [0, 255]], so no need to add a separate - # check for that here. - return "".join(chr(b) for b in bytes_) + # Note: The bytes() call below will throw exception if any + # elements of bytes_ is outside of the range [0, 255]], so no need + # to add a separate check for that here. + return bytes(bytes_) # Used to indicate that the gRPC error Status object returned by the server has # an incorrect format. @@ -267,7 +272,7 @@ def import_p4info_names(self): key = (obj_type, suffix) self.p4info_obj_map[key] = obj suffix_count[key] += 1 - for key, c in suffix_count.items(): + for key, c in list(suffix_count.items()): if c > 1: del self.p4info_obj_map[key] @@ -402,21 +407,24 @@ def add_to(self, mf_id, mk): mf = mk.add() mf.field_id = mf_id mf.lpm.prefix_len = self.pLen - mf.lpm.value = '' + assert isinstance(self.v, bytes) + orig_v_list = list(self.v) + mod_v_list = [] # P4Runtime now has strict rules regarding ternary matches: in the # case of LPM, trailing bits in the value (after prefix) must be set # to 0. - first_byte_masked = self.pLen / 8 + first_byte_masked = self.pLen // 8 for i in range(first_byte_masked): - mf.lpm.value += self.v[i] - if first_byte_masked == len(self.v): + mod_v_list.append(orig_v_list[i]) + if first_byte_masked == len(orig_v_list): + mf.lpm.value = bytes(mod_v_list) return r = self.pLen % 8 - mf.lpm.value += chr( - ord(self.v[first_byte_masked]) & (0xff << (8 - r))) - for i in range(first_byte_masked + 1, len(self.v)): - mf.lpm.value += '\x00' + mod_v_list.append(orig_v_list[first_byte_masked] & (0xff << (8 - r))) + for i in range(first_byte_masked + 1, len(orig_v_list)): + mod_v_list.append(0) + mf.lpm.value = bytes(mod_v_list) class Ternary(MF): def __init__(self, name, v, mask): diff --git a/proto/ptf/bmv2/gen_bmv2_config.py b/proto/ptf/bmv2/gen_bmv2_config.py index e1854d94..5e913f89 100755 --- a/proto/ptf/bmv2/gen_bmv2_config.py +++ b/proto/ptf/bmv2/gen_bmv2_config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2013-present Barefoot Networks, Inc. # diff --git a/proto/ptf/l3_host_fwd/test/l3_host_fwd.py b/proto/ptf/l3_host_fwd/test/l3_host_fwd.py index 9161c0dc..f8d379d0 100644 --- a/proto/ptf/l3_host_fwd/test/l3_host_fwd.py +++ b/proto/ptf/l3_host_fwd/test/l3_host_fwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2013-present Barefoot Networks, Inc. # @@ -27,7 +27,7 @@ from google.rpc import code_pb2 -from base_test import P4RuntimeTest, autocleanup, stringify, ipv4_to_binary +from base_test import P4RuntimeTest, autocleanup, stringify, ipv4_to_binary, mac_to_binary class L3HostFwdTest(P4RuntimeTest): pass @@ -36,13 +36,15 @@ class FwdTest(L3HostFwdTest): @autocleanup def runTest(self): ip_dst_addr = "10.0.0.1" - ip_dst_addr_str = ipv4_to_binary(ip_dst_addr) + ip_dst_addr_bin = ipv4_to_binary(ip_dst_addr) ig_port = self.swports(1) eg_port = self.swports(2) # port is 9-bit in v1model, i.e. 2 bytes eg_port_str = stringify(eg_port, 2) - smac = "\xee\xcd\x00\x7e\x70\x00" - dmac = "\xee\x30\xca\x9d\x1e\x00" + smac = "ee:cd:00:7e:70:00" + dmac = "ee:30:ca:9d:1e:00" + smac_bin = mac_to_binary(smac) + dmac_bin = mac_to_binary(dmac) # we do not care about the src mac address or the src IP address pkt = testutils.simple_tcp_packet( @@ -54,9 +56,9 @@ def runTest(self): # add a forwarding entry self.send_request_add_entry_to_action( - "l3_host_fwd", [self.Exact("hdr.ipv4.dst_addr", ip_dst_addr_str)], + "l3_host_fwd", [self.Exact("hdr.ipv4.dst_addr", ip_dst_addr_bin)], "set_nexthop", - [("port", eg_port_str), ("smac", smac), ("dmac", dmac)]) + [("port", eg_port_str), ("smac", smac_bin), ("dmac", dmac_bin)]) # check that the entry is hit and that no other packets are received exp_pkt = testutils.simple_tcp_packet( @@ -67,18 +69,21 @@ def runTest(self): class DupEntryTest(L3HostFwdTest): @autocleanup def runTest(self): - ip_dst_addr_str = "\x0a\x00\x00\x01" + ip_dst_addr = "10.0.0.1" + ip_dst_addr_bin = ipv4_to_binary(ip_dst_addr) eg_port = self.swports(2) eg_port_str = stringify(eg_port, 2) - smac = "\xee\xcd\x00\x7e\x70\x00" - dmac = "\xee\x30\xca\x9d\x1e\x00" + smac = "ee:cd:00:7e:70:00" + dmac = "ee:30:ca:9d:1e:00" + smac_bin = mac_to_binary(smac) + dmac_bin = mac_to_binary(dmac) def add_entry_once(): self.send_request_add_entry_to_action( "l3_host_fwd", - [self.Exact("hdr.ipv4.dst_addr", ip_dst_addr_str)], + [self.Exact("hdr.ipv4.dst_addr", ip_dst_addr_bin)], "set_nexthop", - [("port", eg_port_str), ("smac", smac), ("dmac", dmac)]) + [("port", eg_port_str), ("smac", smac_bin), ("dmac", dmac_bin)]) add_entry_once() with self.assertP4RuntimeError(): @@ -87,28 +92,31 @@ def add_entry_once(): class BadMatchKeyTest(L3HostFwdTest): @autocleanup def runTest(self): - ip_dst_addr_str = "\x0a\x00\x00\x01" - bad_ip_dst_addr_str = "\x0a\x00\x00" # missing one byte + ip_dst_addr = "10.0.0.1" + ip_dst_addr_bin = ipv4_to_binary(ip_dst_addr) + bad_ip_dst_addr_bin = ip_dst_addr_bin[0:3] # missing one byte eg_port = self.swports(2) eg_port_str = stringify(eg_port, 2) - smac = "\xee\xcd\x00\x7e\x70\x00" - dmac = "\xee\x30\xca\x9d\x1e\x00" + smac = "ee:cd:00:7e:70:00" + dmac = "ee:30:ca:9d:1e:00" + smac_bin = mac_to_binary(smac) + dmac_bin = mac_to_binary(dmac) # missing one byte with self.assertP4RuntimeError(code_pb2.INVALID_ARGUMENT): self.send_request_add_entry_to_action( "l3_host_fwd", - [self.Exact("hdr.ipv4.dst_addr", bad_ip_dst_addr_str)], + [self.Exact("hdr.ipv4.dst_addr", bad_ip_dst_addr_bin)], "set_nexthop", - [("port", eg_port_str), ("smac", smac), ("dmac", dmac)]) + [("port", eg_port_str), ("smac", smac_bin), ("dmac", dmac_bin)]) # unexpected match type with self.assertP4RuntimeError(code_pb2.INVALID_ARGUMENT): self.send_request_add_entry_to_action( "l3_host_fwd", - [self.Lpm("hdr.ipv4.dst_addr", ip_dst_addr_str, 24)], + [self.Lpm("hdr.ipv4.dst_addr", ip_dst_addr_bin, 24)], "set_nexthop", - [("port", eg_port_str), ("smac", smac), ("dmac", dmac)]) + [("port", eg_port_str), ("smac", smac_bin), ("dmac", dmac_bin)]) class BadChecksumTest(L3HostFwdTest): @autocleanup diff --git a/proto/ptf/ptf_runner.py b/proto/ptf/ptf_runner.py index 3ba632cc..e710de76 100755 --- a/proto/ptf/ptf_runner.py +++ b/proto/ptf/ptf_runner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2013-present Barefoot Networks, Inc. # @@ -53,7 +53,7 @@ def check_ifaces(ifaces): ''' Checks that required interfaces exist. ''' - ip_out = subprocess.check_output(['ip', 'link']) + ip_out = subprocess.check_output(['ip', 'link'], text=True) # 'ip link' returns a list of interfaces as # : : ... # : @: ... @@ -105,7 +105,7 @@ def run_test(config_path, p4info_path, grpc_addr, device_id, iface_name = entry["iface_name"] port_map[p4_port] = iface_name - if not check_ifaces(port_map.values()): + if not check_ifaces(list(port_map.values())): error("Some interfaces are missing") return False @@ -117,7 +117,7 @@ def run_test(config_path, p4info_path, grpc_addr, device_id, os.environ['PYTHONPATH'] += ":" + pypath else: os.environ['PYTHONPATH'] = pypath - for iface_idx, iface_name in port_map.items(): + for iface_idx, iface_name in list(port_map.items()): ifaces.extend(['-i', '{}@{}'.format(iface_idx, iface_name)]) cmd = ['ptf'] cmd.extend(['--test-dir', ptfdir]) From 7391c5a21af0358ac82e53139a57c943bed4cff8 Mon Sep 17 00:00:00 2001 From: Andy Fingerhut Date: Sat, 3 Apr 2021 11:42:57 -0400 Subject: [PATCH 2/4] One more change to update python -> python3 executable name in README --- proto/ptf/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/ptf/README.md b/proto/ptf/README.md index 636d8c84..6bdfdfd9 100644 --- a/proto/ptf/README.md +++ b/proto/ptf/README.md @@ -46,7 +46,7 @@ inject and receive test packets. 3. Running the PTF tests (in a second terminal) - sudo python ptf_runner.py \ + sudo python3 ptf_runner.py \ --device-config config.bin --p4info p4info.proto.txt \ --ptfdir l3_host_fwd/test/ --port-map bmv2/port_map.json From 4ec8fe952090e23edde26ade72ef190af7074249 Mon Sep 17 00:00:00 2001 From: Andy Fingerhut Date: Tue, 6 Apr 2021 04:46:02 -0700 Subject: [PATCH 3/4] An alternate implementation of stringify() --- proto/ptf/base_test.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/proto/ptf/base_test.py b/proto/ptf/base_test.py index 58431645..9c14f117 100644 --- a/proto/ptf/base_test.py +++ b/proto/ptf/base_test.py @@ -48,29 +48,24 @@ def __get__(self, instance, owner): return partial(self.func, instance, *(self.args or ()), **(self.keywords or {})) -# Convert integer (with length) to binary byte string -# Equivalent to Python 3.2 int.to_bytes -# See -# https://stackoverflow.com/questions/16022556/has-python-3-to-bytes-been-back-ported-to-python-2-7 -# TODO: When P4Runtime implementation is ready for it, use -# minimum-length byte sequences to represent integers. For unsigned -# integers, this should only require removing the zfill() call below. -def stringify(n, length): +def stringify(n, length=0): """Take a non-negative integer 'n' as the first parameter, and a non-negative integer 'length' in units of _bytes_ as the second - parameter. Return a string with binary contents expected by the - Python P4Runtime client operations. If 'n' does not fit in - 'length' bytes, it is represented in the fewest number of bytes it - does fit into without loss of precision. It always returns a - string at least one byte long, even if value=width=0.""" + parameter (it defaults to 0 if not provided). Return a string + with binary contents expected by the Python P4Runtime client + operations. If 'n' does not fit in 'length' bytes, it is + represented in the fewest number of bytes it does fit into without + loss of precision. It always returns a string at least one byte + long, even if n=length=0.""" if length == 0 and n == 0: length = 1 - while True: - try: - s = n.to_bytes(length, byteorder='big') - return s - except OverflowError: - length = length + 1 + else: + n_size_bits = n.bit_length() + n_size_bytes = (n_size_bits + 7) // 8 + if n_size_bytes > length: + length = n_size_bytes + s = n.to_bytes(length, byteorder='big') + return s def ipv4_to_binary(addr): """Take an argument 'addr' containing an IPv4 address written as a @@ -272,7 +267,7 @@ def import_p4info_names(self): key = (obj_type, suffix) self.p4info_obj_map[key] = obj suffix_count[key] += 1 - for key, c in list(suffix_count.items()): + for key, c in suffix_count.items(): if c > 1: del self.p4info_obj_map[key] From 57f5ae6b2e9bdfa9366a23f1778d0b8f2349dcd0 Mon Sep 17 00:00:00 2001 From: Andy Fingerhut Date: Tue, 6 Apr 2021 04:52:23 -0700 Subject: [PATCH 4/4] Remove another unnecessary list() call --- proto/ptf/ptf_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/ptf/ptf_runner.py b/proto/ptf/ptf_runner.py index e710de76..e3b97c9e 100755 --- a/proto/ptf/ptf_runner.py +++ b/proto/ptf/ptf_runner.py @@ -117,7 +117,7 @@ def run_test(config_path, p4info_path, grpc_addr, device_id, os.environ['PYTHONPATH'] += ":" + pypath else: os.environ['PYTHONPATH'] = pypath - for iface_idx, iface_name in list(port_map.items()): + for iface_idx, iface_name in port_map.items(): ifaces.extend(['-i', '{}@{}'.format(iface_idx, iface_name)]) cmd = ['ptf'] cmd.extend(['--test-dir', ptfdir])