From 7c2435ef76e5f2d9fac44b241e4a54113f6eafbf Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Tue, 23 May 2023 10:37:47 +0100 Subject: tools: ynl: Use dict of predefined Structs to decode scalar types Use a dict of predefined Struct() objects to decode scalar types in native, big or little endian format. This removes the repetitive code for the scalar variants and ensures all the signed variants are supported. Signed-off-by: Donald Hunter Signed-off-by: David S. Miller --- tools/net/ynl/lib/ynl.py | 101 +++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 57 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index aa77bcae4807..6185ba27f2e7 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -1,10 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +from collections import namedtuple import functools import os import random import socket import struct +from struct import Struct import yaml from .nlspec import SpecFamily @@ -76,10 +78,17 @@ class NlError(Exception): class NlAttr: - type_formats = { 'u8' : ('B', 1), 's8' : ('b', 1), - 'u16': ('H', 2), 's16': ('h', 2), - 'u32': ('I', 4), 's32': ('i', 4), - 'u64': ('Q', 8), 's64': ('q', 8) } + ScalarFormat = namedtuple('ScalarFormat', ['native', 'big', 'little']) + type_formats = { + 'u8' : ScalarFormat(Struct('B'), Struct("B"), Struct("B")), + 's8' : ScalarFormat(Struct('b'), Struct("b"), Struct("b")), + 'u16': ScalarFormat(Struct('H'), Struct(">H"), Struct("h"), Struct("I"), Struct("i"), Struct("Q"), Struct("q"), Struct("" if byte_order == "big-endian" else "<" - return "" + return format.big if byte_order == "big-endian" \ + else format.little + return format.native - def as_u8(self): - return struct.unpack("B", self.raw)[0] - - def as_u16(self, byte_order=None): - endian = NlAttr.format_byte_order(byte_order) - return struct.unpack(f"{endian}H", self.raw)[0] - - def as_u32(self, byte_order=None): - endian = NlAttr.format_byte_order(byte_order) - return struct.unpack(f"{endian}I", self.raw)[0] - - def as_u64(self, byte_order=None): - endian = NlAttr.format_byte_order(byte_order) - return struct.unpack(f"{endian}Q", self.raw)[0] + def as_scalar(self, attr_type, byte_order=None): + format = self.get_format(attr_type, byte_order) + return format.unpack(self.raw)[0] def as_strz(self): return self.raw.decode('ascii')[:-1] @@ -115,17 +116,17 @@ class NlAttr: return self.raw def as_c_array(self, type): - format, _ = self.type_formats[type] - return list({ x[0] for x in struct.iter_unpack(format, self.raw) }) + format = self.get_format(type) + return [ x[0] for x in format.iter_unpack(self.raw) ] def as_struct(self, members): value = dict() offset = 0 for m in members: # TODO: handle non-scalar members - format, size = self.type_formats[m.type] - decoded = struct.unpack_from(format, self.raw, offset) - offset += size + format = self.get_format(m.type) + decoded = format.unpack_from(self.raw, offset) + offset += format.size value[m.name] = decoded[0] return value @@ -184,11 +185,11 @@ class NlMsg: if extack.type == Netlink.NLMSGERR_ATTR_MSG: self.extack['msg'] = extack.as_strz() elif extack.type == Netlink.NLMSGERR_ATTR_MISS_TYPE: - self.extack['miss-type'] = extack.as_u32() + self.extack['miss-type'] = extack.as_scalar('u32') elif extack.type == Netlink.NLMSGERR_ATTR_MISS_NEST: - self.extack['miss-nest'] = extack.as_u32() + self.extack['miss-nest'] = extack.as_scalar('u32') elif extack.type == Netlink.NLMSGERR_ATTR_OFFS: - self.extack['bad-attr-offs'] = extack.as_u32() + self.extack['bad-attr-offs'] = extack.as_scalar('u32') else: if 'unknown' not in self.extack: self.extack['unknown'] = [] @@ -272,11 +273,11 @@ def _genl_load_families(): fam = dict() for attr in gm.raw_attrs: if attr.type == Netlink.CTRL_ATTR_FAMILY_ID: - fam['id'] = attr.as_u16() + fam['id'] = attr.as_scalar('u16') elif attr.type == Netlink.CTRL_ATTR_FAMILY_NAME: fam['name'] = attr.as_strz() elif attr.type == Netlink.CTRL_ATTR_MAXATTR: - fam['maxattr'] = attr.as_u32() + fam['maxattr'] = attr.as_scalar('u32') elif attr.type == Netlink.CTRL_ATTR_MCAST_GROUPS: fam['mcast'] = dict() for entry in NlAttrs(attr.raw): @@ -286,7 +287,7 @@ def _genl_load_families(): if entry_attr.type == Netlink.CTRL_ATTR_MCAST_GRP_NAME: mcast_name = entry_attr.as_strz() elif entry_attr.type == Netlink.CTRL_ATTR_MCAST_GRP_ID: - mcast_id = entry_attr.as_u32() + mcast_id = entry_attr.as_scalar('u32') if mcast_name and mcast_id is not None: fam['mcast'][mcast_name] = mcast_id if 'name' in fam and 'id' in fam: @@ -304,9 +305,9 @@ class GenlMsg: self.fixed_header_attrs = dict() for m in fixed_header_members: - format, size = NlAttr.type_formats[m.type] - decoded = struct.unpack_from(format, nl_msg.raw, offset) - offset += size + format = NlAttr.get_format(m.type) + decoded = format.unpack_from(nl_msg.raw, offset) + offset += format.size self.fixed_header_attrs[m.name] = decoded[0] self.raw = nl_msg.raw[offset:] @@ -381,21 +382,13 @@ class YnlFamily(SpecFamily): attr_payload += self._add_attr(attr['nested-attributes'], subname, subvalue) elif attr["type"] == 'flag': attr_payload = b'' - elif attr["type"] == 'u8': - attr_payload = struct.pack("B", int(value)) - elif attr["type"] == 'u16': - endian = NlAttr.format_byte_order(attr.byte_order) - attr_payload = struct.pack(f"{endian}H", int(value)) - elif attr["type"] == 'u32': - endian = NlAttr.format_byte_order(attr.byte_order) - attr_payload = struct.pack(f"{endian}I", int(value)) - elif attr["type"] == 'u64': - endian = NlAttr.format_byte_order(attr.byte_order) - attr_payload = struct.pack(f"{endian}Q", int(value)) elif attr["type"] == 'string': attr_payload = str(value).encode('ascii') + b'\x00' elif attr["type"] == 'binary': attr_payload = value + elif attr['type'] in NlAttr.type_formats: + format = NlAttr.get_format(attr['type'], attr.byte_order) + attr_payload = format.pack(int(value)) else: raise Exception(f'Unknown type at {space} {name} {value} {attr["type"]}') @@ -434,22 +427,16 @@ class YnlFamily(SpecFamily): if attr_spec["type"] == 'nest': subdict = self._decode(NlAttrs(attr.raw), attr_spec['nested-attributes']) decoded = subdict - elif attr_spec['type'] == 'u8': - decoded = attr.as_u8() - elif attr_spec['type'] == 'u16': - decoded = attr.as_u16(attr_spec.byte_order) - elif attr_spec['type'] == 'u32': - decoded = attr.as_u32(attr_spec.byte_order) - elif attr_spec['type'] == 'u64': - decoded = attr.as_u64(attr_spec.byte_order) elif attr_spec["type"] == 'string': decoded = attr.as_strz() elif attr_spec["type"] == 'binary': decoded = self._decode_binary(attr, attr_spec) elif attr_spec["type"] == 'flag': decoded = True + elif attr_spec["type"] in NlAttr.type_formats: + decoded = attr.as_scalar(attr_spec['type'], attr_spec.byte_order) else: - raise Exception(f'Unknown {attr.type} {attr_spec["name"]} {attr_spec["type"]}') + raise Exception(f'Unknown {attr_spec["type"]} with name {attr_spec["name"]}') if not attr_spec.is_multi: rsp[attr_spec['name']] = decoded @@ -555,8 +542,8 @@ class YnlFamily(SpecFamily): fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: value = vals.pop(m.name) - format, _ = NlAttr.type_formats[m.type] - msg += struct.pack(format, value) + format = NlAttr.get_format(m.type) + msg += format.pack(value) for name, value in vals.items(): msg += self._add_attr(op.attr_set.name, name, value) msg = _genl_msg_finalize(msg) -- cgit From bddd2e561b0ad5ca42e16fb26a20fc806d521912 Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Tue, 23 May 2023 10:37:48 +0100 Subject: tools: ynl: Handle byte-order in struct members Add support for byte-order in struct members in the genetlink-legacy spec. Signed-off-by: Donald Hunter Acked-by: Jakub Kicinski Signed-off-by: David S. Miller --- tools/net/ynl/lib/ynl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 6185ba27f2e7..39a2296c0003 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -124,7 +124,7 @@ class NlAttr: offset = 0 for m in members: # TODO: handle non-scalar members - format = self.get_format(m.type) + format = self.get_format(m.type, m.byte_order) decoded = format.unpack_from(self.raw, offset) offset += format.size value[m.name] = decoded[0] @@ -305,7 +305,7 @@ class GenlMsg: self.fixed_header_attrs = dict() for m in fixed_header_members: - format = NlAttr.get_format(m.type) + format = NlAttr.get_format(m.type, m.byte_order) decoded = format.unpack_from(nl_msg.raw, offset) offset += format.size self.fixed_header_attrs[m.name] = decoded[0] @@ -542,7 +542,7 @@ class YnlFamily(SpecFamily): fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: value = vals.pop(m.name) - format = NlAttr.get_format(m.type) + format = NlAttr.get_format(m.type, m.byte_order) msg += format.pack(value) for name, value in vals.items(): msg += self._add_attr(op.attr_set.name, name, value) -- cgit From 081e8df6819997eae236f75dd52f0c147c4be939 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 24 May 2023 10:07:12 -0700 Subject: tools: ynl: avoid dict errors on older Python versions Python 3.9.0 or newer supports combining dicts() with |, but older versions of Python are still used in the wild (e.g. on CentOS 8, which goes EoL May 31, 2024). With Python 3.6.8 we get: TypeError: unsupported operand type(s) for |: 'dict' and 'dict' Use older syntax. Tested with non-legacy families only. Fixes: f036d936ca57 ("tools: ynl: Add fixed-header support to ynl") Reviewed-by: Simon Horman Reviewed-by: Donald Hunter Tested-by: Donald Hunter Link: https://lore.kernel.org/r/20230524170712.2036128-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/ynl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index aa77bcae4807..3144f33196be 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -591,8 +591,9 @@ class YnlFamily(SpecFamily): print('Unexpected message: ' + repr(gm)) continue - rsp.append(self._decode(gm.raw_attrs, op.attr_set.name) - | gm.fixed_header_attrs) + rsp_msg = self._decode(gm.raw_attrs, op.attr_set.name) + rsp_msg.update(gm.fixed_header_attrs) + rsp.append(rsp_msg) if not rsp: return None -- cgit From 5ac18889bde04ed2d4f2559da01e9b160234525c Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Sat, 27 May 2023 14:31:05 +0100 Subject: tools: ynl: Initialise fixed headers to 0 in genetlink-legacy This eliminates the need for e.g. --json '{"dp-ifindex":0}' which is not too big a deal for ovs but will get tiresome for fixed header structs that have many members. Signed-off-by: Donald Hunter Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/ynl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 39a2296c0003..85ee6a4bee72 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -541,7 +541,7 @@ class YnlFamily(SpecFamily): if op.fixed_header: fixed_header_members = self.consts[op.fixed_header].members for m in fixed_header_members: - value = vals.pop(m.name) + value = vals.pop(m.name) if m.name in vals else 0 format = NlAttr.get_format(m.type, m.byte_order) msg += format.pack(value) for name, value in vals.items(): -- cgit From 313a7a808ca8ca0fe08e2175eb145479bd86937e Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Sat, 27 May 2023 14:31:06 +0100 Subject: tools: ynl: Support enums in struct members in genetlink-legacy Support decoding scalars as enums in struct members for genetlink-legacy specs. Signed-off-by: Donald Hunter Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/ynl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 85ee6a4bee72..0692293447ad 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -412,7 +412,11 @@ class YnlFamily(SpecFamily): def _decode_binary(self, attr, attr_spec): if attr_spec.struct_name: - decoded = attr.as_struct(self.consts[attr_spec.struct_name]) + members = self.consts[attr_spec.struct_name] + decoded = attr.as_struct(members) + for m in members: + if m.enum: + self._decode_enum(decoded, m) elif attr_spec.sub_type: decoded = attr.as_c_array(attr_spec.sub_type) else: -- cgit From d8eea68d913c20aabb3a97bc1e9ba61407a9e872 Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Fri, 23 Jun 2023 21:19:27 +0100 Subject: tools: ynl: add display-hint support to ynl Add support to the ynl tool for rendering output based on display-hint properties. Signed-off-by: Donald Hunter Link: https://lore.kernel.org/r/20230623201928.14275-3-donald.hunter@gmail.com Signed-off-by: Jakub Kicinski --- tools/net/ynl/lib/ynl.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) (limited to 'tools/net/ynl/lib/ynl.py') diff --git a/tools/net/ynl/lib/ynl.py b/tools/net/ynl/lib/ynl.py index 3b343d6cbbc0..1b3a36fbb1c3 100644 --- a/tools/net/ynl/lib/ynl.py +++ b/tools/net/ynl/lib/ynl.py @@ -8,6 +8,8 @@ import socket import struct from struct import Struct import yaml +import ipaddress +import uuid from .nlspec import SpecFamily @@ -105,6 +107,20 @@ class NlAttr: else format.little return format.native + @classmethod + def formatted_string(cls, raw, display_hint): + if display_hint == 'mac': + formatted = ':'.join('%02x' % b for b in raw) + elif display_hint == 'hex': + formatted = bytes.hex(raw, ' ') + elif display_hint in [ 'ipv4', 'ipv6' ]: + formatted = format(ipaddress.ip_address(raw)) + elif display_hint == 'uuid': + formatted = str(uuid.UUID(bytes=raw)) + else: + formatted = raw + return formatted + def as_scalar(self, attr_type, byte_order=None): format = self.get_format(attr_type, byte_order) return format.unpack(self.raw)[0] @@ -124,10 +140,16 @@ class NlAttr: offset = 0 for m in members: # TODO: handle non-scalar members - format = self.get_format(m.type, m.byte_order) - decoded = format.unpack_from(self.raw, offset) - offset += format.size - value[m.name] = decoded[0] + if m.type == 'binary': + decoded = self.raw[offset:offset+m['len']] + offset += m['len'] + elif m.type in NlAttr.type_formats: + format = self.get_format(m.type, m.byte_order) + [ decoded ] = format.unpack_from(self.raw, offset) + offset += format.size + if m.display_hint: + decoded = self.formatted_string(decoded, m.display_hint) + value[m.name] = decoded return value def __repr__(self): @@ -385,7 +407,7 @@ class YnlFamily(SpecFamily): elif attr["type"] == 'string': attr_payload = str(value).encode('ascii') + b'\x00' elif attr["type"] == 'binary': - attr_payload = value + attr_payload = bytes.fromhex(value) elif attr['type'] in NlAttr.type_formats: format = NlAttr.get_format(attr['type'], attr.byte_order) attr_payload = format.pack(int(value)) @@ -421,6 +443,8 @@ class YnlFamily(SpecFamily): decoded = attr.as_c_array(attr_spec.sub_type) else: decoded = attr.as_bin() + if attr_spec.display_hint: + decoded = NlAttr.formatted_string(decoded, attr_spec.display_hint) return decoded def _decode(self, attrs, space): -- cgit