Skip to content

Commit

Permalink
T6895: Merge the hsflowd-based sFlow and uacctd-based sFlow
Browse files Browse the repository at this point in the history
  • Loading branch information
@natali-rs1985
natali-rs1985 committed Jan 17, 2025
1 parent 99d0c7a commit c223fd4
Show file tree
Hide file tree
Showing 10 changed files with 118 additions and 243 deletions.
23 changes: 0 additions & 23 deletions data/templates/pmacct/uacctd.conf.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,6 @@ imt_mem_pools_number: 169
{% set _ = plugin.append('nfprobe['~ nf_server_key ~ ']') %}
{% endfor %}
{% endif %}
{% if sflow.server is vyos_defined %}
{% for server in sflow.server %}
{% set sf_server_key = 'sf_' ~ server | dot_colon_to_dash %}
{% set _ = plugin.append('sfprobe[' ~ sf_server_key ~ ']') %}
{% endfor %}
{% endif %}
{% if disable_imt is not defined %}
{% set _ = plugin.append('memory') %}
{% endif %}
Expand Down Expand Up @@ -61,20 +55,3 @@ nfprobe_timeouts[{{ nf_server_key }}]: expint={{ netflow.timeout.expiry_interval

{% endfor %}
{% endif %}

{% if sflow.server is vyos_defined %}
# sFlow servers
{% for server, server_config in sflow.server.items() %}
{# # prevent pmacct syntax error when using IPv6 flow collectors #}
{% set sf_server_key = 'sf_' ~ server | dot_colon_to_dash %}
sfprobe_receiver[{{ sf_server_key }}]: {{ server | bracketize_ipv6 }}:{{ server_config.port }}
sfprobe_agentip[{{ sf_server_key }}]: {{ sflow.agent_address }}
{% if sflow.sampling_rate is vyos_defined %}
sampling_rate[{{ sf_server_key }}]: {{ sflow.sampling_rate }}
{% endif %}
{% if sflow.source_address is vyos_defined %}
sfprobe_source_ip[{{ sf_server_key }}]: {{ sflow.source_address | bracketize_ipv6 }}
{% endif %}

{% endfor %}
{% endif %}
3 changes: 3 additions & 0 deletions data/templates/sflow/hsflowd.conf.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ sflow {
pcap { dev={{ iface }} }
{% endfor %}
{% endif %}
{% if enable_egress is vyos_defined %}
psample { group=1 egress=on }
{% endif %}
{% if drop_monitor_limit is vyos_defined %}
dropmon { limit={{ drop_monitor_limit }} start=on sw=on hw=off }
{% endif %}
Expand Down
2 changes: 1 addition & 1 deletion interface-definitions/include/version/flow-accounting-version.xml.i
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
<!-- include start from include/version/flow-accounting-version.xml.i -->
<syntaxVersion component='flow-accounting' version='1'></syntaxVersion>
<syntaxVersion component='flow-accounting' version='2'></syntaxVersion>
<!-- include end -->
67 changes: 0 additions & 67 deletions interface-definitions/system_flow-accounting.xml.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,73 +362,6 @@
</node>
</children>
</node>
<node name="sflow">
<properties>
<help>sFlow settings</help>
</properties>
<children>
<leafNode name="agent-address">
<properties>
<help>sFlow agent IPv4 address</help>
<completionHelp>
<list>auto</list>
<script>${vyos_completion_dir}/list_local_ips.sh --ipv4</script>
</completionHelp>
<valueHelp>
<format>ipv4</format>
<description>sFlow IPv4 agent address</description>
</valueHelp>
<constraint>
<validator name="ipv4-address"/>
</constraint>
</properties>
</leafNode>
<leafNode name="sampling-rate">
<properties>
<help>sFlow sampling-rate</help>
<valueHelp>
<format>u32</format>
<description>Sampling rate (1 in N packets)</description>
</valueHelp>
<constraint>
<validator name="numeric" argument="--range 0-4294967295"/>
</constraint>
</properties>
</leafNode>
<tagNode name="server">
<properties>
<help>sFlow destination server</help>
<valueHelp>
<format>ipv4</format>
<description>IPv4 server to export sFlow</description>
</valueHelp>
<valueHelp>
<format>ipv6</format>
<description>IPv6 server to export sFlow</description>
</valueHelp>
<constraint>
<validator name="ip-address"/>
</constraint>
</properties>
<children>
<leafNode name="port">
<properties>
<help>sFlow port number</help>
<valueHelp>
<format>u32:1025-65535</format>
<description>sFlow port number</description>
</valueHelp>
<constraint>
<validator name="numeric" argument="--range 1025-65535"/>
</constraint>
</properties>
<defaultValue>6343</defaultValue>
</leafNode>
</children>
</tagNode>
#include <include/source-address-ipv4-ipv6.xml.i>
</children>
</node>
#include <include/interface/vrf.xml.i>
</children>
</node>
Expand Down
6 changes: 6 additions & 0 deletions interface-definitions/system_sflow.xml.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,12 @@
</leafNode>
</children>
</tagNode>
<leafNode name="enable-egress">
<properties>
<help>Enable egress sampling</help>
<valueless/>
</properties>
</leafNode>
#include <include/interface/vrf.xml.i>
</children>
</node>
Expand Down
107 changes: 1 addition & 106 deletions smoketest/scripts/cli/test_system_flow-accounting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,111 +97,6 @@ def test_basic(self):
self.assertIn(f'syslog: {syslog}', uacctd)
self.assertIn(f'plugins: memory', uacctd)

def test_sflow(self):
sampling_rate = '4000'
source_address = '192.0.2.1'
dummy_if = 'dum3841'
agent_address = '192.0.2.2'

sflow_server = {
'1.2.3.4' : { },
'5.6.7.8' : { 'port' : '6000' },
}

self.cli_set(['interfaces', 'dummy', dummy_if, 'address', agent_address + '/32'])
self.cli_set(['interfaces', 'dummy', dummy_if, 'address', source_address + '/32'])
self.cli_set(base_path + ['disable-imt'])

# You need to configure at least one interface for flow-accounting
with self.assertRaises(ConfigSessionError):
self.cli_commit()
for interface in Section.interfaces('ethernet'):
self.cli_set(base_path + ['interface', interface])


# You need to configure at least one sFlow or NetFlow protocol, or not
# set "disable-imt" for flow-accounting
with self.assertRaises(ConfigSessionError):
self.cli_commit()

self.cli_set(base_path + ['sflow', 'agent-address', agent_address])
self.cli_set(base_path + ['sflow', 'sampling-rate', sampling_rate])
self.cli_set(base_path + ['sflow', 'source-address', source_address])
for server, server_config in sflow_server.items():
self.cli_set(base_path + ['sflow', 'server', server])
if 'port' in server_config:
self.cli_set(base_path + ['sflow', 'server', server, 'port', server_config['port']])

# commit changes
self.cli_commit()

uacctd = read_file(uacctd_conf)

# when 'disable-imt' is not configured on the CLI it must be present
self.assertNotIn(f'imt_path: /tmp/uacctd.pipe', uacctd)
self.assertNotIn(f'imt_mem_pools_number: 169', uacctd)
self.assertNotIn(f'plugins: memory', uacctd)

for server, server_config in sflow_server.items():
plugin_name = server.replace('.', '-')
if 'port' in server_config:
self.assertIn(f'sfprobe_receiver[sf_{plugin_name}]: {server}', uacctd)
else:
self.assertIn(f'sfprobe_receiver[sf_{plugin_name}]: {server}:6343', uacctd)

self.assertIn(f'sfprobe_agentip[sf_{plugin_name}]: {agent_address}', uacctd)
self.assertIn(f'sampling_rate[sf_{plugin_name}]: {sampling_rate}', uacctd)
self.assertIn(f'sfprobe_source_ip[sf_{plugin_name}]: {source_address}', uacctd)

self.cli_delete(['interfaces', 'dummy', dummy_if])

def test_sflow_ipv6(self):
sampling_rate = '100'
sflow_server = {
'2001:db8::1' : { },
'2001:db8::2' : { 'port' : '6000' },
}

self.cli_set(base_path + ['disable-imt'])

# You need to configure at least one interface for flow-accounting
with self.assertRaises(ConfigSessionError):
self.cli_commit()
for interface in Section.interfaces('ethernet'):
self.cli_set(base_path + ['interface', interface])


# You need to configure at least one sFlow or NetFlow protocol, or not
# set "disable-imt" for flow-accounting
with self.assertRaises(ConfigSessionError):
self.cli_commit()

self.cli_set(base_path + ['sflow', 'sampling-rate', sampling_rate])
for server, server_config in sflow_server.items():
self.cli_set(base_path + ['sflow', 'server', server])
if 'port' in server_config:
self.cli_set(base_path + ['sflow', 'server', server, 'port', server_config['port']])

# commit changes
self.cli_commit()

uacctd = read_file(uacctd_conf)

# when 'disable-imt' is not configured on the CLI it must be present
self.assertNotIn(f'imt_path: /tmp/uacctd.pipe', uacctd)
self.assertNotIn(f'imt_mem_pools_number: 169', uacctd)
self.assertNotIn(f'plugins: memory', uacctd)

for server, server_config in sflow_server.items():
tmp_srv = server
tmp_srv = tmp_srv.replace(':', '-')

if 'port' in server_config:
self.assertIn(f'sfprobe_receiver[sf_{tmp_srv}]: {bracketize_ipv6(server)}', uacctd)
else:
self.assertIn(f'sfprobe_receiver[sf_{tmp_srv}]: {bracketize_ipv6(server)}:6343', uacctd)
self.assertIn(f'sampling_rate[sf_{tmp_srv}]: {sampling_rate}', uacctd)

def test_netflow(self):
engine_id = '33'
max_flows = '667'
Expand Down Expand Up @@ -288,8 +183,8 @@ def test_netflow(self):

self.assertIn(f'nfprobe_timeouts[nf_{tmp_srv}]: expint={tmo_expiry}:general={tmo_flow}:icmp={tmo_icmp}:maxlife={tmo_max}:tcp.fin={tmo_tcp_fin}:tcp={tmo_tcp_generic}:tcp.rst={tmo_tcp_rst}:udp={tmo_udp}', uacctd)


self.cli_delete(['interfaces', 'dummy', dummy_if])


if __name__ == '__main__':
unittest.main(verbosity=2)
33 changes: 33 additions & 0 deletions smoketest/scripts/cli/test_system_sflow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,39 @@ def test_sflow(self):
for interface in Section.interfaces('ethernet'):
self.assertIn(f'pcap {{ dev={interface} }}', hsflowd)

def test_sflow_ipv6(self):
sampling_rate = '100'
default_polling = '30'
default_port = '6343'
sflow_server = {
'2001:db8::1': {},
'2001:db8::2': {'port': '8023'},
}

for interface in Section.interfaces('ethernet'):
self.cli_set(base_path + ['interface', interface])

self.cli_set(base_path + ['sampling-rate', sampling_rate])
for server, server_config in sflow_server.items():
self.cli_set(base_path + ['server', server])
if 'port' in server_config:
self.cli_set(base_path + ['server', server, 'port', server_config['port']])

# commit changes
self.cli_commit()

# verify configuration
hsflowd = read_file(hsflowd_conf)

self.assertIn(f'sampling={sampling_rate}', hsflowd)
self.assertIn(f'polling={default_polling}', hsflowd)

for server, server_config in sflow_server.items():
if 'port' in server_config:
self.assertIn(f'collector {{ ip = {server} udpport = {server_config["port"]} }}', hsflowd)
else:
self.assertIn(f'collector {{ ip = {server} udpport = {default_port} }}', hsflowd)

def test_vrf(self):
interface = 'eth0'
server = '192.0.2.1'
Expand Down
6 changes: 6 additions & 0 deletions smoketest/scripts/system/test_kernel_options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,5 +128,11 @@ def test_container_cpu(self):
tmp = re.findall(f'{option}=(y|m)', self._config_data)
self.assertTrue(tmp)

def test_psample_enabled(self):
# Psample must be enabled in the OS Kernel to enable egress flow for hsflowd
for option in ['CONFIG_PSAMPLE']:
tmp = re.findall(f'{option}=y', self._config_data)
self.assertTrue(tmp)

if __name__ == '__main__':
unittest.main(verbosity=2)
53 changes: 7 additions & 46 deletions src/conf_mode/system_flow-accounting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import re

from sys import exit
from ipaddress import ip_address

from vyos.config import Config
from vyos.config import config_dict_merge
Expand Down Expand Up @@ -159,9 +158,9 @@ def get_config(config=None):

# delete individual flow type defaults - should only be added if user
# sets this feature
for flow_type in ['sflow', 'netflow']:
if flow_type not in flow_accounting and flow_type in default_values:
del default_values[flow_type]
flow_type = 'netflow'
if flow_type not in flow_accounting and flow_type in default_values:
del default_values[flow_type]

flow_accounting = config_dict_merge(default_values, flow_accounting)

Expand All @@ -171,9 +170,9 @@ def verify(flow_config):
if not flow_config:
return None

# check if at least one collector is enabled
if 'sflow' not in flow_config and 'netflow' not in flow_config and 'disable_imt' in flow_config:
raise ConfigError('You need to configure at least sFlow or NetFlow, ' \
# check if collector is enabled
if 'netflow' not in flow_config and 'disable_imt' in flow_config:
raise ConfigError('You need to configure NetFlow, ' \
'or not set "disable-imt" for flow-accounting!')

# Check if at least one interface is configured
Expand All @@ -185,45 +184,7 @@ def verify(flow_config):
for interface in flow_config['interface']:
verify_interface_exists(flow_config, interface, warning_only=True)

# check sFlow configuration
if 'sflow' in flow_config:
# check if at least one sFlow collector is configured
if 'server' not in flow_config['sflow']:
raise ConfigError('You need to configure at least one sFlow server!')

# check that all sFlow collectors use the same IP protocol version
sflow_collector_ipver = None
for server in flow_config['sflow']['server']:
if sflow_collector_ipver:
if sflow_collector_ipver != ip_address(server).version:
raise ConfigError("All sFlow servers must use the same IP protocol")
else:
sflow_collector_ipver = ip_address(server).version

# check if vrf is defined for Sflow
verify_vrf(flow_config)
sflow_vrf = None
if 'vrf' in flow_config:
sflow_vrf = flow_config['vrf']

# check agent-id for sFlow: we should avoid mixing IPv4 agent-id with IPv6 collectors and vice-versa
for server in flow_config['sflow']['server']:
if 'agent_address' in flow_config['sflow']:
if ip_address(server).version != ip_address(flow_config['sflow']['agent_address']).version:
raise ConfigError('IPv4 and IPv6 addresses can not be mixed in "sflow agent-address" and "sflow '\
'server". You need to set the same IP version for both "agent-address" and '\
'all sFlow servers')

if 'agent_address' in flow_config['sflow']:
tmp = flow_config['sflow']['agent_address']
if not is_addr_assigned(tmp, sflow_vrf):
raise ConfigError(f'Configured "sflow agent-address {tmp}" does not exist in the system!')

# Check if configured sflow source-address exist in the system
if 'source_address' in flow_config['sflow']:
if not is_addr_assigned(flow_config['sflow']['source_address'], sflow_vrf):
tmp = flow_config['sflow']['source_address']
raise ConfigError(f'Configured "sflow source-address {tmp}" does not exist on the system!')
verify_vrf(flow_config)

# check NetFlow configuration
if 'netflow' in flow_config:
Expand Down
Loading

0 comments on commit c223fd4

Please sign in to comment.