Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry pick 3316 and 3299 #77

Merged
merged 3 commits into from
Jun 21, 2024
Merged

Cherry pick 3316 and 3299 #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
90fd38f
Add full configuration validation. (#3316)
xincunli-sonic May 17, 2024
621fc64
Cherry pick 3299
xincunli-sonic Jun 14, 2024
c643f68
Add missing import
xincunli-sonic Jun 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 51 additions & 10 deletions config/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@

from collections import OrderedDict
from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope
from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError
from minigraph import parse_device_desc_xml, minigraph_encoder
from natsort import natsorted
from portconfig import get_child_ports
from socket import AF_INET, AF_INET6
from sonic_py_common import device_info, multi_asic
from sonic_py_common.interface import get_interface_table_name, get_port_table_name, get_intf_longname
from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
from utilities_common import util_base
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
from utilities_common.db import Db
Expand Down Expand Up @@ -1085,19 +1087,54 @@ def validate_gre_type(ctx, _, value):
def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path):
scope, changes = scope_changes
# Replace localhost to DEFAULT_NAMESPACE which is db definition of Host
if scope.lower() == "localhost" or scope == "":
if scope.lower() == HOST_NAMESPACE or scope == "":
scope = multi_asic.DEFAULT_NAMESPACE
scope_for_log = scope if scope else "localhost"

scope_for_log = scope if scope else HOST_NAMESPACE
try:
# Call apply_patch with the ASIC-specific changes and predefined parameters
GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes), config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path)
GenericUpdater(scope=scope).apply_patch(jsonpatch.JsonPatch(changes),
config_format,
verbose,
dry_run,
ignore_non_yang_tables,
ignore_path)
results[scope_for_log] = {"success": True, "message": "Success"}
log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}")
except Exception as e:
results[scope_for_log] = {"success": False, "message": str(e)}
log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}")


def validate_patch(patch):
try:
command = ["show", "runningconfiguration", "all"]
proc = subprocess.Popen(command, text=True, stdout=subprocess.PIPE)
all_running_config, returncode = proc.communicate()
if returncode:
log.log_notice(f"Fetch all runningconfiguration failed as output:{all_running_config}")
return False

# Structure validation and simulate apply patch.
all_target_config = patch.apply(json.loads(all_running_config))

# Verify target config by YANG models
target_config = all_target_config.pop(HOST_NAMESPACE) if multi_asic.is_multi_asic() else all_target_config
target_config.pop("bgpraw", None)
if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
return False

if multi_asic.is_multi_asic():
for asic in multi_asic.get_namespace_list():
target_config = all_target_config.pop(asic)
target_config.pop("bgpraw", None)
if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
return False

return True
except Exception as e:
raise GenericConfigUpdaterError(f"Validate json patch: {patch} failed due to:{e}")

# This is our main entrypoint - the main 'config' command
@click.group(cls=clicommon.AbbreviationGroup, context_settings=CONTEXT_SETTINGS)
@click.pass_context
Expand Down Expand Up @@ -1296,6 +1333,9 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
patch_as_json = json.loads(text)
patch = jsonpatch.JsonPatch(patch_as_json)

if not validate_patch(patch):
raise GenericConfigUpdaterError(f"Failed validating patch:{patch}")

results = {}
config_format = ConfigFormat[format.upper()]
# Initialize a dictionary to hold changes categorized by scope
Expand All @@ -1318,7 +1358,8 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
# Empty case to force validate YANG model.
if not changes_by_scope:
asic_list = [multi_asic.DEFAULT_NAMESPACE]
asic_list.extend(multi_asic.get_namespace_list())
if multi_asic.is_multi_asic():
asic_list.extend(multi_asic.get_namespace_list())
for asic in asic_list:
changes_by_scope[asic] = []

Expand All @@ -1331,7 +1372,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i

if failures:
failure_messages = '\n'.join([f"- {failed_scope}: {results[failed_scope]['message']}" for failed_scope in failures])
raise Exception(f"Failed to apply patch on the following scopes:\n{failure_messages}")
raise GenericConfigUpdaterError(f"Failed to apply patch on the following scopes:\n{failure_messages}")

log.log_notice(f"Patch applied successfully for {patch}.")
click.secho("Patch applied successfully.", fg="cyan", underline=True)
Expand Down Expand Up @@ -1538,9 +1579,9 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
file_input = read_json_file(file)

platform = file_input.get("DEVICE_METADATA", {}).\
get("localhost", {}).get("platform")
get(HOST_NAMESPACE, {}).get("platform")
mac = file_input.get("DEVICE_METADATA", {}).\
get("localhost", {}).get("mac")
get(HOST_NAMESPACE, {}).get("mac")

if not platform or not mac:
log.log_warning("Input file does't have platform or mac. platform: {}, mac: {}"
Expand Down Expand Up @@ -1905,8 +1946,8 @@ def override_config_table(db, input_config_db, dry_run):
if multi_asic.is_multi_asic() and len(config_input):
# Golden Config will use "localhost" to represent host name
if ns == DEFAULT_NAMESPACE:
if "localhost" in config_input.keys():
ns_config_input = config_input["localhost"]
if HOST_NAMESPACE in config_input.keys():
ns_config_input = config_input[HOST_NAMESPACE]
else:
click.secho("Wrong config format! 'localhost' not found in host config! cannot override.. abort")
sys.exit(1)
Expand Down
32 changes: 14 additions & 18 deletions generic_config_updater/change_applier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

print_to_console = False


def set_verbose(verbose=False):
global print_to_console, logger

Expand All @@ -34,11 +35,12 @@ def log_error(m):
logger.log(logger.LOG_PRIORITY_ERROR, m, print_to_console)


def get_config_db(namespace=multi_asic.DEFAULT_NAMESPACE):
config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace)
def get_config_db(scope=multi_asic.DEFAULT_NAMESPACE):
config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=scope)
config_db.connect()
return config_db


def set_config(config_db, tbl, key, data):
config_db.set_entry(tbl, key, data)

Expand All @@ -61,11 +63,9 @@ class DryRunChangeApplier:
def __init__(self, config_wrapper):
self.config_wrapper = config_wrapper


def apply(self, change):
self.config_wrapper.apply_change_to_config_db(change)


def remove_backend_tables_from_config(self, data):
return data

Expand All @@ -74,9 +74,9 @@ class ChangeApplier:

updater_conf = None

def __init__(self, namespace=multi_asic.DEFAULT_NAMESPACE):
self.namespace = namespace
self.config_db = get_config_db(self.namespace)
def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE):
self.scope = scope
self.config_db = get_config_db(self.scope)
self.backend_tables = [
"BUFFER_PG",
"BUFFER_PROFILE",
Expand All @@ -86,7 +86,6 @@ def __init__(self, namespace=multi_asic.DEFAULT_NAMESPACE):
with open(UPDATER_CONF_FILE, "r") as s:
ChangeApplier.updater_conf = json.load(s)


def _invoke_cmd(self, cmd, old_cfg, upd_cfg, keys):
# cmd is in the format as <package/module name>.<method name>
#
Expand All @@ -98,7 +97,6 @@ def _invoke_cmd(self, cmd, old_cfg, upd_cfg, keys):

return method_to_call(old_cfg, upd_cfg, keys)


def _services_validate(self, old_cfg, upd_cfg, keys):
lst_svcs = set()
lst_cmds = set()
Expand All @@ -124,7 +122,6 @@ def _services_validate(self, old_cfg, upd_cfg, keys):
log_debug("service invoked: {}".format(cmd))
return 0


def _upd_data(self, tbl, run_tbl, upd_tbl, upd_keys):
for key in set(run_tbl.keys()).union(set(upd_tbl.keys())):
run_data = run_tbl.get(key, None)
Expand All @@ -135,20 +132,17 @@ def _upd_data(self, tbl, run_tbl, upd_tbl, upd_keys):
upd_keys[tbl][key] = {}
log_debug("Patch affected tbl={} key={}".format(tbl, key))


def _report_mismatch(self, run_data, upd_data):
log_error("run_data vs expected_data: {}".format(
str(jsondiff.diff(run_data, upd_data))[0:40]))


def apply(self, change):
run_data = self._get_running_config()
upd_data = prune_empty_table(change.apply(copy.deepcopy(run_data)))
upd_keys = defaultdict(dict)

for tbl in sorted(set(run_data.keys()).union(set(upd_data.keys()))):
self._upd_data(tbl, run_data.get(tbl, {}),
upd_data.get(tbl, {}), upd_keys)
self._upd_data(tbl, run_data.get(tbl, {}), upd_data.get(tbl, {}), upd_keys)

ret = self._services_validate(run_data, upd_data, upd_keys)
if not ret:
Expand All @@ -168,9 +162,9 @@ def remove_backend_tables_from_config(self, data):

def _get_running_config(self):
_, fname = tempfile.mkstemp(suffix="_changeApplier")
if self.namespace:
cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.namespace]

if self.scope:
cmd = ['sonic-cfggen', '-d', '--print-data', '-n', self.scope]
else:
cmd = ['sonic-cfggen', '-d', '--print-data']

Expand All @@ -181,7 +175,9 @@ def _get_running_config(self):
return_code = result.returncode
if return_code:
os.remove(fname)
raise GenericConfigUpdaterError(f"Failed to get running config for namespace: {self.namespace}, Return code: {return_code}, Error: {err}")
raise GenericConfigUpdaterError(
f"Failed to get running config for scope: {self.scope}," +
f"Return code: {return_code}, Error: {err}")

run_data = {}
try:
Expand Down
Loading
Loading