From eebd26ba118a3524182767554ea5c0d2c9e0cb1c Mon Sep 17 00:00:00 2001 From: kishanps Date: Thu, 15 Dec 2022 16:23:15 -0800 Subject: [PATCH] Add util functions for QoS and buffer profiles overriding and getting gNMIConfig. --- tests/qos/BUILD.bazel | 3 +- tests/qos/frontpanel_qos_test.cc | 3 +- tests/qos/qos_test_util.cc | 98 +++++++++++++++++++++++++++++++- tests/qos/qos_test_util.h | 43 +++++++++++++- 4 files changed, 140 insertions(+), 7 deletions(-) diff --git a/tests/qos/BUILD.bazel b/tests/qos/BUILD.bazel index 27affe5f..d4ea36c2 100644 --- a/tests/qos/BUILD.bazel +++ b/tests/qos/BUILD.bazel @@ -187,11 +187,10 @@ cc_library( "//lib/gnmi:gnmi_helper", "//lib/gnmi:openconfig_cc_proto", "//lib/utils:json_utils", - "//thinkit:generic_testbed", + "//thinkit:mirror_testbed", "//thinkit/proto:generic_testbed_cc_proto", "@com_github_gnmi//proto/gnmi:gnmi_cc_proto", "@com_github_gnmi//proto/gnmi:gnmi_cc_grpc_proto", - "@com_github_nlohmann_json//:nlohmann_json", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/tests/qos/frontpanel_qos_test.cc b/tests/qos/frontpanel_qos_test.cc index 0ca658d1..a52a525b 100644 --- a/tests/qos/frontpanel_qos_test.cc +++ b/tests/qos/frontpanel_qos_test.cc @@ -124,8 +124,7 @@ absl::StatusOr ConfigureFairBufferConfigForPortUntilEndOfScope( buffer_config_by_queue_name; // TODO: Read queue names from switch instead of // hard-coding them here. - for (absl::string_view queue_name : - {"LLQ1", "LLQ2", "BE1", "AF1", "AF2", "AF3", "AF4", "NC1"}) { + for (absl::string_view queue_name : kAllQueuesNames) { buffer_config_by_queue_name[queue_name] = BufferParameters{ .dedicated_buffer = 0, .use_shared_buffer = true, diff --git a/tests/qos/qos_test_util.cc b/tests/qos/qos_test_util.cc index 28a3796b..6fc0c1af 100644 --- a/tests/qos/qos_test_util.cc +++ b/tests/qos/qos_test_util.cc @@ -1,5 +1,7 @@ #include "tests/qos/qos_test_util.h" +#include + #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -7,7 +9,6 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" -#include "absl/strings/strip.h" #include "absl/strings/substitute.h" #include "absl/time/clock.h" #include "absl/time/time.h" @@ -15,7 +16,6 @@ #include "gutil/collections.h" #include "gutil/proto.h" #include "gutil/status.h" -#include "include/nlohmann/json.hpp" #include "lib/gnmi/gnmi_helper.h" #include "lib/gnmi/openconfig.pb.h" #include "lib/utils/json_utils.h" @@ -337,6 +337,18 @@ absl::Status SetSchedulerPolicyParameters( return absl::OkStatus(); } +absl::Status ReplaceCpuSchedulerPolicyParametersForAllQueues( + gnmi::gNMI::StubInterface &gnmi_stub, SchedulerParameters scheduler_param) { + ASSIGN_OR_RETURN(std::string scheduler_policy_name, + GetSchedulerPolicyNameByEgressPort("CPU", gnmi_stub)); + absl::flat_hash_map scheduler_params; + for (absl::string_view queue_name : kAllQueuesNames) { + scheduler_params[queue_name] = scheduler_param; + } + return SetSchedulerPolicyParameters(scheduler_policy_name, scheduler_params, + gnmi_stub); +} + absl::StatusOr> GetSchedulerPolicyWeightsByQueue(absl::string_view scheduler_policy_name, gnmi::gNMI::StubInterface &gnmi) { @@ -590,4 +602,86 @@ absl::Status SetBufferConfigParameters( return absl::OkStatus(); } +absl::Status DisablePuntRateLimits(gnmi::gNMI::StubInterface &gnmi_stub) { + // Effectively Disabling punt rate limiting on both control switch and SUT + // by setting both committed and peak information rate to 1 million pkt/s and + // both committed and excess burst rate to 1'000 pkts. + constexpr int64_t k1Million = 1'000'000; + constexpr int64_t k1Thousand = 1'000; + SchedulerParameters scheduler_params = SchedulerParameters{ + .committed_information_rate = k1Million, + .committed_burst_size = k1Thousand, + .peak_information_rate = k1Million, + .excess_burst_size = k1Thousand, + }; + RETURN_IF_ERROR(ReplaceCpuSchedulerPolicyParametersForAllQueues( + gnmi_stub, scheduler_params)); + return absl::OkStatus(); +} + +absl::Status +UpdateBufferAllocationForAllCpuQueues(gnmi::gNMI::StubInterface &gnmi_stub, + int buffer_size) { + absl::flat_hash_map + buffer_config_by_queue_name; + for (absl::string_view queue_name : kAllQueuesNames) { + buffer_config_by_queue_name[queue_name] = BufferParameters{ + .dedicated_buffer = 0, + .use_shared_buffer = true, + .shared_buffer_limit_type = "openconfig-qos:STATIC", + .dynamic_limit_scaling_factor = 0, + .shared_static_limit = buffer_size, + }; + } + ASSIGN_OR_RETURN(const std::string buffer_profile_name, + GetBufferAllocationProfileByEgressPort("CPU", gnmi_stub)); + RETURN_IF_ERROR(SetBufferConfigParameters( + buffer_profile_name, buffer_config_by_queue_name, gnmi_stub)); + return absl::OkStatus(); +} + +absl::Status +EffectivelyDisablePuntLimitsForSwitch(SwitchRoleToDisablePuntFlowQoS role, + thinkit::MirrorTestbed &testbed) { + std::unique_ptr gnmi_stub; + switch (role) { + case SwitchRoleToDisablePuntFlowQoS::kSwitchUnderTest: { + ASSIGN_OR_RETURN(gnmi_stub, testbed.Sut().CreateGnmiStub()); + break; + } + case SwitchRoleToDisablePuntFlowQoS::kControlSwitch: { + ASSIGN_OR_RETURN(gnmi_stub, testbed.ControlSwitch().CreateGnmiStub()); + break; + } + } + std::string switch_role_name = SwtichRoleToDisableQoSToString(role); + + ASSIGN_OR_RETURN(auto gnmi_config_before_qos_and_buffer_change, + pins_test::GetGnmiConfig(*gnmi_stub)); + RETURN_IF_ERROR(testbed.Environment().StoreTestArtifact( + absl::StrCat(switch_role_name, + "_gnmi_config_before_qos_and_buffer_change.pb.txt"), + gnmi_config_before_qos_and_buffer_change)); + RETURN_IF_ERROR(pins_test::DisablePuntRateLimits(*gnmi_stub)); + LOG(INFO) << "Disabling punt rate limits complete for " << switch_role_name; + + // The default buffer size for CPU on most switches is 18'432. We use 4x of + // the current value to prevent packet loss. + constexpr int kBufferSizeForCpuInBype = 18432 * 4; + RETURN_IF_ERROR(pins_test::UpdateBufferAllocationForAllCpuQueues( + *gnmi_stub, kBufferSizeForCpuInBype)); + + LOG(INFO) << "Update buffer allocation for all CPU queues complete for " + << switch_role_name << ". All CPU queues now have " + << kBufferSizeForCpuInBype << " bytes for their buffers."; + + ASSIGN_OR_RETURN(auto gnmi_config_after_qos_and_buffer_change, + pins_test::GetGnmiConfig(*gnmi_stub)); + RETURN_IF_ERROR(testbed.Environment().StoreTestArtifact( + absl::StrCat(switch_role_name, + "_gnmi_config_after_qos_and_buffer_change.pb.txt"), + gnmi_config_after_qos_and_buffer_change)); + return absl::OkStatus(); +} + } // namespace pins_test diff --git a/tests/qos/qos_test_util.h b/tests/qos/qos_test_util.h index 3f46dff9..07d11207 100644 --- a/tests/qos/qos_test_util.h +++ b/tests/qos/qos_test_util.h @@ -13,7 +13,7 @@ #include "lib/gnmi/openconfig.pb.h" #include "proto/gnmi/gnmi.grpc.pb.h" #include "proto/gnmi/gnmi.pb.h" -#include "thinkit/generic_testbed.h" +#include "thinkit/mirror_testbed.h" #include "thinkit/proto/generic_testbed.pb.h" namespace pins_test { @@ -24,6 +24,26 @@ namespace pins_test { // 10 seconds. constexpr absl::Duration kMaxQueueCounterUpdateTime = absl::Seconds(25); +// TODO read the queue names from the switch instead of using +// hand-coded names. +constexpr std::array kAllQueuesNames = { + "LLQ1", "LLQ2", "BE1", "AF1", "AF2", "AF3", "AF4", "NC1"}; + +enum class SwitchRoleToDisablePuntFlowQoS { + kControlSwitch, + kSwitchUnderTest, +}; + +inline std::string +SwtichRoleToDisableQoSToString(SwitchRoleToDisablePuntFlowQoS role) { + switch (role) { + case SwitchRoleToDisablePuntFlowQoS::kControlSwitch: + return "control_switch"; + case SwitchRoleToDisablePuntFlowQoS::kSwitchUnderTest: + return "switch_under_test"; + } +} + // These are the counters we track in these tests. struct QueueCounters { int64_t num_packets_transmitted = 0; @@ -129,6 +149,10 @@ absl::Status SetSchedulerPolicyParameters( gnmi::gNMI::StubInterface &gnmi, absl::Duration convergence_timeout = absl::Seconds(10)); +// Updates all CPU queues' scheduler policy parameters with `scheduler_params`. +absl::Status UpdateSchedulerPolicyParametersForAllCpuQueues( + gnmi::gNMI::StubInterface &gnmi_stub, SchedulerParameters scheduler_params); + // Reads the weights of all round-robin schedulers belonging to the given // scheduler policy from the state path, and returns them keyed by the name of // the queue they apply to. @@ -204,6 +228,23 @@ absl::Status SetBufferConfigParameters( gnmi::gNMI::StubInterface &gnmi, absl::Duration convergence_timeout = absl::Seconds(10)); +// Disables punt rate limits on all CPU queues for the switch `gnmi_stub` +// connects to. +absl::Status DisablePuntRateLimits(gnmi::gNMI::StubInterface &gnmi_stub); + +// Updates buffer allocation profile for all CPU queues with `buffer_size` for +// the switch `gnmi_stub` connects to. +absl::Status +UpdateBufferAllocationForAllCpuQueues(gnmi::gNMI::StubInterface &gnmi_stub, + int buffer_size); + +// Disables QoS limits for punting for switch with `role`. +// Scheduler policies and buffer allocation will be set to very high value to +// effectively remove limits for punting. +absl::Status +EffectivelyDisablePuntLimitsForSwitch(SwitchRoleToDisablePuntFlowQoS role, + thinkit::MirrorTestbed &testbed); + } // namespace pins_test #endif // PINS_TESTS_QOS_QOS_TEST_UTIL_H_