From d26fa80bd61df454cac3495e1df8eafa21877e24 Mon Sep 17 00:00:00 2001 From: Vladimir Sokolovsky Date: Tue, 10 Sep 2024 17:53:33 -0500 Subject: [PATCH] Added bf-upgrade environment Supported ATF/UEFI capsule update, BMC/CEC firmware upgrade and NIC firmware golden image upgrade. Signed-off-by: Vladimir Sokolovsky --- bf-release.spec | 3 + debian/rules | 1 + src/bf-upgrade | 160 ++++++--- src/bf-upgrade.env/atf-uefi | 19 ++ src/bf-upgrade.env/bmc | 649 ++++++++++++++++++++++++++++++++++++ src/bf-upgrade.env/common | 52 +++ src/bf-upgrade.env/nic-fw | 150 +++++++++ 7 files changed, 984 insertions(+), 50 deletions(-) create mode 100644 src/bf-upgrade.env/atf-uefi create mode 100644 src/bf-upgrade.env/bmc create mode 100644 src/bf-upgrade.env/common create mode 100644 src/bf-upgrade.env/nic-fw diff --git a/bf-release.spec b/bf-release.spec index 901b14d..c8f4a23 100644 --- a/bf-release.spec +++ b/bf-release.spec @@ -140,6 +140,7 @@ install -m 0644 src/mlnx-ovs.conf %{buildroot}/etc/mellanox install -d %{buildroot}/etc/acpi/actions/ install -m 0755 src/rebootcontrol %{buildroot}/etc/acpi/actions/ install -m 0755 src/bf-upgrade %{buildroot}/etc/acpi/actions/ +cp -a src/bf-upgrade.env %{buildroot}/etc/acpi/actions/ install -d %{buildroot}/etc/acpi/events/ install -m 0644 src/mlnx-powerconf %{buildroot}/etc/acpi/events/ @@ -321,6 +322,8 @@ fi %dir /etc/acpi/actions/ /etc/acpi/actions/rebootcontrol /etc/acpi/actions/bf-upgrade +%dir /etc/acpi/actions/bf-upgrade.env +/etc/acpi/actions/bf-upgrade.env/* %dir /etc/systemd/logind.conf.d/ /etc/systemd/logind.conf.d/lid.conf diff --git a/debian/rules b/debian/rules index 0943244..6ede64e 100755 --- a/debian/rules +++ b/debian/rules @@ -90,6 +90,7 @@ endif dh_installdirs -p$(pname) etc/acpi/actions install -m 0755 src/rebootcontrol debian/$(pname)/etc/acpi/actions install -m 0755 src/bf-upgrade debian/$(pname)/etc/acpi/actions + cp -a src/bf-upgrade.env debian/$(pname)/etc/acpi/actions dh_installdirs -p$(pname) etc/acpi/events install -m 0644 src/mlnx-powerconf debian/$(pname)/etc/acpi/events install -m 0644 src/mlnx-lidconf debian/$(pname)/etc/acpi/events diff --git a/src/bf-upgrade b/src/bf-upgrade index d3d9ae1..46882fc 100755 --- a/src/bf-upgrade +++ b/src/bf-upgrade @@ -1,4 +1,26 @@ #!/bin/bash +############################################################################### +# +# Copyright 2024 NVIDIA Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +############################################################################### # Maximum idle time in seconds to decide no more data from BOOTFIFO. IDLE_MAX=5 @@ -6,30 +28,37 @@ IDLE_MAX=5 # Source a configuration if exists. [ -f /etc/bf-upgrade.conf ] && source /etc/bf-upgrade.conf +SCRIPTS_DIR=$(dirname $0) + + [ ! -e /proc/acpi/button/lid/LID/state ] && exit 0 BOOTFIFO="/sys/bus/platform/devices/MLNXBF04:00/driver/bootfifo" if [ ! -e "$BOOTFIFO" ]; then BOOTFIFO="/sys/devices/platform/MLNXBF04:00/bootfifo" - [ ! -e "$BOOTFIFO" ] && exit -1 + [ ! -e "$BOOTFIFO" ] && exit 1 fi +WDIR=${WDIR:-"/tmp/bfb"} +mkdir -p ${WDIR} + # Only handles upgrade in closed state. -state=`cat /proc/acpi/button/lid/LID/state | awk '{print $2}' 2>/dev/null` -[ $state != "closed" ] && exit 0 +state=$(cat /proc/acpi/button/lid/LID/state | awk '{print $2}' 2>/dev/null) +[ "$state" != "closed" ] && exit 0 # Add a delay for the boot-fifo to be filled. sleep $IDLE_MAX # Example code to fetch the upgrade bfb. # !!!Use eMMC or NVME to avoid running out of memory in NIC mode.!!! -UPGRADE_IMAGE=/tmp/upgrade.bfb +UPGRADE_IMAGE=${WDIR}/upgrade.bfb rm -f ${UPGRADE_IMAGE}* 2>/dev/null + idle_cnt=0 while [ $idle_cnt -lt $IDLE_MAX ]; do cat "$BOOTFIFO" > ${UPGRADE_IMAGE}.tmp - filesize=`du -b ${UPGRADE_IMAGE}.tmp | awk '{print $1}'` - if [ -z "$filesize" -o ."$filesize" == ."0" ]; then + filesize=$(du -b ${UPGRADE_IMAGE}.tmp | awk '{print $1}') + if [[ -z "$filesize" || ."$filesize" == ."0" ]]; then # Done if no more data in 5 seconds. idle_cnt=$((idle_cnt + 1)) sleep 1 @@ -39,58 +68,89 @@ while [ $idle_cnt -lt $IDLE_MAX ]; do fi done -# Get the upgrade image. -rm -f ${UPGRADE_IMAGE}.tmp 2>/dev/null -cd /tmp -rm -f dump-upgrade-image-v0 2>/dev/null -mlx-mkbfb -x -n upgrade-image-v0 ${UPGRADE_IMAGE} +/bin/rm -f ${UPGRADE_IMAGE}.tmp +cd ${WDIR} +/bin/rm -f dump-* + +mlx-mkbfb -x ${UPGRADE_IMAGE} rm -f ${UPGRADE_IMAGE} -if [ ! -e upgrade-image-v0 ]; then - bfrshlog "Error: No upgrade image found" - exit 1 +# dump-boot-args-v0 - bf.cfg +# dump-capsule-v0 - BSP upgrade capsule +# dump-image-v0 - BMC firmware image +# dump-upgrade-image-v0 - CEC upgrade image +# dump-ramdisk-v0 - NIC firmware Golden Image + +if ( bash -n dump-boot-args-v0 ); then + bfrshlog "Found bf.cfg" + . dump-boot-args-v0 +fi + +# Include scripts that provide upgrade infrastructure +if (bash -n ${SCRIPTS_DIR}/bf-upgrade.env/common 2>/dev/null); then + . ${SCRIPTS_DIR}/bf-upgrade.env/common +fi + +if (bash -n ${SCRIPTS_DIR}/bf-upgrade.env/atf-uefi 2>/dev/null); then + . ${SCRIPTS_DIR}/bf-upgrade.env/atf-uefi fi -case "$(file --brief --mime-type upgrade-image-v0)" in - application/gzip) - app=gzip - ;; - application/zstd) - app=zstd - ;; - *) - app=gzip - ;; -esac - -mkdir image -cd image -$app -d < ../upgrade-image-v0 | cpio -id - -install_sh=$(/bin/ls -1 */install.sh 2> /dev/null) - -if [ -z $install_sh ]; then - echo "Error: No installation script found" - cd .. - /bin/rm -rf image - echo "upgrade-image-v0 not supported" - exit 1 +if (bash -n ${SCRIPTS_DIR}/bf-upgrade.env/nic-fw 2>/dev/null); then + . ${SCRIPTS_DIR}/bf-upgrade.env/nic-fw fi -mount --bind /dev dev -mount --bind /sys sys -mount --bind /proc proc +if (bash -n ${SCRIPTS_DIR}/bf-upgrade.env/bmc 2>/dev/null); then + . ${SCRIPTS_DIR}/bf-upgrade.env/bmc +fi -if [ -e /etc/bf.cfg ]; then - cp /etc/bf.cfg etc/ +if [ "$UPDATE_ATF_UEFI" == "yes" ]; then + if [ -e dump-capsule-v0 ]; then + if function_exists update_atf_uefi; then + update_atf_uefi $(readlink -f dump-capsule-v0) + else + log "ERROR: BSP upgrade function does not exist" + fi + else + log "ERROR: BSP upgrade capsule dump-capsule-v0 was not found" + UPDATE_ATF_UEFI="no" + fi fi -chroot . /$install_sh +if [ "$UPDATE_BMC_FW" == "yes" ]; then + if [ -e dump-image-v0 ]; then + BMC_IMAGE=$(readlink -f dump-image-v0) + else + log "ERROR: BMC firmware dump-image-v0 was not found" + UPDATE_BMC_FW="no" + fi +fi + +if [ "$UPDATE_CEC_FW" == "yes" ]; then + if [ -e dump-upgrade-image-v0 ]; then + CEC_IMAGE=$(readlink -f dump-upgrade-image-v0) + else + log "ERROR: CEC firmware dump-upgrade-image-v0 was not found" + UPDATE_CEC_FW="no" + fi +fi + +if [ "$UPDATE_NIC_FW_GOLDEN_IMAGE" == "yes" ]; then + if [ -e dump-ramdisk-v0 ]; then + NIC_FW_GOLDEN_IMAGE=$(readlink -f dump-ramdisk-v0) + else + # log "ERROR: NIC firmware golden image dump-upgrade-image-v0 was not found" + UPDATE_NIC_FW_GOLDEN_IMAGE="no" + fi +fi + +if function_exists update_atf_uefi; then + bmc_components_update +else + log "ERROR: BMC upgrade function does not exist" +fi -umount /proc -umount /sys -umount /dev +cd ${WDIR} +/bin/rm -f dump-* -bfrshlog "Runtime upgrade finished" -sleep 3 -reboot -f +log "Runtime upgrade finished" +sleep 3 \ No newline at end of file diff --git a/src/bf-upgrade.env/atf-uefi b/src/bf-upgrade.env/atf-uefi new file mode 100644 index 0000000..30bbdd7 --- /dev/null +++ b/src/bf-upgrade.env/atf-uefi @@ -0,0 +1,19 @@ +update_atf_uefi() +{ + if function_exists pre_update_atf_uefi; then + log "INFO: Running pre_update_atf_uefi from bf.cfg" + pre_update_atf_uefi + fi + + if [ ! -d /sys/firmware/efi/efivars ]; then + mount -t efivarfs none /sys/firmware/efi/efivars + fi + + ilog "Updating ATF/UEFI:" + ilog "$(bfrec --capsule $1 2>&1)" + + if function_exists post_update_atf_uefi; then + log "INFO: Running post_update_atf_uefi from bf.cfg" + post_update_atf_uefi + fi +} diff --git a/src/bf-upgrade.env/bmc b/src/bf-upgrade.env/bmc new file mode 100644 index 0000000..af5446d --- /dev/null +++ b/src/bf-upgrade.env/bmc @@ -0,0 +1,649 @@ +BMC_IP=${BMC_IP:-"192.168.240.1"} +BMC_PORT=${BMC_PORT:-"443"} +BMC_USER=${BMC_USER:-"root"} +DEFAULT_BMC_PASSWORD="0penBmc" +TMP_BMC_PASSWORD="Nvidia_12345!" +RESET_BMC_PASSWORD=0 +BMC_PASSWORD=${BMC_PASSWORD:-""} +BMC_SSH_USER=${BMC_SSH_USER:-"$BMC_USER"} +BMC_SSH_PASSWORD=${BMC_SSH_PASSWORD:-"$BMC_PASSWORD"} +NEW_BMC_PASSWORD=${NEW_BMC_PASSWORD:-""} +UEFI_PASSWORD=${UEFI_PASSWORD:-""} +NEW_UEFI_PASSWORD=${NEW_UEFI_PASSWORD:-""} +OOB_IP=${OOB_IP:-"192.168.240.2"} +OOB_NETPREFIX=${OOB_NETPREFIX:-"29"} +BMC_IP_TIMEOUT=${BMC_IP_TIMEOUT:-600} +BMC_TASK_TIMEOUT=${BMC_TASK_TIMEOUT:-"1800"} +UPDATE_BMC_FW=${UPDATE_BMC_FW:-"yes"} +BMC_REBOOT=${BMC_REBOOT:-"no"} +CEC_REBOOT=${CEC_REBOOT:-"no"} +FIELD_MODE_SET=0 +UPDATE_CEC_FW=${UPDATE_CEC_FW:-"yes"} +BMC_INSTALLED_VERSION="" +BMC_MIN_MULTIPART_VERSION="24.04" +CEC_MIN_RESET_VERSION="00.02.0180.0000" +UPDATE_DPU_GOLDEN_IMAGE=${UPDATE_DPU_GOLDEN_IMAGE:-"yes"} +UPDATE_NIC_FW_GOLDEN_IMAGE=${UPDATE_NIC_FW_GOLDEN_IMAGE:-"yes"} + +FORCE_BMC_FW_INSTALL=${FORCE_BMC_FW_INSTALL:-"no"} +FORCE_CEC_INSTALL=${FORCE_CEC_INSTALL:-"no"} + +BMC_CREDENTIALS="'{\"username\":\"$BMC_USER\", \"password\":\"${BMC_PASSWORD}\"}'" +BMC_LINK_UP="no" +BMC_FIRMWARE_UPDATED="no" +export BMC_TOKEN="" +export task_id="" +export task_state="" +export task_status="" + +SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" +SCP="scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR" + +mlx_mkcap="/usr/lib/firmware/mellanox/boot/capsule/scripts/mlx-mkcap" + +skip_bmc() +{ + log "WARN Skipping BMC components upgrade." + RC=$((RC+1)) + UPDATE_BMC_FW="no" + UPDATE_CEC_FW="no" + UPDATE_DPU_GOLDEN_IMAGE="no" + UPDATE_NIC_FW_GOLDEN_IMAGE="no" +} + +wait_for_bmc_ip() +{ + SECONDS=0 + while ! (ping -c 3 $BMC_IP > /dev/null 2>&1) + do + sleep 10 + if [ $SECONDS -gt $BMC_IP_TIMEOUT ]; then + if ! (ping -c 3 $BMC_IP > /dev/null 2>&1); then + rlog "ERR Failed to access BMC" + ilog "- ERROR: Failed to access $BMC_IP after $SECONDS sec." + RC=$((RC+1)) + fi + fi + done + sleep 60 +} + +create_vlan() +{ + ilog "Creating VLAN 4040" + if [ ! -d "/sys/bus/platform/drivers/mlxbf_gige" ]; then + ilog "- ERROR: mlxbf_gige driver is not loaded" + RC=$((RC+1)) + return + fi + OOB_IF=$(ls -1 "/sys/bus/platform/drivers/mlxbf_gige/MLNXBF17:00/net") + ilog "Configuring VLAN id 4040 on ${OOB_IF}. This operation may take up to $BMC_IP_TIMEOUT seconds" + SECONDS=0 + while ! ip link show vlan4040 2> /dev/null | grep -w ''; do + if [ $SECONDS -gt $BMC_IP_TIMEOUT ]; then + rlog "ERR Failed to create VLAN." + ilog "- ERROR: Failed to create VLAN interface after $SECONDS sec. All the BMC related operations will be skipped." + skip_bmc + return + fi + ip link add link ${OOB_IF} name vlan4040 type vlan id 4040 + output=$(dhclient vlan4040 2>&1) + rc=$? + ilog "$output" + if [ $rc -ne 0 ]; then + ilog "dhclient failed" + ilog "Configuring static IP: ${OOB_IP}/${OOB_NETPREFIX} for vlan4040" + ip addr add ${OOB_IP}/${OOB_NETPREFIX} brd + dev vlan4040 + fi + ip link set dev ${OOB_IF} up + ip link set dev vlan4040 up + sleep 1 + done + while ! ping -c 3 $BMC_IP; do + if [ $SECONDS -gt $BMC_IP_TIMEOUT ]; then + rlog "ERR Failed to access BMC" + ilog "- ERROR: Failed to access $BMC_IP after $SECONDS sec." + skip_bmc + return + fi + sleep 1 + done + ilog "$(ip link show vlan4040)" + BMC_LINK_UP="yes" +} + +prepare_sshpass_environment() +{ + if [ ! -f /dev/pts/ptmx ]; then + echo "none /dev/pts devpts defaults 0 0" >> /etc/fstab + mount /dev/pts + fi + if [ ! -f /etc/passwd ]; then + echo "root:x:0:0:root:/root:/bin/bash" >> /etc/passwd + fi +} + +get_field_mode() +{ + mode=$(ipmitool raw 0x32 0x68 2> /dev/null | tr -d ' ') + return $mode +} + +set_field_mode() +{ + hvalue=$1 + + ilog "Setting Field Mode to $hvalue" + ilog "$(ipmitool raw 0x32 0x67 ${hvalue} 2>&1)" +} + +get_bmc_token() +{ + cmd=$(echo curl -sSk -H \"Content-Type: application/json\" -X POST https://${BMC_IP}/login -d $BMC_CREDENTIALS) + BMC_TOKEN=$(eval $cmd | jq -r ' .token') + if [[ -z "$BMC_TOKEN" || "$BMC_TOKEN" == "null" ]]; then + rlog "ERR Failed to get BMC token. Check BMC user/password" + ilog "- ERROR: Failed to get BMC token using command: $cmd. Check BMC user/password." + RC=$((RC+1)) + return 1 + fi + return 0 +} + +change_uefi_password() +{ + UEFI_CREDENTIALS="'{\"Attributes\":{\"CurrentUefiPassword\":\"$UEFI_PASSWORD\",\"UefiPassword\":\"${NEW_UEFI_PASSWORD}\"}}'" + cmd=$(echo curl -sSk -u $BMC_USER:"$BMC_PASSWORD" -H \"Content-Type: application/json\" -X PATCH https://${BMC_IP}/redfish/v1/Systems/Bluefield/Bios/Settings -d $UEFI_CREDENTIALS) + output=$(eval $cmd) + status=$(echo $output | jq '."@Message.ExtendedInfo"[0].Message') + if [ "$status" != "\"The request completed successfully."\" ]; then + rlog "ERR Failed to change UEFI password." + ilog "Failed to change UEFI password. Output: $output" + return 1 + fi + + ilog "UEFI password is set for the update. The new password will be activated on the second DPU reboot." + + return 0 +} + +change_bmc_password() +{ + current_password="$1" + new_password="$2" + + NEW_BMC_CREDENTIALS="'{\"Password\":\"${new_password}\"}'" + cmd=$(echo curl -sSk -u $BMC_USER:$current_password -H \"Content-Type: application/json\" -X PATCH https://${BMC_IP}/redfish/v1/AccountService/Accounts/$BMC_USER -d $NEW_BMC_CREDENTIALS) + output=$(eval $cmd) + status=$(echo $output | jq '."@Message.ExtendedInfo"[0].Message') + if [ "$status" != "\"The request completed successfully."\" ]; then + rlog "ERR Failed to change BMC $BMC_USER password." + ilog "Failed to change the password. Output: $output" + return 1 + fi + + ilog "BMC password updated successfully." + + return 0 +} + +get_bmc_public_key() +{ + get_bmc_token + SSH_PUBLIC_KEY=$(ssh-keyscan -t ed25519 ${OOB_IP} 2>&1 | tail -1 | cut -d ' ' -f 2-) + + pk_cmd=$(echo curl -sSk -H \"X-Auth-Token: $BMC_TOKEN\" -H \"Content-Type: application/json\" -X POST -d \'{\"RemoteServerIP\":\"${BMC_IP}\", \"RemoteServerKeyString\":\"$SSH_PUBLIC_KEY\"}\' https://$BMC_IP/redfish/v1/UpdateService/Actions/Oem/NvidiaUpdateService.PublicKeyExchange) + bmc_pk=$(eval $pk_cmd | jq -r ' .Resolution') + if [ "$bmc_pk" != "null" ]; then + echo "$bmc_pk" >> /root/.ssh/authorized_keys + fi +} + +bmc_get_task_id() +{ + get_bmc_token + task_id=$(curl -sSk -H "X-Auth-Token: $BMC_TOKEN" -X GET https://${BMC_IP}/redfish/v1/TaskService/Tasks | jq -r ' .Members' | grep odata.id | tail -1 | awk '{print $NF}' | tr -d '"') + ilog "Task id: $task_id" +} + +wait_bmc_task_complete() +{ + copy_status=$(curl -sSk -u $BMC_USER:$BMC_PASSWORD -X GET https://${BMC_IP}/redfish/v1/Chassis/Bluefield_ERoT | jq -r ' .Oem.Nvidia.BackgroundCopyStatus') + if [ "X$copy_status" != "Xnull" ]; then + if [ "$copy_status" != "Completed" ]; then + ilog "BMC background copy is: $copy_status" + fi + + SECONDS=0 + while [ "$copy_status" != "Completed" ] + do + if [ $SECONDS -gt $BMC_TASK_TIMEOUT ]; then + ilog "- ERROR: BMC copy task timeout" + RC=$((RC+1)) + break + fi + sleep 10 + copy_status=$(curl -sSk -u $BMC_USER:$BMC_PASSWORD -X GET https://${BMC_IP}/redfish/v1/Chassis/Bluefield_ERoT | jq -r ' .Oem.Nvidia.BackgroundCopyStatus') + done + fi + + bmc_get_task_id + if [ -z "${task_id}" ]; then + ilog "No active BMC task" + return + fi + output=$(mktemp) + #Check upgrade progress (%). + get_bmc_token + curl -sSk -H "X-Auth-Token: $BMC_TOKEN" https://${BMC_IP}${task_id} > $output + percent_done=$(cat $output | jq -r ' .PercentComplete') + SECONDS=0 + while [ "$percent_done" != "100" ]; do + if [ "$percent_done" == "null" ]; then + ilog "- ERROR: There is no task with task id: $task_id" + RC=$((RC+1)) + break + fi + if [ $SECONDS -gt $BMC_TASK_TIMEOUT ]; then + ilog "- ERROR: BMC task $task_id timeout" + RC=$((RC+1)) + break + fi + get_bmc_token + curl -sSk -H "X-Auth-Token: $BMC_TOKEN" https://${BMC_IP}${task_id} > $output + percent_done=$(cat $output | jq -r ' .PercentComplete') + task_state=$(cat $output | jq -r ' .TaskState') + if [ "$task_state" == "Exception" ]; then + ilog "- ERROR: BMC task $task_id exception" + RC=$((RC+1)) + break + fi + sleep 10 + done + task_state=$(jq '.TaskState' $output | tr -d '"') + task_status=$(jq '.TaskStatus' $output | tr -d '"') + + if [ "$task_state$task_status" != "CompletedOK" ]; then + echo "BMC task failed:" >> $LOG + cat $output >> $LOG + RC=$((RC+1)) + fi + /bin/rm -f $output +} + +update_bmc_fw() +{ + wait_bmc_task_complete + log "Updating BMC firmware" + #Set upload image from local BFB storage (or tempfs). + image="${BMC_IMAGE}" + if [ -z "$image" ]; then + ilog "- ERROR: Cannot find BMC firmware image" + RC=$((RC+1)) + return + fi + ilog "Found BMC firmware image: $image" + + BMC_IMAGE_VERSION="$(strings -a -t d $image | grep -m 1 BF- | cut -d '-' -f 2- | sed -e 's/ //g')" + if [ -z "$BMC_IMAGE_VERSION" ]; then + ilog "- ERROR: Cannot detect included BMC firmware version" + RC=$((RC+1)) + return + fi + ilog "Provided BMC firmware version: $BMC_IMAGE_VERSION" + + get_bmc_token + + BMC_FIRMWARE_URL=$(curl -sSk -H "X-Auth-Token: $BMC_TOKEN" -X GET https://${BMC_IP}/redfish/v1/UpdateService/FirmwareInventory | grep BMC_Firmware | awk '{print $NF}' | tr -d \") + ilog "- INFO: BMC_FIRMWARE_URL: $BMC_FIRMWARE_URL" + BMC_INSTALLED_VERSION="$(curl -sSk -H "X-Auth-Token: $BMC_TOKEN" -X GET https://${BMC_IP}${BMC_FIRMWARE_URL} | jq -r ' .Version' | grep -o "\([0-9]\+\).\([0-9]\+\)-\([0-9]\+\)")" + if [ -z "$BMC_INSTALLED_VERSION" ]; then + ilog "- ERROR: Cannot detect running BMC firmware version" + RC=$((RC+1)) + return + fi + ilog "Running BMC firmware version: $BMC_INSTALLED_VERSION" + + if [ "${BMC_IMAGE_VERSION}" == "${BMC_INSTALLED_VERSION}" ]; then + if [ "X${FORCE_BMC_FW_INSTALL}" == "Xyes" ]; then + ilog "Installed BMC version is the same as provided. FORCE_BMC_FW_INSTALL is set." + else + ilog "Installed BMC version is the same as provided. Skipping BMC firmware update." + return + fi + fi + + ilog "Proceeding with the BMC firmware update." + + if [[ $(echo -e "${BMC_MIN_MULTIPART_VERSION}\n${BMC_INSTALLED_VERSION}" | sort -V | head -n1) == "${BMC_MIN_MULTIPART_VERSION}" ]]; then + ilog "curl -sSk -u https://${BMC_IP}/redfish/v1/UpdateService/update-multipart -F 'UpdateParameters={\"ForceUpdate\":true};type=application/octet-stream' -F UpdateFile=@${image}" + output=$(curl -sSk -u $BMC_USER:$BMC_PASSWORD https://${BMC_IP}/redfish/v1/UpdateService/update-multipart -F 'UpdateParameters={"ForceUpdate":true};type=application/octet-stream' -F UpdateFile=@${image} 2>&1) + else + ilog "curl -sSk -u -H "Content-Type: application/octet-stream" -X POST -T ${image} https://${BMC_IP}/redfish/v1/UpdateService" + output=$(curl -sSk -u $BMC_USER:$BMC_PASSWORD -H "Content-Type: application/octet-stream" -X POST -T ${image} https://${BMC_IP}/redfish/v1/UpdateService 2>&1) + fi + ilog "BMC Firmware update: $output" + + BMC_FIRMWARE_UPDATED="yes" + + wait_bmc_task_complete + if [ "$BMC_REBOOT" != "yes" ]; then + log "INFO: BMC firmware was updated to: ${BMC_IMAGE_VERSION}. BMC restart is required." + fi +} + +update_cec_fw() +{ + wait_bmc_task_complete + log "Updating CEC firmware" + image="${CEC_IMAGE}" + + if [ -z "$image" ]; then + ilog "- ERROR: Cannot find CEC firmware image" + RC=$((RC+1)) + return + fi + ilog "Found CEC firmware image: $image" + + CEC_IMAGE_VERSION="$(strings -a -t d $image | grep -m 1 cec | cut -d '-' -f 2- | sed -e 's/ //g;s/.n02//')" + if [ -z "$CEC_IMAGE_VERSION" ]; then + # BlueField-2 CEC version format + CEC_IMAGE_VERSION_HEXA="$(echo $image | grep -o '\-\(.*\)_' | grep -o '\([0-9a-fA-F]\+\).\([0-9a-fA-F]\+\)')" + CEC_IMAGE_VERSION=$(printf "%d" 0x${CEC_IMAGE_VERSION_HEXA%*.*}).$(printf "%d" 0x${CEC_IMAGE_VERSION_HEXA##*.}) + fi + if [ -z "$CEC_IMAGE_VERSION" ]; then + ilog "- ERROR: Cannot detect included CEC firmware version" + RC=$((RC+1)) + return + fi + ilog "Provided CEC firmware version: $CEC_IMAGE_VERSION" + + get_bmc_token + + CEC_INSTALLED_VERSION="$(curl -sSk -H "X-Auth-Token: $BMC_TOKEN" -X GET https://${BMC_IP}/redfish/v1/UpdateService/FirmwareInventory/Bluefield_FW_ERoT | jq -r ' .Version' | sed -e "s/[_|-]/./g;s/.n02//")" + if [ -z "$CEC_INSTALLED_VERSION" ]; then + ilog "- ERROR: Cannot detect running CEC firmware version" + RC=$((RC+1)) + return + fi + ilog "Running CEC firmware version: $CEC_INSTALLED_VERSION" + + if [ "${CEC_IMAGE_VERSION}" == "${CEC_INSTALLED_VERSION}" ]; then + if [ "X${FORCE_CEC_INSTALL}" == "Xyes" ]; then + ilog "Installed CEC version is the same as provided. FORCE_CEC_INSTALL is set." + else + ilog "Installed CEC version is the same as provided. Skipping CEC firmware update." + return + fi + fi + + ilog "Proceeding with the CEC firmware update..." + + if [ -z "$BMC_INSTALLED_VERSION" ]; then + BMC_INSTALLED_VERSION="$(curl -sSk -H "X-Auth-Token: $BMC_TOKEN" -X GET https://${BMC_IP}${BMC_FIRMWARE_URL} | jq -r ' .Version' | grep -o "\([0-9]\+\).\([0-9]\+\)-\([0-9]\+\)" | tr -s '-' '.')" + if [ -z "$BMC_INSTALLED_VERSION" ]; then + ilog "- ERROR: Cannot detect running BMC firmware version" + RC=$((RC+1)) + return + fi + fi + + if [[ $(echo -e "${BMC_MIN_MULTIPART_VERSION}\n${BMC_INSTALLED_VERSION}" | sort -V | head -n1) == "${BMC_MIN_MULTIPART_VERSION}" ]]; then + ilog "curl -sSk -u https://${BMC_IP}/redfish/v1/UpdateService/update-multipart -F 'UpdateParameters={\"ForceUpdate\":true};type=application/octet-stream' -F UpdateFile=@${image}" + output=$(curl -sSk -u $BMC_USER:$BMC_PASSWORD https://${BMC_IP}/redfish/v1/UpdateService/update-multipart -F 'UpdateParameters={"ForceUpdate":true};type=application/octet-stream' -F UpdateFile=@${image} 2>&1) + else + ilog "curl -sSk -u -H "Content-Type: application/octet-stream" -X POST -T ${image} https://${BMC_IP}/redfish/v1/UpdateService" + output=$(curl -sSk -u $BMC_USER:$BMC_PASSWORD -H "Content-Type: application/octet-stream" -X POST -T ${image} https://${BMC_IP}/redfish/v1/UpdateService 2>&1) + fi + ilog "CEC Firmware update: $output" + + wait_bmc_task_complete + if [ "$CEC_REBOOT" == "yes" ]; then + if [[ $(echo -e "${CEC_MIN_RESET_VERSION}\n${CEC_INSTALLED_VERSION}" | sort -V | head -n1) == "${CEC_MIN_RESET_VERSION}" ]]; then + log "Rebooting CEC..." + output=$(curl -sSk -u $BMC_USER:"$BMC_PASSWORD" -H "Content-Type: application/json" -X POST -d '{"ResetType": "GracefulRestart"}' https://${BMC_IP}/redfish/v1/Chassis/Bluefield_ERoT/Actions/Chassis.Reset) + status=$(echo $output | jq '."@Message.ExtendedInfo"[0].Message') + if [ "$status" == "\"The request completed successfully."\" ]; then + log "INFO: CEC firmware was updated to ${CEC_IMAGE_VERSION}." + else + rlog "ERR Failed to reset CEC" + ilog "Failed to reset CEC. Output: $output" + log "INFO: CEC firmware was updated to ${CEC_IMAGE_VERSION}. Host power cycle is required" + fi + else + log "INFO: CEC firmware was updated to ${CEC_IMAGE_VERSION}. Host power cycle is required" + fi + fi +} + +bmc_reboot() +{ + log "Rebooting BMC..." + get_bmc_token + curl -sSk -H "X-Auth-Token: $BMC_TOKEN" -H "Content-Type: application/json" -X POST https://${BMC_IP}/redfish/v1/Managers/Bluefield_BMC/Actions/Manager.Reset -d '{"ResetType":"GracefulRestart"}' + sleep 10 + wait_for_bmc_ip +} + +bmc_reboot_from_dpu() +{ + ipmitool mc reset cold + sleep 10 +} + +update_dpu_golden_image() +{ + log "Updating DPU Golden Image" + image=${DPU_GOLDEN_IMAGE} + + if [ -z "$image" ]; then + ilog "DPU golden image was not found" + RC=$((RC+rc)) + return + fi + + prepare_sshpass_environment + + ilog "Found DPU Golden Image: $image" + DPU_GI_IMAGE_VERSION="$(sha256sum $image | awk '{print $1}')" + ilog "Provided DPU Golden Image version: $DPU_GI_IMAGE_VERSION" + + DPU_GI_INSTALLED_VERSION="$(sshpass -p $BMC_SSH_PASSWORD $SSH ${BMC_SSH_USER}@${BMC_IP} dpu_golden_image golden_image_arm -V 2> /dev/null)" + ilog "Installed DPU Golden Image version: $DPU_GI_INSTALLED_VERSION" + + if [ "$DPU_GI_IMAGE_VERSION" == "$DPU_GI_INSTALLED_VERSION" ]; then + ilog "Installed DPU Golden Image version is the same as provided. Skipping DPU Golden Image update." + else + sshpass -p $BMC_SSH_PASSWORD $SCP $image ${BMC_SSH_USER}@${BMC_IP}:/tmp/ + output=$(sshpass -p $BMC_SSH_PASSWORD $SSH ${BMC_SSH_USER}@${BMC_IP} dpu_golden_image golden_image_arm -w /tmp/$(basename $image) 2>&1) + if [ $? -eq 0 ]; then + log "DPU Golden Image installed successfully" + else + log "DPU Golden Image installed failed" + fi + ilog "$output" + fi +} + +update_nic_firmware_golden_image() +{ + log "Updating NIC firmware Golden Image" + image=${NIC_FW_GOLDEN_IMAGE} + + if [ -z "$image" ]; then + ilog "NIC firmware Golden Image for $dpu_part_number was not found" + RC=$((RC+rc)) + return + fi + + prepare_sshpass_environment + + ilog "Found NIC firmware Golden Image: $image" + NIC_GI_IMAGE_VERSION="$(sha256sum $image | awk '{print $1}')" + ilog "Provided NIC firmware Golden Image version: $NIC_GI_IMAGE_VERSION" + + NIC_GI_INSTALLED_VERSION="$(sshpass -p $BMC_SSH_PASSWORD $SSH ${BMC_SSH_USER}@${BMC_IP} dpu_golden_image golden_image_nic -V 2> /dev/null)" + ilog "Installed NIC firmware Golden Image version: $NIC_GI_INSTALLED_VERSION" + + if [ "$NIC_GI_IMAGE_VERSION" == "$NIC_GI_INSTALLED_VERSION" ]; then + ilog "Installed NIC firmware Golden Image version is the same as provided. Skipping NIC firmware Golden Image update." + return + fi + + sshpass -p $BMC_SSH_PASSWORD $SCP $image ${BMC_SSH_USER}@${BMC_IP}:/tmp/ + output=$(sshpass -p $BMC_SSH_PASSWORD $SSH ${BMC_SSH_USER}@${BMC_IP} dpu_golden_image golden_image_nic -w /tmp/$(basename $image) 2>&1) + if [ $? -eq 0 ]; then + log "NIC firmware Golden Image installed successfully" + else + log "NIC firmware Golden Image installed failed" + fi + ilog "$output" +} + +bmc_components_update() +{ + if function_exists pre_bmc_components_update; then + log "INFO: Running pre_bmc_components_update from bf.cfg" + pre_bmc_components_update + fi + + if [[ ! -z "$UEFI_PASSWORD" && ! -z "$NEW_UEFI_PASSWORD" ]]; then + if [[ -z "$BMC_USER" || -z "$BMC_PASSWORD" ]]; then + ilog "BMC_USER and/or BMC_PASSWORD are not defined. Skipping UEFI password change." + else + change_uefi_password + fi + fi + + if [[ "$UPDATE_BMC_FW" == "yes" || "$UPDATE_CEC_FW" == "yes" || "$UPDATE_DPU_GOLDEN_IMAGE" == "yes" || "$UPDATE_NIC_FW_GOLDEN_IMAGE" == "yes" || ! -z "$NEW_BMC_PASSWORD" ]]; then + if [[ -z "$BMC_USER" || -z "$BMC_PASSWORD" ]]; then + ilog "BMC_USER and/or BMC_PASSWORD are not defined. Skipping BMC components upgrade." + skip_bmc + return + else + if [ "$(get_field_mode)" == "01" ]; then + set_field_mode '00' + FIELD_MODE_SET=1 + bmc_reboot_from_dpu + fi + ilog "INFO: Running BMC components update flow" + create_vlan + # get_bmc_public_key + if [ "$BMC_LINK_UP" == "yes" ]; then + if [ ! -z "$NEW_BMC_PASSWORD" ]; then + if change_bmc_password "$BMC_PASSWORD" "$NEW_BMC_PASSWORD"; then + if [ "$BMC_PASSWORD" == "$BMC_SSH_PASSWORD" ]; then + BMC_SSH_PASSWORD="$NEW_BMC_PASSWORD" + fi + BMC_PASSWORD="$NEW_BMC_PASSWORD" + else + skip_bmc + return + fi + elif [ "$BMC_PASSWORD" == "$DEFAULT_BMC_PASSWORD" ]; then + ilog "BMC password has the default value. Changing to the temporary password." + if change_bmc_password "$BMC_PASSWORD" "$TMP_BMC_PASSWORD"; then + BMC_PASSWORD="$TMP_BMC_PASSWORD" + else + skip_bmc + return + fi + RESET_BMC_PASSWORD=1 + fi + BMC_CREDENTIALS="'{\"username\":\"$BMC_USER\", \"password\":\"${BMC_PASSWORD}\"}'" + else + skip_bmc + return + fi + fi + + if ! get_bmc_token; then + skip_bmc + return + fi + + else + return + fi + + if function_exists bmc_custom_action1; then + log "INFO: Running bmc_custom_action1 from bf.cfg" + bmc_custom_action1 + fi + + if [[ "$UPDATE_BMC_FW" == "yes" && "$BMC_LINK_UP" == "yes" ]]; then + update_bmc_fw + fi + + if function_exists bmc_custom_action2; then + log "INFO: Running bmc_custom_action2 from bf.cfg" + bmc_custom_action2 + fi + + if [[ "$UPDATE_CEC_FW" == "yes" && "$BMC_LINK_UP" == "yes" ]]; then + update_cec_fw + fi + + if function_exists bmc_custom_action3; then + log "INFO: Running bmc_custom_action3 from bf.cfg" + bmc_custom_action3 + fi + + if [[ "$UPDATE_BMC_FW" == "yes" && "$BMC_LINK_UP" == "yes" && "$BMC_REBOOT" == "yes" && "$BMC_FIRMWARE_UPDATED" == "yes" ]]; then + bmc_reboot + fi + + if function_exists bmc_custom_action4; then + log "INFO: Running bmc_custom_action4 from bf.cfg" + bmc_custom_action4 + fi + + if [[ "$UPDATE_DPU_GOLDEN_IMAGE" == "yes" && "$BMC_LINK_UP" == "yes" ]]; then + update_dpu_golden_image + fi + + if function_exists bmc_post_dpu_gi; then + log "INFO: Running bmc_post_dpu_gi from bf.cfg" + bmc_post_dpu_gi + fi + + if [[ "$UPDATE_NIC_FW_GOLDEN_IMAGE" == "yes" && "$BMC_LINK_UP" == "yes" ]]; then + if [ -z "${dpu_part_number}" ]; then + log "Cannot identify DPU Part Number. Skipping NIC firmware Golden Image update." + else + update_nic_firmware_golden_image + fi + fi + + if function_exists bmc_post_nic_fw_gi; then + log "INFO: Running bmc_post_nic_fw_gi from bf.cfg" + bmc_post_nic_fw_gi + fi + + if [ "X${BMC_REBOOT}" == "Xyes" ]; then + bmc_reboot + fi + + if [ $RESET_BMC_PASSWORD -eq 1 ]; then + ilog "Reset BMC configuration to default" + output=$(curl -sSk -u $BMC_USER:"$BMC_PASSWORD" -H "Content-Type: application/json" -X POST https://${BMC_IP}/redfish/v1/Managers/Bluefield_BMC/Actions/Manager.ResetToDefaults -d '{"ResetToDefaultsType": "ResetAll"}') + status=$(echo $output | jq '."@Message.ExtendedInfo"[0].Message') + if [ "$status" != "\"The request completed successfully."\" ]; then + rlog "ERR Failed to reset BMC $BMC_USER password." + ilog "Failed to reset BMC $BMC_USER password. Output: $output" + fi + fi + + if [ $FIELD_MODE_SET -eq 1 ]; then + if [ "$(get_field_mode)" == "00" ]; then + set_field_mode '01' + FIELD_MODE_SET=0 + bmc_reboot_from_dpu + fi + fi + + if function_exists post_bmc_components_update; then + log "INFO: Running post_bmc_components_update from bf.cfg" + post_bmc_components_update + fi +} diff --git a/src/bf-upgrade.env/common b/src/bf-upgrade.env/common new file mode 100644 index 0000000..f1da1c0 --- /dev/null +++ b/src/bf-upgrade.env/common @@ -0,0 +1,52 @@ +rshimlog=$(which bfrshlog 2> /dev/null) +RC=0 +err_msg="" +export LC_ALL=C + +logfile=pldm.installation.log +LOG=/var/log/$logfile + +fspath=$(readlink -f "$(dirname $0)") + +cx_pcidev=$(lspci -nD 2> /dev/null | grep 15b3:a2d[26c] | awk '{print $1}' | head -1) +cx_dev_id=$(lspci -nD -s ${cx_pcidev} 2> /dev/null | awk -F ':' '{print strtonum("0x" $NF)}') +pciids=$(lspci -nD 2> /dev/null | grep 15b3:a2d[26c] | awk '{print $1}') +dpu_part_number=$(mstflint -d $cx_pcidev q full | grep "Part Number:" | awk '{print $NF}') +PSID=$(mstflint -d $cx_pcidev q | grep PSID | awk '{print $NF}') + +rlog() +{ + msg=$(echo "$*" | sed 's/INFO://;s/ERROR:/ERR/;s/WARNING:/WARN/') + if [ -n "$rshimlog" ]; then + $rshimlog "$msg" + fi +} + +ilog() +{ + msg="[$(date +%H:%M:%S)] $*" + echo "$msg" >> $LOG + echo "$msg" > /dev/ttyAMA0 + echo "$msg" > /dev/hvc0 +} + +log() +{ + ilog "$*" + rlog "$*" +} + +save_log() +{ + for pw in $(grep "PASSWORD=" $LOG | cut -d '=' -f 2- | sed 's/["'\'']//'g) + do + sed -i -e "s,$pw,xxxxxx,g" $LOG + done + sync +} + +function_exists() +{ + declare -f -F "$1" > /dev/null + return $? +} \ No newline at end of file diff --git a/src/bf-upgrade.env/nic-fw b/src/bf-upgrade.env/nic-fw new file mode 100644 index 0000000..4257a15 --- /dev/null +++ b/src/bf-upgrade.env/nic-fw @@ -0,0 +1,150 @@ +NIC_FW_UPDATE_DONE=0 +FORCE_NIC_FW_UPDATE=${FORCE_NIC_FW_UPDATE:-"no"} +NIC_FW_RESET_REQUIRED=0 +BFB_NIC_FW_UPDATE=0 +NIC_FW_FOUND=0 +FW_UPDATER=/opt/mellanox/mlnx-fw-updater/mlnx_fw_updater.pl +FW_DIR=/opt/mellanox/mlnx-fw-updater/firmware/ +is_nic_mode=${is_nic_mode:-0} + +CHROOT= + +running_nic_fw() +{ + mstflint -d $cx_pcidev q 2>&1 | grep -w 'FW Version:' | awk '{print $NF}' +} + +provided_nic_fw() +{ + if [ ! -z "${PSID}" ]; then + $CHROOT ${FW_DIR}/mlxfwmanager_sriov_dis_aarch64_${cx_dev_id} --list 2> /dev/null | grep -w "${PSID}" | awk '{print $4}' + fi +} + +fw_update() +{ + if [[ -x ${FW_UPDATER} && -d ${FW_DIR} ]]; then + BFB_NIC_FW_UPDATE=1 + NIC_FW_FOUND=1 + else + if [ ! -d /mnt/root ]; then + mount -t $ROOTFS /dev/${root_device} /mnt + bind_partitions + fi + + if [[ -x /mnt/${FW_UPDATER} && -d /mnt/${FW_DIR} ]]; then + CHROOT="chroot /mnt" + NIC_FW_FOUND=1 + fi + fi + + if [ $NIC_FW_FOUND -eq 1 ]; then + if [ "$(running_nic_fw)" == "$(provided_nic_fw)" ]; then + if [ "${FORCE_NIC_FW_UPDATE}" == "yes" ]; then + log "INFO: Installed NIC Firmware is the same as provided. FORCE_NIC_FW_UPDATE is set." + else + log "INFO: Installed NIC Firmware is the same as provided. Skipping NIC Firmware update." + return + fi + fi + + log "INFO: Updating NIC firmware..." + $CHROOT ${FW_UPDATER} --log /tmp/mlnx_fw_update.log -v \ + --force-fw-update \ + --fw-dir ${FW_DIR} > /tmp/mlnx_fw_update.out 2>&1 + rc=$? + sync + if [ -e /tmp/mlnx_fw_update.out ]; then + cat /tmp/mlnx_fw_update.out > /dev/hvc0 + cat /tmp/mlnx_fw_update.out > /dev/ttyAMA0 + cat /tmp/mlnx_fw_update.out >> $LOG + fi + if [ -e /tmp/mlnx_fw_update.log ]; then + cat /tmp/mlnx_fw_update.log > /dev/hvc0 + cat /tmp/mlnx_fw_update.log > /dev/ttyAMA0 + cat /tmp/mlnx_fw_update.log >> $LOG + fi + if [ $rc -ne 0 ] || (grep -q '\-E- Failed' /tmp/mlnx_fw_update.log); then + NIC_FW_UPDATE_PASSED=0 + log "INFO: NIC firmware update failed" + else + NIC_FW_UPDATE_PASSED=1 + log "INFO: NIC firmware update done: $(provided_nic_fw)" + fi + NIC_FW_UPDATE_DONE=1 + else + log "WARNING: NIC Firmware files were not found" + fi +} + +fw_reset() +{ + if [ $is_nic_mode -eq 1 ]; then + log "Run mlxfwreset or reboot from the Host to load new NIC firmware" + return + fi + + /sbin/modprobe -a mlx5_core ib_umad + run_mlnx_bf_configure=0 + if [ -x /mnt/sbin/mlnx_bf_configure ]; then + CHROOT="chroot /mnt" + run_mlnx_bf_configure=1 + elif [ -x /sbin/mlnx_bf_configure ]; then + run_mlnx_bf_configure=1 + else + run_mlnx_bf_configure=0 + fi + + if [ $run_mlnx_bf_configure -eq 1 ]; then + ilog "Running mlnx_bf_configure:" + ilog "$($CHROOT /sbin/mlnx_bf_configure 2>&1)" + fi + + MLXFWRESET_TIMEOUT=${MLXFWRESET_TIMEOUT:-180} + SECONDS=0 + while ! ($CHROOT mlxfwreset -d /dev/mst/mt*_pciconf0 q 2>&1 | grep -w "Driver is the owner" | grep -qw "\-Supported") + do + if [ $SECONDS -gt $MLXFWRESET_TIMEOUT ]; then + log "INFO: NIC Firmware reset is not supported. Host power cycle is required" + return + fi + sleep 1 + done + + log "INFO: Running NIC Firmware reset" + save_log + if [ "X$mode" == "Xmanufacturing" ]; then + log "INFO: Rebooting..." + fi + # Wait for these messages to be pulled by the rshim service + # as mlxfwreset will restart the DPU + sleep 3 + + msg=$($CHROOT mlxfwreset -d /dev/mst/mt*_pciconf0 -y -l 3 --sync 1 r 2>&1) + if [ $? -ne 0 ]; then + log "INFO: NIC Firmware reset failed" + log "INFO: $msg" + else + log "INFO: NIC Firmware reset done" + fi +} + +update_nic_firmware() +{ + if [ $NIC_FW_UPDATE_DONE -eq 0 ]; then + fw_update + fi +} + +reset_nic_firmware() +{ + if [ $NIC_FW_UPDATE_DONE -eq 1 ]; then + if [ $NIC_FW_UPDATE_PASSED -eq 1 ]; then + # Reset NIC FW + mount -t ${ROOTFS} ${ROOT_PARTITION} /mnt + bind_partitions + fw_reset + unmount_partitions + fi + fi +}