Merge branch 'main' into vg-3926861

.ci/Jenkinsfile_nbuprod

@@ -0,0 +1,8 @@
+#!/usr/bin/groovy
+
+// load pipeline functions
+// Requires pipeline-github-lib plugin to load library from github
+@Library('github.com/Mellanox/ci-demo@master')
+def matrix = new com.mellanox.cicd.Matrix()
+
+matrix.main()

.ci/Makefile

@@ -0,0 +1,8 @@
+
+jjb-blsm-release:
+	# IMPORTANT: configure the /etc/jenkins_jobs/jenkins_jobs.ini file to point to the relevant blossom
+	python3 -m jenkins_jobs --conf /etc/jenkins_jobs/jenkins_jobs.ini update $(shell pwd)/pipeline/proj_jjb_release_nbuprod.yaml
+
+jjb-blsm-release-test:
+	# IMPORTANT: configure the /etc/jenkins_jobs/jenkins_jobs.ini file to point to the relevant blossom
+	python3 -m jenkins_jobs --conf /etc/jenkins_jobs/jenkins_jobs.ini test $(shell pwd)/pipeline/proj_jjb_release_nbuprod.yaml

.ci/dockerhub_uploader.py

@@ -60,6 +60,7 @@ def __init__(self, username:str, password:str) -> None:
         self.image = ""
         self.tag = ""
         self.image_path = ""
+        self.meta_data = None
 
     def login(self) -> None:
         """Login to docker-hub used for validation of image tags"""
@@ -80,24 +81,22 @@ def login(self) -> None:
     def parse_image_tags(self):
         """Helper function for parsing image name and tag from manifest"""
         # If image name and tag were passed to the object don't parse it from the metadata
-        if self.image and self.tag:
+        if self.image:
             if '/' in self.image:
                 self.repository = '/'.join(self.image.split('/')[:-1])
                 self.image = self.image.split('/')[-1]
-            return
-        meta_data = None
+
         if not tarfile.is_tarfile(self.image_path):
             log_error(f'{self.image_path} is not a valid tar file')
         with tarfile.open(self.image_path, 'r') as t:
             # get the image manifest to determine image name and tag
-            meta_data = json.load(t.extractfile(r'manifest.json'))
-        if not meta_data:
+            self.meta_data = json.load(t.extractfile(r'manifest.json'))
+        if not self.meta_data:
             log_error(f"Could not extract manifest.json from {self.image_path}")
         try:
             # take the first tag of the first image, there should be only one anyway
-            image_name = meta_data[0]['RepoTags'][0]
-            # validate repo in image metadata
-            if '/' in image_name:
+            image_name = self.meta_data[0]['RepoTags'][0]
+            if '/' in image_name and not self.image:
                 self.repository = '/'.join(image_name.split('/')[:-1])
                 image_name = image_name.split('/')[-1]
             if not self.image:
@@ -143,61 +142,91 @@ def check_image_tag_docker_hub(self, tag=None) -> bool:
                 break
         return False
 
-    def _check_image_locally(self, tag=None) -> bool:
+    def _check_image_locally(self,repo=None, image=None, tag=None) -> bool:
         """helper method to check if image already exists in local docker"""
         if not tag:
             tag = self.tag
-        log_debug(f'starting DockerHubUploader._check_image_locally with image:"{self.image}",tag:"{tag}"')
+        if not image:
+            image = self.image
+        if not repo:
+            repo = self.repository
+        log_debug(f'starting DockerHubUploader._check_image_locally with image:"{repo}/{image}",tag:"{tag}"')
         for img in self.client.images.list():
             for img_name in  img.attrs['RepoTags']:
-                if img_name == f"{self.repository}/{self.image}:{tag}":
+                if img_name == f"{repo}/{image}:{tag}":
                     return True
         return False
 
-    def _delete_image_locally(self, tag=None) -> None:
+    def _delete_image_locally(self,repo = None, image=None, tag=None) -> None:
         """Helper method to remove images from local docker"""
-        log_debug(f'starting DockerHubUploader._delete_image_locally with image:"{self.image}",tag:"{tag}"')
+        log_debug(f'starting DockerHubUploader._delete_image_locally with repo="{repo}", image:"{image}",tag:"{tag}"')
+        if not repo:
+            repo = self.repository
+        if not image:
+            image = self.image
         if not tag:
             tag = self.tag
         try:
-            self.client.images.remove(image=f'{self.repository}/{self.image}:{tag}', force=True)
+            self.client.images.remove(image=f'{repo}/{image}:{tag}', force=True)
         except Exception as e:
-            log_error(f"could not remove {self.repository}/{self.image}:{tag}\n    {e}")
+            log_error(f"could not remove {repo}/{image}:{tag}\n    {e}")
 
     def load_docker_img(self):
         """Load a docker file to docker engine"""
         log_debug(f'starting DockerHubUploader.load_docker_img with image:"{self.image}",tag:"{self.tag}", path:"{self.image_path}"')
         # Check if the image:tag already exists if so try to delete it.
-        if self._check_image_locally():
+        if self._check_image_locally(repo=self.repository, image=self.image, tag=self.tag):
             log_warning(f'{self.image}:{self.tag} was found locally removing it from docker engine.')
             try:
-                self._delete_image_locally()
+                self._delete_image_locally(repo=self.repository, image=self.image, tag=self.tag)
             except Exception as e:
                 log_error(f'Could not delete {self.image}:{self.tag} from local docker.\n    {e}')
+        # check if meta_data name and tag are different from given image and tag
+        meta_repo = '/'.join(self.meta_data[0]['RepoTags'][0].split('/')[:-1])
+        meta_image = self.meta_data[0]['RepoTags'][0].split('/')[-1].split(':')[0]
+        meta_tag = self.meta_data[0]['RepoTags'][0].split(':')[-1]
+        # check if need to re-tag the image
+        new_tag_required = f'{meta_repo}/{meta_image}:{meta_tag}' != f'{self.repository}/{self.image}:{self.tag}'
+        if new_tag_required and self._check_image_locally(repo=meta_repo, image=meta_image, tag=meta_tag):
+            log_warning(f'new_tag_required={new_tag_required} and {meta_repo}/{meta_image}:{meta_tag} was found locally removing it from docker engine.')
+            try:
+                self._delete_image_locally(repo=meta_repo, image=meta_image, tag=meta_tag)
+            except Exception as e:
+                log_error(f'Could not delete {meta_repo}/{meta_image}:{meta_tag} from local docker.\n    {e}')
         if not os.path.exists(self.image_path):
             log_error(f'{self.image_path} doesn\'t exists.')
         with open(self.image_path, 'rb') as img:
             try:
                 _ = self.client.images.load(img)
                 log_info(f'Loaded {self.repository}/{self.image}:{self.tag} to local docker engine.')
+                # re-tag the image if needed
+                if new_tag_required:
+                    self.tag_image(new_tag=self.tag, old_image=meta_image, old_repo=meta_repo, old_tag=meta_tag)
+                    self.cleanup(repo=meta_repo, image=meta_image,tag=meta_tag)
             except Exception as e:
                 log_error(f'Could not load the {self.repository}/{self.image}:{self.tag} to docker.\n    {e}')
                 # tag as latest if latest flag exists
 
-    def tag_image(self, new_tag):
+    def tag_image(self, new_tag, old_repo=None, old_image=None, old_tag=None):
         """Load a docker file to docker engine"""
-        log_debug(f'starting DockerHubUploader.tag_image with image:"{self.image}",current_tag:"{self.tag}", new_tag:"{new_tag}"')
+        if not old_repo:
+            old_repo = self.repository
+        if not old_image:
+            old_image = self.image
+        if not old_tag:
+            old_tag = self.tag
+        log_debug(f'starting DockerHubUploader.tag_image with image:"{old_repo}/{old_image}",current_tag:"{old_tag}", new_tag:"{new_tag}"')
          # check if latest exists locally
-        if self._check_image_locally(new_tag):
+        if self._check_image_locally(repo=self.repository, image=self.image, tag=new_tag):
             log_warning(f'{self.repository}/{self.image}:{new_tag} was found locally removing it from docker engine.')
             try:
-                self._delete_image_locally(self.image, new_tag)
+                self._delete_image_locally(repo=self.repository, image=self.image, tag=new_tag)
             except Exception as e:
                 log_error(f'Could not delete {self.repository}/{self.image}:{new_tag} from local docker.\n    {e}')
         try:
-            img = self.client.images.get(f'{self.repository}/{self.image}:{self.tag}')
+            img = self.client.images.get(f'{old_repo}/{old_image}:{old_tag}')
             img.tag(f'{self.repository}/{self.image}', new_tag)
-            log_info(f'Tagged {self.repository}/{self.image}:{self.tag} as {self.repository}/{self.image}:{new_tag}.')
+            log_info(f'Tagged {old_repo}/{old_image}:{self.tag} as {self.repository}/{self.image}:{new_tag}.')
         except Exception as e:
             log_error(f'Could not tag {self.repository}/{self.image}:{self.tag} as {self.repository}/{self.image}:{new_tag}\n    {e}')
 
@@ -208,7 +237,7 @@ def push_image(self,tag=None) -> None:
         log_debug(f'starting DockerHubUploader.push_image with image:"{self.image}"tag:"{tag}"')
         try:
             result = self.client.images.push(repository=f'{self.repository}/{self.image}', tag=tag,
-                                              auth_config={'username': self.username, 'password': self.password},stream=True)
+                                             auth_config={'username': self.username, 'password': self.password},stream=True)
             for line in result:
                 if 'error' in json.loads(line).keys():
                     log_error(f'Could not push {self.repository}/{self.image}:{tag}\nError from Docker:{line.decode("utf-8")}"')
@@ -217,10 +246,16 @@ def push_image(self,tag=None) -> None:
         except Exception as e:
             log_error(f"Could not push {self.repository}/{self.image}:{tag} to Docker-Hub")
 
-    def cleanup(self, tag=None):
+    def cleanup(self,repo=None, image=None, tag=None):
         """Cleanup function to remove loaded docker images from host"""
-        if self._check_image_locally(tag=tag):
-            self._delete_image_locally(tag=tag)
+        if not repo:
+            repo = self.repository
+        if not image:
+            image = self.image
+        if not tag:
+            tag = self.tag
+        if self._check_image_locally(repo=repo, image=image, tag=tag):
+            self._delete_image_locally(repo=repo, image=image, tag=tag)
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(prog='dockerhub_uploader')
@@ -257,7 +292,7 @@ def cleanup(self, tag=None):
         dhu.load_docker_img()
         dhu.push_image()
     if args.latest:
-        dhu.tag_image('latest')
+        dhu.tag_image(new_tag='latest')
         dhu.push_image('latest')
         dhu.cleanup('latest')
     dhu.cleanup()

.ci/jenkinsfile_hub_uploader.groovy

@@ -18,7 +18,7 @@ pipeline{
                 withCredentials([usernamePassword(credentialsId: '0fbf63c0-4a61-4543-811d-a182df47711b', usernameVariable: 'DH_USER', passwordVariable: 'DH_TOKEN' )]){
                     wrap([$class: 'BuildUser']) {
                         sh '''#!/bin/bash
-                        authorized_users=( "bitkin" "afok" "kobib" "drorl" "tlerner" "omarj" "samerd" "atolikin" "atabachnik" "eylonk" "lennyv" )
+                        authorized_users=( "bitkin" "afok" "kobib" "drorl" "tlerner" "omarj" "samerd" "atolikin" "atabachnik" "eylonk" "lennyv" "asafb" "sspormas" "mkianovsky")
                         if [[ ! "${authorized_users[*]}" == *"${BUILD_USER_ID}"* ]]; then
                             echo "${BUILD_USER_ID} not authorized to upload images to docker hub"
                             echo "Please contact one of the approved users to upload a container: ${authorized_users[*]}"
@@ -46,4 +46,4 @@ pipeline{
             }
         }
     }
-}
+}

.ci/matrix_job_release_nbuprod.yaml

@@ -0,0 +1,36 @@
+---
+job: ufm-plugins
+
+registry_host: harbor.mellanox.com
+registry_path: /swx-storage/ci-demo
+registry_auth: swx-storage
+step_allow_single_selector: true
+
+credentials:
+  # harbor login/password
+  - {credentialsId: '425bb907-c357-4fde-92e0-67854a857b4f', usernameVariable: 'UFM_USER', passwordVariable: 'UFM_PASS'}
+
+runs_on_agents:
+  - nodeLabel: 'SWX-CI-DOCKER'
+
+steps:
+  - name: Build Plugin
+    agentSelector: "{nodeLabel: 'SWX-CI-DOCKER'}"
+    run: |
+      set -x
+      if [ -e "/auto/mswg/release/ufm/plugins/${PLUGIN_NAME}/ufm-plugin-${PLUGIN_NAME}_${PLUGIN_VERSION}*" ];then 
+        echo -e "A path and plugin with this version already exist."
+        echo -e "Path: /auto/mswg/release/ufm/plugins/${PLUGIN_NAME}/ufm-plugin-${PLUGIN_NAME}_${PLUGIN_VERSION}*"
+        exit 1
+      fi
+      mkdir -p /auto/mswg/release/ufm/plugins/${PLUGIN_NAME}
+      cd plugins/${PLUGIN_NAME}/build
+      bash -x ./docker_build.sh ${PLUGIN_VERSION} /auto/mswg/release/ufm/plugins/${PLUGIN_NAME} 
+      BUILD_EXIT_CODE=$?
+      ls /auto/mswg/release/ufm/plugins/${PLUGIN_NAME}
+      if [ $BUILD_EXIT_CODE -eq 1 ];then 
+        echo -e "Error: Docker build failed in ./docker_build.sh"
+      fi
+      exit ${BUILD_EXIT_CODE}
+
+    parallel: false

.ci/pipeline/proj_jjb_release_nbuprod.yaml

@@ -0,0 +1,79 @@
+########## UFM APPLIANCE GEN3 jobs ##################
+- project:
+    name: UFM_PLUGINS_SDK_RELEASE
+    jjb_owner: 'Mickey Kianosvky'
+    jobs:
+        - "UFM_PLUGINS_SDK_RELEASE":
+            jjb_git: git@github.com:Mellanox/ufm_sdk_3.0.git
+            jjb_email: 'mkianovsky@nvidia.com'
+            disabled_var: false
+            type: github
+            branch: main
+
+- job-template:
+    name: "UFM_PLUGINS_SDK_RELEASE"
+    project-type: pipeline
+    disabled: '{obj:disabled_var}'
+    properties:
+        - github:
+             url: "{jjb_git}"
+        - build-discarder:
+            days-to-keep: 15
+            num-to-keep: 15
+        - inject:
+            keep-system-variables: true
+            properties-content: |
+              REPOSITORY={jjb_git}
+    description: Do NOT edit this job through the Web GUI !
+    concurrent: false
+    parameters:
+        - string:
+            name: "sha1"
+            default: '{branch}'
+            description: "What branch to take in ufm_sdk_3.0"
+        - string:
+            name: "PLUGIN_VERSION"
+            default: 'latest'
+            description: "Plugin docker version to create ex: 1.0/1.1/2.0 etc.<br> <p style=color:red>default is latest</p>"
+        - string:
+            name: "EMAIL"
+            default: "{jjb_email}"
+            description: "email notifications <br>"
+        - choice:
+            name: "PLUGIN_NAME"
+            choices:
+                - UFM_NDT_Plugin
+                - advanced_hello_world_plugin
+                - bright_plugin
+                - fluentd_telemetry_plugin
+                - grafana_infiniband_telemetry_plugin
+                - grpc_streamer_plugin
+                - hello_world_plugin
+                - pdr_deterministic_plugin
+                - snmp_receiver_plugin
+                - sysinfo_plugin
+                - ufm_consumer_plugin
+                - ufm_syslog_streaming_plugin
+            description: "On which project to run?"
+        - string:
+            name: "conf_file"
+            default: ".ci/matrix_job_release_nbuprod.yaml"
+            description: "Regex to select job config file. Do not change it"
+    pipeline-scm:
+        scm:
+            - git:
+                url: "{jjb_git}"
+                credentials-id: '0b0ea4b8-2b37-427b-bc3f-b68c41a341f7'
+                branches: ['$sha1']
+                shallow-clone: true
+                refspec: "+refs/pull/*:refs/remotes/origin/pr/*"
+                browser: githubweb
+                browser-url: "{jjb_git}"
+                submodule:
+                    recursive: true
+                    tracking: true
+                    parent-credentials: true
+        script-path: ".ci/Jenkinsfile_nbuprod"
+    wrappers:
+      - timeout:
+          timeout: 30

README.md

@@ -83,3 +83,18 @@ If your plugin directory does not contain a `.ci` directory, the CI process will
 
 The CI pipeline gets triggered based on changes made to the plugins. If changes occur in multiple plugins, the pipeline will not trigger the individual `.ci` directories but instead trigger a default empty CI.
 
+
+# DRP Instructions:
+### UFM_PLUGINS_SDK_RELEASE:
+**Release job URL:** https://nbuprod.blsm.nvidia.com/swx-ufm/job/UFM_PLUGINS_SDK_RELEASE/.
+
+**Release job URL - DRP:** will be synced and updated soon.
+
+**Build instructions:**
+- Go to url.
+- Click on login (top right corner).
+- Login using corp username(without '@nvidia.com') and password.
+- Once logged in you now can build the job.
+- Click on "Build With Parameters"
+- Update the necessary parameter.
+- Click on "Build" and the job will be executed