From b8d1e79eeba90085542a71d1f3755a8f7868606d Mon Sep 17 00:00:00 2001 From: nikhil2611 Date: Sat, 7 Dec 2024 22:53:05 +0530 Subject: [PATCH] updating pipeline to run on expeditor instead of github actions Signed-off-by: nikhil2611 --- .expeditor/build.habitat.yml | 9 ++ .../buildkite/artifact.habitat.test.ps1 | 86 ++++++++++++ .expeditor/buildkite/artifact.habitat.test.sh | 71 ++++++++++ .expeditor/habitat-test.pipeline.yml | 35 +++++ habitat/plan.ps1 | 2 +- habitat/plan.sh | 130 +++++++----------- habitat/tests/test.sh | 3 +- 7 files changed, 251 insertions(+), 85 deletions(-) create mode 100644 .expeditor/build.habitat.yml create mode 100644 .expeditor/buildkite/artifact.habitat.test.ps1 create mode 100644 .expeditor/buildkite/artifact.habitat.test.sh create mode 100644 .expeditor/habitat-test.pipeline.yml diff --git a/.expeditor/build.habitat.yml b/.expeditor/build.habitat.yml new file mode 100644 index 000000000..9c2d3d7a2 --- /dev/null +++ b/.expeditor/build.habitat.yml @@ -0,0 +1,9 @@ +--- +origin: chef + +expeditor: + defaults: + buildkite: + retry: + automatic: + limit: 1 \ No newline at end of file diff --git a/.expeditor/buildkite/artifact.habitat.test.ps1 b/.expeditor/buildkite/artifact.habitat.test.ps1 new file mode 100644 index 000000000..2f7dc420e --- /dev/null +++ b/.expeditor/buildkite/artifact.habitat.test.ps1 @@ -0,0 +1,86 @@ +#!/usr/bin/env powershell + +#Requires -Version 5 +# https://stackoverflow.com/questions/9948517 +# TODO: Set-StrictMode -Version Latest +$PSDefaultParameterValues['*:ErrorAction']='Stop' +$ErrorActionPreference = 'Stop' +$env:HAB_BLDR_CHANNEL = "LTS-2024" +$env:HAB_ORIGIN = 'ci' +$env:CHEF_LICENSE = 'accept-no-persist' +$env:HAB_LICENSE = 'accept-no-persist' +$Plan = 'berkshelf' + +Write-Host "--- system details" +$Properties = 'Caption', 'CSName', 'Version', 'BuildType', 'OSArchitecture' +Get-CimInstance Win32_OperatingSystem | Select-Object $Properties | Format-Table -AutoSize + +Write-Host "--- Installing the version of Habitat required" + +function Stop-HabProcess { + $habProcess = Get-Process hab -ErrorAction SilentlyContinue + if ($habProcess) { + Write-Host "Stopping hab process..." + Stop-Process -Name hab -Force + } +} + +# Installing Habitat +function Install-Habitat { + Write-Host "Downloading and installing Habitat..." + Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.ps1')) +} + +try { + hab --version +} +catch { + Set-ExecutionPolicy Bypass -Scope Process -Force + + Stop-HabProcess + + # Remove the existing hab.exe if it exists and if you have permissions + $habPath = "C:\ProgramData\Habitat\hab.exe" + if (Test-Path $habPath) { + Write-Host "Attempting to remove existing hab.exe..." + Remove-Item $habPath -Force -ErrorAction SilentlyContinue + if (Test-Path $habPath) { + Write-Host "Failed to remove hab.exe, re-running script with elevated permissions." + Start-Process powershell -Verb runAs -ArgumentList "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`"" + exit + } + } + + Install-Habitat +} +finally { + Write-Host ":habicat: I think I have the version I need to build." +} + + +Write-Host "--- Generating fake origin key" +hab origin key generate $env:HAB_ORIGIN + +Write-Host "--- Building $Plan" +$project_root = "$(git rev-parse --show-toplevel)" +Set-Location $project_root + +$env:DO_CHECK=$true; hab pkg build . + +. $project_root/results/last_build.ps1 + +Write-Host "--- Installing $pkg_ident/$pkg_artifact" +hab pkg install -b $project_root/results/$pkg_artifact + +Write-Host "+++ Testing $Plan" + +Push-Location $project_root + +try { + Write-Host "Running unit tests..." + habitat/tests/test.ps1 $pkg_ident +} +finally { + # Ensure we always return to the original directory + Pop-Location +} \ No newline at end of file diff --git a/.expeditor/buildkite/artifact.habitat.test.sh b/.expeditor/buildkite/artifact.habitat.test.sh new file mode 100644 index 000000000..27dc4c9a2 --- /dev/null +++ b/.expeditor/buildkite/artifact.habitat.test.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -eo pipefail + +export HAB_ORIGIN='ci' +export PLAN='berkshelf' +export CHEF_LICENSE="accept-no-persist" +export HAB_LICENSE="accept-no-persist" +export HAB_BLDR_CHANNEL="LTS-2024" + +echo "--- checking if git is installed" +if ! command -v git &> /dev/null; then + echo "Git is not installed. Installing Git..." + sudo yum install -y git +else + echo "Git is already installed." + git --version +fi + +echo "--- add an exception for this directory since detected dubious ownership in repository at /workdir" +git config --global --add safe.directory /workdir + +echo "--- git status for this workdir" +git status + +echo "--- ruby version" +ruby -v + +export project_root="$(git rev-parse --show-toplevel)" +echo "The value for project_root is: $project_root" + +export HAB_NONINTERACTIVE=true +export HAB_NOCOLORING=true +export HAB_STUDIO_SECRET_HAB_NONINTERACTIVE=true + +echo "--- system details" +uname -a + +echo "--- Installing Habitat" +id -a +curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | bash + +echo "--- :key: Generating fake origin key" +hab origin key generate "$HAB_ORIGIN" + + +echo "--- Building $PLAN" +cd "$project_root" +DO_CHECK=true hab pkg build . + +echo "--- Sourcing 'results/last_build.sh'" +if [ -f ./results/last_build.env ]; then + cat ./results/last_build.env + . ./results/last_build.env + export pkg_artifact +fi + +echo "+++ Installing ${pkg_ident:?is undefined}" +echo "++++" +echo $project_root +echo "+++" +hab pkg install -b "${project_root:?is undefined}/results/${pkg_artifact:?is undefined}" + +echo "+++ Testing $PLAN" + +PATH="$(hab pkg path ci/berkshelf)/bin:$PATH" +export PATH +echo "PATH is $PATH" + +echo "--- :mag_right: Testing $PLAN" +${project_root}/habitat/tests/test.sh "$pkg_ident" || error 'failures during test of executables' \ No newline at end of file diff --git a/.expeditor/habitat-test.pipeline.yml b/.expeditor/habitat-test.pipeline.yml new file mode 100644 index 000000000..39493f20c --- /dev/null +++ b/.expeditor/habitat-test.pipeline.yml @@ -0,0 +1,35 @@ +--- +expeditor: + defaults: + buildkite: + timeout_in_minutes: 30 + retry: + automatic: + limit: 1 + +steps: + + - label: ":linux: Validate Habitat Builds of Berkshelf" + commands: + - .expeditor/buildkite/artifact.habitat.test.sh + expeditor: + executor: + docker: + image: ruby:3.1 + privileged: true + + - label: ":windows: Validate Habitat Builds of Berkshelf" + commands: + - .expeditor/buildkite/artifact.habitat.test.ps1 + expeditor: + executor: + docker: + host_os: windows + shell: ["powershell", "-Command"] + image: rubydistros/windows-2019:3.1 + user: 'NT AUTHORITY\SYSTEM' + environment: + - FORCE_FFI_YAJL=ext + - EXPIRE_CACHE=true + - CHEF_LICENSE=accept-no-persist + - CHEF_LICENSE_SERVER=http://hosted-license-service-lb-8000-606952349.us-west-2.elb.amazonaws.com:8000/ diff --git a/habitat/plan.ps1 b/habitat/plan.ps1 index 537901284..034692d0a 100644 --- a/habitat/plan.ps1 +++ b/habitat/plan.ps1 @@ -1,6 +1,7 @@ $ErrorActionPreference = "Stop" $PSDefaultParameterValues['*:ErrorAction']='Stop' +$env:HAB_BLDR_CHANNEL = "LTS-2024" $pkg_name="berkshelf" $pkg_origin="chef" $pkg_version=$(Get-Content "$PLAN_CONTEXT/../VERSION") @@ -48,7 +49,6 @@ function Invoke-Build { gem build berkshelf.gemspec Write-BuildLine " ** Using gem to install" gem install berkshelf-*.gem --no-document - If ($lastexitcode -ne 0) { Exit $lastexitcode } } finally { diff --git a/habitat/plan.sh b/habitat/plan.sh index 237ba2e2f..ed231a9ec 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -1,8 +1,9 @@ -_chef_client_ruby="core/ruby31" +export HAB_BLDR_CHANNEL="LTS-2024" +_chef_client_ruby="core/ruby3_1" pkg_name="berkshelf" -pkg_origin=chef +pkg_origin="chef" pkg_maintainer="The Chef Maintainers " -pkg_description="Manage Chef Infra cookbooks and cookbook dependencies" +pkg_description="Manage Chef Infra cookbooks and cookbook dependencies." pkg_license=('Apache-2.0') pkg_bin_dirs=( bin @@ -10,116 +11,79 @@ pkg_bin_dirs=( ) pkg_build_deps=( core/make + core/bash core/gcc - core/git ) pkg_deps=( $_chef_client_ruby core/coreutils + core/git ) pkg_svc_user=root pkg_version() { - cat "${SRC_PATH}/VERSION" + cat "$SRC_PATH/VERSION" } do_before() { - do_default_before update_pkg_version - # We must wait until we update the pkg_version to use the pkg_version - pkg_filename="${pkg_name}-${pkg_version}.tar.gz" -} - -do_download() { - build_line "Locally creating archive of latest repository commit at ${HAB_CACHE_SRC_PATH}/${pkg_filename}" - # source is in this repo, so we're going to create an archive from the - # appropriate path within the repo and place the generated tarball in the - # location expected by do_unpack - ( cd "${SRC_PATH}" || exit_with "unable to enter hab-src directory" 1 - git archive --prefix="${pkg_name}-${pkg_version}/" --output="${HAB_CACHE_SRC_PATH}/${pkg_filename}" HEAD - ) } -do_verify() { - build_line "Skipping checksum verification on the archive we just created." - return 0 +do_unpack() { + mkdir -pv "$HAB_CACHE_SRC_PATH/$pkg_dirname" + cp -RT "$PLAN_CONTEXT"/.. "$HAB_CACHE_SRC_PATH/$pkg_dirname/" } -# Setup environment variables for Ruby Gems -do_setup_environment() { - push_runtime_env GEM_PATH "${pkg_prefix}/vendor" +do_build() { + echo $(pkg_path_for $_chef_client_ruby) + export GEM_HOME="$pkg_prefix/vendor/gems" - set_runtime_env APPBUNDLER_ALLOW_RVM "true" # prevent appbundler from clearing out the carefully constructed runtime GEM_PATH - set_runtime_env LANG "en_US.UTF-8" - set_runtime_env LC_CTYPE "en_US.UTF-8" + build_line "Setting GEM_PATH=$GEM_HOME" + export GEM_PATH="$GEM_HOME" + bundle config --local without integration deploy maintenance + bundle config --local jobs 4 + bundle config --local retry 5 + bundle config --local silence_root_warning 1 + bundle install --without development --jobs=3 --retry=3 + gem build berkshelf.gemspec } -do_prepare() { - export GEM_HOME="${pkg_prefix}/vendor" - export CPPFLAGS="${CPPFLAGS} ${CFLAGS}" - - ( cd "$CACHE_PATH" - bundle config --local jobs "$(nproc)" - bundle config --local without server docgen maintenance pry travis integration ci chefstyle - bundle config --local shebang "$(pkg_path_for "$_chef_client_ruby")/bin/ruby" - bundle config --local retry 5 - bundle config --local silence_root_warning 1 - ) - - build_line "Setting link for /usr/bin/env to 'coreutils'" - if [ ! -f /usr/bin/env ]; then - ln -s "$(pkg_interpreter_for core/coreutils bin/env)" /usr/bin/env - fi -} +do_install() { + export GEM_HOME="$pkg_prefix/vendor/gems" -# Build the gem from the gemspec file -do_build() { - ( cd "$CACHE_PATH" || exit_with "unable to enter hab-cache directory" 1 - build_line "Installing gem dependencies ..." - bundle install --without development --jobs=3 --retry=3 - build_line "Installing gems from git repos properly ..." - build_line "Installing this project's gems ..." - bundle exec rake install:local - gem install chef-utils chef-config - ) + build_line "Setting GEM_PATH=$GEM_HOME" + export GEM_PATH="$GEM_HOME" + gem install berkshelf-*.gem --no-document + wrap_ruby_berkshelf + set_runtime_env "GEM_PATH" "${pkg_prefix}/vendor/gems" } -# Install the built gem into the package directory -do_install() { - ( cd "$pkg_prefix" || exit_with "unable to enter pkg prefix directory" 1 - export BUNDLE_GEMFILE="${CACHE_PATH}/Gemfile" - build_line "** fixing binstub shebangs" - fix_interpreter "${pkg_prefix}/vendor/bin/*" "$_chef_client_ruby" bin/ruby - export BUNDLE_GEMFILE="${CACHE_PATH}/Gemfile" - for gem in berkshelf; do - build_line "** generating binstubs for $gem with precise version pins" - appbundler $CACHE_PATH $pkg_prefix/bin $gem - done - ) +wrap_ruby_berkshelf() { + local bin="$pkg_prefix/bin/berks" + local real_bin="$GEM_HOME/gems/berkshelf-${pkg_version}/bin/berks" + wrap_bin_with_ruby "$bin" "$real_bin" } -do_after() { - build_line "Trimming the fat ..." +wrap_bin_with_ruby() { + local bin="$1" + local real_bin="$2" + build_line "Adding wrapper $bin to $real_bin" + cat < "$bin" +#!$(pkg_path_for core/bash)/bin/bash +set -e - # We don't need the cache of downloaded .gem files ... - rm -rf "$pkg_prefix/vendor/cache" +# Set binary path that allows berkshelf to use non-Hab pkg binaries +export PATH="/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin:\$PATH" - # We don't need the gem docs. - rm -rf "$pkg_prefix/vendor/doc" - # We don't need to ship the test suites for every gem dependency, - # only Chef's for package verification. - find "$pkg_prefix/vendor/gems" -name spec -type d | grep -v "berkshelf-${pkg_version}" \ - | while read spec_dir; do rm -rf "$spec_dir"; done -} +# Set Ruby paths defined from 'do_setup_environment()' +export GEM_HOME="$pkg_prefix/vendor/gems" +export GEM_PATH="\$GEM_HOME" -do_end() { - if [ "$(readlink /usr/bin/env)" = "$(pkg_interpreter_for core/coreutils bin/env)" ]; then - build_line "Removing the symlink we created for '/usr/bin/env'" - rm /usr/bin/env - fi +exec $(pkg_path_for $_chef_client_ruby)/bin/ruby $real_bin \$@ +EOF + chmod -v 755 "$bin" } -# No additional stripping needed do_strip() { return 0 } \ No newline at end of file diff --git a/habitat/tests/test.sh b/habitat/tests/test.sh index d84a2a7d5..5858ae1ac 100644 --- a/habitat/tests/test.sh +++ b/habitat/tests/test.sh @@ -2,6 +2,7 @@ set -euo pipefail project_root="$(git rev-parse --show-toplevel)" +pkg_ident="$1" # print error message followed by usage and exit error () { @@ -18,6 +19,6 @@ package_version=$(awk -F / '{print $3}' <<<"$pkg_ident") cd "${project_root}" -echo "--- :msg_right: Testing ${pkg_ident} executables" +echo "--- :mag_right: Testing ${pkg_ident} executables" actual_version=$(hab pkg exec "${pkg_ident}" berks -v | sed -E 's/.*: ([0-9]+\.[0-9]+\.[0-9]+).*/\1/') [[ "$package_version" = "$actual_version" ]] || error "Berkshelf version is not the expected version. Expected '$package_version', got '$actual_version'"