Skip to content

Commit

Permalink
Improvements for Xenial
Browse files Browse the repository at this point in the history
 * Better management of the Docker containers
 * New version of chef-server
 * chef-rundeck is now running in a container
  • Loading branch information
c-buisson committed Jun 17, 2017
1 parent 2cb52ff commit 6914df0
Show file tree
Hide file tree
Showing 14 changed files with 183 additions and 104 deletions.
15 changes: 15 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
CHANGELOG
=========

2.5: 2017-06-16
---------------
New Features:
- The script ensures knife_admin_key.tar.gz is really accessible before downloading it from the chef-server container.
- Test if br0 interface is present before installing 'kvm-control' with floating IPs.
- New Docker network range.
- Using chef-rundeck in a container.

Updates:
- Using newer Docker image (v2.3).
- Shell and Ruby lint.
- Output color for the bash scripts.
- New env variable for libvirt.
- chef-server has now is static ip.

2.4: 2017-03-20
---------------
New Features:
Expand Down
Binary file added docker/docker_chef_rundeck.tar.gz
Binary file not shown.
2 changes: 1 addition & 1 deletion install
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ echo " |\/| | /__\` /__\` | / \\ |\\ | / \` / \\ |\\ | | |__) / \\ |"
echo " | | | .__/ .__/ | \\__/ | \\| ___ \\__, \\__/ | \\| | | \\ \\__/ |___"
echo -e "\n\nInstallation is progress...\n"
sudo apt-get -q update
sudo apt-get -y install build-essential ruby2.3 ruby2.3-dev
sudo apt-get -y install build-essential curl uuid-runtime ruby2.3 ruby2.3-dev
echo "Installing: bundler"
sudo gem install bundler --no-ri --no-rdoc
bundle install
Expand Down
81 changes: 54 additions & 27 deletions scripts/check_url.sh
Original file line number Diff line number Diff line change
@@ -1,30 +1,57 @@
#!/bin/bash

URL=$1
CODE=1
OPTION=$1
URL=$2
SECONDS=0
TIMEOUT=$2

echo -e "Blocking until <${URL}> responds...\nTimeout: ${TIMEOUT} seconds."

while [ $CODE -ne 0 ]; do
curl -sfk \
--connect-timeout 3 \
--max-time 5 \
--fail \
--silent \
${URL} >/dev/null

CODE=$?

sleep 2
echo -n "."

if [ $SECONDS -ge $TIMEOUT ]; then
echo "$URL is not available after $SECONDS seconds...stopping the install!"
exit 1
fi

done;

echo -e "\n\e[1m$URL\e[0m is accessible!"
txtred=$(tput setaf 1)
txtbold=$(tput bold)
txtreset=$(tput sgr0)

if [ "$OPTION" == "url" ]; then
TIMEOUT=$3
CODE=1
echo -e "Blocking until <${URL}> responds...\nTimeout: ${TIMEOUT} seconds."

while [ "$CODE" -ne 0 ]; do
curl -sfk \
--connect-timeout 3 \
--max-time 5 \
--fail \
--silent \
"${URL}" >/dev/null

CODE=$?

sleep 2
echo -n "."

if [ "$SECONDS" -ge "$TIMEOUT" ]; then
echo "$txtred$URL is not available after $SECONDS seconds...stopping the install!"
exit 1
fi
done;

elif [ "$OPTION" == "file" ]; then
TARGET_SIZE=$3
TIMEOUT=$4
SIZE=0
echo -e "Blocking until $URL is accessible...\nTimeout: $TIMEOUT seconds."

while [ $SIZE -lt $TARGET_SIZE ]; do
SIZE=$(curl -Isk $URL | grep Content-Length | awk '{print $2}' | tr -d '\r\n')

sleep 2
echo -n "."

if [ "$SECONDS" -ge "$TIMEOUT" ]; then
echo "$txtred$URL is not available after $SECONDS seconds...stopping the install!"
exit 1
fi
done;

else
echo "$txtred$OPTION is not a valid choice. Please use 'url' or 'file'!"
exit 1
fi

echo -e "\n$txtbold$URL$txtreset is accessible!"
10 changes: 6 additions & 4 deletions scripts/create_rd_projects.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
#!/bin/bash

txtbold=$(tput bold)

case "$1" in
"kvm-control" )
project=$(sudo su - rundeck -c "rd projects list |grep $1")
if [[ -z "$project" ]]; then
echo -e "Creating $1 project and jobs!\n"
echo -e "$txtboldCreating $1 project and jobs!\n"
sudo su rundeck -c "rd projects create -p kvm-control"
sudo su rundeck -c "rd jobs load -r -f $2/rundeck_jobs.xml -p kvm-control"
else
Expand All @@ -14,7 +16,7 @@ case "$1" in
"kvm-control_with-Chef" )
project=$(sudo su - rundeck -c "rd projects list |grep $1")
if [[ -z "$project" ]]; then
echo -e "Creating $1 project and jobs!\n"
echo -e "$txtboldCreating $1 project and jobs!\n"
sudo su rundeck -c "rd projects create -p kvm-control_with-Chef -- --resources.source.2.config.url=http://localhost:9980 --resources.source.2.type=url --resources.source.2.config.timeout=60 --resources.source.2.config.cache=false"
sudo su rundeck -c "rd jobs load -r -f $2/chef-rundeck_jobs.xml -p kvm-control_with-Chef"
else
Expand All @@ -24,7 +26,7 @@ case "$1" in
"docker-control" )
project=$(sudo su - rundeck -c "rd projects list |grep $1")
if [[ -z "$project" ]]; then
echo -e "Creating $1 project and jobs!\n"
echo -e "$txtboldCreating $1 project and jobs!\n"
sudo su rundeck -c "rd projects create -p docker-control"
sudo su rundeck -c "rd jobs load -r -f $2/rundeck_jobs.xml -p docker-control"
else
Expand All @@ -34,7 +36,7 @@ case "$1" in
"chef_server-control" )
project=$(sudo su - rundeck -c "rd projects list |grep $1")
if [[ -z "$project" ]]; then
echo -e "Creating $1 project and jobs!\n"
echo -e "$txtboldCreating $1 project and jobs!\n"
sudo su rundeck -c "rd projects create -p chef_server-control"
sudo su rundeck -c "rd jobs load -r -f $2/rundeck_jobs-chef.xml -p chef_server-control"
else
Expand Down
29 changes: 10 additions & 19 deletions scripts/get_and_install.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
#!/bin/bash

txtbold=$(tput bold)

case "$1" in
"rundeck-cli" )
file=$(ls /etc/apt/sources.list.d/ |grep rundeck-cli.list)
file=$(for i in /etc/apt/sources.list.d/*; do echo "$i" |grep rundeck-cli.list; done)
if [[ -z "$file" ]]; then
echo -e "Downloading $1!\n"
echo -e "$txtboldDownloading $1!\n"
echo "deb https://dl.bintray.com/rundeck/rundeck-deb /" | sudo tee -a /etc/apt/sources.list.d/rundeck-cli.list
curl "https://bintray.com/user/downloadSubjectPublicKey?username=bintray" > /tmp/bintray.gpg.key
apt-key add - < /tmp/bintray.gpg.key
Expand All @@ -15,11 +17,11 @@ case "$1" in
fi
;;
"rundeck" )
file=$(ls |grep rundeck-$2-GA.deb)
file=$(for i in *; do echo "$i" |grep rundeck-"$2"-GA.deb; done)
if [[ -z "$file" ]]; then
echo -e "Downloading $1!\n"
wget http://download.rundeck.org/deb/rundeck-$2-GA.deb
dpkg -i rundeck-$2-GA.deb
echo -e "$txtboldDownloading $1!\n"
wget http://download.rundeck.org/deb/rundeck-"$2"-GA.deb
dpkg -i rundeck-"$2"-GA.deb
rm -rf /tmp/rundeck/
cp /etc/rundeck/rundeck-config.properties /etc/rundeck/rundeck-config.properties.backup
cp /etc/rundeck/framework.properties /etc/rundeck/framework.properties.backup
Expand All @@ -28,25 +30,14 @@ case "$1" in
fi
;;
"mysql-connector" )
file=$(ls |grep mysql-connector-java-5.1.40.tar.gz)
file=$(for i in *; do echo "$i" |grep mysql-connector-java-5.1.40.tar.gz; done)
if [[ -z "$file" ]]; then
echo -e "Downloading $1!\n"
echo -e "$txtboldDownloading $1!\n"
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.40.tar.gz
tar -zxf mysql-connector-java-5.1.40.tar.gz -C /tmp/
cp /tmp/mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar /var/lib/rundeck/libext/
else
echo -e "$1 is already on this server!\n"
fi
;;
"chef-rundeck" )
file=$(ls |grep ruby-ffi)
if [[ -z "$file" ]]; then
echo -e "Downloading ffy dependencies and chef-rundeck gem!\n"
wget http://mirrors.kernel.org/ubuntu/pool/universe/r/ruby-ffi/ruby-ffi_1.9.10debian-1build2_amd64.deb http://security.ubuntu.com/ubuntu/pool/universe/r/ruby-ffi-yajl/ruby-ffi-yajl_2.2.3-2_amd64.deb
sudo dpkg -i ruby-ffi*
sudo gem install chef-rundeck
else
echo -e "$1 and its dependencies are already installed on this server!\n"
fi
;;
esac
11 changes: 7 additions & 4 deletions scripts/install_docker.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
#!/bin/bash

echo -e "Installing Docker...\n"
txtbold=$(tput bold)

echo -e "$txtboldInstalling Docker...\n"
sudo apt-get -y install docker.io
sudo adduser rundeck docker
sudo mkdir -p $1
cp docker/rundeck_jobs.xml $1
sudo chown rundeck. -R $1
sudo mkdir -p "$1"
cp docker/rundeck_jobs.xml "$1"
sudo chown rundeck. -R "$1"
sudo docker network create --subnet=172.18.0.0/16 mc_net
24 changes: 24 additions & 0 deletions scripts/install_docker_chef-rundeck.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/bash
txtbold=$(tput bold)
chef_rundeck_present=$(sudo docker ps -a |grep "$1")
chef_rundeck_up=$(sudo docker ps |grep "$1")
chef_rundeck_docker_ip=$2
chef_server_docker_ip=$3
set -e
if [[ -z "$chef_rundeck_present" ]]; then
echo -e "$txtboldCreating image, launching container and start $1\n"
# Create the chef-rundeck Docker image and run it
tar -zxf docker/docker_chef_rundeck.tar.gz
sed -i "s,CHEF_SERVER_CONTAINER_IP,$3,g" chef-rundeck/Dockerfile
sed -i "s,CHEF_SERVER_CONTAINER_IP,$3,g" chef-rundeck/config.rb
cp ~/.chef/admin.pem chef-rundeck/
sudo docker build -t c_rundeck_image chef-rundeck/
sudo docker run --net mc_net --ip $chef_rundeck_docker_ip -d --name $1 -e 'USER=ubuntu' -p 9980:9980 c_rundeck_image
# Get the container's IP and add it to /etc/hosts
grep -q "$chef_rundeck_docker_ip $1" /etc/hosts || echo "$chef_rundeck_docker_ip $1" | sudo tee -a /etc/hosts
elif [[ -z "$chef_rundeck_up" ]]; then
echo -e "Starting $1 container\n"
sudo docker start "$1"
else
echo -e "\nThere is already a Docker container named: $1\nRemove it first and re-run that script if you want a new container!\n"
fi
42 changes: 25 additions & 17 deletions scripts/install_docker_chef-server.sh
Original file line number Diff line number Diff line change
@@ -1,41 +1,49 @@
#!/bin/bash
chef_present=$(sudo docker ps -a |grep $1)
chef_up=$(sudo docker ps |grep $1)
txtbold=$(tput bold)
chef_present=$(sudo docker ps -a |grep "$1")
chef_up=$(sudo docker ps |grep "$1")
chef_docker_ip=$4
set -e
if [[ -z "$chef_present" ]]; then
echo -e "Downloading container and start $1\n"
sudo mkdir -p $3/$1/logs
docker pull cbuisson/chef-server:v2.2
sudo docker run --privileged -e CONTAINER_NAME=$1 -e CHEF_PORT=$2 --name $1 -d -v $3/$1/logs/chef-logs:/var/log -v $3/$1/logs/install-chef-out:/root -p $2:$2 cbuisson/chef-server:v2.2
docker_ip=$(sudo docker inspect -f '{{.NetworkSettings.IPAddress }}' $1)
grep -q "$docker_ip $1" /etc/hosts || echo "$docker_ip $1" | sudo tee -a /etc/hosts
sudo scripts/check_url.sh https://$1:$2/knife_admin_key.tar.gz 600
sudo curl -o $3/$1/knife_admin_key.tar.gz -Ok https://$1:$2/knife_admin_key.tar.gz
echo -e "$txtboldDownloading container and start $1\n"
sudo mkdir -p "$3"/"$1"/logs
# Get the chef-server Docker image and run it
docker pull cbuisson/chef-server:v2.3
sudo docker run --net mc_net --ip $chef_docker_ip --privileged -e CONTAINER_NAME="$1" -e SSL_PORT="$2" --name "$1" -d -v "$3"/"$1"/logs/chef-logs:/var/log -v "$3"/"$1"/logs/install-chef-out:/root -p "$2":"$2" cbuisson/chef-server:v2.3
# Get the container's IP and add it to /etc/hosts
grep -q "$chef_docker_ip $1" /etc/hosts || echo "$chef_docker_ip $1" | sudo tee -a /etc/hosts
# Check the Chef is running
sudo scripts/check_url.sh url https://"$1":"$2" 900
# Check if the archive containg the knife key is accessible and then download it
sudo scripts/check_url.sh file https://"$1":"$2"/knife_admin_key.tar.gz 2500 900
sudo curl -o "$3"/"$1"/knife_admin_key.tar.gz -Ok https://"$1":"$2"/knife_admin_key.tar.gz
sudo DEBIAN_FRONTEND=noninteractive apt-get install -q -y chef
# Create two .chef folders, one for the current user and one for the rundeck user
knife_keys=( /var/lib/rundeck ~ )
for i in "${knife_keys[@]}"
do
mkdir -p $i/.chef
cat > $i/.chef/config.rb << EOL
mkdir -p "$i"/.chef
cat > "$i"/.chef/config.rb << EOL
log_level :info
log_location STDOUT
cache_type 'BasicFile'
node_name 'admin'
client_key '$i/.chef/admin.pem'
chef_server_url 'https://$1:$2/organizations/my_org'
EOL
sudo tar -zxf $3/$1/knife_admin_key.tar.gz -C $i/.chef/
sudo tar -zxf "$3"/"$1"/knife_admin_key.tar.gz -C "$i"/.chef/
done
sudo chown -R rundeck. /var/lib/rundeck/.chef/
knife ssl fetch
knife user list
sudo su - rundeck -c "knife ssl fetch && knife user list"
sudo chown rundeck. -R $3/$1
echo -e "\n\e[1mCreating knife keys for rundeck and $SUDO_USER users!\e[0m"
sudo chown rundeck. -R "$3"/"$1"
echo -e "\n\eCreating knife keys for rundeck and $SUDO_USER users!\e"
sudo chown rundeck. -R /var/lib/rundeck/.chef/
sudo chown $SUDO_USER. -R ~/.chef/
sudo chown "$SUDO_USER". -R ~/.chef/
elif [[ -z "$chef_up" ]]; then
echo -e "Starting $1 container\n"
sudo docker start $1
sudo docker start "$1"
else
echo -e "\nThere is already a Docker container named: $1\nRemove it first and re-run that script if you want a new container!\n"
fi
9 changes: 7 additions & 2 deletions scripts/install_kvm.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#!/bin/bash

sudo apt-get -y install qemu-utils cloud-utils kvm libvirt-bin
sudo mkdir -p $1/{kvm_guests,lib,templates,lists,sources/{iso,cloud_images}}
sudo mkdir -p "$1"/{kvm_guests,lib,templates,lists,sources/{iso,cloud_images}}
echo "Add rundeck user to libvirtd and kvm groups"
sudo adduser rundeck libvirtd && sudo adduser rundeck kvm
kvm_guests=`sudo virsh list`
kvm_guests=$(sudo virsh list)
if [[ -z "$kvm_guests" ]]; then
echo "Restart libvirtd..."
sudo systemctl restart libvirt-bin
Expand All @@ -23,3 +23,8 @@ if [[ $2 == "yes" ]];then
fi
sudo resolvconf -u
fi

# Restart Rundeck
systemctl restart rundeckd
scripts/check_url.sh url http://"$3":4440 60
grep -q "export LIBVIRT_DEFAULT_URI=qemu:///system" /etc/environment || echo "export LIBVIRT_DEFAULT_URI=qemu:///system" | sudo tee -a /etc/environment
19 changes: 9 additions & 10 deletions scripts/install_rundeck.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
scripts/get_and_install.sh rundeck-cli

# Get Rundeck deb and install
apt-get -y install openjdk-8-jre rundeck-cli
scripts/get_and_install.sh rundeck $2
apt-get -y --allow-unauthenticated install openjdk-8-jre rundeck-cli
scripts/get_and_install.sh rundeck "$2"
cp /etc/rundeck/rundeck-config.properties.backup /etc/rundeck/rundeck-config.properties
cp /etc/rundeck/framework.properties.backup /etc/rundeck/framework.properties

Expand All @@ -20,11 +20,11 @@ if [[ $3 == "mysql" ]]; then
apt-get install -q -y mysql-server mysql-client libmysqlclient-dev
# Setup rundeckdb
scripts/get_and_install.sh "mysql-connector"
mysql -u root -p$4 -e "create database rundeckdb"
mysql -u root -p$4 -e "grant ALL on rundeckdb.* to 'rduser'@'localhost' identified by 'rdpasswd';"
mysql -u root -p"$4" -e "create database rundeckdb"
mysql -u root -p"$4" -e "grant ALL on rundeckdb.* to 'rduser'@'localhost' identified by 'rdpasswd';"
sed -i "s,jdbc:h2:file:/var/lib/rundeck/data/rundeckdb;MVCC=true,jdbc:mysql://localhost/rundeckdb?autoReconnect=true,g" /etc/rundeck/rundeck-config.properties
echo -e "dataSource.username=rduser\ndataSource.password=rdpasswd" >> /etc/rundeck/rundeck-config.properties
elif [[ $3 == "postgres" ]]; then
elif [[ "$3" == "postgres" ]]; then
apt-get -y install postgresql libpq-dev
su - postgres -c "createuser pguser -s"
echo -e "local all postgres peer\nlocal all pguser trust\nlocal all all peer\nhost all all 127.0.0.1/32 md5" | sudo tee /etc/postgresql/9.5/main/pg_hba.conf
Expand All @@ -35,19 +35,18 @@ else
fi

# Configure Rundeck
sed -i s,localhost,$1,g /etc/rundeck/framework.properties
sed -i s,localhost,"$1",g /etc/rundeck/framework.properties
sed -i "s,grails.serverURL=http://localhost:4440,grails.serverURL=http://$1:4440,g" /etc/rundeck/rundeck-config.properties
hostname=`hostname`
hostname=$(hostname)
grep -q "$1 $hostname" /etc/hosts || echo "$1 $hostname" | sudo tee -a /etc/hosts
sed -i s,"/var/lib/rundeck:/bin/false","/var/lib/rundeck:/bin/bash",g /etc/passwd
sed -i "s,/var/lib/rundeck:/bin/false,/var/lib/rundeck:/bin/bash,g" /etc/passwd
chown rundeck. /var/lib/rundeck
ls /var/lib/rundeck/.ssh || sudo su rundeck -c "echo -e \"\n\" | ssh-keygen -t rsa -N \"\""
mkdir /var/lib/rundeck/.rd
echo -e "export RD_URL=http://$1:4440\nexport RD_USER=admin\nexport RD_PASSWORD=admin" > /var/lib/rundeck/.rd/rd.conf
chown -R rundeck. /var/lib/rundeck/.rd
echo "rundeck ALL=NOPASSWD: /bin/systemctl reload bind9" > /etc/sudoers.d/rundeck
chmod 440 /etc/sudoers.d/rundeck
rundeck_url=`sudo cat /etc/rundeck/framework.properties |grep framework.server.url |awk '{print $3}'`.chomp
systemctl enable rundeckd
systemctl restart rundeckd
scripts/check_url.sh http://$1:4440 60
scripts/check_url.sh url http://"$1":4440 60
Loading

0 comments on commit 6914df0

Please sign in to comment.