-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsetup-ec2.sh
executable file
·138 lines (109 loc) · 7 KB
/
setup-ec2.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#!/bin/bash
AWS_ACCESS_KEY_ID=""
AWS_SECRET_ACCESS_KEY=""
sshKeyName=""
echo
echo "Starting setup..."
if [ ${#AWS_ACCESS_KEY_ID} -eq 0 ] || [ ${#AWS_SECRET_ACCESS_KEY} -eq 0 ] || [ ${#sshKeyName} -eq 0 ]
then
echo "Please fill in the 3 vars at the top of the file"
else
echo " Receiving elastic search url..."
elasticSearchEndpoint=$(aws es describe-elasticsearch-domain --domain-name "tweetanalyzer" --query "DomainStatus.Endpoint")
if [ ${#elasticSearchEndpoint} -gt 0 ]
then
echo " Checking available subnets..."
subnets=$(aws ec2 describe-subnets --query "Subnets[*].SubnetId")
temp="${subnets%]}"
subnets="${temp#[}"
subnets=$(echo "$subnets" | sed "s/\"//g")
echo " Check available security groups..."
securityGroup=$(aws ec2 describe-security-groups --query "SecurityGroups[?GroupName=='default'].IpPermissions[][UserIdGroupPairs][][][GroupId][]")
temp="${securityGroup%]}"
securityGroup="${temp#[}"
echo
echo "Setting up SQS"
echo " analyised-tweets"
aws sqs create-queue --cli-input-json file://AnalyisedTweetsSQS.json
echo " fetched-tweets"
aws sqs create-queue --cli-input-json file://FetchedTweetsSQS.json
echo
echo " Setting up Analyzer..."
echo " Prepare Userdata..."
dockerRunText=`cat dockerRunAnalyzer.txt`
dockerRunText=${dockerRunText//ACCESSKEY/$AWS_ACCESS_KEY_ID}
dockerRunText=${dockerRunText//SECRETKEY/$AWS_SECRET_ACCESS_KEY}
userdata=$'#!/bin/bash\\n'
userdata="$userdata$dockerRunText"
echo " Setting up LauchConfiguration JSON..."
launchConfigText=`cat LaunchConfigurationsAnalyzer.json`
launchConfigText=${launchConfigText//USERDATAVAR/$userdata}
launchConfigText=${launchConfigText//SECURITYGROUPID/$securityGroup}
launchConfigText=${launchConfigText//SSHKEYNAME/$sshKeyName}
echo $launchConfigText > LaunchConfigurationsAnalyzer_prepared.json
echo " Setting up launch configuration..."
aws autoscaling create-launch-configuration --cli-input-json file://LaunchConfigurationsAnalyzer_prepared.json
echo " Setting up auto scaling group..."
aws autoscaling create-auto-scaling-group --cli-input-json file://AutoScalingGroupsAnalyzer.json
echo " Update auto scaling group..."
aws autoscaling update-auto-scaling-group --auto-scaling-group-name ASE_ASG_Analyzer --vpc-zone-identifier "$subnets"
aws autoscaling create-or-update-tags --tags "ResourceId=ASE_ASG_Analyzer,ResourceType=auto-scaling-group,Key=Type,Value=Analyzer,PropagateAtLaunch=true"
echo " Setting up auto scaling group metrics..."
aws autoscaling enable-metrics-collection --cli-input-json file://AutoScalingGroupMetricsAnalyzer.json
echo " Setting up ScaleoutPolicy..."
scaleOutARN=$(aws autoscaling put-scaling-policy --policy-name my-sqs-scaleout-policy --auto-scaling-group-name ASE_ASG_Analyzer --scaling-adjustment 1 --adjustment-type ChangeInCapacity --query "PolicyARN")
echo " Setting up ScaleinPolicy..."
scaleInARN=$(aws autoscaling put-scaling-policy --policy-name my-sqs-scalein-policy --auto-scaling-group-name ASE_ASG_Analyzer --scaling-adjustment -1 --adjustment-type ChangeInCapacity --query "PolicyARN")
echo " Put alarm metric: scale in..."
temp="${scaleInARN%\"}"
scaleInARN="${temp#\"}"
aws cloudwatch put-metric-alarm --alarm-name RemoveCapacityFromFetchedQueue --metric-name ApproximateNumberOfMessagesVisible --namespace "AWS/SQS" --statistic Average --period 60 --threshold 20 --comparison-operator LessThanOrEqualToThreshold --dimensions Name=QueueName,Value=fetched-tweets --evaluation-periods 1 --alarm-actions $scaleInARN
echo " Put alarm metric: scale out..."
temp="${scaleOutARN%\"}"
scaleOutARN="${temp#\"}"
aws cloudwatch put-metric-alarm --alarm-name AddCapacityToFetchedQueue --metric-name ApproximateNumberOfMessagesVisible --namespace "AWS/SQS" --statistic Average --period 60 --threshold 30 --comparison-operator GreaterThanOrEqualToThreshold --dimensions Name=QueueName,Value=fetched-tweets --evaluation-periods 1 --alarm-actions $scaleOutARN
echo " Analyzer done"
echo
echo " Setting up ES producer..."
echo " Prepare Userdata..."
temp="${elasticSearchEndpoint%\"}"
elasticSearchEndpoint="${temp#\"}"
dockerRunText=`cat dockerRunESproducer.txt`
dockerRunText=${dockerRunText//ACCESSKEY/$AWS_ACCESS_KEY_ID}
dockerRunText=${dockerRunText//SECRETKEY/$AWS_SECRET_ACCESS_KEY}
dockerRunText=${dockerRunText//AWSHOSTURL/$elasticSearchEndpoint}
userdata=$'#!/bin/bash\\n'
userdata="$userdata$dockerRunText"
echo " Setting up LauchConfiguration JSON..."
launchConfigText=`cat LaunchConfigurationsESProducer.json`
launchConfigText=${launchConfigText//USERDATAVAR/$userdata}
launchConfigText=${launchConfigText//SECURITYGROUPID/$securityGroup}
launchConfigText=${launchConfigText//SSHKEYNAME/$sshKeyName}
echo $launchConfigText > LaunchConfigurationsESProducer_prepared.json
echo " Setting up launch configuration..."
aws autoscaling create-launch-configuration --cli-input-json file://LaunchConfigurationsESProducer_prepared.json
echo " Setting up auto scaling group..."
aws autoscaling create-auto-scaling-group --cli-input-json file://AutoScalingGroupsESProducer.json
echo " Update auto scaling group..."
aws autoscaling update-auto-scaling-group --auto-scaling-group-name ASE_ASG_ESProducer --vpc-zone-identifier "$subnets"
aws autoscaling create-or-update-tags --tags "ResourceId=ASE_ASG_ESProducer,ResourceType=auto-scaling-group,Key=Type,Value=ESProducer,PropagateAtLaunch=true"
echo " Setting up auto scaling group metrics..."
aws autoscaling enable-metrics-collection --cli-input-json file://AutoScalingGroupMetricsESProducer.json
echo " Setting up ScaleoutPolicy..."
scaleOutARN=$(aws autoscaling put-scaling-policy --policy-name my-es-sqs-scaleout-policy --auto-scaling-group-name ASE_ASG_ESProducer --scaling-adjustment 1 --adjustment-type ChangeInCapacity --query "PolicyARN")
echo " Setting up ScaleinPolicy..."
scaleInARN=$(aws autoscaling put-scaling-policy --policy-name my-es-sqs-scalein-policy --auto-scaling-group-name ASE_ASG_ESProducer --scaling-adjustment -1 --adjustment-type ChangeInCapacity --query "PolicyARN")
echo " Put alarm metric: scale in..."
temp="${scaleInARN%\"}"
scaleInARN="${temp#\"}"
aws cloudwatch put-metric-alarm --alarm-name RemoveCapacityFromProcessQueue --metric-name ApproximateNumberOfMessagesVisible --namespace "AWS/SQS" --statistic Average --period 60 --threshold 20 --comparison-operator LessThanOrEqualToThreshold --dimensions Name=QueueName,Value=analyised-tweets --evaluation-periods 1 --alarm-actions $scaleInARN
echo " Put alarm metric: scale out..."
temp="${scaleOutARN%\"}"
scaleOutARN="${temp#\"}"
aws cloudwatch put-metric-alarm --alarm-name AddCapacityToProcessQueue --metric-name ApproximateNumberOfMessagesVisible --namespace "AWS/SQS" --statistic Average --period 60 --threshold 30 --comparison-operator GreaterThanOrEqualToThreshold --dimensions Name=QueueName,Value=analyised-tweets --evaluation-periods 1 --alarm-actions $scaleOutARN
echo " ES producer done"
echo "Setup done!"
else
echo "Elastic search url not found. Aborting!"
fi
fi