diff --git a/aws/cloudformation/docker_for_aws.json b/aws/cloudformation/docker_for_aws.json index 28b011c5f..950f6a1e2 100644 --- a/aws/cloudformation/docker_for_aws.json +++ b/aws/cloudformation/docker_for_aws.json @@ -547,6 +547,7 @@ "echo \"localhost: ", {"Ref": "ExternalLoadBalancer"}, "\" >> /var/lib/docker/swarm/elb.config\n", "echo \"default: ", {"Ref": "ExternalLoadBalancer"}, "\" >> /var/lib/docker/swarm/elb.config\n", "export DOCKER_FOR_IAAS_VERSION='", { "Fn::FindInMap" : [ "DockerForAWS", "version", "forAws" ] }, "'\n", + "export LOCAL_IP=$(wget -qO- http://169.254.169.254/latest/meta-data/local-ipv4)\n", "echo '{\"experimental\": true, \"log-driver\": \"awslogs\",\"log-opts\": {\"awslogs-group\":\"", { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "lg" ] ] }, "\" }}' > /etc/docker/daemon.json \n", @@ -555,6 +556,13 @@ "rc-service docker restart\n", "sleep 5\n", + "docker run --log-driver=json-file --name=meta-aws --restart=always -d -p $LOCAL_IP:9024:8080 ", + "-e AWS_REGION='",{ "Ref" : "AWS::Region" }, "' ", + "-e MANAGER_SECURITY_GROUP_ID='",{ "Ref" : "ManagerVpcSG" }, "' ", + "-e WORKER_SECURITY_GROUP_ID='",{ "Ref" : "NodeVpcSG" }, "' ", + "-v /var/run/docker.sock:/var/run/docker.sock ", + "docker4x/meta-aws:$DOCKER_FOR_IAAS_VERSION\n", + "docker run --log-driver=json-file --restart=no -d ", "-e DYNAMODB_TABLE='", { "Ref" : "SwarmDynDBTable" } , "' ", "-e NODE_TYPE='manager' ", diff --git a/aws/cloudformation/docker_for_aws_cloud.json b/aws/cloudformation/docker_for_aws_cloud.json index 5861da236..c76687457 100644 --- a/aws/cloudformation/docker_for_aws_cloud.json +++ b/aws/cloudformation/docker_for_aws_cloud.json @@ -572,6 +572,7 @@ "echo \"localhost: ", {"Ref": "ExternalLoadBalancer"}, "\" >> /var/lib/docker/swarm/elb.config\n", "echo \"default: ", {"Ref": "ExternalLoadBalancer"}, "\" >> /var/lib/docker/swarm/elb.config\n", "export DOCKER_FOR_IAAS_VERSION='", { "Fn::FindInMap" : [ "DockerForAWS", "version", "forAws" ] }, "'\n", + "export LOCAL_IP=$(wget -qO- http://169.254.169.254/latest/meta-data/local-ipv4)\n", "echo '{\"experimental\": true, \"log-driver\": \"awslogs\",\"log-opts\": {\"awslogs-group\":\"", { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "lg" ] ] }, "\" }}' > /etc/docker/daemon.json \n", @@ -580,6 +581,13 @@ "rc-service docker restart\n", "sleep 5\n", + "docker run --log-driver=json-file --name=meta-aws --restart=always -d -p $LOCAL_IP:9024:8080 ", + "-e AWS_REGION='",{ "Ref" : "AWS::Region" }, "' ", + "-e MANAGER_SECURITY_GROUP_ID='",{ "Ref" : "ManagerVpcSG" }, "' ", + "-e WORKER_SECURITY_GROUP_ID='",{ "Ref" : "NodeVpcSG" }, "' ", + "-v /var/run/docker.sock:/var/run/docker.sock ", + "kencochrane/meta:latest\n", + "docker run --log-driver=json-file --restart=no -d ", "-e DYNAMODB_TABLE='", { "Ref" : "SwarmDynDBTable" } , "' ", "-e NODE_TYPE='manager' ", diff --git a/aws/cloudformation/docker_for_aws_ddc.json b/aws/cloudformation/docker_for_aws_ddc.json index 9e9ed0ff9..e6fb6137c 100644 --- a/aws/cloudformation/docker_for_aws_ddc.json +++ b/aws/cloudformation/docker_for_aws_ddc.json @@ -574,6 +574,7 @@ "echo \"localhost: ", {"Ref": "ExternalLoadBalancer"}, "\" >> /var/lib/docker/swarm/elb.config\n", "echo \"default: ", {"Ref": "ExternalLoadBalancer"}, "\" >> /var/lib/docker/swarm/elb.config\n", "export DOCKER_FOR_IAAS_VERSION='", { "Fn::FindInMap" : [ "DockerForAWS", "version", "forAws" ] }, "'\n", + "export LOCAL_IP=$(wget -qO- http://169.254.169.254/latest/meta-data/local-ipv4)\n", "echo '{\"log-driver\": \"awslogs\",\"log-opts\": {\"awslogs-group\":\"", { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName"}, "lg" ] ] }, "\" }}' > /etc/docker/daemon.json \n", @@ -582,6 +583,13 @@ "rc-service docker restart\n", "sleep 5\n", + "docker run --log-driver=json-file --name=meta-aws --restart=always -d -p $LOCAL_IP:9024:8080 ", + "-e AWS_REGION='",{ "Ref" : "AWS::Region" }, "' ", + "-e MANAGER_SECURITY_GROUP_ID='",{ "Ref" : "ManagerVpcSG" }, "' ", + "-e WORKER_SECURITY_GROUP_ID='",{ "Ref" : "NodeVpcSG" }, "' ", + "-v /var/run/docker.sock:/var/run/docker.sock ", + "kencochrane/meta:latest\n", + "docker run --log-driver=json-file --restart=no -d ", "-e DYNAMODB_TABLE='", { "Ref" : "SwarmDynDBTable" } , "' ", "-e NODE_TYPE='manager' ", diff --git a/aws/dockerfiles/Dockerfile.guide b/aws/dockerfiles/Dockerfile.guide index ca4213a69..f6e46ef4d 100644 --- a/aws/dockerfiles/Dockerfile.guide +++ b/aws/dockerfiles/Dockerfile.guide @@ -18,11 +18,10 @@ ADD files/guide/crontab.txt /usr/docker/crontab.txt ADD files/guide/watcher.sh /usr/bin ADD files/guide/cleanup.sh /usr/bin ADD files/guide/buoy.sh /usr/bin -ADD files/guide/refresh.sh /usr/bin ADD files/bin/buoy /usr/bin COPY files/guide/entry.sh /entry.sh RUN chmod 755 /usr/bin/watcher.sh /entry.sh /usr/bin/cleanup.sh \ - /usr/bin/buoy /usr/bin/buoy.sh /usr/bin/refresh.sh + /usr/bin/buoy /usr/bin/buoy.sh RUN /usr/bin/crontab /usr/docker/crontab.txt CMD ["/entry.sh"] diff --git a/aws/dockerfiles/files/guide/crontab.txt b/aws/dockerfiles/files/guide/crontab.txt index 2bd29b087..868755040 100644 --- a/aws/dockerfiles/files/guide/crontab.txt +++ b/aws/dockerfiles/files/guide/crontab.txt @@ -2,4 +2,3 @@ */1 * * * * /usr/bin/watcher.sh >> /var/log/docker/watcher.log */5 * * * * /usr/bin/cleanup.sh >> /var/log/docker/cleanup.log 24 * * * * /usr/bin/buoy.sh >> /var/log/docker/buoy.log -42 * * * * /usr/bin/refresh.sh >> /var/log/docker/refresh.log diff --git a/aws/dockerfiles/files/guide/refresh.sh b/aws/dockerfiles/files/guide/refresh.sh deleted file mode 100644 index 1b8dd3415..000000000 --- a/aws/dockerfiles/files/guide/refresh.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# this script refreshes the swarm tokens in dynamodb if they have changed. -if [ "$NODE_TYPE" == "worker" ] ; then - # this doesn't run on workers, only managers. - exit 0 -fi - -# make sure we are not in process of shutting down. -if [ -e /tmp/.shutdown-init ] -then - echo "We are shutting down, no need to continue." - # shutdown has initialized, don't start because we might not be able to finish. - exit 0 -fi - -IS_LEADER=$(docker node inspect self -f '{{ .ManagerStatus.Leader }}') - -if [[ "$IS_LEADER" == "true" ]]; then - # we are the leader, We only need to call once, so we only call from the current leader. - MANAGER=$(aws dynamodb get-item --region $REGION --table-name $DYNAMODB_TABLE --key '{"node_type":{"S": "primary_manager"}}') - MANAGER_IP=$(echo $MANAGER | jq -r '.Item.ip.S') - STORED_MANAGER_TOKEN=$(echo $MANAGER | jq -r '.Item.manager_token.S') - STORED_WORKER_TOKEN=$(echo $MANAGER | jq -r '.Item.worker_token.S') - - MANAGER_TOKEN=$(docker swarm join-token manager -q) - WORKER_TOKEN=$(docker swarm join-token worker -q) - - if [[ "$STORED_MANAGER_TOKEN" != "$MANAGER_TOKEN" ]] || [[ "$STORED_WORKER_TOKEN" != "$WORKER_TOKEN" ]]; then - echo "Swarm tokens changed, updating dynamodb with new tokens" - aws dynamodb update-item \ - --table-name $DYNAMODB_TABLE \ - --region $REGION \ - --key '{"node_type":{"S": "primary_manager"}}' \ - --update-expression 'SET manager_token=:m, worker_token=:w' \ - --expression-attribute-values '{":m": {"S":"'"$MANAGER_TOKEN"'"}, ":w": {"S":"'"$WORKER_TOKEN"'"}}' \ - --return-consumed-capacity TOTAL - fi - -fi diff --git a/aws/dockerfiles/files/guide/watcher.sh b/aws/dockerfiles/files/guide/watcher.sh index c855e0382..7528ff711 100644 --- a/aws/dockerfiles/files/guide/watcher.sh +++ b/aws/dockerfiles/files/guide/watcher.sh @@ -83,10 +83,6 @@ if [ "$NODE_TYPE" == "manager" ] ; then MANAGER=$(aws dynamodb get-item --region $REGION --table-name $DYNAMODB_TABLE --key '{"node_type":{"S": "primary_manager"}}') export CURRENT_MANAGER_IP=$(echo $MANAGER | jq -r '.Item.ip.S') - export MANAGER_TOKEN=$(echo $MANAGER | jq -r '.Item.manager_token.S') - export WORKER_TOKEN=$(echo $MANAGER | jq -r '.Item.worker_token.S') - echo "MANAGER_TOKEN=$MANAGER_TOKEN" - echo "WORKER_TOKEN=$WORKER_TOKEN" echo "Current manager IP = $CURRENT_MANAGER_IP ; my IP = $MYIP" @@ -154,7 +150,7 @@ if [ "$NODE_TYPE" == "manager" ] ; then aws dynamodb put-item \ --table-name $DYNAMODB_TABLE \ --region $REGION \ - --item '{"node_type":{"S": "primary_manager"},"ip": {"S":"'"$NEW_MANAGER_IP"'"},"manager_token": {"S":"'"$MANAGER_TOKEN"'"},"worker_token": {"S":"'"$WORKER_TOKEN"'"}}' \ + --item '{"node_type":{"S": "primary_manager"},"ip": {"S":"'"$NEW_MANAGER_IP"'"}}' \ --return-consumed-capacity TOTAL fi fi diff --git a/aws/dockerfiles/files/init/entry.sh b/aws/dockerfiles/files/init/entry.sh index e7a1e0e7f..c449ce3e3 100644 --- a/aws/dockerfiles/files/init/entry.sh +++ b/aws/dockerfiles/files/init/entry.sh @@ -40,36 +40,68 @@ get_primary_manager_ip() # query dynamodb and get the Ip for the primary manager. MANAGER=$(aws dynamodb get-item --region $REGION --table-name $DYNAMODB_TABLE --key '{"node_type":{"S": "primary_manager"}}') export MANAGER_IP=$(echo $MANAGER | jq -r '.Item.ip.S') - export MANAGER_TOKEN=$(echo $MANAGER | jq -r '.Item.manager_token.S') - export WORKER_TOKEN=$(echo $MANAGER | jq -r '.Item.worker_token.S') - echo "MANAGER_TOKEN=$MANAGER_TOKEN" - echo "WORKER_TOKEN=$WORKER_TOKEN" + echo "MANAGER_IP=$MANAGER_IP" } -get_tokens() +get_manager_token() { - export MANAGER_TOKEN=$(docker swarm join-token manager -q) - export WORKER_TOKEN=$(docker swarm join-token worker -q) - echo "MANAGER_TOKEN=$MANAGER_TOKEN" - echo "WORKER_TOKEN=$WORKER_TOKEN" + if [ -n "$MANAGER_IP" ]; then + export MANAGER_TOKEN=$(wget -qO- http://$MANAGER_IP:9024/token/manager/) + echo "MANAGER_TOKEN=$MANAGER_TOKEN" + else + echo "MANAGER_TOKEN can't be found yet. MANAGER_IP isn't set yet." + fi +} + +get_worker_token() +{ + if [ -n "$MANAGER_IP" ]; then + export WORKER_TOKEN=$(wget -qO- http://$MANAGER_IP:9024/token/worker/) + echo "WORKER_TOKEN=$WORKER_TOKEN" + else + echo "WORKER_TOKEN can't be found yet. MANAGER_IP isn't set yet." + fi } -confirm_primary_ready() +confirm_manager_ready() { n=0 until [ $n -ge 5 ] do get_primary_manager_ip echo "PRIMARY_MANAGER_IP=$MANAGER_IP" + get_manager_token # if Manager IP or manager_token is empty or manager_token is null, not ready yet. # token would be null for a short time between swarm init, and the time the # token is added to dynamodb if [ -z "$MANAGER_IP" ] || [ -z "$MANAGER_TOKEN" ] || [ "$MANAGER_TOKEN" == "null" ]; then - echo "Primary manager Not ready yet, sleep for 60 seconds." + echo "Manager: Primary manager Not ready yet, sleep for 60 seconds." sleep 60 n=$[$n+1] else - echo "Primary manager is ready." + echo "Manager: Primary manager is ready." + break + fi + done +} + +confirm_node_ready() +{ + n=0 + until [ $n -ge 5 ] + do + get_primary_manager_ip + echo "PRIMARY_MANAGER_IP=$MANAGER_IP" + get_worker_token + # if Manager IP or manager_token is empty or manager_token is null, not ready yet. + # token would be null for a short time between swarm init, and the time the + # token is added to dynamodb + if [ -z "$MANAGER_IP" ] || [ -z "$WORKER_TOKEN" ] || [ "$WORKER_TOKEN" == "null" ]; then + echo "Worker: Primary manager Not ready yet, sleep for 60 seconds." + sleep 60 + n=$[$n+1] + else + echo "Worker: Primary manager is ready." break fi done @@ -79,7 +111,7 @@ join_as_secondary_manager() { echo " Secondary Manager" if [ -z "$MANAGER_IP" ] || [ -z "$MANAGER_TOKEN" ] || [ "$MANAGER_TOKEN" == "null" ]; then - confirm_primary_ready + confirm_manager_ready fi echo " MANAGER_IP=$MANAGER_IP" echo " MANAGER_TOKEN=$MANAGER_TOKEN" @@ -146,17 +178,9 @@ setup_manager() # we are the primary, so init the cluster docker swarm init --listen-addr $PRIVATE_IP:2377 --advertise-addr $PRIVATE_IP:2377 # we can now get the tokens. - get_tokens get_swarm_id get_node_id - # update dynamodb with the tokens - aws dynamodb put-item \ - --table-name $DYNAMODB_TABLE \ - --region $REGION \ - --item '{"node_type":{"S": "primary_manager"},"ip": {"S":"'"$PRIVATE_IP"'"},"manager_token": {"S":"'"$MANAGER_TOKEN"'"},"worker_token": {"S":"'"$WORKER_TOKEN"'"}}' \ - --return-consumed-capacity TOTAL - echo " Primary Manager init complete" # send identify message buoy -event=identify -swarm_id=$SWARM_ID -flavor=aws -node_id=$NODE_ID @@ -180,8 +204,8 @@ setup_node() { echo " Setup Node" # setup the node, by joining the swarm. - if [ -z "$MANAGER_IP" ] || [ -z "$WORKER_TOKEN" ] || [ "$MANAGER_TOKEN" == "null" ]; then - confirm_primary_ready + if [ -z "$MANAGER_IP" ] || [ -z "$WORKER_TOKEN" ] || [ "$WORKER_TOKEN" == "null" ]; then + confirm_node_ready fi echo " MANAGER_IP=$MANAGER_IP" # try an connect to the swarm manager. @@ -222,9 +246,11 @@ get_primary_manager_ip # if it is a manager, setup as manager, if not, setup as worker node. if [ "$NODE_TYPE" == "manager" ] ; then echo " It's a Manager, run setup" + get_manager_token setup_manager else echo " It's a worker Node, run setup" + get_worker_token setup_node fi