Perform complete adoptopenjdk.repo installation in install_adoptopenj…

…dk_repo function Change jdk option name to java-version/java_version and make it an int If the correct version of Java is already installed then do not install again Add warning if the java version is being downgraded
nchammas · Oct 30, 2020 · 45e1523 · 45e1523
1 parent 897c4ba
commit 45e1523
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 24 deletions.
diff --git a/flintrock/config.yaml.template b/flintrock/config.yaml.template
@@ -53,5 +53,6 @@ launch:
   num-slaves: 1
   # install-hdfs: True
   # install-spark: False
+  # java-version: 8
 
 debug: false
diff --git a/flintrock/core.py b/flintrock/core.py
@@ -508,40 +508,42 @@ def get_java_major_version(client: paramiko.client.SSHClient):
             tokens = output.split()
             # First line of the output is like: 'openjdk version "1.8.0_252"' or 'openjdk version "11.0.7" 2020-04-14'
             # Get the version string and strip out the first two parts of the
-            # version as a tuple: (1, 8)
+            # version as an int: 7, 8, 9, 10...
             if len(tokens) >= 3:
                 version_parts = tokens[2].strip('"').split(".")
                 if len(version_parts) >= 2:
                     if version_parts[0] == "1":
                         # Java 6, 7 or 8
-                        return version_parts[1]
+                        return int(version_parts[1])
                     else:
                         # Java 9+
-                        return version_parts[0]
+                        return int(version_parts[0])
         except SSHError:
             pass
 
     return None
 
 
-def ensure_java(client: paramiko.client.SSHClient, jdk: str):
+def ensure_java(client: paramiko.client.SSHClient, java_version: int):
     host = client.get_transport().getpeername()[0]
     java_major_version = get_java_major_version(client)
 
+    if java_major_version and java_major_version == java_version:
+        logger.info("Java {j} is already installed, skipping Java install".format(j=java_major_version))
+    elif java_major_version and java_major_version > java_version:
+        logger.warning("Downgrading existing Java {j} installation".format(j=java_major_version))
+
     # sudo yum install -y java-1.8.0-openjdk
     # sudo amazon-linux-extras install -y java-openjdk11
-    if not java_major_version or java_major_version < jdk:
-        logger.info("[{h}] Installing Java {j}...".format(h=host, j=jdk))
+    elif not java_major_version or java_major_version < java_version:
+        logger.info("[{h}] Installing Java {j}...".format(h=host, j=java_version))
 
         install_adoptopenjdk_repo(client)
-        java_package = "adoptopenjdk-{j}-hotspot".format(j=jdk)
+        java_package = "adoptopenjdk-{j}-hotspot".format(j=java_version)
         ssh_check_output(
             client=client,
             command="""
                 set -e
-
-                # Set up the AdoptOpenJDK yum repo
-                sudo mv /tmp/adoptopenjdk.repo /etc/yum.repos.d/
                 
                 # Install Java first to protect packages that depend on Java from being removed.
                 sudo yum install -q -y {jp}
@@ -559,13 +561,19 @@ def ensure_java(client: paramiko.client.SSHClient, jdk: str):
 
 def install_adoptopenjdk_repo(client):
     """
-    Copies the adoptopenjdk.repo file to the /tmp directory on the client.
-    It must be sudo mv'd to /etc/yum.repos.d/ later.
+    Installs the adoptopenjdk.repo file into /etc/yum.repos.d/
     """
     with client.open_sftp() as sftp:
         sftp.put(
             localpath=os.path.join(SCRIPTS_DIR, 'adoptopenjdk.repo'),
             remotepath='/tmp/adoptopenjdk.repo')
+    ssh_check_output(
+        client=client,
+        command="""
+            # Use sudo to install the repo file
+            sudo mv /tmp/adoptopenjdk.repo /etc/yum.repos.d/        
+        """
+    )
 
 
 def setup_node(
@@ -575,7 +583,7 @@ def setup_node(
         # can be looked up by host and reused?
         ssh_client: paramiko.client.SSHClient,
         services: list,
-        jdk: str,
+        java_version: int,
         cluster: FlintrockCluster):
     """
     Setup a new node.
@@ -617,7 +625,7 @@ def setup_node(
     cluster.storage_dirs.root = storage_dirs['root']
     cluster.storage_dirs.ephemeral = storage_dirs['ephemeral']
 
-    ensure_java(ssh_client, jdk)
+    ensure_java(ssh_client, java_version)
 
     for service in services:
         try:
@@ -635,7 +643,7 @@ def setup_node(
 def provision_cluster(
         *,
         cluster: FlintrockCluster,
-        jdk: str,
+        java_version: int,
         services: list,
         user: str,
         identity_file: str):
@@ -644,7 +652,7 @@ def provision_cluster(
     """
     partial_func = functools.partial(
         provision_node,
-        jdk=jdk,
+        java_version=java_version,
         services=services,
         user=user,
         identity_file=identity_file,
@@ -685,7 +693,7 @@ def provision_cluster(
 
 def provision_node(
         *,
-        jdk: str,
+        java_version: int,
         services: list,
         user: str,
         host: str,
@@ -708,7 +716,7 @@ def provision_node(
         setup_node(
             ssh_client=client,
             services=services,
-            jdk=jdk,
+            java_version=java_version,
             cluster=cluster)
         for service in services:
             service.configure(
@@ -779,7 +787,10 @@ def add_slaves_node(
 
     with client:
         if is_new_host:
-            setup_node(ssh_client=client, services=services, cluster=cluster)
+            setup_node(
+                ssh_client=client,
+                services=services,
+                cluster=cluster)
 
         for service in services:
             service.configure(

diff --git a/flintrock/ec2.py b/flintrock/ec2.py
@@ -787,7 +787,7 @@ def launch(
         *,
         cluster_name,
         num_slaves,
-        jdk,
+        java_version,
         services,
         assume_yes,
         key_name,
@@ -934,7 +934,7 @@ def launch(
 
         provision_cluster(
             cluster=cluster,
-            jdk=jdk,
+            java_version=java_version,
             services=services,
             user=user,
             identity_file=identity_file)

diff --git a/flintrock/flintrock.py b/flintrock/flintrock.py
@@ -253,7 +253,7 @@ def cli(cli_context, config, provider, debug):
 @cli.command()
 @click.argument('cluster-name')
 @click.option('--num-slaves', type=click.IntRange(min=1), required=True)
-@click.option('--jdk', type=click.IntRange(min=8), default=8)
+@click.option('--java-version', type=click.IntRange(min=8), default=8)
 @click.option('--install-hdfs/--no-install-hdfs', default=False)
 @click.option('--hdfs-version', default='2.8.5')
 @click.option('--hdfs-download-source',
@@ -324,7 +324,7 @@ def launch(
         cli_context,
         cluster_name,
         num_slaves,
-        jdk,
+        java_version,
         install_hdfs,
         hdfs_version,
         hdfs_download_source,
@@ -438,7 +438,7 @@ def launch(
         cluster = ec2.launch(
             cluster_name=cluster_name,
             num_slaves=num_slaves,
-            jdk=jdk,
+            java_version=java_version,
             services=services,
             assume_yes=assume_yes,
             key_name=ec2_key_name,