From 1dbc95d4724805b767ad1c0bd05aca13654c76ca Mon Sep 17 00:00:00 2001 From: Hal Rosenberg Date: Wed, 8 Jul 2020 18:23:07 -0400 Subject: [PATCH 1/5] Start of adding test of storage classes Signed-off-by: Hal Rosenberg --- runWeathervane.pl | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/runWeathervane.pl b/runWeathervane.pl index 99534ca7..58596099 100755 --- a/runWeathervane.pl +++ b/runWeathervane.pl @@ -102,6 +102,7 @@ sub parseConfigFile { my @k8sConfigFiles; my $dockerNamespace; + my %storageClassNames; while () { if ($_ =~ /^\s*"kubeconfigFile"\s*\:\s*"(.*)"\s*,/) { if ((! -e $1) || (! -f $1)) { @@ -114,6 +115,10 @@ sub parseConfigFile { } } elsif ($_ =~ /^\s*"dockerNamespace"\s*\:\s*"(.*)"\s*,/) { $dockerNamespace = $1; + } elsif ($_ =~ /StorageClass"\s*\:\s*"(.*)"\s*,/) { + # ToDo: Need to associate storage classes with kubeconfig and context so that can + # correctly try creating a pvc/pv + $storageClassNames{$1} = 1; } elsif ($_ =~ /useLoadBalancer/) { print "The useLoadBalancer parameter has been replaced with the appIngressMethod parameter. " . "You must update your configuration file.\n" . @@ -129,11 +134,43 @@ sub parseConfigFile { exit 1; } - my @return = (\@k8sConfigFiles, $dockerNamespace); + my @return = (\@k8sConfigFiles, $dockerNamespace, \%storageClassNames); return \@return; } +sub checkStorageClasses { + my ($storageClassNamesRef) = @_; + + my $pvcYamlString = <<"END"; +{ + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "test-claim", + "annotations": { + "volume.beta.kubernetes.io/storage-class": "storageClassNameHere" + } + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "1Mi" + } + } + } +} +END + # ToDo: Need to loop through clusters and then storage classes in the cluster + foreach my $storageClassName (keys %$storageClassNamesRef) { + + + } + +} sub parseKubeconfigFile { my ($configFileName) = @_; @@ -313,6 +350,9 @@ sub forceLicenseAccept { my $retRef = parseConfigFile($configFile); my $k8sConfigFilesRef = $retRef->[0]; my $dockerNamespace = $retRef->[1]; +my $storageClassNamesRef = $retRef->[2]; + +checkStorageClasses($storageClassNamesRef); my $k8sConfigMountString = ""; foreach my $k8sConfig (@$k8sConfigFilesRef) { From 1e14b15b6a6bbda8d39ba25159fed077b3e38ffb Mon Sep 17 00:00:00 2001 From: Hal Rosenberg Date: Thu, 9 Jul 2020 22:30:42 -0400 Subject: [PATCH 2/5] Add a test to make sure that weathervane can create pvs Signed-off-by: Hal Rosenberg --- runWeathervane.pl | 162 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 124 insertions(+), 38 deletions(-) diff --git a/runWeathervane.pl b/runWeathervane.pl index 58596099..6b839040 100755 --- a/runWeathervane.pl +++ b/runWeathervane.pl @@ -98,27 +98,78 @@ sub parseConfigFile { my ($configFileName) = @_; # Read in the config file - open( CONFIGFILE, "<$configFileName" ) or die "Couldn't open configuration file $configFileName: $!\n"; + open( my $configFile, "<$configFileName" ) or die "Couldn't open configuration file $configFileName: $!\n"; my @k8sConfigFiles; + my %clusterNameToKubeconfig; my $dockerNamespace; - my %storageClassNames; - while () { - if ($_ =~ /^\s*"kubeconfigFile"\s*\:\s*"(.*)"\s*,/) { - if ((! -e $1) || (! -f $1)) { - print "The kubeconfigFile $1 must exist and be a regular file.\n"; - usage(); - exit 1; + my $topLevelAppInstanceCluster = ""; + my %topLevelStorageClassNames; + # Parsing the config file manually to avoid requiring the JSON package + while (<$configFile>) { + if ($_ =~ /^\s*"kubernetesClusters"\s*\:\s*\[/) { + # Fill out a hash with clustername -> [kubeconfig, context]) + while (<$configFile>) { + if ($_ =~ /^\s*{/) { + my $name = ""; + my $kubeconfigFileName = "~/.kube/config"; + my $kubeconfigContext = ""; + while (<$configFile>) { + if ($_ =~ /^\s*"kubeconfigFile"\s*\:\s*"(.*)"\s*,/) { + $kubeconfigFileName = $1; + if ((! -e $kubeconfigFileName) || (! -f $kubeconfigFileName)) { + print "The kubeconfigFile $kubeconfigFileName must exist and be a regular file.\n"; + usage(); + exit 1; + } + if (!($kubeconfigFileName ~~ @k8sConfigFiles)) { + push(@k8sConfigFiles, $kubeconfigFileName); + } + } elsif ($_ =~ /^\s*"kubeconfigContext"\s*\:\s*"(.*)"\s*,/) { + $kubeconfigContext = $1; + } elsif ($_ =~ /^\s*"name"\s*\:\s*"(.*)"\s*,/) { + $name = $1; + } elsif ($_ =~ /^\s*\}/) { + last; + } + } + $clusterNameToKubeconfig{$name} = [$kubeconfigFileName, $kubeconfigContext]; + } elsif ($_ =~ /^\s*\]/) { + last; + } + } + } elsif ($_ =~ /^\s*"workloads"\s*\:\s*\[/) { + # Don't parse inside workloads + my $numOpenBrackets = 1; + while (<$configFile>) { + if ($_ =~ /\[/) { + $numOpenBrackets++; + } elsif ($_ =~ /\]/) { + $numOpenBrackets--; + if ($numOpenBrackets == 0) { + last; + } + } } - if (!($1 ~~ @k8sConfigFiles)) { - push(@k8sConfigFiles, $1); + } elsif ($_ =~ /^\s*"appInstances"\s*\:\s*\[/) { + # Don't parse inside appInstances + my $numOpenBrackets = 1; + while (<$configFile>) { + if ($_ =~ /\[/) { + $numOpenBrackets++; + } elsif ($_ =~ /\]/) { + $numOpenBrackets--; + if ($numOpenBrackets == 0) { + last; + } + } } + } elsif ($_ =~ /StorageClass"\s*\:\s*"(.*)"\s*,/) { + $topLevelStorageClassNames{$1} = 1; + } elsif ($_ =~ /appInstanceCluster"\s*\:\s*"(.*)"\s*,/) { + $topLevelAppInstanceCluster = $1; } elsif ($_ =~ /^\s*"dockerNamespace"\s*\:\s*"(.*)"\s*,/) { $dockerNamespace = $1; - } elsif ($_ =~ /StorageClass"\s*\:\s*"(.*)"\s*,/) { - # ToDo: Need to associate storage classes with kubeconfig and context so that can - # correctly try creating a pvc/pv - $storageClassNames{$1} = 1; } elsif ($_ =~ /useLoadBalancer/) { print "The useLoadBalancer parameter has been replaced with the appIngressMethod parameter. " . "You must update your configuration file.\n" . @@ -126,50 +177,83 @@ sub parseConfigFile { exit(1); } } - close CONFIGFILE; - + close $configFile; + if (!$dockerNamespace) { print "You must specify the dockerNamespace parameter in configuration file $configFileName.\n"; usage(); exit 1; } - - my @return = (\@k8sConfigFiles, $dockerNamespace, \%storageClassNames); + + my %clusterToStorageClassNames; + if ($topLevelAppInstanceCluster) { + $clusterToStorageClassNames{$topLevelAppInstanceCluster} = \%topLevelStorageClassNames; + } + + my @return = (\@k8sConfigFiles, $dockerNamespace, \%clusterNameToKubeconfig, \%clusterToStorageClassNames); return \@return; } sub checkStorageClasses { - my ($storageClassNamesRef) = @_; + my ($clusterNameToKubeconfigRef, $clusterToStorageClassNamesRef) = @_; my $pvcYamlString = <<"END"; { - "kind": "PersistentVolumeClaim", - "apiVersion": "v1", - "metadata": { - "name": "test-claim", - "annotations": { - "volume.beta.kubernetes.io/storage-class": "storageClassNameHere" + \\\"kind\\\": \\\"PersistentVolumeClaim\\\", + \\\"apiVersion\\\": \\\"v1\\\", + \\\"metadata\\\": { + \\\"name\\\": \\\"weathervane-test-claim\\\", + \\\"annotations\\\": { + \\\"volume.beta.kubernetes.io/storage-class\\\": \\\"storageClassNameHere\\\" } }, - "spec": { - "accessModes": [ - "ReadWriteOnce" + \\\"spec\\\": { + \\\"accessModes\\\": [ + \\\"ReadWriteOnce\\\" ], - "resources": { - "requests": { - "storage": "1Mi" + \\\"resources\\\": { + \\\"requests\\\": { + \\\"storage\\\": \\\"1Mi\\\" } } } } END - # ToDo: Need to loop through clusters and then storage classes in the cluster - foreach my $storageClassName (keys %$storageClassNamesRef) { - - - } + foreach my $clusterName (keys $clusterToStorageClassNamesRef) { + my $kubeconfigFileName = $clusterNameToKubeconfigRef->{$clusterName}->[0]; + my $kubeconfigContext = $clusterNameToKubeconfigRef->{$clusterName}->[1]; + foreach my $storageClassName (keys %{$clusterToStorageClassNamesRef->{$clusterName}}) { + # Delete any old PVC with name weathervane-test-claim + my $out = `kubectl --kubeconfig=$kubeconfigFileName --context=$kubeconfigContext delete pvc weathervane-test-claim 2>&1`; + + # Create a PVC + my $pvcYamlStringCopy = $pvcYamlString; + $pvcYamlStringCopy =~ s/storageClassNameHere/$storageClassName/; + $out = `echo "$pvcYamlStringCopy" | kubectl --kubeconfig=$kubeconfigFileName --context=$kubeconfigContext apply -f -`; + + # Check 5 times for the status to equal Bound and exit if not sucessful + my $retries = 5; + my $status; + do { + $status = `kubectl --kubeconfig=$kubeconfigFileName --context=$kubeconfigContext get pvc weathervane-test-claim -o=jsonpath='{.status.phase}'`; + chomp($status); + $retries--; + if ($status ne "Bound") { + if ($retries == 0) { + $out = `kubectl --kubeconfig=$kubeconfigFileName --context=$kubeconfigContext delete pvc weathervane-test-claim 2>&1`; + die "Weathervane is unable to create a persistant volume using storage class $storageClassName in kubernetesCluster $clusterName.\n" . + "Check the configuration of your cluster to ensure that the storage class exists and can provision persistent volumes.\n"; + } + sleep 5; + } + } while (($status ne "Bound") && ($retries > 0)); + + # Delete the PVC + $out = `kubectl --kubeconfig=$kubeconfigFileName --context=$kubeconfigContext delete pvc weathervane-test-claim 2>&1`; + } + } } sub parseKubeconfigFile { my ($configFileName) = @_; @@ -350,9 +434,11 @@ sub forceLicenseAccept { my $retRef = parseConfigFile($configFile); my $k8sConfigFilesRef = $retRef->[0]; my $dockerNamespace = $retRef->[1]; -my $storageClassNamesRef = $retRef->[2]; +my $clusterNameToKubeconfigRef = $retRef->[2]; +my $clusterToStorageClassNamesRef = $retRef->[3]; -checkStorageClasses($storageClassNamesRef); +checkStorageClasses($clusterNameToKubeconfigRef, $clusterToStorageClassNamesRef); +exit 0; my $k8sConfigMountString = ""; foreach my $k8sConfig (@$k8sConfigFilesRef) { From 4ed935517ef82715d4dc34c2f50fcb6a4384ab78 Mon Sep 17 00:00:00 2001 From: Hal Rosenberg Date: Thu, 9 Jul 2020 22:31:30 -0400 Subject: [PATCH 3/5] Add a test to make sure that weathervane can create pvs Signed-off-by: Hal Rosenberg --- runWeathervane.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/runWeathervane.pl b/runWeathervane.pl index 6b839040..58b7339f 100755 --- a/runWeathervane.pl +++ b/runWeathervane.pl @@ -438,7 +438,6 @@ sub forceLicenseAccept { my $clusterToStorageClassNamesRef = $retRef->[3]; checkStorageClasses($clusterNameToKubeconfigRef, $clusterToStorageClassNamesRef); -exit 0; my $k8sConfigMountString = ""; foreach my $k8sConfig (@$k8sConfigFilesRef) { From d6ec0486ee7e02d6291a6884203dca47fc14a9f9 Mon Sep 17 00:00:00 2001 From: Hal Rosenberg Date: Fri, 10 Jul 2020 12:08:39 -0400 Subject: [PATCH 4/5] Add flag to allow skipping the pv tests Signed-off-by: Hal Rosenberg --- runWeathervane.pl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/runWeathervane.pl b/runWeathervane.pl index 58b7339f..60dda6b3 100755 --- a/runWeathervane.pl +++ b/runWeathervane.pl @@ -18,6 +18,7 @@ package Weathervane; my $tmpDir = ''; my $backgroundScript = ''; my $mapSsh = ''; +my $skipPvTest = ''; my $fixedConfigsFile = ""; my $scriptPeriodSec = 60; my $help = ''; @@ -37,6 +38,7 @@ package Weathervane; 'fixedConfigsFile=s' => \$fixedConfigsFile, 'scriptPeriod=i' => \$scriptPeriodSec, 'mapSsh!' => \$mapSsh, + 'skipPvTest!' => \$skipPvTest, 'help!' => \$help, ); @@ -87,6 +89,9 @@ sub usage { print " from another script. Only needs to be specified on the first run in a given directory.\n"; print " default value: None. If no value is specified the user is prompted to accept the\n"; print " license terms.\n"; + print "--skipPvTest: Causes the scipt to skip testing whether Weathervane can dynamically allocate \n"; + print " persistant volumes in the storage classes defined in the configuration file.\n"; + print " default value: False"; print "--help: Displays this text.\n"; print "\n"; print "To pass command-line parameters to the Weathervane run harness, enter them following two dashes\n"; @@ -437,7 +442,9 @@ sub forceLicenseAccept { my $clusterNameToKubeconfigRef = $retRef->[2]; my $clusterToStorageClassNamesRef = $retRef->[3]; -checkStorageClasses($clusterNameToKubeconfigRef, $clusterToStorageClassNamesRef); +if (!$skipPvTest) { + checkStorageClasses($clusterNameToKubeconfigRef, $clusterToStorageClassNamesRef); +} my $k8sConfigMountString = ""; foreach my $k8sConfig (@$k8sConfigFilesRef) { From 4b90f596320d64fa37787592be52e736e86e444b Mon Sep 17 00:00:00 2001 From: Hal Rosenberg Date: Fri, 10 Jul 2020 15:05:12 -0400 Subject: [PATCH 5/5] skip comments when parsing the config file Signed-off-by: Hal Rosenberg --- runWeathervane.pl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/runWeathervane.pl b/runWeathervane.pl index 60dda6b3..8eec2bf1 100755 --- a/runWeathervane.pl +++ b/runWeathervane.pl @@ -112,15 +112,21 @@ sub parseConfigFile { my %topLevelStorageClassNames; # Parsing the config file manually to avoid requiring the JSON package while (<$configFile>) { - if ($_ =~ /^\s*"kubernetesClusters"\s*\:\s*\[/) { + if ($_ =~ /^\s*#/) { + next; + } elsif ($_ =~ /^\s*"kubernetesClusters"\s*\:\s*\[/) { # Fill out a hash with clustername -> [kubeconfig, context]) while (<$configFile>) { - if ($_ =~ /^\s*{/) { + if ($_ =~ /^\s*#/) { + next; + } elsif ($_ =~ /^\s*{/) { my $name = ""; my $kubeconfigFileName = "~/.kube/config"; my $kubeconfigContext = ""; while (<$configFile>) { - if ($_ =~ /^\s*"kubeconfigFile"\s*\:\s*"(.*)"\s*,/) { + if ($_ =~ /^\s*#/) { + next; + } elsif ($_ =~ /^\s*"kubeconfigFile"\s*\:\s*"(.*)"\s*,/) { $kubeconfigFileName = $1; if ((! -e $kubeconfigFileName) || (! -f $kubeconfigFileName)) { print "The kubeconfigFile $kubeconfigFileName must exist and be a regular file.\n"; @@ -147,7 +153,9 @@ sub parseConfigFile { # Don't parse inside workloads my $numOpenBrackets = 1; while (<$configFile>) { - if ($_ =~ /\[/) { + if ($_ =~ /^\s*#/) { + next; + } elsif ($_ =~ /\[/) { $numOpenBrackets++; } elsif ($_ =~ /\]/) { $numOpenBrackets--; @@ -160,7 +168,9 @@ sub parseConfigFile { # Don't parse inside appInstances my $numOpenBrackets = 1; while (<$configFile>) { - if ($_ =~ /\[/) { + if ($_ =~ /^\s*#/) { + next; + } elsif ($_ =~ /\[/) { $numOpenBrackets++; } elsif ($_ =~ /\]/) { $numOpenBrackets--;