Skip to content

Commit

Permalink
vdk-server: Wire up external containers (Git, Docker) with the service (
Browse files Browse the repository at this point in the history 
#194)

* vdk-server: Wire up external containers (Git and Docker) with the service

Updated the Helm installation to account for the previously installed Git
and Docker containers. This includes obtaining and passing the IP of the
Git container directly because the Job Builder fails to resolve it.

Reduced the number of replicas of the control service and the database to 1.

Added missing dependencies to setup.py.

Signed-off-by: Tsvetomir Palashki <[email protected]>

* vdk-server: improve the installer code

* Rename a local variable to avoid name shadowing
* Properly call the Docker client's close method

Signed-off-by: Tsvetomir Palashki <[email protected]>
  • Loading branch information
@tpalashki
tpalashki authored Sep 7, 2021
1 parent 346d11b commit dc199db
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 29 deletions.
2 changes: 1 addition & 1 deletion projects/vdk-core/plugins/vdk-server/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
description="Versatile Data Kit SDK plugin that facilitates the installation of a local Control Service.",
long_description=pathlib.Path("README.md").read_text(),
long_description_content_type="text/markdown",
install_requires=["vdk-core"],
install_requires=["vdk-core", "docker", "kubernetes"],
package_dir={"": "src"},
packages=setuptools.find_namespace_packages(where="src"),
include_package_data=True,
Expand Down
121 changes: 93 additions & 28 deletions projects/vdk-core/plugins/vdk-server/src/taurus/vdk/installer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import sys
import requests
from kubernetes import client, config, utils
from taurus.vdk.core.errors import BaseVdkError
from taurus.vdk.core.errors import BaseVdkError, ErrorMessage

log = logging.getLogger(__name__)

Expand All @@ -36,12 +36,14 @@ class Installer(object):
git_server_admin_email = "[email protected]"
git_server_repository_name = "vdk-git-repo"

def __init__(self):
self.__current_directory = self.get_current_directory()

def install(self):
"""
Installs all necessary components and configurations.
"""
log.info(f"Starting installation of Versatile Data Kit Control Service")
self.__current_directory = self.get_current_directory()
self.__create_docker_registry_container()
if self.__create_git_server_container():
self.__configure_git_server_with_error_handling()
Expand All @@ -50,6 +52,7 @@ def install(self):
self.__create_kind_cluster()
self.__connect_container_to_kind_network(self.docker_registry_container_name)
self.__connect_container_to_kind_network(self.git_server_container_name)
self.__git_server_ip = self.__resolve_container_ip(self.git_server_container_name)
self.__configure_kind_local_docker_registry()
self.__install_helm_chart()
log.info(f"Versatile Data Kit Control Service installed successfully")
Expand All @@ -63,24 +66,25 @@ def uninstall(self):
self.__delete_git_server_container()
self.__delete_docker_registry_container()

def get_current_directory(self) -> pathlib.Path:
@staticmethod
def get_current_directory() -> pathlib.Path:
return pathlib.Path(__file__).parent.resolve()

def __create_docker_registry_container(self):
"""
Creates a Docker registry container with name specified by docker_registry_name,
unless a container with this name already exists.
"""
client = docker.from_env()
docker_client = docker.from_env()
try:
# Check if a container with that name already exists by inspecting it;
# If the inspection throws an exception, the container does not exist and we
# proceed with creating it
client.api.inspect_container(self.docker_registry_container_name)
docker_client.api.inspect_container(self.docker_registry_container_name)
except:
try:
# docker run -d --restart=always -p "127.0.0.1:${docker_registry_port}:5000" --name "${docker_registry_name}" registry:2
client.containers.run("registry:2",
docker_client.containers.run("registry:2",
detach=True,
restart_policy={"Name": "always"},
name=self.docker_registry_container_name,
Expand All @@ -89,7 +93,7 @@ def __create_docker_registry_container(self):
log.error(
f"Error: Failed to create Docker registry container {self.docker_registry_container_name}. {str(ex)}")
finally:
client.close
docker_client.close()

def __delete_docker_registry_container(self):
self.__delete_container(self.docker_registry_container_name)
Expand All @@ -99,15 +103,15 @@ def __delete_container(container_name: str):
"""
Deletes the Docker registry container with the specified name.
"""
client = docker.from_env()
docker_client = docker.from_env()
try:
client.api.inspect_container(container_name)
client.api.stop(container_name)
client.api.remove_container(container_name)
docker_client.api.inspect_container(container_name)
docker_client.api.stop(container_name)
docker_client.api.remove_container(container_name)
except Exception as ex:
log.error(f"Error: Failed to remove Docker container {container_name}. {str(ex)}")
finally:
client.close
docker_client.close()

def __restart_git_server_container(self):
self.__restart_container(self.git_server_container_name)
Expand All @@ -117,14 +121,14 @@ def __restart_container(container_name: str):
"""
Restarts the container with the specified name.
"""
client = docker.from_env()
docker_client = docker.from_env()
try:
client.api.inspect_container(container_name)
client.api.restart(container_name)
docker_client.api.inspect_container(container_name)
docker_client.api.restart(container_name)
except Exception as ex:
log.info(f"Failed to restart Docker container {container_name}. {str(ex)}")
finally:
client.close
docker_client.close()

def __create_git_server_container(self) -> bool:
"""
Expand All @@ -133,25 +137,25 @@ def __create_git_server_container(self) -> bool:
Returns true if the container did not exist and was created successfully; otherwise, false.
"""
client = docker.from_env()
docker_client = docker.from_env()
try:
# Check if a container with that name already exists by inspecting it;
# If the inspection throws an exception, the container does not exist and we
# proceed with creating it
client.api.inspect_container(self.git_server_container_name)
docker_client.api.inspect_container(self.git_server_container_name)
return False
except:
try:
# docker run --name=vdk-git-server -p 10022:22 -p 10080:3000 -p 10081:80 gogs/gogs:0.12
client.containers.run("gogs/gogs:0.12",
docker_client.containers.run("gogs/gogs:0.12",
detach=True,
name=self.git_server_container_name,
ports={'22/tcp': '10022', '3000/tcp': '10080', '80/tcp': '10081'})
return True
except Exception as ex:
log.error(f"Error: Failed to create Git server container {self.git_server_container_name}. {str(ex)}")
finally:
client.close
docker_client.close()

def __delete_git_server_container(self):
self.__delete_container(self.git_server_container_name)
Expand All @@ -162,13 +166,45 @@ def __connect_container_to_kind_network(container_name: str):
Connects a Docker container to the Kind cluster network.
If the container is already connected, an info message is logged.
"""
client = docker.from_env()
docker_client = docker.from_env()
try:
client.api.connect_container_to_network(container_name, "kind")
# docker network connect "kind" "{container_name}"
docker_client.api.connect_container_to_network(container_name, "kind")
except Exception as ex:
log.info(ex)
finally:
client.close
docker_client.close()

def __resolve_container_ip(self, container_name):
"""
Returns the IP of the Docker container with the specified name, registered within the 'kind' network.
The IP is obtained by inspecting the configuration of the 'kind' network.
"""
docker_client = docker.from_env()
try:
# Find the id of the "kind" network
# docker network ls
networks = docker_client.api.networks()
kind_network = next((n for n in networks if n['Name'] == 'kind'), None)
# Find the "kind" network configuration
# docker network inspect "{kind_net_id}"
kind_network_details = docker_client.api.inspect_network(kind_network['Id'])
# Extract the container's IP
containers = kind_network_details['Containers']
container_id = next((c for c in containers if containers[c]['Name'] == container_name), None)
if container_id:
return self.__remove_ip_subnet_mask(containers[container_id]['IPv4Address'])
except Exception as ex:
log.info(ex)
finally:
docker_client.close()

@staticmethod
def __remove_ip_subnet_mask(ip: str) -> str:
pos = ip.find('/')
if pos == -1:
return ip
return ip[:pos]

def __configure_git_server_with_error_handling(self):
"""
Expand Down Expand Up @@ -270,7 +306,9 @@ def __transform_file(input_file_name, output_file_name, transformation):
with open(output_file_name, "w") as output_file:
output_file.write(transformed_content)
except IOError as ex:
raise BaseVdkError(f"Failed to transform file {input_file_name} into {output_file_name}. {str(ex)}")
# TODO: fill in what/why/etc for the error message
raise BaseVdkError(
ErrorMessage(f"Failed to transform file {input_file_name} into {output_file_name}. {str(ex)}"))

def __transform_template(self, content: str) -> str:
return content.format(docker_registry_name=self.docker_registry_container_name,
Expand Down Expand Up @@ -333,13 +371,40 @@ def __configure_kind_local_docker_registry(self):
log.info(ex)

def __install_helm_chart(self):
"""
Install the VDK Control Service's Helm Chart with all necessary configurations.
"""
try:
# helm repo add vdk-gitlab https://gitlab.com/api/v4/projects/28814611/packages/helm/stable
# helm repo update
# helm install my-release vdk-gitlab/pipelines-control-service
subprocess.run(["helm", "repo", "add", self.helm_repo_local_name, self.helm_repo_url])
subprocess.run(["helm", "repo", "update"])
subprocess.run(["helm", "install", self.helm_installation_name, self.helm_chart_name])
# helm install vdk vdk-gitlab/pipelines-control-service \
# --set deploymentGitUrl=vdk-git-server/vdkuser/vdk-repo.git \
# ...
# Note: The Git server is referenced by IP rather than directly by name; the reason for this
# is that, currently, the Git server name cannot be resolved within the Job Builder container.
# The reason for this is unknown, but is suspected to be related to the Kaniko image that is
# used as a base.
subprocess.run(['helm', 'repo', 'add', self.helm_repo_local_name, self.helm_repo_url])
subprocess.run(['helm', 'repo', 'update'])
subprocess.run(['helm', 'install', self.helm_installation_name, self.helm_chart_name,
# '--wait',
# '--debug',
'--set', 'resources.limits.memory=1G',
'--set', 'cockroachdb.statefulset.replicas=1',
'--set', 'replicas=1',
'--set', 'deploymentBuilderImage.tag=1.2', # TODO: Remove when service adopts 1.2
'--set', 'deploymentGitBranch=master',
'--set', 'deploymentDockerRegistryType=generic',
'--set', f'deploymentDockerRepository={self.docker_registry_container_name}:5000',
'--set', f'proxyRepositoryURL=localhost:5000',
'--set', f'deploymentGitUrl={self.__git_server_ip}/{self.git_server_admin_user}/{self.git_server_repository_name}.git',
'--set', f'deploymentGitUsername={self.git_server_admin_user}',
'--set', f'deploymentGitPassword={self.git_server_admin_password}',
'--set', f'uploadGitReadWriteUsername={self.git_server_admin_user}',
'--set', f'uploadGitReadWritePassword={self.git_server_admin_password}',
'--set', 'extraEnvVars.GIT_SSL_ENABLED=false',
'--set', 'extraEnvVars.DATAJOBS_DEPLOYMENT_BUILDER_EXTRAARGS=--insecure',
])
except Exception as ex:
log.error(f"Failed to install Helm chart. Make sure you have Helm installed. {str(ex)}")

Expand Down

0 comments on commit dc199db

Please sign in to comment.