Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update azure version endpoint #1147

Merged
merged 2 commits into from
Sep 7, 2022
Merged

Update azure version endpoint #1147

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fb4fac
update version endpoints
Aug 18, 2022
995c619
Merge branch 'develop' into azure-cloud-details
Aug 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 126 additions & 93 deletions hubblestack/grains/cloud_details.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,21 @@

log = logging.getLogger(__name__)


def get_cloud_details():
"""
Gather all cloud details and return them, along with the fieldnames
"""
grains = {}

aws = _get_aws_details()
if not aws.get('cloud_details'):
if not aws.get("cloud_details"):
log.debug("Unable to fetch AWS details. Now trying to fetch Azure details")
azure = _get_azure_details()
if not azure.get('cloud_details'):
if not azure.get("cloud_details"):
log.debug("Unable to fetch Azure details. Now trying to fetch GCP details")
gcp = _get_gcp_details()
if not gcp.get('cloud_details'):
if not gcp.get("cloud_details"):
log.debug("Unable to fetch details from AWS/Azure/GCP. Please verify the instance settings.")
else:
log.debug("Fetched instance metadata from GCP")
Expand All @@ -41,36 +42,44 @@ def _get_aws_details():
"""
ret = {}
aws_extra = {}
aws = {'cloud_instance_id': None, 'cloud_account_id': None, 'cloud_type': 'aws'}
proxies = {'http': None}
aws = {"cloud_instance_id": None, "cloud_account_id": None, "cloud_type": "aws"}
proxies = {"http": None}

try:
ttl_header = {'X-aws-ec2-metadata-token-ttl-seconds': '300'}
token_url = 'http://169.254.169.254/latest/api/token'
ttl_header = {"X-aws-ec2-metadata-token-ttl-seconds": "300"}
token_url = "http://169.254.169.254/latest/api/token"
token_request = requests.put(token_url, headers=ttl_header, timeout=3, proxies=proxies)
if token_request.status_code == requests.codes.ok:
token = token_request.text
aws_token_header = {'X-aws-ec2-metadata-token': token}
response = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document',
headers=aws_token_header, timeout=3, proxies=proxies)
aws_token_header = {"X-aws-ec2-metadata-token": token}
response = requests.get(
"http://169.254.169.254/latest/dynamic/instance-identity/document",
headers=aws_token_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
res = response.json()
aws['cloud_account_id'] = res.get('accountId', 'unknown')
aws["cloud_account_id"] = res.get("accountId", "unknown")
# AWS account id is always an integer number
# So if it's an aws machine it must be a valid integer number
# Else it will throw an Exception
#
# Once it's a confirmed aws account, pad the account id to 12 digits to account for leading zeros.
# https://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html#awsdocs-filter-selector:~:text=A%2012%2Ddigit%20number
id = int(aws['cloud_account_id'])
aws['cloud_account_id'] = f"{id:0>12}"
cloud_account_id = int(aws["cloud_account_id"])
aws["cloud_account_id"] = f"{cloud_account_id:0>12}"
else:
raise ValueError("Error while fetching AWS account id. Got status code: %s " % (response.status_code))

response = requests.get('http://169.254.169.254/latest/meta-data/instance-id',
headers=aws_token_header, timeout=3, proxies=proxies)
response = requests.get(
"http://169.254.169.254/latest/meta-data/instance-id",
headers=aws_token_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
aws['cloud_instance_id'] = response.text
aws["cloud_instance_id"] = response.text
else:
raise ValueError("Error while fetching AWS instance id. Got status code: %s " % (response.status_code))
else:
Expand All @@ -80,33 +89,45 @@ def _get_aws_details():
aws = None
if aws:
try:
aws_extra['cloud_private_ip'] = res.get('privateIp')
aws_extra['cloud_instance_type'] = res.get('instanceType')
aws_extra['cloud_availability_zone'] = res.get('availabilityZone')
aws_extra['cloud_ami_id'] = res.get('imageId')
aws_extra['cloud_region'] = res.get('region')
r = requests.get('http://169.254.169.254/latest/meta-data/public-hostname',
headers=aws_token_header, timeout=3, proxies=proxies)
if r.status_code == requests.codes.ok:
aws_extra['cloud_public_hostname'] = r.text
r = requests.get('http://169.254.169.254/latest/meta-data/public-ipv4',
headers=aws_token_header, timeout=3, proxies=proxies)
if r.status_code == requests.codes.ok:
aws_extra['cloud_public_ipv4'] = r.text
r = requests.get('http://169.254.169.254/latest/meta-data/local-hostname',
headers=aws_token_header, timeout=3, proxies=proxies)
if r.status_code == requests.codes.ok:
aws_extra['cloud_private_hostname'] = r.text
aws_extra["cloud_private_ip"] = res.get("privateIp")
aws_extra["cloud_instance_type"] = res.get("instanceType")
aws_extra["cloud_availability_zone"] = res.get("availabilityZone")
aws_extra["cloud_ami_id"] = res.get("imageId")
aws_extra["cloud_region"] = res.get("region")
response = requests.get(
"http://169.254.169.254/latest/meta-data/public-hostname",
headers=aws_token_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
aws_extra["cloud_public_hostname"] = response.text
response = requests.get(
"http://169.254.169.254/latest/meta-data/public-ipv4",
headers=aws_token_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
aws_extra["cloud_public_ipv4"] = response.text
response = requests.get(
"http://169.254.169.254/latest/meta-data/local-hostname",
headers=aws_token_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
aws_extra["cloud_private_hostname"] = response.text
for key in list(aws_extra):
if not aws_extra[key]:
aws_extra.pop(key)

except (requests.exceptions.RequestException, ValueError) as e:
log.debug(e)
aws_extra = None
ret['cloud_details'] = aws
ret["cloud_details"] = aws
if aws_extra:
ret['cloud_details_extra'] = aws_extra
ret["cloud_details_extra"] = aws_extra
return ret


Expand All @@ -116,89 +137,103 @@ def _get_azure_details():
"""
ret = {}
azure_extra = {}
azure = {'cloud_instance_id': None, 'cloud_account_id': None, 'cloud_type': 'azure'}
azure_header = {'Metadata': 'true'}
proxies = {'http': None}
azure = {"cloud_instance_id": None, "cloud_account_id": None, "cloud_type": "azure"}
azure_header = {"Metadata": "true"}
proxies = {"http": None}

try:
# Reminder: rev the api version for access to more details
response = requests.get(
'http://169.254.169.254/metadata/instance/compute?api-version=2017-08-01',
headers=azure_header, timeout=3, proxies=proxies)
"http://169.254.169.254/metadata/instance/compute?api-version=2021-02-01",
headers=azure_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
instance_info = response.json()
azure['cloud_instance_id'] = instance_info['vmId']
azure['cloud_account_id'] = instance_info['subscriptionId']
azure["cloud_instance_id"] = instance_info["vmId"]
azure["cloud_account_id"] = instance_info["subscriptionId"]
else:
raise ValueError("Error while fetching Azure instance metadata. Got status code: %s " % (response.status_code))
raise ValueError(
"Error while fetching Azure instance metadata. Got status code: %s " % (response.status_code)
)
except (requests.exceptions.RequestException, ValueError) as e:
log.debug(e)
# Not on an Azure box
azure = None

if azure:
try:
azure_extra['cloud_resource_group_name'] = instance_info['resourceGroupName']
azure_extra['cloud_location'] = instance_info['location']
azure_extra['cloud_name'] = instance_info['name']
azure_extra['cloud_image_offer'] = instance_info['offer']
azure_extra['cloud_os_type'] = instance_info['osType']
azure_extra['cloud_image_publisher'] = instance_info['publisher']
azure_extra['cloud_tags'] = instance_info['tags']
azure_extra['cloud_image_version'] = instance_info['version']
azure_extra['cloud_size'] = instance_info['vmSize']
azure_extra["cloud_resource_group_name"] = instance_info["resourceGroupName"]
azure_extra["cloud_location"] = instance_info["location"]
azure_extra["cloud_name"] = instance_info["name"]
azure_extra["cloud_image_offer"] = instance_info["offer"]
azure_extra["cloud_os_type"] = instance_info["osType"]
azure_extra["cloud_image_publisher"] = instance_info["publisher"]
azure_extra["cloud_tags"] = instance_info["tags"]
azure_extra["cloud_image_version"] = instance_info["version"]
azure_extra["cloud_size"] = instance_info["vmSize"]
response = requests.get(
'http://169.254.169.254/metadata/instance/network/interface?api-version=2017-08-01',
headers=azure_header, timeout=3, proxies=proxies)
"http://169.254.169.254/metadata/instance/network/interface?api-version=2021-02-01",
headers=azure_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
interface_list = response.json()
for counter, value in enumerate(interface_list):
grain_name_private_ipv4 = "cloud_interface_{0}_private_ipv4".format(counter)
azure_extra[grain_name_private_ipv4] = value['ipv4']['ipAddress'][0][
'privateIpAddress']
azure_extra[grain_name_private_ipv4] = value["ipv4"]["ipAddress"][0]["privateIpAddress"]

grain_name_public_ipv4 = "cloud_interface_{0}_public_ipv4".format(counter)
azure_extra[grain_name_public_ipv4] = value['ipv4']['ipAddress'][0][
'publicIpAddress']
azure_extra[grain_name_public_ipv4] = value["ipv4"]["ipAddress"][0]["publicIpAddress"]

grain_name_mac = "cloud_interface_{0}_mac_address".format(counter)
azure_extra[grain_name_mac] = value['macAddress']
azure_extra[grain_name_mac] = value["macAddress"]

for key in list(azure_extra):
if not azure_extra[key]:
azure_extra.pop(key)
else:
raise ValueError("Error while fetching metadata for Azure instance: %s. Got status code: %s" % (azure['cloud_instance_id'], response.status_code))
raise ValueError(
"Error while fetching metadata for Azure instance: %s. Got status code: %s"
% (azure["cloud_instance_id"], response.status_code)
)
except (requests.exceptions.RequestException, ValueError) as e:
log.debug(e)
azure_extra = None

if azure:
ret['cloud_details'] = azure
ret["cloud_details"] = azure
if azure_extra:
ret['cloud_details_extra'] = azure_extra
ret["cloud_details_extra"] = azure_extra
return ret


def _get_gcp_details():
# Gather google compute platform information if present
ret = {}
gcp_extra = {}
gcp = {'cloud_instance_id': None, 'cloud_account_id': None, 'cloud_type': 'gcp'}
gcp_header = {'Metadata-Flavor': 'Google'}
proxies = {'http': None}
gcp = {"cloud_instance_id": None, "cloud_account_id": None, "cloud_type": "gcp"}
gcp_header = {"Metadata-Flavor": "Google"}
proxies = {"http": None}
metadata = {}

try:
response = requests.get(
'http://metadata.google.internal/computeMetadata/v1/?recursive=true',
headers=gcp_header, timeout=3, proxies=proxies)
"http://metadata.google.internal/computeMetadata/v1/?recursive=true",
headers=gcp_header,
timeout=3,
proxies=proxies,
)
if response.status_code == requests.codes.ok:
metadata = response.json()
gcp['cloud_instance_id'] = metadata['instance']['id']
gcp['cloud_account_id'] = metadata['project']['numericProjectId']
gcp["cloud_instance_id"] = metadata["instance"]["id"]
gcp["cloud_account_id"] = metadata["project"]["numericProjectId"]
else:
raise ValueError("Error while fetching GCP instance metadata. Got status code: %s" % (response.status_code))
raise ValueError(
"Error while fetching GCP instance metadata. Got status code: %s" % (response.status_code)
)

except (requests.exceptions.RequestException, KeyError, ValueError) as e:
log.debug(e)
Expand All @@ -216,47 +251,45 @@ def _get_gcp_details():
gcp_extra = None

if gcp:
ret['cloud_details'] = gcp
ret["cloud_details"] = gcp
if gcp_extra:
ret['cloud_details_extra'] = gcp_extra
ret["cloud_details_extra"] = gcp_extra
return ret


def _build_gcp_extra(metadata):
""" Helper function to build the gcp extra dict """
gcp_extra= {}
gcp_extra['cloud_project_id']=metadata['project']['projectId']
gcp_extra['cloud_name'] = metadata['instance']['name']
gcp_extra['cloud_hostname'] = metadata['instance']['hostname']
gcp_extra['cloud_zone'] = metadata['instance']['zone']
gcp_extra['cloud_image'] = metadata['instance']['image']
gcp_extra['cloud_machine_type'] = metadata['instance']['machineType']
gcp_extra['cloud_tags'] = metadata['instance']['tags']
interface_list = metadata['instance']['networkInterfaces']
"""Helper function to build the gcp extra dict"""
gcp_extra = {}
gcp_extra["cloud_project_id"] = metadata["project"]["projectId"]
gcp_extra["cloud_name"] = metadata["instance"]["name"]
gcp_extra["cloud_hostname"] = metadata["instance"]["hostname"]
gcp_extra["cloud_zone"] = metadata["instance"]["zone"]
gcp_extra["cloud_image"] = metadata["instance"]["image"]
gcp_extra["cloud_machine_type"] = metadata["instance"]["machineType"]
gcp_extra["cloud_tags"] = metadata["instance"]["tags"]
interface_list = metadata["instance"]["networkInterfaces"]

for counter, value in enumerate(interface_list):
grain_name_network = "cloud_interface_{0}_network".format(counter)
gcp_extra[grain_name_network] = value['network']
gcp_extra[grain_name_network] = value["network"]

grain_name_ip = "cloud_interface_{0}_ip".format(counter)
gcp_extra[grain_name_ip] = value['ip']
gcp_extra[grain_name_ip] = value["ip"]

grain_name_subnetmask = "cloud_interface_{0}_subnetmask".format(counter)
gcp_extra[grain_name_subnetmask] = value['subnetmask']
gcp_extra[grain_name_subnetmask] = value["subnetmask"]

grain_name_mac = "cloud_interface_{0}_mac_address".format(counter)
gcp_extra[grain_name_mac] = value['mac']
gcp_extra[grain_name_mac] = value["mac"]

grain_name_forwardedips = "cloud_interface_{0}_forwarded_ips".format(counter)
gcp_extra[grain_name_forwardedips] = ','.join(value['forwardedIps'])
gcp_extra[grain_name_forwardedips] = ",".join(value["forwardedIps"])

grain_name_targetips = "cloud_interface_{0}_target_ips".format(counter)
gcp_extra[grain_name_targetips] = ','.join(value['targetInstanceIps'])
gcp_extra[grain_name_targetips] = ",".join(value["targetInstanceIps"])

grain_name_accessconfig_external_ips = "cloud_interface_{0}_" \
"accessconfigs_external_ips".format(counter)
external_ips_list = [item['externalIp'] for item in value['accessConfigs'] if
'externalIp' in item]
gcp_extra[grain_name_accessconfig_external_ips] = ','.join(external_ips_list)
grain_name_accessconfig_external_ips = "cloud_interface_{0}_" "accessconfigs_external_ips".format(counter)
external_ips_list = [item["externalIp"] for item in value["accessConfigs"] if "externalIp" in item]
gcp_extra[grain_name_accessconfig_external_ips] = ",".join(external_ips_list)

return gcp_extra