Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding support for Google Secret Manager for issue 543 #578

Merged
merged 5 commits into from
May 22, 2024
+697 −0
Merged

Adding support for Google Secret Manager for issue 543 #578

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
76eb024
Adding support for Google Secret Manager for issue 543
dcostakos Jun 21, 2023
953b06f
updated lookuup plugin based on comment https://github.com/ansible-co…
dcostakos Jun 22, 2023
375b317
Merge branch 'ansible-collections:master' into gcp_secret_manager
dcostakos Jul 14, 2023
3ce29db
updated plugsins based on feedback, fixed linting and documentation e…
dcostakos Jul 14, 2023
40d2c9a
updated plugsins based on feedback, fixed linting and documentation e…
dcostakos Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 126 additions & 87 deletions plugins/lookup/gcp_secret_manager.py
View file
Open in desktop
Copy link

@tze-dev tze-dev Jun 22, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for implementing this - it's awesome to see this feature being worked on.

In terms of the code for this lookup, is there a reason of not leveraging ansible_collections.google.cloud.plugins.module_utils.gcp_utils as a helper for handling the authentication workflow? IMO it would simplify the code and also gain the benefit of being able to use the new OAUTH token as well - recently added by this PR - #574.

I have used it in my private Collection and tested working fine. Code snippet here - you can add the additional env handling, but the heavy lifting of the authentication workflows and API requests are taken care of by gcp_utils

try:
    import os
    import requests
    import json
    import base64
except ImportError:
    pass

try:
    from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import (
        GcpSession,
    )
    HAS_GOOGLE_CLOUD_COLLECTION = True
except ImportError:
    HAS_GOOGLE_CLOUD_COLLECTION = False

display = Display()

class GcpMockModule(object):
    def __init__(self, params):
        self.params = params

    def fail_json(self, *args, **kwargs):
        raise AnsibleError(kwargs["msg"])

    def raise_for_status(self, response):
        try:
            response.raise_for_status()
        except getattr(requests.exceptions, "RequestException"):
            self.fail_json(msg="GCP returned error: %s" % response.json())


class GcpSecretLookup:
    def run(self, variables=None, **kwargs):
        params = {
            "project": kwargs.get("project", None),
            "secret": kwargs.get("secret", None),
            "version": kwargs.get("version", "latest"),
            "auth_kind": kwargs.get("auth_kind", None),
            "service_account_file": kwargs.get("service_account_file", None),
            "service_account_email": kwargs.get("service_account_email", None),
            "access_token": kwargs.get("access_token", None), # added for https://github.com/ansible-collections/google.cloud/pull/574
            "scopes": kwargs.get("scopes", None),
        }
        if not params["scopes"]:
            params["scopes"] = ["https://www.googleapis.com/auth/cloud-platform"]
        fake_module = GcpMockModule(params)
        result = self.get_secret(fake_module)
        return [base64.b64decode(result)]

    def get_secret(self, module):
        auth = GcpSession(module, "secretmanager")
        url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{secret}/versions/{version}:access".format(
            **module.params
        )
        response = auth.get(url)
        return response.json()['payload']['data']

class LookupModule(LookupBase):
    def run(self, terms, variables=None, **kwargs):
        if not HAS_GOOGLE_CLOUD_COLLECTION:
            raise AnsibleError(
                "gcp_secret lookup needs a supported version of the google.cloud collection installed. Use `ansible-galaxy collection install google.cloud` to install it"
            )
        return GcpSecretLookup().run(terms, variables=variables, **kwargs)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you so much for this suggestion. I'll integrate these changes.

Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@
options:
key:
description:
- the key of the secret to look up in Secret Manager
- the name of the secret to look up in Secret Manager
type: str
required: True
aliases:
- name
- secret
- secret_id
project:
description:
- The name of the google cloud project
Expand Down Expand Up @@ -57,11 +61,30 @@
- defaults to OS env variable GCP_SERVICE_ACCOUNT_INFO if not present
type: jsonarg
required: False
errors:
access_token:
description:
- support for GCP Access Token
- defaults to OS env variable GCP_ACCESS_TOKEN if not present
type: str
required: False
on_error:
description:
- how to handle errors
choices: ['strict','warn','ignore']
default: strict
- strict means raise an exception
- warn means warn, and return none
- ignore means just return none
type: str
required: False
choices:
- 'strict'
- 'warn'
- 'ignore'
default: 'strict'
scopes:
description:
- Authenticaiton scopes for Google Secret Manager
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
- Authenticaiton scopes for Google Secret Manager
- Authentication scopes for Google Secret Manager

type: list
default: ["https://www.googleapis.com/auth/cloud-platform"]
'''

EXAMPLES = '''
Expand Down Expand Up @@ -99,6 +122,8 @@


from ansible.plugins.lookup import LookupBase
from ansible.errors import AnsibleError
from ansible.utils.display import Display

try:
import requests
Expand All @@ -107,103 +132,117 @@
HAS_REQUESTS = False

try:
import google.auth
from google.oauth2 import service_account
from google.auth.transport.requests import AuthorizedSession
HAS_GOOGLE_LIBRARIES = True
from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import (
GcpSession,
)
HAS_GOOGLE_CLOUD_COLLECTION = True
except ImportError:
HAS_GOOGLE_LIBRARIES = False
HAS_GOOGLE_CLOUD_COLLECTION = False

from ansible_collections.google.cloud.plugins.module_utils.gcp_utils import GcpSession, GcpRequest
from ansible.errors import AnsibleError
from ansible.utils.display import Display

class GcpLookupException(Exception):
pass

class GcpMockModule(object):
def __init__(self, params):
self.params = params

def fail_json(self, *args, **kwargs):
raise AnsibleError(kwargs["msg"])

def raise_for_status(self, response):
try:
response.raise_for_status()
except getattr(requests.exceptions, "RequestException"):
self.fail_json(msg="GCP returned error: %s" % response.json())

class LookupModule(LookupBase):
def run(self, terms, variables, **kwargs):
def run(self, terms=None, variables=None, **kwargs):
self._display = Display()
if not HAS_GOOGLE_CLOUD_COLLECTION:
raise AnsibleError(
"gcp_secret lookup needs a supported version of the google.cloud collection installed. Use `ansible-galaxy collection install google.cloud` to install it"
)
self.set_options(var_options=variables, direct=kwargs)
self.scopes = ["https://www.googleapis.com/auth/cloud-platform"]
self._validate()
self.service_acct_creds = self._credentials()
session = AuthorizedSession(self.service_acct_creds)
response = session.get("https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{key}/versions/{version}:access".format(**self.get_options()))
if response.status_code == 200:
result_data = response.json()
secret_value = base64.b64decode(result_data['payload']['data'])
return [ secret_value ]
params = {
"key": self.get_option("key"),
"version": self.get_option("version"),
"access_token": self.get_option("access_token"),
"scopes": self.get_option("scopes"),
"on_error": self.get_option("on_error")
}

params['name'] = params['key']

# support GCP_* env variables for some parameters
for param in ["project", "auth_kind", "service_account_file", "service_account_info", "service_account_email", "access_token"]:
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As you're now using gcp_utils to handle authentication, this part is no longer needed here. This is because in gcp_utils there is already mechanism handling the fallback to env.

params[param] = self.fallback_from_env(param)

self._display.vvv(msg=f"Module Parameters: {params}")
fake_module = GcpMockModule(params)
result = self.get_secret(fake_module)
return [base64.b64decode(result)]

def fallback_from_env(self, arg):
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As per above - this is already being handled by gcp_utils

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once again, thank you for reviewing my code. Much appreciated.

There is indeed that functionality within the GcpModule class (in the init method for example) here . however, in your example, and the one I committed, we don't actually use that class only GcpSession.

I don't believe I can reuse the GcpModule class here in my lookup plugin. I did actually try this for grins and I got this error when attempting to call my super class constructor:
TypeError: AnsibleModule.__init__() got multiple values for argument 'argument_spec'

Not sure this is possible? What do you think?

And once again, thank you so much.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dcostakos you're right. Sorry, I got mixed up between the module and lookup helper as I just created a PR for a new module.
Your env handling is absolutely fine.

Thanks for doing this. Hope your PR gets merged soon, and we can all start using the default GCP collection rather than maintaining separate collections.

if self.get_option(arg):
return self.get_option(arg)
else:
if self.get_option('errors') == 'warn':
self.warn(f"secret request returned bad status: {response.status_code} {response.json()}")
return [ '' ]
elif self.get_option('error') == 'ignore':
return [ '' ]
else:
raise AnsibleError(f"secret request returned bad status: {response.status_code} {response.json()}")

def _validate(self):
if HAS_GOOGLE_LIBRARIES == False:
raise AnsibleError("Please install the google-auth library")
env_name = f"GCP_{arg.upper()}"
if env_name in os.environ:
self.set_option(arg, os.environ[env_name])
return self.get_option(arg)

if HAS_REQUESTS == False:
raise AnsibleError("Please install the requests library")

# set version to the latest version because
# we can't be sure that "latest" is always going
# to be set if secret versions get disabled
# see https://issuetracker.google.com/issues/286489671
def get_latest_version(self, module, auth):
url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions?filter=state:ENABLED".format(
**module.params
)
response = auth.get(url)
self._display.vvv(msg=f"List Version Response: {response.status_code} for {response.request.url}: {response.json()}")
if response.status_code != 200:
self.raise_error(module, f"unable to list versions of secret {response.status_code}")
version_list = response.json()
if "versions" in version_list:
return sorted(version_list['versions'], key=lambda d: d['name'])[-1]['name'].split('/')[-1]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does not return the latest version when you have more than 10. The problem is that it sorts the paths before getting the version, so somepath/12 is smaller than somepath/9, so it always returns 9 as the latest version.

Suggested change
return sorted(version_list['versions'], key=lambda d: d['name'])[-1]['name'].split('/')[-1]
versions_numbers = []
for version in version_list['versions']:
versions_numbers.append(version['name'].split('/')[-1])
return sorted(versions_numbers, key=int)[-1]

else:
self.raise_error(module, f"Unable to list secret versions via {response.request.url}: {response.json()}")


def raise_error(self, module, msg):
if module.params['on_error'] == 'strict':
raise GcpLookupException(msg)
elif module.params['on_error'] == 'warn':
self._display.warning(msg)

if self.get_option('key') == None:
raise AnsibleError("'key' is a required parameter")
return None

if self.get_option('version') == None:
self.set_option('version', 'latest')

self._set_from_env('project', 'GCP_PROJECT', True)
self._set_from_env('auth_kind', 'GCP_AUTH_KIND', True)
self._set_from_env('service_account_email', 'GCP_SERVICE_ACCOUNT_EMAIL')
self._set_from_env('service_account_file', 'GCP_SERVICE_ACCOUNT_FILE')
self._set_from_env('service_account_info', 'GCP_SERVICE_ACCOUNT_INFO')
def get_secret(self, module):
auth = GcpSession(module, "secretmanager")
if module.params['version'] == "latest":
module.params['calc_version'] = self.get_latest_version(module, auth)
else:
module.params['calc_version'] = module.params['version']

# there was an error listing secret versions
if module.params['calc_version'] is None:
return ''

url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets/{name}/versions/{calc_version}:access".format(
**module.params
)
response = auth.get(url)
self._display.vvv(msg=f"Response: {response.status_code} for {response.request.url}: {response.json()}")
if response.status_code != 200:
self.raise_error(module, f"Failed to lookup secret value via {response.request.url} {response.status_code}")
return ''

def _set_from_env(self, var=None, env_name=None, raise_on_empty=False):
if self.get_option(var) == None:
if env_name is not None and env_name in os.environ:
fallback = os.environ[env_name]
self.set_option(var, fallback)

if self.get_option(var) == None and raise_on_empty:
msg = f"No key '{var}' provided"
if env_name is not None:
msg += f" and no fallback to env['{env_name}'] available"
raise AnsibleError(msg)

def _credentials(self):
cred_type = self.get_option('auth_kind')

if cred_type == 'application':
credentials, project_id = google.auth.default(scopes=self.scopes)
return credentials

if cred_type == 'serviceaccount':
if self.get_option('service_account_file') is not None:
path = os.path.realpath(os.path.expanduser(self.get_option('service_account_file')))
try:
svc_acct_creds = service_account.Credentials.from_service_account_file(path)
except OSError as e:
raise GcpLookupException("Unable to read service_account_file at %s: %s" % (path, e.strerror))

elif self.get_option('service_account_contents') is not None:
try:
info = json.loads(self.get_option('service_account_contents'))
except json.decoder.JSONDecodeError as e:
raise GcpLookupException("Unable to decode service_account_contents as JSON: %s" % e)

svc_acct_creds = service_account.Credentials.from_service_account_info(info)
else:
raise GcpLookupException('Service Account authentication requires setting either service_account_file or service_account_contents')

return svc_acct_creds.with_scopes(self.scopes)

if cred_type == 'machineaccount':
self.svc_acct_creds = google.auth.compute_engine.Credentials(self.service_account_email)
return self.svc_acct_creds

raise GcpLookupException("Credential type '%s' not implemented" % cred_type)
return response.json()['payload']['data']



Expand Down
22 changes: 19 additions & 3 deletions plugins/modules/gcp_secret_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@
description:
- Name of the secret to be used
type: str
aliases:
- key
- secret
- secret_id
value:
description:
- The secret value that the secret should have
Expand All @@ -80,6 +84,13 @@
- "all" is also acceptable on delete (which will delete all versions of a secret)
type: str
default: 'latest'
labels:
description:
- A set of key-value pairs to assign as labels to asecret
- only used in creation
- Note that the "value" piece of a label must contain only readable chars
type: dict
required: False
'''

EXAMPLES='''
Expand Down Expand Up @@ -120,7 +131,6 @@
version: all
state: absent

- name: Get
'''

RETURN = '''
Expand Down Expand Up @@ -258,10 +268,14 @@ def snake_to_camel(snake):
def create_secret(module):
# build the payload
payload = { "replication": { "automatic": {} } }
if module.params['labels']:
payload['labels'] = module.params['labels']

url = "https://secretmanager.googleapis.com/v1/projects/{project}/secrets".format(**module.params)
auth = get_auth(module)
post_response = auth.post(url, body=payload, params={'secretId': module.params['name']})
# validate create
module.raise_for_status(post_response)
return update_secret(module)

def update_secret(module):
Expand Down Expand Up @@ -343,13 +357,15 @@ def main():
module = GcpModule(
argument_spec=dict(
state=dict(default='present', choices=['present', 'absent'], type='str'),
name=dict(required=True, type='str', aliases=['key', 'secret']),
name=dict(required=True, type='str', aliases=['key', 'secret', 'secret_id']),
value=dict(required=False, type='str'),
version=dict(required=False, type='str', default='latest'),
return_value=dict(required=False, type='bool', default=True)
return_value=dict(required=False, type='bool', default=True),
labels=dict(required=False, type='dict', default=dict())
)
)


if not module.params['scopes']:
module.params['scopes'] = ["https://www.googleapis.com/auth/cloud-platform"]

Expand Down