feat: Support URI sources in write_files module (#5505)

This change adds an optional `source` key to the `write_files` module,
allowing users to specify a URI from which to load file contents. This
facilitates more flexible multi-part configurations, as file contents
can be managed via external sources such as independent Git
repositories.

Fixes GH-5500

cloudinit/config/cc_write_files.py

@@ -9,8 +9,9 @@
 import base64
 import logging
 import os
+from typing import Optional
 
-from cloudinit import util
+from cloudinit import url_helper, util
 from cloudinit.cloud import Cloud
 from cloudinit.config import Config
 from cloudinit.config.schema import MetaSchema
@@ -44,7 +45,8 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
             name,
         )
         return
-    write_files(name, filtered_files, cloud.distro.default_owner)
+    ssl_details = util.fetch_ssl_details(cloud.paths)
+    write_files(name, filtered_files, cloud.distro.default_owner, ssl_details)
 
 
 def canonicalize_extraction(encoding_type):
@@ -72,7 +74,7 @@ def canonicalize_extraction(encoding_type):
     return [TEXT_PLAIN_ENC]
 
 
-def write_files(name, files, owner: str):
+def write_files(name, files, owner: str, ssl_details: Optional[dict] = None):
     if not files:
         return
 
@@ -86,8 +88,23 @@ def write_files(name, files, owner: str):
             )
             continue
         path = os.path.abspath(path)
-        extractions = canonicalize_extraction(f_info.get("encoding"))
-        contents = extract_contents(f_info.get("content", ""), extractions)
+        # Read content from provided URL, if any, or decode from inline
+        contents = read_url_or_decode(
+            f_info.get("source", None),
+            ssl_details,
+            f_info.get("content", None),
+            f_info.get("encoding", None),
+        )
+        if contents is None:
+            LOG.warning(
+                "No content could be loaded for entry %s in module %s;"
+                " skipping",
+                i + 1,
+                name,
+            )
+            continue
+        # Only create the file if content exists. This will not happen, for
+        # example, if the URL fails and no inline content was provided
         (u, g) = util.extract_usergroup(f_info.get("owner", owner))
         perms = decode_perms(f_info.get("permissions"), DEFAULT_PERMS)
         omode = "ab" if util.get_cfg_option_bool(f_info, "append") else "wb"
@@ -118,6 +135,43 @@ def decode_perms(perm, default):
         return default
 
 
+def read_url_or_decode(source, ssl_details, content, encoding):
+    url = None if source is None else source.get("uri", None)
+    use_url = bool(url)
+    # Special case: empty URL and content. Write a blank file
+    if content is None and not use_url:
+        return ""
+    # Fetch file content from source URL, if provided
+    result = None
+    if use_url:
+        try:
+            # NOTE: These retry parameters are arbitrarily chosen defaults.
+            # They have no significance, and may be changed if appropriate
+            result = url_helper.read_file_or_url(
+                url,
+                headers=source.get("headers", None),
+                retries=3,
+                sec_between=3,
+                ssl_details=ssl_details,
+            ).contents
+        except Exception:
+            util.logexc(
+                LOG,
+                'Failed to retrieve contents from source "%s"; falling back to'
+                ' data from "contents" key',
+                url,
+            )
+            use_url = False
+    # If inline content is provided, and URL is not provided or is
+    # inaccessible, parse the former
+    if content is not None and not use_url:
+        # NOTE: This is not simply an "else"! Notice that `use_url` can change
+        # in the previous "if" block
+        extractions = canonicalize_extraction(encoding)
+        result = extract_contents(content, extractions)
+    return result
+
+
 def extract_contents(contents, extraction_types):
     result = contents
     for t in extraction_types:

cloudinit/config/cc_write_files_deferred.py

@@ -39,4 +39,5 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
             name,
         )
         return
-    write_files(name, filtered_files, cloud.distro.default_owner)
+    ssl_details = util.fetch_ssl_details(cloud.paths)
+    write_files(name, filtered_files, cloud.distro.default_owner, ssl_details)

cloudinit/config/schemas/schema-cloud-config-v1.json

@@ -3386,6 +3386,28 @@
                 "default": "''",
                 "description": "Optional content to write to the provided ``path``. When content is present and encoding is not 'text/plain', decode the content prior to writing. Default: ``''``"
               },
+              "source": {
+                "type": "object",
+                "description": "Optional specification for content loading from an arbitrary URI",
+                "additionalProperties": false,
+                "properties": {
+                  "uri": {
+                    "type": "string",
+                    "format": "uri",
+                    "description": "URI from which to load file content. If loading fails repeatedly, ``content`` is used instead."
+                  },
+                  "headers": {
+                    "type": "object",
+                    "description": "Optional HTTP headers to accompany load request, if applicable",
+                    "additionalProperties": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "required": [
+                  "uri"
+                ]
+              },
               "owner": {
                 "type": "string",
                 "default": "root:root",

doc/module-docs/cc_runcmd/example1.yaml

@@ -4,4 +4,3 @@ runcmd:
 - [sh, -xc, 'echo $(date) '': hello world!''']
 - [sh, -c, echo "=========hello world'========="]
 - ls -l /root
-- [wget, 'http://example.org', -O, /tmp/index.html]

doc/module-docs/cc_write_files/data.yaml

@@ -3,8 +3,9 @@ cc_write_files:
     Write out arbitrary content to files, optionally setting permissions.
     Parent folders in the path are created if absent. Content can be specified
     in plain text or binary. Data encoded with either base64 or binary gzip
-    data can be specified and will be decoded before being written. For empty
-    file creation, content can be omitted.
+    data can be specified and will be decoded before being written. Data can
+    also be loaded from an arbitrary URI. For empty file creation, content can
+    be omitted.
 
     .. note::
        If multi-line data is provided, care should be taken to ensure it
@@ -36,5 +37,10 @@ cc_write_files:
       Example 5: Defer writing the file until after the package (Nginx) is
       installed and its user is created.
     file: cc_write_files/example5.yaml
+  - comment: >
+      Example 6: Retrieve file contents from a URI source, rather than inline.
+      Especially useful with an external config-management repo, or for large
+      binaries.
+    file: cc_write_files/example6.yaml
   name: Write Files
   title: Write arbitrary files

doc/module-docs/cc_write_files/example6.yaml

@@ -0,0 +1,9 @@
+#cloud-config
+write_files:
+- source:
+    uri: https://gitlab.example.com/some_ci_job/artifacts/hello
+    headers:
+      Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==
+      User-Agent: cloud-init on myserver.example.com
+  path: /usr/bin/hello
+  permissions: '0755'

tests/unittests/config/test_cc_write_files.py

@@ -9,6 +9,7 @@
 import tempfile
 
 import pytest
+import responses
 
 from cloudinit import util
 from cloudinit.config.cc_write_files import decode_perms, handle, write_files
@@ -84,6 +85,16 @@ def test_simple(self):
         )
         self.assertEqual(util.load_text_file(filename), expected)
 
+    def test_empty(self):
+        self.patchUtils(self.tmp)
+        filename = "/tmp/my.file"
+        write_files(
+            "test_empty",
+            [{"path": filename}],
+            self.owner,
+        )
+        self.assertEqual(util.load_text_file(filename), "")
+
     def test_append(self):
         self.patchUtils(self.tmp)
         existing = "hello "
@@ -167,6 +178,71 @@ def test_handle_plain_text(self):
             "Unknown encoding type text/plain", self.logs.getvalue()
         )
 
+    def test_file_uri(self):
+        self.patchUtils(self.tmp)
+        src_path = "/tmp/file-uri"
+        dst_path = "/tmp/file-uri-target"
+        content = "asdf"
+        util.write_file(src_path, content)
+        cfg = {
+            "write_files": [
+                {
+                    "source": {"uri": "file://" + src_path},
+                    "path": dst_path,
+                }
+            ]
+        }
+        cc = self.tmp_cloud("ubuntu")
+        handle("ignored", cfg, cc, [])
+        self.assertEqual(
+            util.load_text_file(src_path), util.load_text_file(dst_path)
+        )
+
+    @responses.activate
+    def test_http_uri(self):
+        self.patchUtils(self.tmp)
+        path = "/tmp/http-uri-target"
+        url = "http://hostname/path"
+        content = "more asdf"
+        responses.add(responses.GET, url, content)
+        cfg = {
+            "write_files": [
+                {
+                    "source": {
+                        "uri": url,
+                        "headers": {
+                            "foo": "bar",
+                            "blah": "blah",
+                        },
+                    },
+                    "path": path,
+                }
+            ]
+        }
+        cc = self.tmp_cloud("ubuntu")
+        handle("ignored", cfg, cc, [])
+        self.assertEqual(content, util.load_text_file(path))
+
+    def test_uri_fallback(self):
+        self.patchUtils(self.tmp)
+        src_path = "/tmp/INVALID"
+        dst_path = "/tmp/uri-fallback-target"
+        content = "asdf"
+        util.del_file(src_path)
+        cfg = {
+            "write_files": [
+                {
+                    "source": {"uri": "file://" + src_path},
+                    "content": content,
+                    "encoding": "text/plain",
+                    "path": dst_path,
+                }
+            ]
+        }
+        cc = self.tmp_cloud("ubuntu")
+        handle("ignored", cfg, cc, [])
+        self.assertEqual(content, util.load_text_file(dst_path))
+
     def test_deferred(self):
         self.patchUtils(self.tmp)
         file_path = "/tmp/deferred.file"
@@ -249,6 +325,12 @@ class TestWriteFilesSchema:
                     "write_files": [
                         {
                             "append": False,
+                            "source": {
+                                "uri": "http://a.com/a",
+                                "headers": {
+                                    "Authorization": "Bearer SOME_TOKEN"
+                                },
+                            },
                             "content": "a",
                             "encoding": "text/plain",
                             "owner": "jeff",

tools/.github-cla-signers

@@ -110,6 +110,7 @@ licebmi
 linitio
 LKHN
 lkundrak
+LRitzdorf
 lucasmoura
 lucendio
 lungj