Skip to content

Commit

Permalink
Add signature checking to install-debs.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@akoeplinger
akoeplinger committed Jan 5, 2025
1 parent 4e6b9f8 commit e74cb5a
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 19 deletions.
20 changes: 10 additions & 10 deletions eng/common/cross/build-rootfs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,14 @@ elif [[ "$__CodeName" == "haiku" ]]; then
popd
rm -rf "$__RootfsDir/tmp"
elif [[ -n "$__CodeName" ]]; then
__UpdateOptions=
if [[ "$__SkipSigCheck" == "0" ]]; then
__Keyring="$__Keyring --force-check-gpg"
else
__Keyring=
__UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
fi

if [[ "$__SkipEmulation" == "1" ]]; then
if [[ -z "$AR" ]]; then
if command -v ar &>/dev/null; then
Expand All @@ -800,26 +808,18 @@ elif [[ -n "$__CodeName" ]]; then
PYTHON=${PYTHON_EXECUTABLE:-python3}

# shellcheck disable=SC2086,SC2046
echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
echo running "$PYTHON" "$__CrossDir/install-debs.py" $__Keyring --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
$(echo $suites | xargs -n 1 | xargs -I {} echo -n "--suite {} ") \
$__UbuntuPackages

# shellcheck disable=SC2086,SC2046
"$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
"$PYTHON" "$__CrossDir/install-debs.py" $__Keyring --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
$(echo $suites | xargs -n 1 | xargs -I {} echo -n "--suite {} ") \
$__UbuntuPackages

exit 0
fi

__UpdateOptions=
if [[ "$__SkipSigCheck" == "0" ]]; then
__Keyring="$__Keyring --force-check-gpg"
else
__Keyring=
__UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
fi

# shellcheck disable=SC2086
echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"

Expand Down
82 changes: 73 additions & 9 deletions eng/common/cross/install-debs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import asyncio
import aiohttp
import gzip
import hashlib
import os
import re
import shutil
Expand All @@ -16,7 +17,7 @@
from collections import deque
from functools import cmp_to_key

async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60):
async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, timeout=60, checksum=None):
"""Asynchronous file download with retries."""
attempt = 0
while attempt < max_retries:
Expand All @@ -25,6 +26,13 @@ async def download_file(session, url, dest_path, max_retries=3, retry_delay=2, t
if response.status == 200:
with open(dest_path, "wb") as f:
content = await response.read()

# verify checksum if provided
if checksum:
sha256 = hashlib.sha256(content).hexdigest()
if sha256 != checksum:
raise Exception(f"SHA256 mismatch for {url}: expected {checksum}, got {sha256}")

f.write(content)
print(f"Downloaded {url} at {dest_path}")
return
Expand All @@ -51,22 +59,21 @@ async def download_deb_files_parallel(mirror, packages, tmp_dir):
if filename:
url = f"{mirror}/{filename}"
dest_path = os.path.join(tmp_dir, os.path.basename(filename))
tasks.append(asyncio.create_task(download_file(session, url, dest_path)))
tasks.append(asyncio.create_task(download_file(session, url, dest_path, checksum=info.get("SHA256"))))

await asyncio.gather(*tasks)

async def download_package_index_parallel(mirror, arch, suites):
async def download_package_index_parallel(mirror, arch, suites, check_sig, keyring):
"""Download package index files for specified suites and components entirely in memory."""
tasks = []
timeout = aiohttp.ClientTimeout(total=60)

async with aiohttp.ClientSession(timeout=timeout) as session:
for suite in suites:
for component in ["main", "universe"]:
url = f"{mirror}/dists/{suite}/{component}/binary-{arch}/Packages.gz"
tasks.append(fetch_and_decompress(session, url))
tasks.append(fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring))

results = await asyncio.gather(*tasks, return_exceptions=True)
results = await asyncio.gather(*tasks)

merged_content = ""
for result in results:
Expand All @@ -77,21 +84,74 @@ async def download_package_index_parallel(mirror, arch, suites):

return merged_content

async def fetch_and_decompress(session, url):
async def fetch_and_decompress(session, mirror, arch, suite, component, check_sig, keyring):
"""Fetch and decompress the Packages.gz file."""

path = f"{component}/binary-{arch}/Packages.gz"
url = f"{mirror}/dists/{suite}/{path}"

try:
async with session.get(url) as response:
if response.status == 200:
compressed_data = await response.read()
decompressed_data = gzip.decompress(compressed_data).decode('utf-8')
print(f"Downloaded index: {url}")

if check_sig:
# Verify the package index against the sha256 recorded in the Release file
release_file_content = await fetch_release_file(session, mirror, suite, keyring)
packages_sha = parse_release_file(release_file_content, path)

sha256 = hashlib.sha256(compressed_data).hexdigest()
if sha256 != packages_sha:
raise Exception(f"SHA256 mismatch for {path}: expected {packages_sha}, got {sha256}")
print(f"Checksum verified for {path}")

return decompressed_data
else:
print(f"Skipped index: {url} (doesn't exist)")
return None
except Exception as e:
print(f"Error fetching {url}: {e}")

async def fetch_release_file(session, mirror, suite, keyring):
"""Fetch Release and Release.gpg files and verify the signature."""

release_url = f"{mirror}/dists/{suite}/Release"
release_gpg_url = f"{mirror}/dists/{suite}/Release.gpg"

with tempfile.NamedTemporaryFile() as release_file, tempfile.NamedTemporaryFile() as release_gpg_file:
await download_file(session, release_url, release_file.name)
await download_file(session, release_gpg_url, release_gpg_file.name)

if keyring != '':
keyring_arg = f"--keyring {args.keyring}"

print("Verifying signature of Release with Release.gpg.")
verify_command = f"gpg {keyring_arg} --verify {release_gpg_file.name} {release_file.name}"
result = subprocess.run(verify_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

if result.returncode != 0:
raise Exception(f"Signature verification failed: {result.stderr.decode('utf-8')}")

print("Signature verified successfully.")

with open(release_file.name) as f: return f.read()

def parse_release_file(content, path):
"""Parses the Release file and returns sha256 checksum of the specified path."""

# data looks like this:
# <checksum> <size> <path>
matches = re.findall(r'^ (\S*) +(\S*) +(\S*)$', content, re.MULTILINE)

for entry in matches:
# the file has both md5 and sha256 checksums, we want sha256 which has a length of 64
if entry[2] == path and len(entry[0]) == 64:
return entry[0]

raise Exception(f"Could not find checksum for {path} in Release file.")

def parse_debian_version(version):
"""Parse a Debian package version into epoch, upstream version, and revision."""
match = re.match(r'^(?:(\d+):)?([^-]+)(?:-(.+))?$', version)
Expand Down Expand Up @@ -171,13 +231,15 @@ def parse_package_index(content):
filename = fields.get("Filename")
depends = fields.get("Depends")
provides = fields.get("Provides", None)
sha256 = fields.get("SHA256")

# Only update if package_name is not in packages or if the new version is higher
if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0:
packages[package_name] = {
"Version": version,
"Filename": filename,
"Depends": depends
"Depends": depends,
"SHA256": sha256
}

# Update aliases if package provides any alternatives
Expand Down Expand Up @@ -301,6 +363,8 @@ def finalize_setup(rootfsdir):
parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.')
parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)")
parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)")
parser.add_argument("--force-check-gpg", required=False, action='store_true', help="Verify the packages against signatures in Release file.")
parser.add_argument("--keyring", required=False, default='', help="Keyring file to check signature of Release file.")
parser.add_argument("packages", nargs="+", help="List of package names to be installed.")

args = parser.parse_args()
Expand All @@ -324,7 +388,7 @@ def finalize_setup(rootfsdir):

print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}")

package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite))
package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite, args.force_check_gpg, args.keyring))

packages_info, aliases = parse_package_index(package_index_content)

Expand Down

0 comments on commit e74cb5a

Please sign in to comment.