Skip to content

Commit

Permalink
Merge pull request #10932 from MikeMcQuaid/json_schema
Browse files Browse the repository at this point in the history 
github_packages: various improvements.
  • Loading branch information
@MikeMcQuaid
MikeMcQuaid authored Mar 26, 2021
2 parents 6362f3e + 169a1ba commit c3f90e9
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 26 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
**/vendor/bundle/ruby/*/gems/thread_safe-*/lib/thread_safe/util

# Ignore dependencies we don't wish to vendor
**/vendor/bundle/ruby/*/gems/addressable-*/
**/vendor/bundle/ruby/*/gems/ast-*/
**/vendor/bundle/ruby/*/gems/bootsnap-*/
**/vendor/bundle/ruby/*/gems/bundler-*/
Expand Down Expand Up @@ -119,6 +120,7 @@
**/vendor/bundle/ruby/*/gems/powerpack-*/
**/vendor/bundle/ruby/*/gems/psych-*/
**/vendor/bundle/ruby/*/gems/pry-*/
**/vendor/bundle/ruby/*/gems/public_suffix-*/
**/vendor/bundle/ruby/*/gems/racc-*/
**/vendor/bundle/ruby/*/gems/rainbow-*/
**/vendor/bundle/ruby/*/gems/rdiscount-*/
Expand Down
120 changes: 94 additions & 26 deletions Library/Homebrew/github_packages.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,65 @@ def upload_bottles(bottles_hash)
skopeo = Formula["skopeo"].opt_bin/"skopeo"
end

# TODO: these dependencies are installed but cannot be required automatically.
Homebrew.install_gem!("public_suffix")
Homebrew.install_gem!("addressable")
Homebrew.install_gem!("json-schema")
require "json-schema"

load_schemas!

bottles_hash.each do |formula_name, bottle_hash|
upload_bottle(user, token, skopeo, formula_name, bottle_hash)
end
end

private

IMAGE_CONFIG_SCHEMA_URI = "https://opencontainers.org/schema/image/config"
IMAGE_INDEX_SCHEMA_URI = "https://opencontainers.org/schema/image/index"
IMAGE_LAYOUT_SCHEMA_URI = "https://opencontainers.org/schema/image/layout"
IMAGE_MANIFEST_SCHEMA_URI = "https://opencontainers.org/schema/image/manifest"

def load_schemas!
schema_uri("content-descriptor",
"https://opencontainers.org/schema/image/content-descriptor.json")
schema_uri("defs", %w[
https://opencontainers.org/schema/defs.json
https://opencontainers.org/schema/descriptor/defs.json
https://opencontainers.org/schema/image/defs.json
https://opencontainers.org/schema/image/descriptor/defs.json
https://opencontainers.org/schema/image/index/defs.json
https://opencontainers.org/schema/image/manifest/defs.json
])
schema_uri("defs-descriptor", %w[
https://opencontainers.org/schema/descriptor.json
https://opencontainers.org/schema/defs-descriptor.json
https://opencontainers.org/schema/descriptor/defs-descriptor.json
https://opencontainers.org/schema/image/defs-descriptor.json
https://opencontainers.org/schema/image/descriptor/defs-descriptor.json
https://opencontainers.org/schema/image/index/defs-descriptor.json
https://opencontainers.org/schema/image/manifest/defs-descriptor.json
https://opencontainers.org/schema/index/defs-descriptor.json
])
schema_uri("config-schema", IMAGE_CONFIG_SCHEMA_URI)
schema_uri("image-index-schema", IMAGE_INDEX_SCHEMA_URI)
schema_uri("image-layout-schema", IMAGE_LAYOUT_SCHEMA_URI)
schema_uri("image-manifest-schema", IMAGE_MANIFEST_SCHEMA_URI)
end

def schema_uri(basename, uris)
url = "https://raw.githubusercontent.com/opencontainers/image-spec/master/schema/#{basename}.json"
out, = curl_output(url)
json = JSON.parse(out)

Array(uris).each do |uri|
schema = JSON::Schema.new(json, uri)
schema.uri = uri
JSON::Validator.add_schema(schema)
end
end

def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
_, org, repo, = *bottle_hash["bottle"]["root_url"].match(URL_REGEX)

Expand All @@ -64,8 +116,9 @@ def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
end
version_rebuild = "#{version}#{rebuild}"
root = Pathname("#{formula_name}-#{version_rebuild}")
FileUtils.rm_rf root

write_oci_layout(root)
write_image_layout(root)

blobs = root/"blobs/sha256"
blobs.mkpath
Expand Down Expand Up @@ -129,7 +182,7 @@ def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
Utils.safe_popen_read("gunzip", "--stdout", "--decompress", local_file),
)

config_json_sha256, config_json_size = write_config(platform_hash, tar_sha256, blobs)
config_json_sha256, config_json_size = write_image_config(platform_hash, tar_sha256, blobs)

created_time = tab.source_modified_time
created_time ||= Time.now
Expand All @@ -147,7 +200,7 @@ def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
annotations_hash.delete(key) if value.blank?
end

manifest_json_sha256, manifest_json_size = write_hash(blobs, {
image_manifest = {
schemaVersion: 2,
config: {
mediaType: "application/vnd.oci.image.config.v1+json",
Expand All @@ -159,21 +212,24 @@ def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
digest: "sha256:#{tar_gz_sha256}",
size: File.size(local_file),
annotations: {
"org.opencontainers.image.title": local_file,
"org.opencontainers.image.title" => local_file,
},
}],
annotations: annotations_hash,
})
}
JSON::Validator.validate!(IMAGE_MANIFEST_SCHEMA_URI, image_manifest)
manifest_json_sha256, manifest_json_size = write_hash(blobs, image_manifest)

{
mediaType: "application/vnd.oci.image.manifest.v1+json",
digest: "sha256:#{manifest_json_sha256}",
size: manifest_json_size,
platform: platform_hash,
mediaType: "application/vnd.oci.image.manifest.v1+json",
digest: "sha256:#{manifest_json_sha256}",
size: manifest_json_size,
platform: platform_hash,
annotations: {},
}
end

index_json_sha256, index_json_size = write_index(manifests, blobs)
index_json_sha256, index_json_size = write_image_index(manifests, blobs)

write_index_json(index_json_sha256, index_json_size, root)

Expand All @@ -186,8 +242,10 @@ def upload_bottle(user, token, skopeo, formula_name, bottle_hash)
])
end

def write_oci_layout(root)
write_hash(root, { imageLayoutVersion: "1.0.0" }, "oci-layout")
def write_image_layout(root)
image_layout = { imageLayoutVersion: "1.0.0" }
JSON::Validator.validate!(IMAGE_LAYOUT_SCHEMA_URI, image_layout)
write_hash(root, image_layout, "oci-layout")
end

def write_tar_gz(local_file, blobs)
Expand All @@ -197,37 +255,47 @@ def write_tar_gz(local_file, blobs)
tar_gz_sha256
end

def write_config(platform_hash, tar_sha256, blobs)
write_hash(blobs, platform_hash.merge({
def write_image_config(platform_hash, tar_sha256, blobs)
image_config = platform_hash.merge({
rootfs: {
type: "layers",
diff_ids: ["sha256:#{tar_sha256}"],
},
}))
})
JSON::Validator.validate!(IMAGE_CONFIG_SCHEMA_URI, image_config)
write_hash(blobs, image_config)
end

def write_index(manifests, blobs)
write_hash(blobs, {
def write_image_index(manifests, blobs)
image_index = {
schemaVersion: 2,
manifests: manifests,
})
annotations: {},
}
JSON::Validator.validate!(IMAGE_INDEX_SCHEMA_URI, image_index)
write_hash(blobs, image_index)
end

def write_index_json(index_json_sha256, index_json_size, root)
write_hash(root, {
index_json = {
schemaVersion: 2,
manifests: [{
mediaType: "application/vnd.oci.image.index.v1+json",
digest: "sha256:#{index_json_sha256}",
size: index_json_size,
mediaType: "application/vnd.oci.image.index.v1+json",
digest: "sha256:#{index_json_sha256}",
size: index_json_size,
annotations: {},
}],
}, "index.json")
annotations: {},
}
JSON::Validator.validate!(IMAGE_INDEX_SCHEMA_URI, index_json)
write_hash(root, index_json, "index.json")
end

def write_hash(directory, hash, _filename = nil)
json = hash.to_json
def write_hash(directory, hash, filename = nil)
json = JSON.pretty_generate(hash)
sha256 = Digest::SHA256.hexdigest(json)
path = directory/sha256
filename ||= sha256
path = directory/filename
path.unlink if path.exist?
path.write(json)

Expand Down
2 changes: 2 additions & 0 deletions Library/Homebrew/sorbet/rbi/todo.rbi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
# typed: strong
module ::StackProf; end
module DependencyCollector::Compat; end
module JSON::Schema; end
module JSON::Validator; end
module OS::Mac::Version::NULL; end
module T::InterfaceWrapper::Helpers; end
module T::Private::Abstract::Hooks; end
Expand Down

0 comments on commit c3f90e9

Please sign in to comment.