Skip to content

Commit

Permalink
Prune image configs of manifests V2 schema 2
Browse files Browse the repository at this point in the history 
Renamed LayerNode to ComponentNode because it now identifies image
configs as well. The ComponentNode still needs to contain an information
whether it represents config or layer for proper logging.

The ComponentNode can now be connected to ImageNode or ImageStreamNode
using either ImageLayerEdge or ImageConfigEdge.

Signed-off-by: Michal Minář <[email protected]>
  • Loading branch information
Michal Minář committed Sep 9, 2016
1 parent 12e5235 commit 919493f
Show file tree
Hide file tree
Showing 7 changed files with 619 additions and 100 deletions.
4 changes: 2 additions & 2 deletions pkg/cmd/admin/top/graph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func addImagesToGraph(g graph.Graph, images *imageapi.ImageList) {
// schema v2 does not have that problem.
for i := len(image.DockerImageLayers) - 1; i >= 0; i-- {
layer := image.DockerImageLayers[i]
layerNode := imagegraph.EnsureImageLayerNode(g, layer.Name)
layerNode := imagegraph.EnsureImageComponentLayerNode(g, layer.Name)
edgeKind := ImageLayerEdgeKind
if !topLayerAdded && layer.Name != digest.DigestSha256EmptyTar && layer.Name != digestSHA256GzippedEmptyTar {
edgeKind = ImageTopLayerEdgeKind
Expand Down Expand Up @@ -138,7 +138,7 @@ func markParentsInGraph(g graph.Graph) {
for _, in := range imageNodes {
// find image's top layer, should be just one
for _, e := range g.OutboundEdges(in, ImageTopLayerEdgeKind) {
layerNode, _ := e.To().(*imagegraph.ImageLayerNode)
layerNode, _ := e.To().(*imagegraph.ImageComponentNode)
// find image's containing this layer but not being their top layer
for _, ed := range g.InboundEdges(layerNode, ImageLayerEdgeKind) {
childNode, _ := ed.From().(*imagegraph.ImageNode)
Expand Down
34 changes: 29 additions & 5 deletions pkg/image/graph/nodes/nodes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,12 +161,36 @@ func FindOrCreateSyntheticImageStreamNode(g osgraph.MutableUniqueGraph, is *imag
).(*ImageStreamNode)
}

// EnsureImageLayerNode adds a graph node for the layer if it does not already exist.
func EnsureImageLayerNode(g osgraph.MutableUniqueGraph, layer string) graph.Node {
return osgraph.EnsureUnique(g,
ImageLayerNodeName(layer),
func ensureImageComponentNode(g osgraph.MutableUniqueGraph, name string, t ImageComponentType) graph.Node {
node := osgraph.EnsureUnique(g,
ImageComponentNodeName(name),
func(node osgraph.Node) graph.Node {
return &ImageLayerNode{node, layer}
return &ImageComponentNode{
Node: node,
Component: name,
Type: t,
}
},
)

// If at least one image referers to the blob as its config, treat it as a config even if it is a layer of
// some other image.
if t == ImageComponentTypeConfig {
cn := node.(*ImageComponentNode)
if cn.Type != ImageComponentTypeConfig {
cn.Type = ImageComponentTypeConfig
}
}

return node
}

// EnsureImageComponentLayerNode adds a graph node for the image config if it does not already exist.
func EnsureImageComponentConfigNode(g osgraph.MutableUniqueGraph, name string) graph.Node {
return ensureImageComponentNode(g, name, ImageComponentTypeConfig)
}

// EnsureImagecomponentLayerNode adds a graph node for the image layer if it does not already exist.
func EnsureImageComponentLayerNode(g osgraph.MutableUniqueGraph, name string) graph.Node {
return ensureImageComponentNode(g, name, ImageComponentTypeLayer)
}
35 changes: 24 additions & 11 deletions pkg/image/graph/nodes/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,15 @@ import (
imageapi "github.com/openshift/origin/pkg/image/api"
)

type ImageComponentType string

const (
ImageComponentNodeKind = "ImageComponent"

ImageComponentTypeConfig ImageComponentType = `Config`
ImageComponentTypeLayer ImageComponentType = `Layer`
)

var (
ImageStreamNodeKind = reflect.TypeOf(imageapi.ImageStream{}).Name()
ImageNodeKind = reflect.TypeOf(imageapi.Image{}).Name()
Expand All @@ -16,7 +25,6 @@ var (

// non-api types
DockerRepositoryNodeKind = reflect.TypeOf(imageapi.DockerImageReference{}).Name()
ImageLayerNodeKind = "ImageLayer"
)

func ImageStreamNodeName(o *imageapi.ImageStream) osgraph.UniqueName {
Expand Down Expand Up @@ -185,23 +193,28 @@ func (*ImageNode) Kind() string {
return ImageNodeKind
}

func ImageLayerNodeName(layer string) osgraph.UniqueName {
return osgraph.UniqueName(fmt.Sprintf("%s|%s", ImageLayerNodeKind, layer))
func ImageComponentNodeName(name string) osgraph.UniqueName {
return osgraph.UniqueName(fmt.Sprintf("%s|%s", ImageComponentNodeKind, name))
}

type ImageLayerNode struct {
type ImageComponentNode struct {
osgraph.Node
Layer string
Component string
Type ImageComponentType
}

func (n ImageComponentNode) Object() interface{} {
return n.Component
}

func (n ImageLayerNode) Object() interface{} {
return n.Layer
func (n ImageComponentNode) String() string {
return string(ImageComponentNodeName(n.Component))
}

func (n ImageLayerNode) String() string {
return string(ImageLayerNodeName(n.Layer))
func (n *ImageComponentNode) Describe() string {
return fmt.Sprintf("Image%s|%s", n.Type, n.Component)
}

func (*ImageLayerNode) Kind() string {
return ImageLayerNodeKind
func (*ImageComponentNode) Kind() string {
return ImageComponentNodeKind
}
116 changes: 71 additions & 45 deletions pkg/image/prune/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,12 @@ const (
// not keep an ImageNode from being a candidate for pruning.
WeakReferencedImageEdgeKind = "WeakReferencedImage"

// ReferencedImageConfigEdgeKind defines an edge from an ImageStreamNode or an
// ImageNode to an ImageComponentNode.
ReferencedImageConfigEdgeKind = "ReferencedImageConfig"

// ReferencedImageLayerEdgeKind defines an edge from an ImageStreamNode or an
// ImageNode to an ImageLayerNode.
// ImageNode to an ImageComponentNode.
ReferencedImageLayerEdgeKind = "ReferencedImageLayer"
)

Expand Down Expand Up @@ -128,7 +132,7 @@ type PrunerOptions struct {
RegistryURL string
}

// Pruner knows how to prune images and layers.
// Pruner knows how to prune istags, images, layers and image configs.
type Pruner interface {
// Prune uses imagePruner, streamPruner, layerLinkPruner, blobPruner, and
// manifestPruner to remove images that have been identified as candidates
Expand Down Expand Up @@ -305,9 +309,16 @@ func addImagesToGraph(g graph.Graph, images *imageapi.ImageList, algorithm prune
glog.V(4).Infof("Adding image %q to graph", image.Name)
imageNode := imagegraph.EnsureImageNode(g, image)

if len(image.DockerImageConfig) > 0 {
configName := image.DockerImageMetadata.ID
glog.V(4).Infof("Adding image config %q to graph", configName)
configNode := imagegraph.EnsureImageComponentConfigNode(g, configName)
g.AddEdge(imageNode, configNode, ReferencedImageConfigEdgeKind)
}

for _, layer := range image.DockerImageLayers {
glog.V(4).Infof("Adding image layer %q to graph", layer.Name)
layerNode := imagegraph.EnsureImageLayerNode(g, layer.Name)
layerNode := imagegraph.EnsureImageComponentLayerNode(g, layer.Name)
g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind)
}
}
Expand Down Expand Up @@ -377,14 +388,20 @@ func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, li
glog.V(4).Infof("Adding edge (kind=%s) from %q to %q", kind, imageStreamNode.UniqueName(), imageNode.UniqueName())
g.AddEdge(imageStreamNode, imageNode, kind)

glog.V(4).Infof("Adding stream->layer references")
glog.V(4).Infof("Adding stream->(layer|config) references")
// add stream -> layer references so we can prune them later
for _, s := range g.From(imageNode) {
if g.Kind(s) != imagegraph.ImageLayerNodeKind {
cn, ok := s.(*imagegraph.ImageComponentNode)
if !ok {
continue
}
glog.V(4).Infof("Adding reference from stream %q to layer %q", stream.Name, s.(*imagegraph.ImageLayerNode).Layer)
g.AddEdge(imageStreamNode, s, ReferencedImageLayerEdgeKind)

glog.V(4).Infof("Adding reference from stream %q to %s", stream.Name, cn.Describe())
if cn.Type == imagegraph.ImageComponentTypeConfig {
g.AddEdge(imageStreamNode, s, ReferencedImageConfigEdgeKind)
} else {
g.AddEdge(imageStreamNode, s, ReferencedImageLayerEdgeKind)
}
}
}
}
Expand Down Expand Up @@ -663,26 +680,28 @@ func subgraphWithoutPrunableImages(g graph.Graph, prunableImageIDs graph.NodeSet
)
}

// calculatePrunableLayers returns the list of prunable layers.
func calculatePrunableLayers(g graph.Graph) []*imagegraph.ImageLayerNode {
prunable := []*imagegraph.ImageLayerNode{}
// calculatePrunableImageComponents returns the list of prunable image components.
func calculatePrunableImageComponents(g graph.Graph) (configs []*imagegraph.ImageComponentNode, layers []*imagegraph.ImageComponentNode) {

nodes := g.Nodes()
for i := range nodes {
layerNode, ok := nodes[i].(*imagegraph.ImageLayerNode)
cn, ok := nodes[i].(*imagegraph.ImageComponentNode)
if !ok {
continue
}

glog.V(4).Infof("Examining layer %q", layerNode.Layer)

if layerIsPrunable(g, layerNode) {
glog.V(4).Infof("Layer %q is prunable", layerNode.Layer)
prunable = append(prunable, layerNode)
glog.V(4).Infof("Examining %v", cn)
if imageComponentIsPrunable(g, cn) {
glog.V(4).Infof("%v is prunable", cn)
if cn.Type == imagegraph.ImageComponentTypeConfig {
configs = append(configs, cn)
} else {
layers = append(layers, cn)
}
}
}

return prunable
return configs, layers
}

// pruneStreams removes references from all image streams' status.tags entries
Expand Down Expand Up @@ -776,8 +795,8 @@ func (p *pruner) determineRegistry(imageNodes []*imagegraph.ImageNode) (string,
}

// Run identifies images eligible for pruning, invoking imagePruner for each image, and then it identifies
// image layers eligible for pruning, invoking layerLinkPruner for each registry URL that has
// layers that can be pruned.
// image configs and layers eligible for pruning, invoking layerLinkPruner for each registry URL that has
// layers or configs that can be pruned.
func (p *pruner) Prune(
imagePruner ImageDeleter,
streamPruner ImageStreamDeleter,
Expand All @@ -804,13 +823,14 @@ func (p *pruner) Prune(

prunableImageNodes, prunableImageIDs := calculatePrunableImages(p.g, imageNodes)
graphWithoutPrunableImages := subgraphWithoutPrunableImages(p.g, prunableImageIDs)
prunableLayers := calculatePrunableLayers(graphWithoutPrunableImages)
prunableConfigs, prunableLayers := calculatePrunableImageComponents(graphWithoutPrunableImages)

errs := []error{}

errs = append(errs, pruneStreams(p.g, prunableImageNodes, streamPruner)...)
errs = append(errs, pruneLayers(p.g, p.registryClient, registryURL, prunableLayers, layerLinkPruner)...)
errs = append(errs, pruneBlobs(p.g, p.registryClient, registryURL, prunableLayers, blobPruner)...)
errs = append(errs, pruneImageComponents(p.g, p.registryClient, registryURL, prunableLayers, layerLinkPruner)...)
errs = append(errs, pruneImageComponents(p.g, p.registryClient, registryURL, prunableConfigs, layerLinkPruner)...)
errs = append(errs, pruneBlobs(p.g, p.registryClient, registryURL, append(prunableConfigs, prunableLayers...), blobPruner)...)
errs = append(errs, pruneManifests(p.g, p.registryClient, registryURL, prunableImageNodes, manifestPruner)...)

if len(errs) > 0 {
Expand All @@ -824,24 +844,24 @@ func (p *pruner) Prune(
return kerrors.NewAggregate(errs)
}

// layerIsPrunable returns true if the layer is not referenced by any images.
func layerIsPrunable(g graph.Graph, layerNode *imagegraph.ImageLayerNode) bool {
for _, predecessor := range g.To(layerNode) {
glog.V(4).Infof("Examining layer predecessor %#v", predecessor)
// imageComponentIsPrunable returns true if the image component is not referenced by any images.
func imageComponentIsPrunable(g graph.Graph, cn *imagegraph.ImageComponentNode) bool {
for _, predecessor := range g.To(cn) {
glog.V(4).Infof("Examining predecessor %#v of image config %v", predecessor, cn)
if g.Kind(predecessor) == imagegraph.ImageNodeKind {
glog.V(4).Infof("Layer has an image predecessor")
glog.V(4).Infof("Config %v has an image predecessor", cn)
return false
}
}

return true
}

// streamLayerReferences returns a list of ImageStreamNodes that reference a
// given ImageLayerNode.
func streamLayerReferences(g graph.Graph, layerNode *imagegraph.ImageLayerNode) []*imagegraph.ImageStreamNode {
// streamReferencingImageComponent returns a list of ImageStreamNodes that reference a
// given ImageComponentNode.
func streamsReferencingImageComponent(g graph.Graph, cn *imagegraph.ImageComponentNode) []*imagegraph.ImageStreamNode {
ret := []*imagegraph.ImageStreamNode{}
for _, predecessor := range g.To(layerNode) {
for _, predecessor := range g.To(cn) {
if g.Kind(predecessor) != imagegraph.ImageStreamNodeKind {
continue
}
Expand All @@ -851,28 +871,28 @@ func streamLayerReferences(g graph.Graph, layerNode *imagegraph.ImageLayerNode)
return ret
}

// pruneLayers invokes layerLinkDeleter.DeleteLayerLink for each repository layer link to
// pruneImageComponents invokes layerLinkDeleter.DeleteLayerLink for each repository layer link to
// be deleted from the registry.
func pruneLayers(
func pruneImageComponents(
g graph.Graph,
registryClient *http.Client,
registryURL string,
layerNodes []*imagegraph.ImageLayerNode,
imageComponents []*imagegraph.ImageComponentNode,
layerLinkDeleter LayerLinkDeleter,
) []error {
errs := []error{}

for _, layerNode := range layerNodes {
// get streams that reference layer
streamNodes := streamLayerReferences(g, layerNode)
for _, cn := range imageComponents {
// get streams that reference config
streamNodes := streamsReferencingImageComponent(g, cn)

for _, streamNode := range streamNodes {
stream := streamNode.ImageStream
streamName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name)

glog.V(4).Infof("Pruning registry=%q, repo=%q, layer=%q", registryURL, streamName, layerNode.Layer)
if err := layerLinkDeleter.DeleteLayerLink(registryClient, registryURL, streamName, layerNode.Layer); err != nil {
errs = append(errs, fmt.Errorf("error pruning repo %q layer link %q: %v", streamName, layerNode.Layer, err))
glog.V(4).Infof("Pruning registry=%q, repo=%q, %s", registryURL, streamName, cn.Describe())
if err := layerLinkDeleter.DeleteLayerLink(registryClient, registryURL, streamName, cn.Component); err != nil {
errs = append(errs, fmt.Errorf("error pruning layer link %s in repo %q: %v", cn.Component, streamName, err))
}
}
}
Expand All @@ -882,13 +902,19 @@ func pruneLayers(

// pruneBlobs invokes blobPruner.DeleteBlob for each blob to be deleted from the
// registry.
func pruneBlobs(g graph.Graph, registryClient *http.Client, registryURL string, layerNodes []*imagegraph.ImageLayerNode, blobPruner BlobDeleter) []error {
func pruneBlobs(
g graph.Graph,
registryClient *http.Client,
registryURL string,
componentNodes []*imagegraph.ImageComponentNode,
blobPruner BlobDeleter,
) []error {
errs := []error{}

for _, layerNode := range layerNodes {
glog.V(4).Infof("Pruning registry=%q, blob=%q", registryURL, layerNode.Layer)
if err := blobPruner.DeleteBlob(registryClient, registryURL, layerNode.Layer); err != nil {
errs = append(errs, fmt.Errorf("error pruning blob %q: %v", layerNode.Layer, err))
for _, cn := range componentNodes {
glog.V(4).Infof("Pruning registry=%q, blob=%q", registryURL, cn.Component)
if err := blobPruner.DeleteBlob(registryClient, registryURL, cn.Component); err != nil {
errs = append(errs, fmt.Errorf("error pruning blob %q: %v", cn.Component, err))
}
}

Expand Down
Loading

0 comments on commit 919493f

Please sign in to comment.