Merge pull request #1584 from weaveworks/1563-read-docker-networks

Add docker networks to the Overlay Topology

probe/docker/registry.go

@@ -39,7 +39,8 @@ type Registry interface {
 	Stop()
 	LockedPIDLookup(f func(func(int) Container))
 	WalkContainers(f func(Container))
-	WalkImages(f func(*docker_client.APIImages))
+	WalkImages(f func(docker_client.APIImages))
+	WalkNetworks(f func(docker_client.Network))
 	WatchContainerUpdates(ContainerUpdateWatcher)
 	GetContainer(string) (Container, bool)
 	GetContainerByPrefix(string) (Container, bool)
@@ -61,14 +62,16 @@ type registry struct {
 	watchers        []ContainerUpdateWatcher
 	containers      *radix.Tree
 	containersByPID map[int]Container
-	images          map[string]*docker_client.APIImages
+	images          map[string]docker_client.APIImages
+	networks        []docker_client.Network
 }
 
 // Client interface for mocking.
 type Client interface {
 	ListContainers(docker_client.ListContainersOptions) ([]docker_client.APIContainers, error)
 	InspectContainer(string) (*docker_client.Container, error)
 	ListImages(docker_client.ListImagesOptions) ([]docker_client.APIImages, error)
+	ListNetworks() ([]docker_client.Network, error)
 	AddEventListener(chan<- *docker_client.APIEvents) error
 	RemoveEventListener(chan *docker_client.APIEvents) error
 
@@ -97,7 +100,7 @@ func NewRegistry(interval time.Duration, pipes controls.PipeClient, collectStats
 	r := &registry{
 		containers:      radix.New(),
 		containersByPID: map[int]Container{},
-		images:          map[string]*docker_client.APIImages{},
+		images:          map[string]docker_client.APIImages{},
 
 		client:       client,
 		pipes:        pipes,
@@ -172,6 +175,11 @@ func (r *registry) listenForEvents() bool {
 		return true
 	}
 
+	if err := r.updateNetworks(); err != nil {
+		log.Errorf("docker registry: %s", err)
+		return true
+	}
+
 	otherUpdates := time.Tick(r.interval)
 	for {
 		select {
@@ -187,6 +195,10 @@ func (r *registry) listenForEvents() bool {
 				log.Errorf("docker registry: %s", err)
 				return true
 			}
+			if err := r.updateNetworks(); err != nil {
+				log.Errorf("docker registry: %s", err)
+				return true
+			}
 
 		case ch := <-r.quit:
 			r.Lock()
@@ -217,7 +229,8 @@ func (r *registry) reset() {
 
 	r.containers = radix.New()
 	r.containersByPID = map[int]Container{}
-	r.images = map[string]*docker_client.APIImages{}
+	r.images = map[string]docker_client.APIImages{}
+	r.networks = r.networks[:0]
 }
 
 func (r *registry) updateContainers() error {
@@ -242,15 +255,28 @@ func (r *registry) updateImages() error {
 	r.Lock()
 	defer r.Unlock()
 
-	for i := range images {
-		image := &images[i]
+	for _, image := range images {
 		r.images[trimImageID(image.ID)] = image
 	}
 
 	return nil
 }
 
+func (r *registry) updateNetworks() error {
+	networks, err := r.client.ListNetworks()
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	r.networks = networks
+	r.Unlock()
+
+	return nil
+}
+
 func (r *registry) handleEvent(event *docker_client.APIEvents) {
+	// TODO: Send shortcut reports on networks being created/destroyed?
 	switch event.Status {
 	case CreateEvent, RenameEvent, StartEvent, DieEvent, DestroyEvent, PauseEvent, UnpauseEvent, NetworkConnectEvent, NetworkDisconnectEvent:
 		r.updateContainerState(event.ID, stateAfterEvent(event.Status))
@@ -400,7 +426,7 @@ func (r *registry) GetContainerImage(id string) (*docker_client.APIImages, bool)
 
 // WalkImages runs f on every image of running containers the registry
 // knows of.  f may be run on the same image more than once.
-func (r *registry) WalkImages(f func(*docker_client.APIImages)) {
+func (r *registry) WalkImages(f func(docker_client.APIImages)) {
 	r.RLock()
 	defer r.RUnlock()
 
@@ -414,6 +440,16 @@ func (r *registry) WalkImages(f func(*docker_client.APIImages)) {
 	})
 }
 
+// WalkNetworks runs f on every network the registry knows of.
+func (r *registry) WalkNetworks(f func(docker_client.Network)) {
+	r.RLock()
+	defer r.RUnlock()
+
+	for _, network := range r.networks {
+		f(network)
+	}
+}
+
 // ImageNameWithoutVersion splits the image name apart, returning the name
 // without the version, if possible
 func ImageNameWithoutVersion(name string) string {

probe/docker/registry_test.go

@@ -82,6 +82,7 @@ type mockDockerClient struct {
 	apiContainers []client.APIContainers
 	containers    map[string]*client.Container
 	apiImages     []client.APIImages
+	networks      []client.Network
 	events        []chan<- *client.APIEvents
 }
 
@@ -107,6 +108,12 @@ func (m *mockDockerClient) ListImages(client.ListImagesOptions) ([]client.APIIma
 	return m.apiImages, nil
 }
 
+func (m *mockDockerClient) ListNetworks() ([]client.Network, error) {
+	m.RLock()
+	defer m.RUnlock()
+	return m.networks, nil
+}
+
 func (m *mockDockerClient) AddEventListener(events chan<- *client.APIEvents) error {
 	m.Lock()
 	defer m.Unlock()
@@ -244,13 +251,22 @@ var (
 			"imgfoo2": "bar2",
 		},
 	}
+	network1 = client.Network{
+		ID:    "deadbeef",
+		Name:  "network1",
+		Scope: "local",
+		IPAM: client.IPAMOptions{
+			Config: []client.IPAMConfig{{Subnet: "5.6.7.8/24"}},
+		},
+	}
 )
 
 func newMockClient() *mockDockerClient {
 	return &mockDockerClient{
 		apiContainers: []client.APIContainers{apiContainer1},
 		containers:    map[string]*client.Container{"ping": container1},
 		apiImages:     []client.APIImages{apiImage1},
+		networks:      []client.Network{network1},
 	}
 }
 
@@ -284,9 +300,17 @@ func allContainers(r docker.Registry) []docker.Container {
 	return result
 }
 
-func allImages(r docker.Registry) []*client.APIImages {
-	result := []*client.APIImages{}
-	r.WalkImages(func(i *client.APIImages) {
+func allImages(r docker.Registry) []client.APIImages {
+	result := []client.APIImages{}
+	r.WalkImages(func(i client.APIImages) {
+		result = append(result, i)
+	})
+	return result
+}
+
+func allNetworks(r docker.Registry) []client.Network {
+	result := []client.Network{}
+	r.WalkNetworks(func(i client.Network) {
 		result = append(result, i)
 	})
 	return result
@@ -307,11 +331,19 @@ func TestRegistry(t *testing.T) {
 		}
 
 		{
-			want := []*client.APIImages{&apiImage1}
+			want := []client.APIImages{apiImage1}
 			test.Poll(t, 100*time.Millisecond, want, func() interface{} {
 				return allImages(registry)
 			})
 		}
+
+		{
+			want := []client.Network{network1}
+			test.Poll(t, 100*time.Millisecond, want, func() interface{} {
+				return allNetworks(registry)
+			})
+		}
+
 	})
 }
 

probe/docker/reporter.go

@@ -7,14 +7,16 @@ import (
 	docker_client "github.com/fsouza/go-dockerclient"
 
 	"github.com/weaveworks/scope/probe"
+	"github.com/weaveworks/scope/probe/host"
 	"github.com/weaveworks/scope/report"
 )
 
 // Keys for use in Node
 const (
-	ImageID          = "docker_image_id"
-	ImageName        = "docker_image_name"
-	ImageLabelPrefix = "docker_image_label_"
+	ImageID           = "docker_image_id"
+	ImageName         = "docker_image_name"
+	ImageLabelPrefix  = "docker_image_label_"
+	OverlayPeerPrefix = "docker_peer_"
 )
 
 // Exposed for testing
@@ -145,6 +147,7 @@ func (r *Reporter) Report() (report.Report, error) {
 	result := report.MakeReport()
 	result.Container = result.Container.Merge(r.containerTopology(localAddrs))
 	result.ContainerImage = result.ContainerImage.Merge(r.containerImageTopology())
+	result.Overlay = result.Overlay.Merge(r.overlayTopology())
 	return result, nil
 }
 
@@ -223,7 +226,7 @@ func (r *Reporter) containerImageTopology() report.Topology {
 		WithMetadataTemplates(ContainerImageMetadataTemplates).
 		WithTableTemplates(ContainerImageTableTemplates)
 
-	r.registry.WalkImages(func(image *docker_client.APIImages) {
+	r.registry.WalkImages(func(image docker_client.APIImages) {
 		imageID := trimImageID(image.ID)
 		nodeID := report.MakeContainerImageNodeID(imageID)
 		node := report.MakeNodeWith(nodeID, map[string]string{
@@ -241,6 +244,21 @@ func (r *Reporter) containerImageTopology() report.Topology {
 	return result
 }
 
+func (r *Reporter) overlayTopology() report.Topology {
+	localSubnets := []string{}
+	r.registry.WalkNetworks(func(network docker_client.Network) {
+		if network.Scope == "local" {
+			for _, config := range network.IPAM.Config {
+				localSubnets = append(localSubnets, config.Subnet)
+			}
+		}
+	})
+	peerID := OverlayPeerPrefix + r.hostID
+	node := report.MakeNode(report.MakeOverlayNodeID(peerID)).WithSets(
+		report.MakeSets().Add(host.LocalNetworks, report.MakeStringSet(localSubnets...)))
+	return report.MakeTopology().AddNode(node)
+}
+
 // Docker sometimes prefixes ids with a "type" annotation, but it renders a bit
 // ugly and isn't necessary, so we should strip it off
 func trimImageID(id string) string {

probe/docker/reporter_test.go

@@ -6,12 +6,14 @@ import (
 	client "github.com/fsouza/go-dockerclient"
 
 	"github.com/weaveworks/scope/probe/docker"
+	"github.com/weaveworks/scope/probe/host"
 	"github.com/weaveworks/scope/report"
 )
 
 type mockRegistry struct {
 	containersByPID map[int]docker.Container
-	images          map[string]*client.APIImages
+	images          map[string]client.APIImages
+	networks        []client.Network
 }
 
 func (r *mockRegistry) Stop() {}
@@ -28,12 +30,18 @@ func (r *mockRegistry) WalkContainers(f func(docker.Container)) {
 	}
 }
 
-func (r *mockRegistry) WalkImages(f func(*client.APIImages)) {
+func (r *mockRegistry) WalkImages(f func(client.APIImages)) {
 	for _, i := range r.images {
 		f(i)
 	}
 }
 
+func (r *mockRegistry) WalkNetworks(f func(client.Network)) {
+	for _, i := range r.networks {
+		f(i)
+	}
+}
+
 func (r *mockRegistry) WatchContainerUpdates(_ docker.ContainerUpdateWatcher) {}
 
 func (r *mockRegistry) GetContainer(_ string) (docker.Container, bool) { return nil, false }
@@ -46,20 +54,25 @@ func (r *mockRegistry) GetContainerImage(id string) (*client.APIImages, bool) {
 }
 
 var (
+	imageID              = "baz"
 	mockRegistryInstance = &mockRegistry{
 		containersByPID: map[int]docker.Container{
 			2: &mockContainer{container1},
 		},
-		images: map[string]*client.APIImages{
-			"baz": &apiImage1,
+		images: map[string]client.APIImages{
+			imageID: apiImage1,
 		},
+		networks: []client.Network{network1},
 	}
 )
 
 func TestReporter(t *testing.T) {
-	var controlProbeID = "a1b2c3d4"
+	var (
+		controlProbeID = "a1b2c3d4"
+		hostID         = "host1"
+	)
 
-	containerImageNodeID := report.MakeContainerImageNodeID("baz")
+	containerImageNodeID := report.MakeContainerImageNodeID(imageID)
 	rpt, err := docker.NewReporter(mockRegistryInstance, "host1", controlProbeID, nil).Report()
 	if err != nil {
 		t.Fatal(err)
@@ -76,7 +89,7 @@ func TestReporter(t *testing.T) {
 		for k, want := range map[string]string{
 			docker.ContainerID:    "ping",
 			docker.ContainerName:  "pong",
-			docker.ImageID:        "baz",
+			docker.ImageID:        imageID,
 			report.ControlProbeID: controlProbeID,
 		} {
 			if have, ok := node.Latest.Lookup(k); !ok || have != want {
@@ -103,7 +116,7 @@ func TestReporter(t *testing.T) {
 		}
 
 		for k, want := range map[string]string{
-			docker.ImageID:                      "baz",
+			docker.ImageID:                      imageID,
 			docker.ImageName:                    "bang",
 			docker.ImageLabelPrefix + "imgfoo1": "bar1",
 			docker.ImageLabelPrefix + "imgfoo2": "bar2",
@@ -118,4 +131,20 @@ func TestReporter(t *testing.T) {
 			t.Errorf("Container images should not have any controls")
 		}
 	}
+
+	// Reporter should add a container network
+	{
+		peerID := docker.OverlayPeerPrefix + hostID
+		overlayNodeID := report.MakeOverlayNodeID(peerID)
+		node, ok := rpt.Overlay.Nodes[overlayNodeID]
+		if !ok {
+			t.Fatalf("Expected report to have overlay node  %q, but not found", overlayNodeID)
+		}
+
+		want := "5.6.7.8/24"
+		if have, ok := node.Sets.Lookup(host.LocalNetworks); !ok || len(have) != 1 || have[0] != want {
+			t.Fatalf("Expected node to have exactly local network %v but found %v", want, have)
+		}
+
+	}
 }