From 9a3b603744919f12f8e63f7660877ff8aa69ee7c Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Tue, 6 Feb 2018 13:08:57 +0000 Subject: [PATCH] Move DNS name mapping from endpoint to report --- probe/endpoint/connection_tracker.go | 22 ++++++++---- render/detailed/connections.go | 14 ++++---- render/endpoint.go | 2 +- render/id.go | 26 +++++++------- report/report.go | 53 +++++++++++++++++++++++++++- 5 files changed, 89 insertions(+), 28 deletions(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index 762afdfffa..e755f77a6d 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -219,23 +219,33 @@ func (t *connectionTracker) addConnection(rpt *report.Report, incoming bool, ft ) rpt.Endpoint = rpt.Endpoint.AddNode(fromNode.WithAdjacent(toNode.ID)) rpt.Endpoint = rpt.Endpoint.AddNode(toNode) + t.addDNS(rpt, ft.fromAddr) + t.addDNS(rpt, ft.toAddr) } func (t *connectionTracker) makeEndpointNode(namespaceID string, addr string, port uint16, extra map[string]string) report.Node { portStr := strconv.Itoa(int(port)) node := report.MakeNodeWith(report.MakeEndpointNodeID(t.conf.HostID, namespaceID, addr, portStr), nil) - if names := t.conf.DNSSnooper.CachedNamesForIP(addr); len(names) > 0 { - node = node.WithSet(SnoopedDNSNames, report.MakeStringSet(names...)) - } - if names, err := t.reverseResolver.get(addr); err == nil && len(names) > 0 { - node = node.WithSet(ReverseDNSNames, report.MakeStringSet(names...)) - } if extra != nil { node = node.WithLatests(extra) } return node } +// Add DNS record for address to report, if not already there +func (t *connectionTracker) addDNS(rpt *report.Report, addr string) { + if _, found := rpt.DNS[addr]; !found { + forward := t.conf.DNSSnooper.CachedNamesForIP(addr) + record := report.DNSRecord{ + Forward: report.MakeStringSet(forward...), + } + if names, err := t.reverseResolver.get(addr); err == nil && len(names) > 0 { + record.Reverse = report.MakeStringSet(names...) + } + rpt.DNS[addr] = record + } +} + func (t *connectionTracker) Stop() error { if t.ebpfTracker != nil { t.ebpfTracker.stop() diff --git a/render/detailed/connections.go b/render/detailed/connections.go index e41dbd4013..d3c3df8292 100644 --- a/render/detailed/connections.go +++ b/render/detailed/connections.go @@ -73,7 +73,7 @@ func newConnectionCounters() *connectionCounters { return &connectionCounters{counted: map[string]struct{}{}, counts: map[connection]int{}} } -func (c *connectionCounters) add(outgoing bool, localNode, remoteNode, localEndpoint, remoteEndpoint report.Node) { +func (c *connectionCounters) add(rpt report.Report, outgoing bool, localNode, remoteNode, localEndpoint, remoteEndpoint report.Node) { // We identify connections by their source endpoint, pre-NAT, to // ensure we only count them once. srcEndpoint, dstEndpoint := remoteEndpoint, localEndpoint @@ -94,10 +94,10 @@ func (c *connectionCounters) add(outgoing bool, localNode, remoteNode, localEndp return } // For internet nodes we break out individual addresses - if conn.remoteAddr, ok = internetAddr(remoteNode, remoteEndpoint); !ok { + if conn.remoteAddr, ok = internetAddr(rpt, remoteNode, remoteEndpoint); !ok { return } - if conn.localAddr, ok = internetAddr(localNode, localEndpoint); !ok { + if conn.localAddr, ok = internetAddr(rpt, localNode, localEndpoint); !ok { return } @@ -105,7 +105,7 @@ func (c *connectionCounters) add(outgoing bool, localNode, remoteNode, localEndp c.counts[conn]++ } -func internetAddr(node report.Node, ep report.Node) (string, bool) { +func internetAddr(rpt report.Report, node report.Node, ep report.Node) (string, bool) { if !render.IsInternetNode(node) { return "", true } @@ -113,7 +113,7 @@ func internetAddr(node report.Node, ep report.Node) (string, bool) { if !ok { return "", false } - if name, found := render.DNSFirstMatch(ep, func(string) bool { return true }); found { + if name, found := render.DNSFirstMatch(rpt, ep, func(string) bool { return true }); found { // we show the "most important" name only, since we don't have // space for more addr = fmt.Sprintf("%s (%s)", name, addr) @@ -171,7 +171,7 @@ func incomingConnectionsSummary(topologyID string, r report.Report, n report.Nod for _, remoteEndpoint := range endpointChildrenOf(node) { for _, localEndpointID := range remoteEndpoint.Adjacency.Intersection(localEndpointIDs) { localEndpointID = canonicalEndpointID(localEndpointIDCopies, localEndpointID) - counts.add(false, n, node, r.Endpoint.Nodes[localEndpointID], remoteEndpoint) + counts.add(r, false, n, node, r.Endpoint.Nodes[localEndpointID], remoteEndpoint) } } } @@ -203,7 +203,7 @@ func outgoingConnectionsSummary(topologyID string, r report.Report, n report.Nod for _, localEndpoint := range localEndpoints { for _, remoteEndpointID := range localEndpoint.Adjacency.Intersection(remoteEndpointIDs) { remoteEndpointID = canonicalEndpointID(remoteEndpointIDCopies, remoteEndpointID) - counts.add(true, n, node, localEndpoint, r.Endpoint.Nodes[remoteEndpointID]) + counts.add(r, true, n, node, localEndpoint, r.Endpoint.Nodes[remoteEndpointID]) } } } diff --git a/render/endpoint.go b/render/endpoint.go index 113ab7eaaf..2e610b1f83 100644 --- a/render/endpoint.go +++ b/render/endpoint.go @@ -36,7 +36,7 @@ func (e mapEndpoints) Render(rpt report.Report) Nodes { // Nodes without a hostid are mapped to pseudo nodes, if // possible. if _, ok := n.Latest.Lookup(report.HostNodeID); !ok { - if id, ok := pseudoNodeID(n, local); ok { + if id, ok := pseudoNodeID(rpt, n, local); ok { ret.addChild(n, id, Pseudo) continue } diff --git a/render/id.go b/render/id.go index 3b8eda0a69..cc2a00904f 100644 --- a/render/id.go +++ b/render/id.go @@ -3,7 +3,6 @@ package render import ( "strings" - "github.com/weaveworks/scope/probe/endpoint" "github.com/weaveworks/scope/report" ) @@ -62,13 +61,13 @@ func NewDerivedPseudoNode(id string, node report.Node) report.Node { return output } -func pseudoNodeID(n report.Node, local report.Networks) (string, bool) { +func pseudoNodeID(rpt report.Report, n report.Node, local report.Networks) (string, bool) { _, addr, _, ok := report.ParseEndpointNodeID(n.ID) if !ok { return "", false } - if id, ok := externalNodeID(n, addr, local); ok { + if id, ok := externalNodeID(rpt, n, addr, local); ok { return id, ok } @@ -78,11 +77,11 @@ func pseudoNodeID(n report.Node, local report.Networks) (string, bool) { } // figure out if a node should be considered external and returns an ID which can be used to create a pseudo node -func externalNodeID(n report.Node, addr string, local report.Networks) (string, bool) { +func externalNodeID(rpt report.Report, n report.Node, addr string, local report.Networks) (string, bool) { // First, check if it's a known service and emit a a specific node if it // is. This needs to be done before checking IPs since known services can // live in the same network, see https://github.com/weaveworks/scope/issues/2163 - if hostname, found := DNSFirstMatch(n, isKnownService); found { + if hostname, found := DNSFirstMatch(rpt, n, isKnownService); found { return ServiceNodeIDPrefix + hostname, true } @@ -103,20 +102,21 @@ func externalNodeID(n report.Node, addr string, local report.Networks) (string, } // DNSFirstMatch returns the first DNS name where match() returns -// true, from a prioritized list of snooped and reverse-resolved DNS -// names associated with node n. -func DNSFirstMatch(n report.Node, match func(name string) bool) (string, bool) { - // we rely on Sets being sorted, to make selection for display more +// true, from snooped and reverse-resolved DNS names. +func DNSFirstMatch(rpt report.Report, n report.Node, match func(name string) bool) (string, bool) { + _, addr, _, ok := report.ParseEndpointNodeID(n.ID) + if !ok { + return "", false + } + // we rely on StringSets being sorted, to make selection for display more // deterministic // prioritize snooped names - snoopedNames, _ := n.Sets.Lookup(endpoint.SnoopedDNSNames) - for _, hostname := range snoopedNames { + for _, hostname := range rpt.DNS[addr].Forward { if match(hostname) { return hostname, true } } - reverseNames, _ := n.Sets.Lookup(endpoint.ReverseDNSNames) - for _, hostname := range reverseNames { + for _, hostname := range rpt.DNS[addr].Reverse { if match(hostname) { return hostname, true } diff --git a/report/report.go b/report/report.go index 2cac43de1d..d9fff1b37e 100644 --- a/report/report.go +++ b/report/report.go @@ -65,6 +65,21 @@ var topologyNames = []string{ SwarmService, } +type DNSRecord struct { + Forward StringSet `json:"forward,omitempty"` + Reverse StringSet `json:"reverse,omitempty"` +} + +type DNSRecords map[string]DNSRecord + +func (r DNSRecords) Copy() DNSRecords { + cp := make(DNSRecords, len(r)) + for k, v := range r { + cp[k] = v + } + return cp +} + // Report is the core data type. It's produced by probes, and consumed and // stored by apps. It's composed of multiple topologies, each representing // a different (related, but not equivalent) view of the network. @@ -151,6 +166,8 @@ type Report struct { // their status endpoints. Edges are present. Overlay Topology + DNS DNSRecords + // Sampling data for this report. Sampling Sampling @@ -242,6 +259,8 @@ func MakeReport() Report { WithShape(Heptagon). WithLabel("service", "services"), + DNS: DNSRecords{}, + Sampling: Sampling{}, Window: 0, Plugins: xfer.MakePluginSpecs(), @@ -252,6 +271,7 @@ func MakeReport() Report { // Copy returns a value copy of the report. func (r Report) Copy() Report { newReport := Report{ + DNS: r.DNS.Copy(), Sampling: r.Sampling, Window: r.Window, Plugins: r.Plugins.Copy(), @@ -371,7 +391,7 @@ func (r Report) Validate() error { // // This for now creates node's LatestControls from Controls. func (r Report) Upgrade() Report { - return r.upgradeLatestControls().upgradePodNodes().upgradeNamespaces() + return r.upgradeLatestControls().upgradePodNodes().upgradeNamespaces().upgradeDNSRecords() } func (r Report) upgradeLatestControls() Report { @@ -469,6 +489,37 @@ func (r Report) upgradeNamespaces() Report { return r } +// Note in-place modification of Endpoint.Nodes +func (r Report) upgradeDNSRecords() Report { + if len(r.DNS) > 0 { + return r + } + dns := make(DNSRecords) + for endpointID, endpoint := range r.Endpoint.Nodes { + _, addr, _, ok := ParseEndpointNodeID(endpointID) + snoopedNames, foundS := endpoint.Sets.Lookup(SnoopedDNSNames) + reverseNames, foundR := endpoint.Sets.Lookup(ReverseDNSNames) + if ok && (foundS || foundR) { + // Add address and names to report-level data - we assume + // all endpoints with the same address have equally good data + if _, found := dns[addr]; ok && !found { + dns[addr] = DNSRecord{Forward: snoopedNames, Reverse: reverseNames} + } + // Now remove the entries from Sets, to save work during merging + // First, the case where these are the only entries in Sets - just blank it + if (foundS && foundR && endpoint.Sets.Size() == 2) || endpoint.Sets.Size() == 1 { + endpoint.Sets = MakeSets() + } else { + // Do it the hard way + endpoint.Sets = endpoint.Sets.Delete(SnoopedDNSNames).Delete(ReverseDNSNames) + } + r.Endpoint.Nodes[endpointID] = endpoint + } + } + r.DNS = dns + return r +} + // BackwardCompatible returns a new backward-compatible report. // // This for now creates node's Controls from LatestControls.