diff --git a/auth/basic.go b/auth/basic.go index 438f13bdf..084812fb5 100644 --- a/auth/basic.go +++ b/auth/basic.go @@ -21,18 +21,18 @@ func newBasicAuth(cfg config.BasicAuth) (AuthScheme, error) { log.Println("[WARN] Error processing a line in an htpasswd file:", err) } - stat, err := os.Stat(cfg.File) - if err != nil { - return nil, err - } - cfg.ModTime = stat.ModTime() - secrets, err := htpasswd.New(cfg.File, htpasswd.DefaultSystems, bad) if err != nil { return nil, err } if cfg.Refresh > 0 { + stat, err := os.Stat(cfg.File) + if err != nil { + return nil, err + } + cfg.ModTime = stat.ModTime() + go func() { ticker := time.NewTicker(cfg.Refresh).C for range ticker { diff --git a/config/config.go b/config/config.go index 5c84c8ad5..8e95ed24e 100644 --- a/config/config.go +++ b/config/config.go @@ -118,6 +118,7 @@ type Registry struct { Static Static File File Consul Consul + Custom Custom Timeout time.Duration Retry time.Duration } @@ -153,6 +154,17 @@ type Consul struct { ServiceMonitors int } +type Custom struct { + Host string + Path string + QueryParams string + Scheme string + CheckTLSSkipVerify bool + PollingInterval time.Duration + NoRouteHTML string + Timeout time.Duration +} + type Tracing struct { TracingEnabled bool CollectorType string diff --git a/config/default.go b/config/default.go index 913e34999..7afd0c903 100644 --- a/config/default.go +++ b/config/default.go @@ -66,6 +66,16 @@ var defaultConfig = &Config{ CheckDeregisterCriticalServiceAfter: "90m", ChecksRequired: "one", }, + Custom: Custom{ + Host: "", + Scheme: "https", + CheckTLSSkipVerify: false, + PollingInterval: 5, + NoRouteHTML: "", + Timeout: 10, + Path: "", + QueryParams: "", + }, Timeout: 10 * time.Second, Retry: 500 * time.Millisecond, }, diff --git a/config/load.go b/config/load.go index 3f0a71f75..ba700947a 100644 --- a/config/load.go +++ b/config/load.go @@ -200,6 +200,15 @@ func load(cmdline, environ, envprefix []string, props *properties.Properties) (c f.StringVar(&cfg.Tracing.SpanHost, "tracing.SpanHost", defaultConfig.Tracing.SpanHost, "Host:Port info to add to spans") f.BoolVar(&cfg.GlobMatchingDisabled, "glob.matching.disabled", defaultConfig.GlobMatchingDisabled, "Disable Glob Matching on routes, one of [true, false]") + f.StringVar(&cfg.Registry.Custom.Host, "registry.custom.host", defaultConfig.Registry.Custom.Host, "custom back end hostname/port") + f.StringVar(&cfg.Registry.Custom.Scheme, "registry.custom.scheme", defaultConfig.Registry.Custom.Scheme, "custom back end scheme - http/https") + f.StringVar(&cfg.Registry.Custom.NoRouteHTML, "registry.custom.noroutehtml", defaultConfig.Registry.Custom.NoRouteHTML, "path to file for HTML returned when no route is found") + f.BoolVar(&cfg.Registry.Custom.CheckTLSSkipVerify, "registry.custom.checkTLSSkipVerify", defaultConfig.Registry.Custom.CheckTLSSkipVerify, "custom back end check TLS verification") + f.DurationVar(&cfg.Registry.Custom.Timeout, "registry.custom.timeout", defaultConfig.Registry.Custom.Timeout, "timeout for API request to custom back end") + f.DurationVar(&cfg.Registry.Custom.PollingInterval, "registry.custom.pollinginterval", defaultConfig.Registry.Custom.PollingInterval, "polling interval for API request to custom back end") + f.StringVar(&cfg.Registry.Custom.Path, "registry.custom.path", defaultConfig.Registry.Custom.Path, "custom back end path in the URL") + f.StringVar(&cfg.Registry.Custom.QueryParams, "registry.custom.queryparams", defaultConfig.Registry.Custom.QueryParams, "custom back end query parameters in the URL") + // deprecated flags var proxyLogRoutes string f.StringVar(&proxyLogRoutes, "proxy.log.routes", "", "deprecated. use log.routes.format instead") diff --git a/config/load_test.go b/config/load_test.go index 869fbb041..057dd9e74 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -666,6 +666,55 @@ func TestLoad(t *testing.T) { return cfg }, }, + { + args: []string{"-registry.custom.host", "localhost:8080"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.Host = "localhost:8080" + return cfg + }, + }, + { + args: []string{"-registry.custom.scheme", "https"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.Scheme = "https" + return cfg + }, + }, + { + args: []string{"-registry.custom.checkTLSSkipVerify", "true"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.CheckTLSSkipVerify = true + return cfg + }, + }, + { + args: []string{"-registry.custom.timeout", "5s"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.Timeout = 5 * time.Second + return cfg + }, + }, + { + args: []string{"-registry.custom.pollinginterval", "5s"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.PollingInterval = 5 * time.Second + return cfg + }, + }, + { + args: []string{"-registry.custom.path", "test"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.Path = "test" + return cfg + }, + }, + { + args: []string{"-registry.custom.queryparams", "test=1"}, + cfg: func(cfg *Config) *Config { + cfg.Registry.Custom.QueryParams = "test=1" + return cfg + }, + }, { args: []string{"-log.access.format", "foobar"}, cfg: func(cfg *Config) *Config { diff --git a/docs/content/ref/registry.backend.md b/docs/content/ref/registry.backend.md index 28b8aa538..6b81eae6d 100644 --- a/docs/content/ref/registry.backend.md +++ b/docs/content/ref/registry.backend.md @@ -3,7 +3,23 @@ title: "registry.backend" --- `registry.backend` configures which backend is used. -Supported backends are: `consul`, `static`, `file`. +Supported backends are: `consul`, `static`, `file`, `custom`. If custom is used fabio makes an api +call to a remote system expecting the below json response + +```json +[ + { + "cmd": "string", + "service": "string", + "src": "string", + "dest": "string", + "weight": float, + "tags": ["string"], + "opts": {"string":"string"} + } +] +``` + The default is diff --git a/docs/content/ref/registry.custom.checkTLSSkipVerify.md b/docs/content/ref/registry.custom.checkTLSSkipVerify.md new file mode 100644 index 000000000..0fe6bb449 --- /dev/null +++ b/docs/content/ref/registry.custom.checkTLSSkipVerify.md @@ -0,0 +1,9 @@ +--- +title: "registry.custom.checkTLSSkipVerify" +--- + +`registry.custom.checkTLSSkipVerify` disables the TLS validation for the API call + +The default is + + registry.custom.checkTLSSkipVerify = false \ No newline at end of file diff --git a/docs/content/ref/registry.custom.host.md b/docs/content/ref/registry.custom.host.md new file mode 100644 index 000000000..010b8cf69 --- /dev/null +++ b/docs/content/ref/registry.custom.host.md @@ -0,0 +1,9 @@ +--- +title: "registry.custom.host" +--- + +`registry.custom.host` configures the host:port for fabio to make the API call + +The default is + + registry.custom.host = \ No newline at end of file diff --git a/docs/content/ref/registry.custom.path.md b/docs/content/ref/registry.custom.path.md new file mode 100644 index 000000000..3ba6d7047 --- /dev/null +++ b/docs/content/ref/registry.custom.path.md @@ -0,0 +1,15 @@ +--- +title: "registry.custom.path" +--- + + `registry.custom.path` is the path used in the custom back end API Call + + The path does not need to contain the initial '/' + + Example: + + registry.custom.path = api/v1/ + + The default is + + registry.custom.path = \ No newline at end of file diff --git a/docs/content/ref/registry.custom.pollinginterval.md b/docs/content/ref/registry.custom.pollinginterval.md new file mode 100644 index 000000000..39099dbe4 --- /dev/null +++ b/docs/content/ref/registry.custom.pollinginterval.md @@ -0,0 +1,9 @@ +--- +title: "registry.custom.pollinginterval" +--- + +`registry.custom.pollinginterval` is the length of time between API calls + +The default is + + registry.custom.pollinginterval = 10s \ No newline at end of file diff --git a/docs/content/ref/registry.custom.queryparams.md b/docs/content/ref/registry.custom.queryparams.md new file mode 100644 index 000000000..f55da7e47 --- /dev/null +++ b/docs/content/ref/registry.custom.queryparams.md @@ -0,0 +1,16 @@ +--- +title: "registry.custom.queryparams" +--- + + `registry.custom.queryparams` is the query parameters used in the custom back + end API Call + + Multiple query parameters should be separated with an & + + Example: + + registry.custom.queryparams = foo=bar&bar=foo + + The default is + + registry.custom.queryparams = \ No newline at end of file diff --git a/docs/content/ref/registry.custom.scheme.md b/docs/content/ref/registry.custom.scheme.md new file mode 100644 index 000000000..aa987a460 --- /dev/null +++ b/docs/content/ref/registry.custom.scheme.md @@ -0,0 +1,10 @@ +--- +title: "registry.custom.scheme" +--- + +`registry.custom.scheme` configures the scheme use to make the API call +must be one of `http`, `https` + +The default is + + registry.custom.scheme = https \ No newline at end of file diff --git a/docs/content/ref/registry.custom.timeout.md b/docs/content/ref/registry.custom.timeout.md new file mode 100644 index 000000000..d511b7a19 --- /dev/null +++ b/docs/content/ref/registry.custom.timeout.md @@ -0,0 +1,9 @@ +--- +title: "registry.custom.timeout" +--- + +`registry.custom.timeout` controls the timeout for the API call + +The default is + + registry.custom.timeout = 5s \ No newline at end of file diff --git a/fabio.iml b/fabio.iml new file mode 100644 index 000000000..8021953ed --- /dev/null +++ b/fabio.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/fabio.properties b/fabio.properties index 52e2e8d0a..3ec4c04c2 100644 --- a/fabio.properties +++ b/fabio.properties @@ -592,7 +592,20 @@ # registry.backend configures which backend is used. -# Supported backends are: consul, static, file +# Supported backends are: consul, static, file, custom +# if custom is used fabio makes an api call to a remote system +# expecting the below json response +# [ +# { +# "cmd": "string", +# "service": "string", +# "src": "string", +# "dest": "string", +# "weight": float, +# "tags": ["string"], +# "opts": {"string":"string"} +# } +# ] # # The default is # @@ -814,6 +827,68 @@ # registry.consul.serviceMonitors = 1 +# registry.custom.host configures the host:port for fabio to make the API call +# +# The default is +# +# registry.custom.host = + + +# registry.custom.scheme configures the scheme use to make the API call +# must be one of http, https +# +# The default is +# +# registry.custom.scheme = https + + +# registry.custom.checkTLSSkipVerify disables the TLS validation for the API call +# +# The default is +# +# registry.custom.checkTLSSkipVerify = false + + +# registry.custom.timeout controls the timeout for the API call +# +# The default is +# +# registry.custom.timeout = 5s + + +# registry.custom.pollinginterval is the length of time between API calls +# +# The default is +# +#registry.custom.pollinginterval = 10s + + +# registry.custom.path is the path used in the custom back end API Call +# +# The path does not need to contain the initial '/' +# +# Example: +# +# registry.custom.path = api/v1/ +# +# The default is +# +# registry.custom.path = + +# registry.custom.queryparams is the query parameters used in the custom back +# end API Call +# +# Multiple query parameters should be separated with an & +# +# Example: +# +# registry.custom.queryparams = foo=bar&bar=foo +# +# The default is +# +# registry.custom.queryparams = + + # glob.matching.disabled disables glob matching on route lookups # If glob matching is enabled there is a performance decrease # for every route lookup. At a large number of services (> 500) this diff --git a/main.go b/main.go index 9a79dbb5c..73c91b8c4 100644 --- a/main.go +++ b/main.go @@ -5,18 +5,6 @@ import ( "crypto/tls" "encoding/json" "fmt" - "io" - "log" - "net" - "net/http" - "os" - "runtime" - "runtime/debug" - "strings" - "sync" - "sync/atomic" - "time" - "github.com/fabiolb/fabio/admin" "github.com/fabiolb/fabio/auth" "github.com/fabiolb/fabio/cert" @@ -29,6 +17,7 @@ import ( "github.com/fabiolb/fabio/proxy/tcp" "github.com/fabiolb/fabio/registry" "github.com/fabiolb/fabio/registry/consul" + "github.com/fabiolb/fabio/registry/custom" "github.com/fabiolb/fabio/registry/file" "github.com/fabiolb/fabio/registry/static" "github.com/fabiolb/fabio/route" @@ -37,6 +26,17 @@ import ( "github.com/pkg/profile" dmp "github.com/sergi/go-diff/diffmatchpatch" "google.golang.org/grpc" + "io" + "log" + "net" + "net/http" + "os" + "runtime" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" ) // version contains the version number @@ -399,6 +399,8 @@ func initBackend(cfg *config.Config) { registry.Default, err = static.NewBackend(&cfg.Registry.Static) case "consul": registry.Default, err = consul.NewBackend(&cfg.Registry.Consul) + case "custom": + registry.Default, err = custom.NewBackend(&cfg.Registry.Custom) default: exit.Fatal("[FATAL] Unknown registry backend ", cfg.Registry.Backend) } @@ -423,46 +425,64 @@ func initBackend(cfg *config.Config) { func watchBackend(cfg *config.Config, first chan bool) { var ( - last string - svccfg string - mancfg string + last string + svccfg string + mancfg string + customBE string once sync.Once ) - svc := registry.Default.WatchServices() - man := registry.Default.WatchManual() - - for { - select { - case svccfg = <-svc: - case mancfg = <-man: + switch cfg.Registry.Backend { + //Custom Back End. Gets JSON from Remote Backend that contains a slice of route.RouteDef. It loads the route table + //Directly from that input + case "custom": + svc := registry.Default.WatchServices() + for { + customBE = <-svc + if customBE != "OK" { + log.Printf("[ERROR] error during update from custom back end - %s", customBE) + } + once.Do(func() { close(first) }) } + //All other back ends + default: + svc := registry.Default.WatchServices() + man := registry.Default.WatchManual() - // manual config overrides service config - // order matters - next := svccfg + "\n" + mancfg - if next == last { - continue - } + for { + select { + case svccfg = <-svc: + case mancfg = <-man: + } - aliases, err := route.ParseAliases(next) - if err != nil { - log.Printf("[WARN]: %s", err) - } - registry.Default.Register(aliases) + // manual config overrides service config + // order matters + next := svccfg + "\n" + mancfg + if next == last { + continue + } - t, err := route.NewTable(next) - if err != nil { - log.Printf("[WARN] %s", err) - continue + aliases, err := route.ParseAliases(next) + if err != nil { + log.Printf("[WARN]: %s", err) + } + registry.Default.Register(aliases) + + t, err := route.NewTable(next) + if err != nil { + log.Printf("[WARN] %s", err) + continue + } + route.SetTable(t) + logRoutes(t, last, next, cfg.Log.RoutesFormat) + last = next + + once.Do(func() { close(first) }) } - route.SetTable(t) - logRoutes(t, last, next, cfg.Log.RoutesFormat) - last = next - once.Do(func() { close(first) }) } + } func watchNoRouteHTML(cfg *config.Config) { diff --git a/registry/custom/backend.go b/registry/custom/backend.go new file mode 100644 index 000000000..80596bb33 --- /dev/null +++ b/registry/custom/backend.go @@ -0,0 +1,57 @@ +package custom + +import ( + "github.com/fabiolb/fabio/config" + "github.com/fabiolb/fabio/registry" + "log" +) + +type be struct { + cfg *config.Custom +} + +func NewBackend(cfg *config.Custom) (registry.Backend, error) { + return &be{cfg}, nil +} + +func (b *be) Register(services []string) error { + return nil +} + +func (b *be) Deregister(serviceName string) error { + return nil +} + +func (b *be) DeregisterAll() error { + return nil +} + +func (b *be) ManualPaths() ([]string, error) { + return nil, nil +} + +func (b *be) ReadManual(string) (value string, version uint64, err error) { + return "", 0, nil +} + +func (b *be) WriteManual(path string, value string, version uint64) (ok bool, err error) { + return false, nil +} + +func (b *be) WatchServices() chan string { + + log.Printf("[INFO] custom: Using custom routes from %s", b.cfg.Host) + ch := make(chan string, 1) + go customRoutes(b.cfg, ch) + return ch +} + +func (b *be) WatchManual() chan string { + return make(chan string) +} + +func (b *be) WatchNoRouteHTML() chan string { + ch := make(chan string, 1) + ch <- b.cfg.NoRouteHTML + return ch +} diff --git a/registry/custom/custom.go b/registry/custom/custom.go new file mode 100644 index 000000000..e724ed279 --- /dev/null +++ b/registry/custom/custom.go @@ -0,0 +1,82 @@ +package custom + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "github.com/fabiolb/fabio/config" + "github.com/fabiolb/fabio/route" + "log" + "net/http" + "time" +) + +func customRoutes(cfg *config.Custom, ch chan string) { + + var Routes *[]route.RouteDef + var trans *http.Transport + var URL string + + if !cfg.CheckTLSSkipVerify { + trans = &http.Transport{} + + } else { + trans = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + client := &http.Client{ + Transport: trans, + Timeout: cfg.Timeout, + } + + if cfg.QueryParams != "" { + URL = fmt.Sprintf("%s://%s/%s?%s", cfg.Scheme, cfg.Host, cfg.Path, cfg.QueryParams) + } else { + URL = fmt.Sprintf("%s://%s/%s", cfg.Scheme, cfg.Host, cfg.Path) + } + + req, err := http.NewRequest("GET", URL, nil) + if err != nil { + log.Printf("[ERROR] Can not generate new HTTP request") + } + req.Close = true + + for { + log.Printf("[DEBUG] Custom Registry starting request %s \n", time.Now()) + resp, err := client.Do(req) + if err != nil { + ch <- fmt.Sprintf("Error Sending HTTPs Request To Custom be - %s -%s", URL, err.Error()) + time.Sleep(cfg.PollingInterval) + continue + } + + if resp.StatusCode != 200 { + ch <- fmt.Sprintf("Error Non-200 return (%v) from -%s", resp.StatusCode, URL) + time.Sleep(cfg.PollingInterval) + continue + } + log.Printf("[DEBUG] Custom Registry begin decoding json %s \n", time.Now()) + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&Routes) + if err != nil { + ch <- fmt.Sprintf("Error decoding request - %s -%s", URL, err.Error()) + time.Sleep(cfg.PollingInterval) + continue + } + + log.Printf("[DEBUG] Custom Registry building table %s \n", time.Now()) + t, err := route.NewTableCustom(Routes) + if err != nil { + ch <- fmt.Sprintf("Error generating new table - %s", err.Error()) + } + log.Printf("[DEBUG] Custom Registry building table complete %s \n", time.Now()) + route.SetTable(t) + log.Printf("[DEBUG] Custom Registry table set complete %s \n", time.Now()) + ch <- "OK" + time.Sleep(cfg.PollingInterval) + + } + +} diff --git a/registry/custom/custom_test.go b/registry/custom/custom_test.go new file mode 100644 index 000000000..8ce21603d --- /dev/null +++ b/registry/custom/custom_test.go @@ -0,0 +1,95 @@ +package custom + +import ( + "encoding/json" + "fmt" + "github.com/fabiolb/fabio/config" + "github.com/fabiolb/fabio/route" + "net/http" + "testing" + "time" +) + +func TestCustomRoutes(t *testing.T) { + + var resp string + cfg := config.Custom{ + Host: "localhost:8080", + Path: "test", + Scheme: "http", + CheckTLSSkipVerify: false, + PollingInterval: 3 * time.Second, + Timeout: 3 * time.Second, + } + + ch := make(chan string, 1) + + mux := http.NewServeMux() + mux.HandleFunc("/test", handleTest) + server := &http.Server{ + Addr: "localhost:8080", + Handler: mux, + } + go server.ListenAndServe() + time.Sleep(3 * time.Second) + defer server.Close() + + go customRoutes(&cfg, ch) + + resp = <-ch + + if resp != "OK" { + fmt.Printf("Failed to get routes for custom backend - %s", resp) + t.FailNow() + } + + return + +} + +func handleTest(w http.ResponseWriter, r *http.Request) { + + var routes []route.RouteDef + var tags = []string{"tag1", "tag2"} + var opts = make(map[string]string) + opts["tlsskipverify"] = "true" + opts["proto"] = "http" + + var route1 = route.RouteDef{ + Cmd: "route add", + Service: "service1", + Src: "app.com", + Dst: "http://10.1.1.1:8080", + Weight: 0.50, + Tags: tags, + Opts: opts, + } + + var route2 = route.RouteDef{ + Cmd: "route add", + Service: "service1", + Src: "app.com", + Dst: "http://10.1.1.2:8080", + Weight: 0.50, + Tags: tags, + Opts: opts, + } + var route3 = route.RouteDef{ + Cmd: "route add", + Service: "service2", + Src: "app.com", + Dst: "http://10.1.1.3:8080", + Weight: 0.25, + Tags: tags, + Opts: opts, + } + + routes = append(routes, route1) + routes = append(routes, route2) + routes = append(routes, route3) + + rt, _ := json.Marshal(routes) + + w.Write(rt) + +} diff --git a/route/table.go b/route/table.go index 4e12fc182..13c874289 100644 --- a/route/table.go +++ b/route/table.go @@ -109,6 +109,7 @@ func hostpath(prefix string) (host string, path string) { } func NewTable(s string) (t Table, err error) { + defs, err := Parse(s) if err != nil { return nil, err @@ -133,6 +134,27 @@ func NewTable(s string) (t Table, err error) { return t, nil } +func NewTableCustom(defs *[]RouteDef) (t Table, err error) { + + t = make(Table) + for _, d := range *defs { + switch d.Cmd { + case RouteAddCmd: + err = t.addRoute(&d) + case RouteDelCmd: + err = t.delRoute(&d) + case RouteWeightCmd: + err = t.weighRoute(&d) + default: + err = fmt.Errorf("route: invalid command: %s", d.Cmd) + } + if err != nil { + return nil, err + } + } + return t, nil +} + // addRoute adds a new route prefix -> target for the given service. func (t Table) addRoute(d *RouteDef) error { host, path := hostpath(d.Src) diff --git a/route/table_test.go b/route/table_test.go index 824693c48..cf6fd182b 100644 --- a/route/table_test.go +++ b/route/table_test.go @@ -639,3 +639,67 @@ func TestTableLookup(t *testing.T) { } } } + +func TestNewTableCustom(t *testing.T) { + + var routes []RouteDef + var tags = []string{"tag1", "tag2"} + var opts = make(map[string]string) + opts["tlsskipverify"] = "true" + opts["proto"] = "http" + + var route1 = RouteDef{ + Cmd: "route add", + Service: "service1", + Src: "app.com", + Dst: "http://10.1.1.1:8080", + Weight: 0.50, + Tags: tags, + Opts: opts, + } + var route2 = RouteDef{ + Cmd: "route add", + Service: "service1", + Src: "app.com", + Dst: "http://10.1.1.2:8080", + Weight: 0.50, + Tags: tags, + Opts: opts, + } + var route3 = RouteDef{ + Cmd: "route add", + Service: "service2", + Src: "app.com", + Dst: "http://10.1.1.3:8080", + Weight: 0.25, + Tags: tags, + Opts: opts, + } + + routes = append(routes, route1) + routes = append(routes, route2) + routes = append(routes, route3) + + table, err := NewTableCustom(&routes) + + if err != nil { + fmt.Printf("Got error from NewTableCustom - %s", err.Error()) + t.FailNow() + } + + tableString := table.String() + if !strings.Contains(tableString, route1.Dst) { + fmt.Printf("Table Missing Destination %s -- Table -- %s", route1.Dst, tableString) + t.FailNow() + } + + if !strings.Contains(tableString, route2.Dst) { + fmt.Printf("Table Missing Destination %s -- Table -- %s", route1.Dst, tableString) + t.FailNow() + } + + if !strings.Contains(tableString, route3.Dst) { + fmt.Printf("Table Missing Destination %s -- Table -- %s", route1.Dst, tableString) + t.FailNow() + } +}