feat(agent): Watch for deleted files

Currently, the agent will recognize when config files are deleted and
then block, waiting for them to re-appear. This will trigger a reload.
This is not likely the intended behavior, instead a user wants to remove
a configuration and have telegraf remove that configuration from the
running Telegraf.

Note this only works on watched directories. If a user specifies a
specific file to load and then deletes it, Telegraf will fail to start
as that file is now missing.

In order to avoid any additional error messages on reload, the list of
known configs needs to update as well. Therefore, right before the
reload we go get a new list of files based on the CLI options + reading
any configuration directories.

cmd/telegraf/telegraf.go

@@ -174,6 +174,12 @@ func (t *Telegraf) reloadLoop() error {
 			case sig := <-signals:
 				if sig == syscall.SIGHUP {
 					log.Println("I! Reloading Telegraf config")
+					// May need to update the list of known config files
+					// if a delete or create occured. That way on the reload
+					// we ensure we watch the correct files.
+					if err := t.getConfigFiles(); err != nil {
+						log.Println("E! Error loading config files: ", err)
+					}
 					<-reload
 					reload <- true
 				}
@@ -220,11 +226,6 @@ func (t *Telegraf) watchLocalConfig(signals chan os.Signal, fConfig string) {
 			log.Println("I! Config file overwritten")
 		} else {
 			log.Println("W! Config file deleted")
-			if err := watcher.BlockUntilExists(&mytomb); err != nil {
-				log.Printf("E! Cannot watch for config: %s\n", err.Error())
-				return
-			}
-			log.Println("I! Config file appeared")
 		}
 	case <-changes.Truncated:
 		log.Println("I! Config file truncated")
@@ -279,17 +280,28 @@ func (t *Telegraf) loadConfiguration() (*config.Config, error) {
 	// If no other options are specified, load the config file and run.
 	c := config.NewConfig()
 	c.Agent.Quiet = t.quiet
+	c.Agent.ConfigURLRetryAttempts = t.configURLRetryAttempts
 	c.OutputFilters = t.outputFilters
 	c.InputFilters = t.inputFilters
 	c.SecretStoreFilters = t.secretstoreFilters
 
+	if err := t.getConfigFiles(); err != nil {
+		return c, err
+	}
+	if err := c.LoadAll(t.configFiles...); err != nil {
+		return c, err
+	}
+	return c, nil
+}
+
+func (t *Telegraf) getConfigFiles() error {
 	var configFiles []string
 
 	configFiles = append(configFiles, t.config...)
 	for _, fConfigDirectory := range t.configDir {
 		files, err := config.WalkDirectory(fConfigDirectory)
 		if err != nil {
-			return c, err
+			return err
 		}
 		configFiles = append(configFiles, files...)
 	}
@@ -298,17 +310,13 @@ func (t *Telegraf) loadConfiguration() (*config.Config, error) {
 	if len(configFiles) == 0 {
 		defaultFiles, err := config.GetDefaultConfigPath()
 		if err != nil {
-			return nil, fmt.Errorf("unable to load default config paths: %w", err)
+			return fmt.Errorf("unable to load default config paths: %w", err)
 		}
 		configFiles = append(configFiles, defaultFiles...)
 	}
 
-	c.Agent.ConfigURLRetryAttempts = t.configURLRetryAttempts
 	t.configFiles = configFiles
-	if err := c.LoadAll(configFiles...); err != nil {
-		return c, err
-	}
-	return c, nil
+	return nil
 }
 
 func (t *Telegraf) runAgent(ctx context.Context, reloadConfig bool) error {