Feat/ameliore les logs

.github/workflows/ci.yml

@@ -116,6 +116,12 @@ jobs:
             -type f \
             -exec rm -v {} +
 
+      - name: Crée le fichier de configuration minimal
+        run: |
+          touch config.toml 
+          echo "[log]" >> config.toml
+          echo "level = \"error\"" >> config.toml
+
       - run: ./tests/test-cli.sh
 
       - run: ./tests/test-prune-entities.sh

config-sample.toml

@@ -16,5 +16,8 @@ FIRST_BATCH = "1802"
 MRthreads = 2
 chunkByteSize = 1000000000
 
+[log]
+level = "error"
+
 [export]
 path = "path/export"

lib/engine/csv.go

@@ -73,15 +73,14 @@ func writeLinesToCSV(key string, tuples map[string]marshal.Tuple) {
 }
 
 func openFile(key string, tuple marshal.Tuple) *csv.Writer {
-	logger := slog.Default().With(slog.Any("tuple", tuple))
 	file, found := csvFiles[tuple.Type()]
 	if found {
 		return csv.NewWriter(file)
 	}
 	var err error
 	fullFilename := prepareFilename(key, tuple.Type())
 
-	logger = logger.With(slog.String("filename", fullFilename))
+	logger := slog.Default().With(slog.String("filename", fullFilename))
 	file, err = os.OpenFile(fullFilename, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644)
 	csvFiles[tuple.Type()] = file
 	if err != nil {
@@ -94,7 +93,7 @@ func openFile(key string, tuple marshal.Tuple) *csv.Writer {
 	writer := csv.NewWriter(file)
 	headers := tuple.Headers()
 	logger.Info(
-		"write headers",
+		"écrit les headers",
 		slog.Any("headers", headers),
 	)
 	err = writer.Write(headers)
@@ -108,15 +107,22 @@ func openFile(key string, tuple marshal.Tuple) *csv.Writer {
 	return writer
 }
 
-func prepareFilename(key string, s string) string {
+func prepareFilename(key string, filetype string) string {
 	rootPath := exportPath
-	filename := string(s) + ".csv"
+	filename := filetype + ".csv"
 	if viper.IsSet("export.path") {
 		rootPath = viper.GetString("export.path")
 	}
 	exportPath := filepath.Join(rootPath, key)
 	createExportFolder(exportPath)
-	return filepath.Join(exportPath, filename)
+	filename = filepath.Join(exportPath, filename)
+	slog.Debug(
+		"le nom de fichier est généré",
+		slog.String("key", key),
+		slog.String("type", filetype),
+		slog.String("filename", filename),
+	)
+	return filename
 }
 
 func createExportFolder(path string) {

lib/marshal/parser.go

@@ -3,7 +3,7 @@ package marshal
 import (
 	"context"
 	"errors"
-	"fmt"
+	"log/slog"
 	"time"
 
 	"github.com/spf13/viper"
@@ -78,7 +78,10 @@ func ParseFile(path base.BatchFile, parser Parser, batch *base.AdminBatch, cache
 	tracker := NewParsingTracker()
 	fileType := parser.GetFileType()
 	filePath := path.Prefix() + viper.GetString("APP_DATA") + path.FilePath()
+	parserLog := slog.Default().With(slog.String("filename", filePath), slog.String("filetype", fileType))
+	parserLog.Info("c'est parti")
 	err := runParserOnFile(filePath, parser, batch, cache, &tracker, outputChannel)
+	parserLog.Info("c'est fini")
 	if err != nil {
 		tracker.AddFatalError(err)
 	}
@@ -129,8 +132,8 @@ func parseTuplesFromLine(lineResult ParsedLineResult, filter *SirenFilter, track
 
 // LogProgress affiche le numéro de ligne en cours de parsing, toutes les 2s.
 func LogProgress(lineNumber *int) (stop context.CancelFunc) {
-	return base.Cron(time.Second*2, func() {
-		fmt.Printf("Reading csv line %d\n", *lineNumber)
+	return base.Cron(time.Minute*1, func() {
+		slog.Info("Lis une ligne du fichier csv", slog.Int("line", *lineNumber))
 	})
 }
 

lib/urssaf/effectif.go

@@ -2,7 +2,6 @@ package urssaf
 
 import (
 	"encoding/csv"
-	"log/slog"
 	"os"
 	"strconv"
 	"time"
@@ -21,7 +20,6 @@ type Effectif struct {
 }
 
 func (effectif Effectif) Headers() []string {
-	slog.Warn("le tag `col` n'est pas défini sur tous les champs de l'Effectif Urssaf")
 	return []string{
 		"siret",
 		"compte",

logger.go

@@ -1,38 +1,68 @@
 package main
 
 import (
+	"errors"
 	"log/slog"
 	"os"
 	"runtime/debug"
+	"strings"
+
+	"github.com/spf13/viper"
 )
 
 var loglevel *slog.LevelVar
 
-func LogLevel(newLevel slog.LevelVar) {
-	loglevel = &newLevel
-}
-
-func init() {
+func initLogger() {
 	loglevel = new(slog.LevelVar)
-	loglevel.Set(slog.LevelWarn)
+	loglevel.Set(slog.LevelInfo)
 	handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
 		Level: loglevel,
 		//AddSource: true,
 	})
 
-	parentLogger := slog.New(
-		handler)
+	parentLogger := slog.New(handler)
 	buildInfo, _ := debug.ReadBuildInfo()
 	sha1 := GitCommit
 	appLogger := parentLogger.With(
 		slog.Group("app", slog.String("sha1", sha1)),
 	)
 	slog.SetDefault(appLogger)
 
+	level, err := parseLogLevel(viper.GetString("log.level"))
+	if err != nil {
+		slog.Warn("erreur de log level", slog.Any("error", err))
+	}
+	loglevel.Set(level)
+
 	slog.Info(
 		"initialisation",
 		slog.String("go", buildInfo.GoVersion),
 		slog.String("path", buildInfo.Path),
 		slog.Any("any", buildInfo.Settings),
+		slog.Any("level", loglevel),
 	)
 }
+
+func parseLogLevel(logLevel string) (slog.Level, error) {
+	switch strings.ToUpper(logLevel) {
+	case "DEBUG":
+		return slog.LevelDebug, nil
+	case "INFO":
+		return slog.LevelInfo, nil
+	case "WARN":
+		return slog.LevelWarn, nil
+	case "ERROR":
+		return slog.LevelError, nil
+	default:
+		return slog.LevelInfo, errors.New("log level inconnu : '" + logLevel + "'")
+	}
+}
+
+func ConfigureLogLevel(logLevel string) {
+	var level, err = parseLogLevel(logLevel)
+	if err != nil {
+		slog.Warn("Erreur de configuration sur le loglevel", slog.String("cause", err.Error()))
+		return
+	}
+	loglevel.Set(level)
+}

main.go

@@ -26,6 +26,7 @@ func connectDB() {
 // main Fonction Principale
 func main() {
 	initConfig()
+	initLogger()
 	exitCode := runCLI(os.Args...)
 	os.Exit(exitCode)
 }
@@ -62,6 +63,10 @@ func initConfig() {
 	viper.AddConfigPath("/etc/opensignauxfaibles")
 	viper.AddConfigPath("$HOME/.opensignauxfaibles")
 	viper.AddConfigPath(".")
+	err := viper.ReadInConfig()
+	if err != nil {
+		slog.Error("erreur à la lecture de la config toml", slog.Any("error", err))
+	}
 	marshal.SetGitCommit(GitCommit)
 }