Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

handle new predictions format #304

Merged
merged 3 commits into from
Jan 17, 2025
Merged

handle new predictions format #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b3e7414
handle new predictions format
cmarcoin Jan 10, 2025
b5722be
add sireneUL logs
Jan 16, 2025
5e07886
add prediction logs
Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ db_url = "postgres://postgres:toto@localhost:5432/datapilogs"
par l'exécution des tests. C'est pratique dans le cas où de nouvelles features nécessitent de modifier les golden sources.
__ATTENTION : __Ce cas de figure est rare. Il ne faut pas utiliser ce flag si l'on est pas sûr que le code est correct.

### Configuration des tests sur macOS
Si vous développez en local sur macOS avec architecture arm64, il est possible que les tests ne fonctionnent pas.
Pour résoudre ce problème, il faut ajouter la variable d'environnement `GOARCH=amd64` avant de lancer les tests. Et il faut également s'assurer que la version de go utilisée est la version 1.21.13 sans quoi certaines librairies utilisées dans les tests ne fonctionneront pas.
```bash
go1.21.13 env -w GOARCH=amd64
```


## Gestion des droits
### zone d'attribution
Expand Down
4 changes: 2 additions & 2 deletions pkg/core/entreprise.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,8 @@ type EtablissementScore struct {
Alert string `json:"alert"`
MacroRadar map[string]float64 `json:"macroRadar,omitempty"`
ExplSelection *EtablissementScoreExplSelection `json:"explSelection,omitempty"`
MacroExpl map[string]float64 `json:"-"`
MicroExpl map[string]float64 `json:"-"`
MacroExpl map[string]float64 `json:"macroExpl,omitempty"`
MicroExpl map[string]interface{} `json:"microExpl,omitempty"`
AlertPreRedressements string `json:"alertPreRedressements"`
Redressements []string `json:"redressements"`
}
Expand Down
12 changes: 8 additions & 4 deletions pkg/ops/imports/imports.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,25 +515,29 @@ func importUnitesLegales(ctx context.Context) error {
if viper.GetString("source.sireneULPath") == "" || viper.GetString("source.geoSirenePath") == "" {
return utils.NewJSONerror(http.StatusConflict, "not supported, missing parameters in server configuration")
}
slog.Info("Truncate entreprise table", slog.String("status", "start"))

slog.Info("Truncate entreprise table and drop index", slog.String("status", "start"))
err := TruncateEntreprise(ctx)
if err != nil {
return err
}
slog.Info("Truncate entreprise table", slog.String("status", "end"))

err = DropEntrepriseIndex(ctx)
if err != nil {
return err
}
slog.Info("Truncate entreprise table and drop index", slog.String("status", "end"))

slog.Info("Insert sireneUL", slog.String("status", "start"))
err = InsertSireneUL(ctx)
if err != nil {
return err
}
slog.Info("Insert sireneUL", slog.String("status", "end"))

return CreateEntrepriseIndex(ctx)
slog.Info("Create entreprise index", slog.String("status", "start"))
err = CreateEntrepriseIndex(ctx)
slog.Info("Create entreprise index", slog.String("status", "end"))
return err
}

func importStockEtablissement(ctx context.Context) error {
Expand Down
44 changes: 29 additions & 15 deletions pkg/ops/imports/predictions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,9 @@ type score struct {
SelectConcerning [][]string `json:"select_concerning"`
SelectReassuring [][]string `json:"select_reassuring"`
} `json:"expl_selection"`
MacroExpl map[string]float64 `json:"macro_expl"`
MicroExpl map[string]float64 `json:"micro_expl"`
MacroRadar map[string]float64 `json:"macro_radar"`
AlertPreRedressements string `json:"alert_pre_redressements"`
MacroExpl map[string]float64 `json:"macro_expl"`
MicroExpl map[string]interface{} `json:"micro_expl"`
AlertPreRedressements string `json:"alert_pre_redressements"`
}

type scoreFile struct {
Expand All @@ -63,9 +62,10 @@ type scoreFile struct {
SelectConcerning [][]string `json:"selectConcerning"`
SelectReassuring [][]string `json:"selectReassuring"`
} `json:"explSelection"`
MacroRadar map[string]float64 `json:"macroRadar"`
Redressements []string `json:"redressements"`
AlertPreRedressements string `json:"alertPreRedressements"`
MacroExpl map[string]float64 `json:"macroExpl"`
MicroExpl map[string]interface{} `json:"microExpl"`
Redressements []string `json:"redressements"`
AlertPreRedressements string `json:"alertPreRedressements"`
}

func refreshVtablesHandler(c *gin.Context) {
Expand All @@ -75,6 +75,7 @@ func refreshVtablesHandler(c *gin.Context) {

func importPredictions(batchNumber string, algo string) error {
filename := viper.GetString("source.listPath")

file, err := os.Open(filename)
if err != nil {
return errors.New("open file: " + err.Error())
Expand Down Expand Up @@ -108,33 +109,40 @@ func importPredictions(batchNumber string, algo string) error {
algo text,
expl_selection_concerning jsonb default '{}',
expl_selection_reassuring jsonb default '{}',
macro_radar jsonb default '{}',
macro_expl jsonb default '{}',
micro_expl jsonb default '{}',
alert_pre_redressements text,
redressements text[] default '{}'
);`)
if err != nil {
return errors.New("create tmp_score: " + err.Error())
}

slog.Info("Inserting into tmp_score", slog.String("status", "start"))
batch := &pgx.Batch{}
for _, s := range scores {
queueScoreToBatch(s, batchNumber, batch)
}
slog.Info("Inserting into tmp_score", slog.String("status", "end"))
slog.Info("Inserting into score", slog.String("status", "start"))
batch.Queue(`insert into score
(siret, siren, libelle_liste, batch, algo, periode,
score, diff, alert, expl_selection_concerning,
expl_selection_reassuring, macro_radar,
expl_selection_reassuring, macro_expl, micro_expl,
redressements, alert_pre_redressements)
select
e.siret, t.siren, t.libelle_liste, batch, $1,
$2, score, diff, alert, expl_selection_concerning,
expl_selection_reassuring, macro_radar,
expl_selection_reassuring, macro_expl, micro_expl,
redressements, alert_pre_redressements
from tmp_score t
inner join etablissement e on e.siren = t.siren and e.siege`, algo, now)
slog.Info("Inserting into score", slog.String("status", "end"))

slog.Info("Inserting into liste", slog.String("status", "start"))
batch.Queue(`insert into liste (libelle, batch, algo) values ($1, $2, $3)`, toLibelle(batchNumber), batchNumber, algo)
batch.Queue("drop table if exists tmp_score;")
slog.Info("Inserting into liste", slog.String("status", "end"))
results := tx.SendBatch(context.Background(), batch)
err = results.Close()

Expand All @@ -145,6 +153,7 @@ func importPredictions(batchNumber string, algo string) error {
if err != nil {
return errors.New("commit: " + err.Error())
}

return nil
}

Expand Down Expand Up @@ -192,8 +201,8 @@ func sqlDeletePredictions(ctx context.Context, tx pgx.Tx, batchNumber string, al

func queueScoreToBatch(s scoreFile, batchNumber string, batch *pgx.Batch) {
sqlScore := `insert into tmp_score (siren, libelle_liste, batch, algo, score, diff, alert,
expl_selection_concerning, expl_selection_reassuring, macro_radar, alert_pre_redressements, redressements)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`
expl_selection_concerning, expl_selection_reassuring, macro_expl, micro_expl, alert_pre_redressements, redressements)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`

if s.ExplSelection.SelectConcerning == nil {
s.ExplSelection.SelectConcerning = make([][]string, 0)
Expand All @@ -202,9 +211,13 @@ func queueScoreToBatch(s scoreFile, batchNumber string, batch *pgx.Batch) {
s.ExplSelection.SelectReassuring = make([][]string, 0)
}

if s.MacroRadar == nil {
s.MacroRadar = make(map[string]float64)
if s.MacroExpl == nil {
s.MacroExpl = make(map[string]float64)
}
if s.MicroExpl == nil {
s.MicroExpl = make(map[string]interface{})
}

if s.Redressements == nil {
s.Redressements = make([]string, 0)
}
Expand All @@ -219,7 +232,8 @@ func queueScoreToBatch(s scoreFile, batchNumber string, batch *pgx.Batch) {
s.Alert,
s.ExplSelection.SelectConcerning,
s.ExplSelection.SelectReassuring,
s.MacroRadar,
s.MacroExpl,
s.MicroExpl,
s.AlertPreRedressements,
s.Redressements,
)
Expand Down
Binary file modified test/data/getEtablissement-VAF-03889351603035.json.gz
View file
Open in desktop
Binary file not shown.
Binary file modified test/data/getEtablissement-VaF-02176220713665.json.gz
View file
Open in desktop
Binary file not shown.
Binary file modified test/data/getEtablissement-vAF-02052333785743.json.gz
View file
Open in desktop
Binary file not shown.
Binary file modified test/data/getEtablissement-vaF-00785301892202.json.gz
View file
Open in desktop
Binary file not shown.
Loading