From fa94a5739c40c54e9440249bdfdd278d3b022705 Mon Sep 17 00:00:00 2001 From: Noureddine Date: Wed, 23 Oct 2024 16:35:55 +0000 Subject: [PATCH 1/3] add score to report --- bin/sequencer_report | 37 +++++++++++++++++++------------- etc/sequencer_report.md.template | 8 +++---- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/bin/sequencer_report b/bin/sequencer_report index af64db4f0d..c68e1a7dd4 100755 --- a/bin/sequencer_report +++ b/bin/sequencer_report @@ -23,7 +23,6 @@ STAGES = ["stable", "beta", "preview", "alpha"] # stages to consider for a pass STAGES_FOR_PASS = ["stable", "beta"] -DEFAULT_SCORE = 1 REFERENCE_SEQUENCES_DIR = "validator/sequences" CHECKMARK = "✓" CROSS = "✕" @@ -41,7 +40,13 @@ def first_occurence(iteratable: Iterable, predicate: Callable) -> int: raise ValueError() -@dataclass(unsafe_hash=True, eq=True, order=True) +def sanitize(string: str): + """Sanitize string for safe displaying""" + sanitized = string.replace("\n", ";") + return sanitized + + +@dataclass(kw_only=True,unsafe_hash=True, eq=True, order=True) class TestResult: """Container for test result.""" @@ -51,7 +56,8 @@ class TestResult: result: str = "" stage: str = "" message: str = "" - score: int = DEFAULT_SCORE + score: int = 0 + total: int = 0 def passed(self): if self.result == "pass": @@ -394,6 +400,10 @@ class SequencerReport: self.site_path, "cloud_iot_config.json" ) + self.gateway = ( + self.metadata.get("gateway", {}).get("gateway_id") + ) + def has_stage(self, stage: str): """Checks if sequencer report has any tests at given stage.""" all_features = [x[stage] for x in self.features.values()] @@ -408,14 +418,16 @@ class SequencerReport: features[feature] = copy.deepcopy(stages_template) for name, result in sequences["sequences"].items(): results[name] = TestResult( - feature, - name, - result.get("summary", ""), - result["result"], - result["stage"], - self.sanitize(result["status"]["message"]), + bucket=feature, + name=name, + description=result.get("summary", ""), + result=result["result"], + stage=result["stage"], + message=sanitize(result["status"]["message"]), + score=result["scoring"]["value"], + total=result["scoring"]["total"], ) - features[feature][result["stage"]].add(result["result"], DEFAULT_SCORE) + features[feature][result["stage"]].add(result["result"], result["scoring"]["value"]) features[feature][result["stage"]].tests.append(result) self.results = { @@ -444,11 +456,6 @@ class SequencerReport: for f in self.features } - def sanitize(self, string: str): - """Sanitize string for safe displaying""" - sanitized = string.replace("\n", ";") - return sanitized - def __repr__(self): return str(self.results) diff --git a/etc/sequencer_report.md.template b/etc/sequencer_report.md.template index 19f14d248c..00a6a807f5 100644 --- a/etc/sequencer_report.md.template +++ b/etc/sequencer_report.md.template @@ -8,10 +8,10 @@ | Device | {{report.device_id}} | |---|---| -| Site | | | Make | {{report.device_make}} | | Model | {{report.device_model}} | | Software | {{report.device_software|pretty_dict}} | +{% if report.gateway is not none %}| Gateway | {{report.gateway}} |{%- endif %} ## Summary @@ -23,10 +23,10 @@ ## Results -| Bucket | Feature | Stage | Result | Description | -| --- | --- | --- | --- | --- | +| Bucket | Feature | Stage | Score | Result | Description | +| --- | --- | --- | --- | --- | --- | {% for test in report.results.values() -%} -| {{test.bucket}} | {{test.name}} | {{test.stage}} | {{test.result}} | {{test.message}} | +| {{test.bucket}} | {{test.name}} | {{test.stage}} | {{test.score}} | {{test.result}} | {{test.message}} | {% endfor %} ## Schema From dc4130116fd431b644b55453f1f4220303bd2384 Mon Sep 17 00:00:00 2001 From: Noureddine Date: Wed, 23 Oct 2024 19:25:29 +0000 Subject: [PATCH 2/3] fix total scoring mybe --- bin/sequencer_report | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/bin/sequencer_report b/bin/sequencer_report index c68e1a7dd4..383848a273 100755 --- a/bin/sequencer_report +++ b/bin/sequencer_report @@ -42,7 +42,7 @@ def first_occurence(iteratable: Iterable, predicate: Callable) -> int: def sanitize(string: str): """Sanitize string for safe displaying""" - sanitized = string.replace("\n", ";") + sanitized = string.replace("\n", "; ") return sanitized @@ -278,17 +278,23 @@ class TemplateHelper: @dataclass class FeatureStage: - """Container for scored points and total points for a feature and stage.""" + """Container for scored points and total points for a feature and stage. + + Arguments: + scored: Number of points scored for a given feature at a given stage. + total: Total points which could possibly have attained. + stage: The stage e.g. BETA, STABLE, etc. + tests: List of tests for this feature at the given stage. + """ scored: int = 0 total: int = 0 stage: str = "" tests: list = field(default_factory=list) - def add(self, result, score): - self.total += score - if result == "pass": - self.scored += score + def add(self, *, score:int, total:int): + self.total += total + self.scored += score def has(self): """Did the sequencer results have this feature & stage combination?.""" @@ -427,7 +433,8 @@ class SequencerReport: score=result["scoring"]["value"], total=result["scoring"]["total"], ) - features[feature][result["stage"]].add(result["result"], result["scoring"]["value"]) + + features[feature][result["stage"]].add(scored=result["scoring"]["value"], total=result["scoring"]["total"]) features[feature][result["stage"]].tests.append(result) self.results = { From b2bf16583e653ce5980a459db6278a42109e06fb Mon Sep 17 00:00:00 2001 From: Noureddine Date: Wed, 23 Oct 2024 19:30:18 +0000 Subject: [PATCH 3/3] fix total scoring --- bin/sequencer_report | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/sequencer_report b/bin/sequencer_report index 383848a273..ad5b6c2798 100755 --- a/bin/sequencer_report +++ b/bin/sequencer_report @@ -292,9 +292,9 @@ class FeatureStage: stage: str = "" tests: list = field(default_factory=list) - def add(self, *, score:int, total:int): - self.total += total - self.scored += score + def add(self, *, test_score:int, test_total:int): + self.total += test_total + self.scored += test_score def has(self): """Did the sequencer results have this feature & stage combination?.""" @@ -434,7 +434,7 @@ class SequencerReport: total=result["scoring"]["total"], ) - features[feature][result["stage"]].add(scored=result["scoring"]["value"], total=result["scoring"]["total"]) + features[feature][result["stage"]].add(test_score=result["scoring"]["value"], test_total=result["scoring"]["total"]) features[feature][result["stage"]].tests.append(result) self.results = {