Qodana stat (#42)

Add a script to convert the data received by the Qodana into the format of the Hyperstyle tool for
analysis and statistics gathering.

src/python/evaluation/common/pandas_util.py

@@ -1,12 +1,12 @@
 import json
 import logging
 from pathlib import Path
-from typing import Any, List, Set, Union
+from typing import Any, Iterable, List, Set, Union
 
 import numpy as np
 import pandas as pd
 from src.python.evaluation.common.csv_util import write_dataframe_to_csv
-from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.common.util import ColumnName
 from src.python.evaluation.common.xlsx_util import create_workbook, remove_sheet, write_dataframe_to_xlsx_sheet
 from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 from src.python.review.application_config import LanguageVersion
@@ -18,15 +18,19 @@
 
 def filter_df_by_language(df: pd.DataFrame, languages: Set[LanguageVersion],
                           column: str = ColumnName.LANG.value) -> pd.DataFrame:
-    return df.loc[df[column].isin(set(map(lambda l: l.value, languages)))]
+    return filter_df_by_iterable_value(df, column, set(map(lambda l: l.value, languages)))
 
 
-def filter_df_by_condition(df: pd.DataFrame, column: str, value: Any) -> pd.DataFrame:
+def filter_df_by_iterable_value(df: pd.DataFrame, column: str, value: Iterable) -> pd.DataFrame:
+    return df.loc[df[column].isin(value)]
+
+
+def filter_df_by_single_value(df: pd.DataFrame, column: str, value: Any) -> pd.DataFrame:
     return df.loc[df[column] == value]
 
 
 def drop_duplicates(df: pd.DataFrame, column: str = ColumnName.CODE.value) -> pd.DataFrame:
-    return df.drop_duplicates(column, keep='last')
+    return df.drop_duplicates(column, keep='last').reset_index(drop=True)
 
 
 # Find all rows and columns where two dataframes are inconsistent.
@@ -100,4 +104,4 @@ def get_issues_from_json(str_json: str) -> List[PenaltyIssue]:
 
 
 def get_issues_by_row(df: pd.DataFrame, row: int) -> List[PenaltyIssue]:
-    return get_issues_from_json(df.iloc[row][EvaluationArgument.TRACEBACK.value])
+    return get_issues_from_json(df.iloc[row][ColumnName.TRACEBACK.value])

src/python/evaluation/common/util.py

@@ -21,6 +21,7 @@ class ColumnName(Enum):
     PENALTY = 'penalty'
     USER = 'user'
     HISTORY = 'history'
+    TRACEBACK = 'traceback'
 
 
 @unique

src/python/evaluation/evaluation_run_tool.py

@@ -94,22 +94,22 @@ def __get_grade_from_traceback(traceback: str) -> str:
 # TODO: calculate grade after it
 def inspect_solutions_df(config: EvaluationConfig, lang_code_dataframe: pd.DataFrame) -> pd.DataFrame:
     report = pd.DataFrame(columns=lang_code_dataframe.columns)
-    report[EvaluationArgument.TRACEBACK.value] = []
+    report[ColumnName.TRACEBACK.value] = []
 
     pandarallel.initialize()
     if config.traceback:
-        report[EvaluationArgument.TRACEBACK.value] = []
+        report[ColumnName.TRACEBACK.value] = []
     try:
-        lang_code_dataframe[EvaluationArgument.TRACEBACK.value] = lang_code_dataframe.parallel_apply(
+        lang_code_dataframe[ColumnName.TRACEBACK.value] = lang_code_dataframe.parallel_apply(
             lambda row: __inspect_row(row[ColumnName.LANG.value],
                                       row[ColumnName.CODE.value],
                                       row[ColumnName.ID.value], config), axis=1)
 
         lang_code_dataframe[ColumnName.GRADE.value] = lang_code_dataframe.parallel_apply(
-            lambda row: __get_grade_from_traceback(row[EvaluationArgument.TRACEBACK.value]), axis=1)
+            lambda row: __get_grade_from_traceback(row[ColumnName.TRACEBACK.value]), axis=1)
 
         if not config.traceback:
-            del lang_code_dataframe[EvaluationArgument.TRACEBACK.value]
+            del lang_code_dataframe[ColumnName.TRACEBACK.value]
         return lang_code_dataframe
 
     except ValueError as e:

src/python/evaluation/inspectors/diffs_between_df.py

@@ -6,7 +6,7 @@
 from src.python.evaluation.common.pandas_util import (
     get_inconsistent_positions, get_issues_by_row, get_solutions_df, get_solutions_df_by_file_path,
 )
-from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.common.util import ColumnName
 from src.python.review.common.file_system import (
     Extension, get_parent_folder, get_restricted_extension, serialize_data_and_write_to_file,
 )
@@ -52,15 +52,15 @@ def find_diffs(old_df: pd.DataFrame, new_df: pd.DataFrame) -> dict:
     diffs = {
         ColumnName.GRADE.value: [],
         ColumnName.DECREASED_GRADE.value: [],
-        EvaluationArgument.TRACEBACK.value: {},
+        ColumnName.TRACEBACK.value: {},
         ColumnName.PENALTY.value: {},
     }
     if ColumnName.USER.value in new_df.columns:
         diffs[ColumnName.USER.value] = len(new_df[ColumnName.USER.value].unique())
     else:
         diffs[ColumnName.USER.value] = 0
     # Keep only diffs in the TRACEBACK column
-    for row, _ in filter(lambda t: t[1] == EvaluationArgument.TRACEBACK.value, inconsistent_positions.index):
+    for row, _ in filter(lambda t: t[1] == ColumnName.TRACEBACK.value, inconsistent_positions.index):
         old_value = old_df.iloc[row][ColumnName.GRADE.value]
         new_value = new_df.iloc[row][ColumnName.GRADE.value]
         old_quality = QualityType(old_value).to_number()
@@ -79,7 +79,7 @@ def find_diffs(old_df: pd.DataFrame, new_df: pd.DataFrame) -> dict:
                 raise ValueError(f'New dataframe contains less issues than old for fragment {id}')
             difference = set(set(new_issues) - set(old_issues))
             if len(difference) > 0:
-                diffs[EvaluationArgument.TRACEBACK.value][fragment_id] = difference
+                diffs[ColumnName.TRACEBACK.value][fragment_id] = difference
 
             # Find issues with influence_in_penalty > 0
             penalty = set(filter(lambda i: i.influence_on_penalty > 0, new_issues))

src/python/evaluation/inspectors/distribute_grades.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from src.python.common.tool_arguments import RunToolArgument
 from src.python.evaluation.common.pandas_util import get_solutions_df, get_solutions_df_by_file_path, write_df_to_file
-from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.common.util import ColumnName
 from src.python.review.common.file_system import Extension, get_parent_folder, get_restricted_extension
 
 CodeToGradesDict = Dict[str, Tuple[str, Optional[str]]]
@@ -35,12 +35,12 @@ def get_code_to_grades_dict(df: pd.DataFrame) -> CodeToGradesDict:
     df.apply(lambda row: __add_grade(code_to_grades_dict,
                                      row[ColumnName.CODE.value],
                                      row[ColumnName.GRADE.value],
-                                     row[EvaluationArgument.TRACEBACK.value]), axis=1)
+                                     row[ColumnName.TRACEBACK.value]), axis=1)
     return code_to_grades_dict
 
 
 def fill_all_solutions_df(all_solutions_df: pd.DataFrame, code_to_grades_dict: CodeToGradesDict) -> pd.DataFrame:
-    all_solutions_df[ColumnName.GRADE.value], all_solutions_df[EvaluationArgument.TRACEBACK.value] = zip(
+    all_solutions_df[ColumnName.GRADE.value], all_solutions_df[ColumnName.TRACEBACK.value] = zip(
         *all_solutions_df[ColumnName.CODE.value].map(lambda code: code_to_grades_dict[code]))
     return all_solutions_df
 

src/python/evaluation/inspectors/filter_issues.py

@@ -5,13 +5,13 @@
 import pandas as pd
 from src.python.common.tool_arguments import RunToolArgument
 from src.python.evaluation.common.pandas_util import get_issues_from_json, get_solutions_df_by_file_path
-from src.python.evaluation.common.util import ColumnName, EvaluationArgument, parse_set_arg
+from src.python.evaluation.common.util import ColumnName, parse_set_arg
 from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 from src.python.review.common.file_system import Extension, get_parent_folder, serialize_data_and_write_to_file
 from src.python.review.inspectors.issue import BaseIssue
 
 
-TRACEBACK = EvaluationArgument.TRACEBACK.value
+TRACEBACK = ColumnName.TRACEBACK.value
 ID = ColumnName.ID.value
 GRADE = ColumnName.GRADE.value
 

src/python/evaluation/inspectors/get_worse_public_examples.py

@@ -5,8 +5,8 @@
 import pandas as pd
 from src.python.common.tool_arguments import RunToolArgument
 from src.python.evaluation.common.csv_util import write_dataframe_to_csv
-from src.python.evaluation.common.pandas_util import filter_df_by_condition, get_solutions_df_by_file_path
-from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.common.pandas_util import filter_df_by_single_value, get_solutions_df_by_file_path
+from src.python.evaluation.common.util import ColumnName
 from src.python.evaluation.inspectors.common.statistics import PenaltyIssue
 from src.python.review.common.file_system import deserialize_data_from_file, Extension, get_parent_folder
 
@@ -32,20 +32,20 @@ def __get_new_inspections(fragment_id_to_issues: Dict[int, List[PenaltyIssue]],
 
 def __get_public_fragments(solutions_df: pd.DataFrame, diffs_dict: dict) -> pd.DataFrame:
     # Keep only public solutions
-    public_fragments = filter_df_by_condition(solutions_df, ColumnName.IS_PUBLIC.value, 'YES')
+    public_fragments = filter_df_by_single_value(solutions_df, ColumnName.IS_PUBLIC.value, 'YES')
     count_inspections_column = 'count_inspections'
     new_inspections_column = 'new_inspections'
 
     # Get only new inspections and count them
-    fragment_id_to_issues = diffs_dict[EvaluationArgument.TRACEBACK.value]
+    fragment_id_to_issues = diffs_dict[ColumnName.TRACEBACK.value]
     public_fragments[new_inspections_column] = public_fragments.apply(
         lambda row: __get_new_inspections(fragment_id_to_issues, row[ColumnName.ID.value]), axis=1)
     public_fragments[count_inspections_column] = public_fragments.apply(
         lambda row: len(row[new_inspections_column].split(',')), axis=1)
 
     public_fragments = public_fragments.sort_values(count_inspections_column, ascending=False)
     # Keep only public columns
-    return public_fragments[[ColumnName.CODE.value, EvaluationArgument.TRACEBACK.value, new_inspections_column]]
+    return public_fragments[[ColumnName.CODE.value, ColumnName.TRACEBACK.value, new_inspections_column]]
 
 
 # TODO: add readme

src/python/evaluation/inspectors/print_inspectors_statistics.py

@@ -4,7 +4,7 @@
 from typing import Dict, List
 
 from src.python.common.tool_arguments import RunToolArgument
-from src.python.evaluation.common.util import ColumnName, EvaluationArgument
+from src.python.evaluation.common.util import ColumnName
 from src.python.evaluation.inspectors.common.statistics import (
     GeneralInspectorsStatistics, IssuesStatistics, PenaltyInfluenceStatistics, PenaltyIssue,
 )
@@ -32,11 +32,11 @@ def configure_arguments(parser: argparse.ArgumentParser) -> None:
 
 
 def has_incorrect_grades(diffs_dict: dict) -> bool:
-    return len(diffs_dict[ColumnName.GRADE.value]) > 0
+    return len(diffs_dict.get(ColumnName.GRADE.value, [])) > 0
 
 
 def has_decreased_grades(diffs_dict: dict) -> bool:
-    return len(diffs_dict[ColumnName.DECREASED_GRADE.value]) > 0
+    return len(diffs_dict.get(ColumnName.DECREASED_GRADE.value, [])) > 0
 
 
 def __gather_issues_stat(issues_stat_dict: Dict[int, List[PenaltyIssue]]) -> IssuesStatistics:
@@ -50,10 +50,10 @@ def __gather_issues_stat(issues_stat_dict: Dict[int, List[PenaltyIssue]]) -> Iss
 
 
 def gather_statistics(diffs_dict: dict) -> GeneralInspectorsStatistics:
-    new_issues_stat = __gather_issues_stat(diffs_dict[EvaluationArgument.TRACEBACK.value])
-    penalty_issues_stat = __gather_issues_stat(diffs_dict[ColumnName.PENALTY.value])
+    new_issues_stat = __gather_issues_stat(diffs_dict.get(ColumnName.TRACEBACK.value, {}))
+    penalty_issues_stat = __gather_issues_stat(diffs_dict.get(ColumnName.PENALTY.value, {}))
     return GeneralInspectorsStatistics(new_issues_stat, penalty_issues_stat,
-                                       PenaltyInfluenceStatistics(diffs_dict[ColumnName.PENALTY.value]))
+                                       PenaltyInfluenceStatistics(diffs_dict.get(ColumnName.PENALTY.value, {})))
 
 
 def main() -> None:
@@ -73,7 +73,7 @@ def main() -> None:
         print('All grades are equal.')
     else:
         print(f'Decreased grades was found in {len(diffs[ColumnName.DECREASED_GRADE.value])} fragments')
-    print(f'{diffs[ColumnName.USER.value]} unique users was found!')
+    print(f'{diffs.get(ColumnName.USER.value, 0)} unique users was found!')
     print(separator)
 
     statistics = gather_statistics(diffs)

src/python/evaluation/qodana/README.md

@@ -1,4 +1,4 @@
-# Dataset label
+# Dataset labelling
 This script allows you to label a dataset using the found [Qodana](https://github.com/JetBrains/Qodana) inspections.
 
 The dataset must contain at least three columns: `id`, `code` and `lang`, where `id` is a unique solution number, `lang` is the language in which the code is written in the `code` column. The `lang` must belong to one of the following values: `java7`, `java8`, `java9`, `java11`, `python3`, `kotlin`. If `lang` is not equal to any of the values, the row will be skipped.
@@ -22,7 +22,7 @@ Run the [dataset_labeling.py](dataset_labeling.py) with the arguments from comma
 
 ---
 
-# Postprocessing
+# Preprocessing
 
 The model that imitates Qodana analysis gets input from a dataset in a special format. 
 This module allows preparing datasets that were graded by [dataset_marking.py](dataset_marking.py) script.
@@ -231,3 +231,81 @@ id   |  code                                  |  lang         |  inspections
 1    |  "// second line from code with id 1"  |  java11       |  0
 
 ```
+
+# Postprocessing
+
+At this stage, you can convert the data received by the Qodana into the format of the Hyperstyle tool for 
+analysis and statistics gathering.
+
+## Convert Qodana inspections into Hyperstyle inspections
+
+This stage allows you to convert the `inspections` column from `csv` marked by Qodana into 
+`traceback` column with the Hyperstyle tool format.
+
+This stage includes:
+- keep only unique code fragments in both datasets (Qodana and Hyperstyle);
+- keep only fragments in both datasets that have same ids and same code fragments;
+- add a `grade` column into Qodana dataset corresponding to the `grade` column from Hyperstyle dataset;
+- add a `traceback` column in the Hyperstyle format into Qodana dataset with inspection from the `inspections` column. 
+
+Please, note that your Qodana input file must be graded by [dataset_labeling.py](dataset_labeling.py) script 
+and have `inspections` column. Your Hyperstyle input file must be graded by [evaluation_run_tool.py](../evaluation_run_tool.py) script 
+and have `traceback` and `grade` columns.
+
+Output files is two new `csv` files.
+
+#### Usage
+
+Run the [convert_to_hyperstyle_inspections.py](convert_to_hyperstyle_inspections.py) with the arguments from command line.
+
+Required arguments:
+
+- `solutions_file_path_hyperstyle` — path to a `csv` file labelled by Hyperstyle;
+- `solutions_file_path_qodana` — path to a `csv` file labelled by Qodana.
+
+Optional arguments:
+Argument | Description
+--- | ---
+|**‑i**, **‑‑issues-to-keep**| Set of issues ids to keep in the dataset separated by comma. By default all issues are deleted. |
+
+The Hyperstyle resulting file will be stored in the same folder with `solutions_file_path_hyperstyle`.
+The Qodana resulting file will be stored in the same folder with `solutions_file_path_qodana`.
+
+An example of the Qodana inspections before and after this processing:
+
+1. Before:
+
+```json
+{
+  "issues": [
+    {
+      "fragment_id": 0,
+      "line": 8,
+      "offset": 8,
+      "length": 10,
+      "highlighted_element": "System.out",
+      "description": "Uses of <code>System.out</code> should probably be replaced with more robust logging #loc",
+      "problem_id": "SystemOutErr"
+    }
+  ]
+}
+```
+
+2. After:
+
+```json
+{
+  "issues": [
+    {
+      "code": "SystemOutErr",
+      "text": "Uses of <code>System.out</code> should probably be replaced with more robust logging #loc",
+      "line": "",
+      "line_number": 8,
+      "column_number": 8,
+      "category": "INFO",
+      "influence_on_penalty": 0
+    }
+  ]
+}
+```
+___