#214 Scan PostgreSQL table columns using query planner statistics (#223)

dataherald/db_scanner/sqlalchemy.py

@@ -89,6 +89,27 @@ def get_processed_column(
                 data_type=str(column["type"]),
                 low_cardinality=False,
             )
+
+        # special case for PostgreSQL table - read query planner statistics from the pg_stats view
+        # TODO doesn't work for views, only tables
+        if db_engine.engine.driver == "psycopg2":
+            # TODO escape table and column names
+            rs = db_engine.engine.execute(
+                f"SELECT n_distinct, most_common_vals::TEXT::TEXT[] FROM pg_catalog.pg_stats WHERE tablename = '{table}' AND attname = '{column['name']}'"
+            ).fetchall()
+            if MIN_CATEGORY_VALUE < rs[0]["n_distinct"] <= MAX_CATEGORY_VALUE:
+                return ColumnDetail(
+                    name=column["name"],
+                    data_type=str(column["type"]),
+                    low_cardinality=True,
+                    categories=rs[0]["most_common_vals"],
+                )
+            return ColumnDetail(
+                name=column["name"],
+                data_type=str(column["type"]),
+                low_cardinality=False,
+            )
+
         try:
             cardinality_query = sqlalchemy.select(
                 [func.distinct(dynamic_meta_table.c[column["name"]])]