feat: add more metadata support (#1182)

* feat: add table custom properties (#1165)

* feat: add table custom properties

* comments

* update package version

* feat: load tag from metastore (#1164)

* feat: load tag from metastore

* comments

* comments

* feat: add column level tag support (#1169)

* feat: add column level tag support

* comments

* set commit False for create_table

* tagsDOM

* use verify_data_column_permission

* ffix linter

* feat: load data table ownership from metastore (#1170)

* load data table owners from metastore

* add tooltip

* fix linter

* comments

* fix circular import

* display name comments

* display name comments

* get owner types comments

* feat: load user groups  (#1176)

* feat: load and sync user groups from metastore

* remove sync group task

* remove and add delta

* refactor

* chore: merge alembic changes (#1180)

* chore: merge alembic changes

* add foregin key name

package.json

@@ -1,6 +1,6 @@
 {
     "name": "querybook",
-    "version": "3.18.0",
+    "version": "3.19.0",
     "description": "A Big Data Webapp",
     "private": true,
     "scripts": {

querybook/migrations/versions/ec2f32c25f34_add_more_metadata_support.py

@@ -0,0 +1,34 @@
+"""add more metadata support
+
+Revision ID: ec2f32c25f34
+Revises: 1b8aba201c94
+Create Date: 2023-02-24 23:46:29.304134
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = 'ec2f32c25f34'
+down_revision = '1b8aba201c94'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('data_table_information', sa.Column('custom_properties', sa.JSON(), nullable=True))
+    op.add_column('data_table_ownership', sa.Column('type', sa.String(length=255), nullable=True))
+    op.add_column('tag_item', sa.Column('column_id', sa.Integer(), nullable=True))
+    op.create_foreign_key('fk_tag_item_data_table_column', 'tag_item', 'data_table_column', ['column_id'], ['id'], ondelete='CASCADE')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('fk_tag_item_data_table_column', 'tag_item', type_='foreignkey')
+    op.drop_column('tag_item', 'column_id')
+    op.drop_column('data_table_ownership', 'type')
+    op.drop_column('data_table_information', 'custom_properties')
+    # ### end Alembic commands ###

querybook/server/const/metastore.py

@@ -1,4 +1,86 @@
 from enum import Enum
+from typing import NamedTuple
+
+
+class DataSchema(NamedTuple):
+    name: str
+
+
+class DataTag(NamedTuple):
+    name: str
+    # below properties will be stored in tag.meta
+    type: str = None
+    description: str = None
+    color: str = None
+
+
+class DataOwnerType(NamedTuple):
+    name: str
+    # It will be rendered as the field label in the detailed table view
+    display_name: str
+    description: str = None
+
+
+class DataOwner(NamedTuple):
+    username: str
+    # If provided, the type here must be one of the type names from metastore loader
+    type: str = None
+
+
+class DataTable(NamedTuple):
+    name: str
+
+    # The type of table, it can be an arbitrary string
+    type: str = None
+
+    # This is the legacy field, which will be replaced by owners field below.
+    owner: str = None
+    # list of owner usernames
+    owners: list[DataOwner] = []
+
+    # description from metastore, expect HTML format
+    description: str = None
+
+    # list of tags
+    tags: list[DataTag] = []
+
+    # Expected in UTC seconds
+    table_created_at: int = None
+    table_updated_at: int = None
+    table_updated_by: str = None
+
+    # size of table
+    data_size_bytes: int = None
+    # Location of the raw file
+    location: str = None
+
+    # Json arrays of partitions
+    partitions: list[str] = []
+    earliest_partitions: list[str] = None
+    latest_partitions: list[str] = None
+
+    # Store the raw info here
+    raw_description: str = None
+
+    # Arrays of partition keys
+    partition_keys: list[str] = []
+
+    # Custom properties
+    custom_properties: dict[str, str] = None
+
+
+class DataColumn(NamedTuple):
+    name: str
+    type: str
+
+    # column comment from sql query when creating the table
+    comment: str = None
+
+    # user edited description from metastore, expect HTML format
+    description: str = None
+
+    # list of column level tags from metastore
+    tags: list[DataTag] = []
 
 
 class DataTableWarningSeverity(Enum):
@@ -18,6 +100,7 @@ class MetadataMode(Enum):
     READ_ONLY = "read_only"
 
     # On saving, metadata will only be written to querybook db. This is the default mode if not specified.
+    # It also indicates that it will not load this metadata from the metastore.
     WRITE_LOCAL = "write_local"
 
     # On saving, metadata will be written back to metastore, as well as querybook db
@@ -37,5 +120,12 @@ class MetastoreLoaderConfig:
     def __init__(self, config: dict[MetadataType, MetadataMode]):
         self._config = {**self._default_config, **config}
 
+    def can_load_external_metadata(self, metadataType: MetadataType) -> bool:
+        """Check if the given metadata type will be loaded from metastore"""
+        return self._config.get(metadataType, MetadataMode.WRITE_LOCAL) in (
+            MetadataMode.READ_ONLY,
+            MetadataMode.WRITE_BACK,
+        )
+
     def to_dict(self):
         return {key.value: value.value for (key, value) in self._config.items()}

querybook/server/const/user.py

@@ -0,0 +1,10 @@
+from typing import NamedTuple
+
+
+class UserGroup(NamedTuple):
+    name: str
+    display_name: str
+    description: str
+    email: str
+    # list of user names
+    members: list[str]

querybook/server/datasources/tag.py

@@ -2,7 +2,10 @@
 
 from app.datasource import register, api_assert
 from app.db import DBSession
-from app.auth.permission import verify_data_table_permission
+from app.auth.permission import (
+    verify_data_table_permission,
+    verify_data_column_permission,
+)
 from logic import tag as logic
 from models.tag import Tag
 
@@ -11,10 +14,18 @@
     "/table/<int:table_id>/tag/",
     methods=["GET"],
 )
-def get_tag_by_table_id(table_id: int):
-    with DBSession() as session:
-        verify_data_table_permission(table_id, session=session)
-        return logic.get_tag_by_table_id(table_id=table_id, session=session)
+def get_tags_by_table_id(table_id: int):
+    verify_data_table_permission(table_id)
+    return logic.get_tags_by_table_id(table_id=table_id)
+
+
+@register(
+    "/column/<int:column_id>/tag/",
+    methods=["GET"],
+)
+def get_tags_by_column_id(column_id: int):
+    verify_data_column_permission(column_id)
+    return logic.get_tags_by_column_id(column_id=column_id)
 
 
 @register(