diff --git a/package.json b/package.json index e07dd16af..48f3dc04c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "querybook", - "version": "3.18.0", + "version": "3.19.0", "description": "A Big Data Webapp", "private": true, "scripts": { diff --git a/querybook/migrations/versions/ec2f32c25f34_add_more_metadata_support.py b/querybook/migrations/versions/ec2f32c25f34_add_more_metadata_support.py new file mode 100644 index 000000000..d877a0bca --- /dev/null +++ b/querybook/migrations/versions/ec2f32c25f34_add_more_metadata_support.py @@ -0,0 +1,34 @@ +"""add more metadata support + +Revision ID: ec2f32c25f34 +Revises: 1b8aba201c94 +Create Date: 2023-02-24 23:46:29.304134 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'ec2f32c25f34' +down_revision = '1b8aba201c94' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('data_table_information', sa.Column('custom_properties', sa.JSON(), nullable=True)) + op.add_column('data_table_ownership', sa.Column('type', sa.String(length=255), nullable=True)) + op.add_column('tag_item', sa.Column('column_id', sa.Integer(), nullable=True)) + op.create_foreign_key('fk_tag_item_data_table_column', 'tag_item', 'data_table_column', ['column_id'], ['id'], ondelete='CASCADE') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('fk_tag_item_data_table_column', 'tag_item', type_='foreignkey') + op.drop_column('tag_item', 'column_id') + op.drop_column('data_table_ownership', 'type') + op.drop_column('data_table_information', 'custom_properties') + # ### end Alembic commands ### diff --git a/querybook/server/const/metastore.py b/querybook/server/const/metastore.py index edcf02d4d..a299f357c 100644 --- a/querybook/server/const/metastore.py +++ b/querybook/server/const/metastore.py @@ -1,4 +1,86 @@ from enum import Enum +from typing import NamedTuple + + +class DataSchema(NamedTuple): + name: str + + +class DataTag(NamedTuple): + name: str + # below properties will be stored in tag.meta + type: str = None + description: str = None + color: str = None + + +class DataOwnerType(NamedTuple): + name: str + # It will be rendered as the field label in the detailed table view + display_name: str + description: str = None + + +class DataOwner(NamedTuple): + username: str + # If provided, the type here must be one of the type names from metastore loader + type: str = None + + +class DataTable(NamedTuple): + name: str + + # The type of table, it can be an arbitrary string + type: str = None + + # This is the legacy field, which will be replaced by owners field below. + owner: str = None + # list of owner usernames + owners: list[DataOwner] = [] + + # description from metastore, expect HTML format + description: str = None + + # list of tags + tags: list[DataTag] = [] + + # Expected in UTC seconds + table_created_at: int = None + table_updated_at: int = None + table_updated_by: str = None + + # size of table + data_size_bytes: int = None + # Location of the raw file + location: str = None + + # Json arrays of partitions + partitions: list[str] = [] + earliest_partitions: list[str] = None + latest_partitions: list[str] = None + + # Store the raw info here + raw_description: str = None + + # Arrays of partition keys + partition_keys: list[str] = [] + + # Custom properties + custom_properties: dict[str, str] = None + + +class DataColumn(NamedTuple): + name: str + type: str + + # column comment from sql query when creating the table + comment: str = None + + # user edited description from metastore, expect HTML format + description: str = None + + # list of column level tags from metastore + tags: list[DataTag] = [] class DataTableWarningSeverity(Enum): @@ -18,6 +100,7 @@ class MetadataMode(Enum): READ_ONLY = "read_only" # On saving, metadata will only be written to querybook db. This is the default mode if not specified. + # It also indicates that it will not load this metadata from the metastore. WRITE_LOCAL = "write_local" # On saving, metadata will be written back to metastore, as well as querybook db @@ -37,5 +120,12 @@ class MetastoreLoaderConfig: def __init__(self, config: dict[MetadataType, MetadataMode]): self._config = {**self._default_config, **config} + def can_load_external_metadata(self, metadataType: MetadataType) -> bool: + """Check if the given metadata type will be loaded from metastore""" + return self._config.get(metadataType, MetadataMode.WRITE_LOCAL) in ( + MetadataMode.READ_ONLY, + MetadataMode.WRITE_BACK, + ) + def to_dict(self): return {key.value: value.value for (key, value) in self._config.items()} diff --git a/querybook/server/const/user.py b/querybook/server/const/user.py new file mode 100644 index 000000000..29c8b8d70 --- /dev/null +++ b/querybook/server/const/user.py @@ -0,0 +1,10 @@ +from typing import NamedTuple + + +class UserGroup(NamedTuple): + name: str + display_name: str + description: str + email: str + # list of user names + members: list[str] diff --git a/querybook/server/datasources/tag.py b/querybook/server/datasources/tag.py index ca7866bc7..542e5366a 100644 --- a/querybook/server/datasources/tag.py +++ b/querybook/server/datasources/tag.py @@ -2,7 +2,10 @@ from app.datasource import register, api_assert from app.db import DBSession -from app.auth.permission import verify_data_table_permission +from app.auth.permission import ( + verify_data_table_permission, + verify_data_column_permission, +) from logic import tag as logic from models.tag import Tag @@ -11,10 +14,18 @@ "/table//tag/", methods=["GET"], ) -def get_tag_by_table_id(table_id: int): - with DBSession() as session: - verify_data_table_permission(table_id, session=session) - return logic.get_tag_by_table_id(table_id=table_id, session=session) +def get_tags_by_table_id(table_id: int): + verify_data_table_permission(table_id) + return logic.get_tags_by_table_id(table_id=table_id) + + +@register( + "/column//tag/", + methods=["GET"], +) +def get_tags_by_column_id(column_id: int): + verify_data_column_permission(column_id) + return logic.get_tags_by_column_id(column_id=column_id) @register( diff --git a/querybook/server/lib/metastore/base_metastore_loader.py b/querybook/server/lib/metastore/base_metastore_loader.py index 34d015e3f..593807a89 100644 --- a/querybook/server/lib/metastore/base_metastore_loader.py +++ b/querybook/server/lib/metastore/base_metastore_loader.py @@ -1,82 +1,44 @@ -from abc import ABCMeta, abstractmethod, abstractclassmethod -import gevent import math -from typing import NamedTuple, List, Dict, Tuple, Optional import traceback +from abc import ABCMeta, abstractclassmethod, abstractmethod +from typing import Dict, List, Optional, Tuple +import gevent from app.db import DBSession, with_session -from const.metastore import MetadataType, MetastoreLoaderConfig -from lib.logger import get_logger - +from const.metastore import ( + DataColumn, + DataOwnerType, + DataTable, + MetadataType, + MetastoreLoaderConfig, +) from lib.form import AllFormField +from lib.logger import get_logger from lib.utils import json from lib.utils.utils import with_exception -from logic.elasticsearch import update_table_by_id, delete_es_table_by_id +from logic.elasticsearch import delete_es_table_by_id, update_table_by_id from logic.metastore import ( + create_column, create_schema, - delete_schema, create_table, - delete_table, create_table_information, - create_column, + create_table_ownerships, delete_column, - iterate_data_schema, - get_table_by_schema_id, + delete_schema, + delete_table, get_column_by_table_id, get_schema_by_name, + get_table_by_schema_id, get_table_by_schema_id_and_name, + iterate_data_schema, ) +from logic.tag import create_column_tags, create_table_tags from .utils import MetastoreTableACLChecker LOG = get_logger(__name__) -class DataSchema(NamedTuple): - name: str - - -class DataTable(NamedTuple): - name: str - - # The type of table, it can be an arbitrary string - type: str = None - owner: str = None - - # description from metastore, expect HTML format - description: str = None - - # Expected in UTC seconds - table_created_at: int = None - table_updated_at: int = None - table_updated_by: str = None - - # size of table - data_size_bytes: int = None - # Location of the raw file - location: str = None - - # Json arrays of partitions - partitions: List = [] - - # Store the raw info here - raw_description: str = None - - # Arrays of partition keys - partition_keys: List[str] = [] - - -class DataColumn(NamedTuple): - name: str - type: str - - # column comment from sql query when creating the table - comment: str = None - - # user edited description from metastore, expect HTML format - description: str = None - - class BaseMetastoreLoader(metaclass=ABCMeta): loader_config: MetastoreLoaderConfig = MetastoreLoaderConfig({}) @@ -100,6 +62,36 @@ def get_metastore_link( """ return None + @classmethod + def get_table_owner_types(cls) -> list[DataOwnerType]: + """Return all the owner types the meatstore supports. + + Override this method if loading table owners from metastore is enabled + and your metastore supports owner types. + + E.g. + [ + DataOwnerType( + name="CREATOR", + display_name="Table Creator", + description="Person who created the table", + ), + DataOwnerType( + name="BUSINESS_OWNER", + display_name="Owners", + description="Person or group who is responsible for business related aspects of the table", + ), + ] + + + The `display_name` will be rendered as the field label in the detailed table view, which is `Owners` by default. + """ + return [ + DataOwnerType( + name=None, display_name="Owners", description="People who own the table" + ) + ] + @with_session def sync_table( self, schema_name: str, table_name: str, session=None @@ -355,15 +347,21 @@ def _create_table_table( location=table.location, column_count=len(columns), schema_id=schema_id, + commit=False, session=session, ).id create_table_information( data_table_id=table_id, description=table.description, - latest_partitions=json.dumps((table.partitions or [])[-10:]), - earliest_partitions=json.dumps((table.partitions or [])[:10]), + latest_partitions=json.dumps( + table.latest_partitions or (table.partitions or [])[-10:] + ), + earliest_partitions=json.dumps( + table.earliest_partitions or (table.partitions or [])[:10] + ), hive_metastore_description=table.raw_description, partition_keys=table.partition_keys, + custom_properties=table.custom_properties, session=session, ) delete_column_not_in_metastore( @@ -371,7 +369,7 @@ def _create_table_table( ) for column in columns: - create_column( + column_id = create_column( name=column.name, type=column.type, comment=column.comment, @@ -379,6 +377,33 @@ def _create_table_table( table_id=table_id, commit=False, session=session, + ).id + + # create tags only if the metastore is configured to sync tags + if self.loader_config.can_load_external_metadata(MetadataType.TAG): + create_column_tags( + column_id=column_id, + tags=column.tags, + commit=False, + session=session, + ) + + # create tags only if the metastore is configured to sync tags + if self.loader_config.can_load_external_metadata(MetadataType.TAG): + create_table_tags( + table_id=table_id, + tags=table.tags, + commit=False, + session=session, + ) + + # load owners if the metastore is configured to sync table owners + if self.loader_config.can_load_external_metadata(MetadataType.OWNER): + create_table_ownerships( + table_id=table_id, + owners=table.owners, + commit=False, + session=session, ) session.commit() update_table_by_id(table_id, session=session) diff --git a/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py b/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py index 7aeac3c93..70044b7f5 100644 --- a/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py +++ b/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py @@ -2,12 +2,9 @@ from typing import Dict, List, Tuple from clients.glue_client import GlueDataCatalogClient -from lib.form import StructFormField, FormField -from lib.metastore.base_metastore_loader import ( - BaseMetastoreLoader, - DataTable, - DataColumn, -) +from const.metastore import DataColumn, DataTable +from lib.form import FormField, StructFormField +from lib.metastore.base_metastore_loader import BaseMetastoreLoader from lib.metastore.loaders.form_fileds import load_partitions_field diff --git a/querybook/server/lib/metastore/loaders/hive_metastore_loader.py b/querybook/server/lib/metastore/loaders/hive_metastore_loader.py index 13c0b6075..c9e78eecd 100644 --- a/querybook/server/lib/metastore/loaders/hive_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/hive_metastore_loader.py @@ -1,13 +1,10 @@ from typing import Dict, List, Tuple -from lib.form import ExpandableFormField, FormField, FormFieldType, StructFormField -from hmsclient.genthrift.hive_metastore.ttypes import NoSuchObjectException from clients.hms_client import HiveMetastoreClient -from lib.metastore.base_metastore_loader import ( - BaseMetastoreLoader, - DataTable, - DataColumn, -) +from const.metastore import DataColumn, DataTable +from hmsclient.genthrift.hive_metastore.ttypes import NoSuchObjectException +from lib.form import ExpandableFormField, FormField, FormFieldType, StructFormField +from lib.metastore.base_metastore_loader import BaseMetastoreLoader from lib.metastore.loaders.form_fileds import load_partitions_field from lib.utils import json as ujson diff --git a/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py b/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py index f634dc721..05de10e54 100644 --- a/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py @@ -1,8 +1,9 @@ from typing import List, Tuple -from lib.utils.utils import DATETIME_TO_UTC +from const.metastore import DataColumn, DataTable from lib.utils import json as ujson -from lib.metastore.base_metastore_loader import DataTable, DataColumn +from lib.utils.utils import DATETIME_TO_UTC + from .sqlalchemy_metastore_loader import SqlAlchemyMetastoreLoader diff --git a/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py b/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py index 485b933ac..25a452d40 100644 --- a/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py @@ -1,12 +1,9 @@ from typing import Dict, List, Tuple -from lib.metastore.base_metastore_loader import ( - BaseMetastoreLoader, - DataTable, - DataColumn, -) -from lib.query_executor.executor_template.templates import sqlalchemy_template +from const.metastore import DataColumn, DataTable +from lib.metastore.base_metastore_loader import BaseMetastoreLoader from lib.query_executor.connection_string.sqlalchemy import create_sqlalchemy_engine +from lib.query_executor.executor_template.templates import sqlalchemy_template class SqlAlchemyMetastoreLoader(BaseMetastoreLoader): diff --git a/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py b/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py index 9869e2eb2..6656581c0 100644 --- a/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py @@ -1,12 +1,11 @@ from typing import Dict, List, Tuple -from lib.form import ExpandableFormField, FormField, StructFormField -from lib.query_executor.clients.hive import HiveClient - from clients.hms_client import HiveMetastoreClient -from lib.metastore.loaders.hive_metastore_loader import HMSMetastoreLoader +from const.metastore import DataColumn, DataTable +from lib.form import ExpandableFormField, FormField, StructFormField from lib.metastore.loaders.form_fileds import load_partitions_field -from lib.metastore.base_metastore_loader import DataTable, DataColumn +from lib.metastore.loaders.hive_metastore_loader import HMSMetastoreLoader +from lib.query_executor.clients.hive import HiveClient class HMSThriftMetastoreLoader(HMSMetastoreLoader): diff --git a/querybook/server/logic/metastore.py b/querybook/server/logic/metastore.py index 0865d00e2..62875394f 100644 --- a/querybook/server/logic/metastore.py +++ b/querybook/server/logic/metastore.py @@ -1,25 +1,30 @@ import datetime -from models.admin import QueryEngineEnvironment -from sqlalchemy import func, and_ -from sqlalchemy.orm import aliased from app.db import with_session from const.elasticsearch import ElasticsearchItem +from const.metastore import DataOwner +from lib.logger import get_logger from lib.sqlalchemy import update_model_fields +from logic.user import get_user_by_name +from models.admin import QueryEngineEnvironment from models.metastore import ( + DataJobMetadata, DataSchema, DataTable, - DataTableInformation, DataTableColumn, + DataTableColumnStatistics, + DataTableInformation, DataTableOwnership, - DataJobMetadata, DataTableQueryExecution, DataTableStatistics, - DataTableColumnStatistics, ) from models.query_execution import QueryExecution +from sqlalchemy import and_, func +from sqlalchemy.orm import aliased from tasks.sync_elasticsearch import sync_elasticsearch +LOG = get_logger(__file__) + @with_session def get_all_schemas( @@ -267,6 +272,7 @@ def create_table_information( earliest_partitions=None, hive_metastore_description=None, partition_keys=[], + custom_properties=None, commit=False, session=None, ): @@ -284,6 +290,7 @@ def create_table_information( earliest_partitions=earliest_partitions, hive_metastore_description=hive_metastore_description, column_info=column_infomation, + custom_properties=custom_properties, ) # The reason that we dont add description direclty in @@ -391,6 +398,33 @@ def create_table_ownership(table_id, uid, commit=True, session=None): return table_ownership +@with_session +def create_table_ownerships( + table_id: int, owners: list[DataOwner] = [], commit=True, session=None +): + """This function is used for loading owners from metastore.""" + # delete all the ownerships of the table first + session.query(DataTableOwnership).filter_by(data_table_id=table_id).delete() + + for owner in owners: + user = get_user_by_name(owner.username, session=session) + if not user: + LOG.error( + f"Failed to find user or group: {owner} when loading table owners." + ) + continue + # add table ownership + table_ownership = DataTableOwnership( + data_table_id=table_id, uid=user.id, type=owner.type + ) + session.add(table_ownership) + + if commit: + session.commit() + else: + session.flush() + + @with_session def delete_table_ownership(table_id, uid, commit=True, session=None): table_ownership = get_table_ownership(table_id=table_id, uid=uid, session=session) diff --git a/querybook/server/logic/tag.py b/querybook/server/logic/tag.py index 947778a8b..fb8296ecd 100644 --- a/querybook/server/logic/tag.py +++ b/querybook/server/logic/tag.py @@ -1,11 +1,13 @@ import datetime + from app.db import with_session -from models.tag import Tag, TagItem +from const.metastore import DataTag from logic.metastore import update_es_tables_by_id +from models.tag import Tag, TagItem @with_session -def get_tag_by_table_id(table_id, session=None): +def get_tags_by_table_id(table_id, session=None): return ( session.query(Tag) .join(TagItem) @@ -15,6 +17,17 @@ def get_tag_by_table_id(table_id, session=None): ) +@with_session +def get_tags_by_column_id(column_id: int, session=None): + return ( + session.query(Tag) + .join(TagItem) + .filter(TagItem.column_id == column_id) + .order_by(Tag.count.desc()) + .all() + ) + + @with_session def get_tags_by_keyword(keyword, limit=10, session=None): return ( @@ -28,19 +41,19 @@ def get_tags_by_keyword(keyword, limit=10, session=None): @with_session -def create_or_update_tag(tag_name, commit=True, session=None): +def create_or_update_tag(tag_name, meta={}, commit=True, session=None): tag = Tag.get(name=tag_name, session=session) if not tag: tag = Tag.create( - {"name": tag_name, "count": 1, "meta": {}}, + {"name": tag_name, "count": 1, "meta": meta}, commit=commit, session=session, ) else: tag = Tag.update( id=tag.id, - fields={"count": tag.count + 1}, + fields={"count": tag.count + 1, "meta": meta}, skip_if_value_none=True, commit=commit, session=session, @@ -89,3 +102,79 @@ def delete_tag_from_table( update_es_tables_by_id(tag_item.table_id) else: session.flush() + + +@with_session +def create_table_tags( + table_id: int, + tags: list[DataTag] = [], + commit=True, + session=None, +): + """This function is used for loading table tags from metastore.""" + # delete all tags from the table + session.query(TagItem).filter_by(table_id=table_id).delete() + + for tag in tags: + meta = { + "type": tag.type, + "tooltip": tag.description, + "color": tag.color, + "admin": True, + } + # filter out properties with none values + meta = {k: v for k, v in meta.items() if v is not None} + + # update or create a new tag if not exist + create_or_update_tag( + tag_name=tag.name, meta=meta, commit=commit, session=session + ) + + # add a new tag_item to associate with the table + TagItem.create( + {"tag_name": tag.name, "table_id": table_id, "uid": None}, + session=session, + ) + + if commit: + session.commit() + else: + session.flush() + + +@with_session +def create_column_tags( + column_id: int, + tags: list[DataTag] = [], + commit=True, + session=None, +): + """This function is used for loading column tags from metastore.""" + # delete all tags from the table + session.query(TagItem).filter_by(column_id=column_id).delete() + + for tag in tags: + meta = { + "type": tag.type, + "tooltip": tag.description, + "color": tag.color, + "admin": True, + } + # filter out properties with none values + meta = {k: v for k, v in meta.items() if v is not None} + + # update or create a new tag if not exist + create_or_update_tag( + tag_name=tag.name, meta=meta, commit=commit, session=session + ) + + # add a new tag_item to associate with the table + TagItem.create( + {"tag_name": tag.name, "column_id": column_id, "uid": None}, + session=session, + ) + + if commit: + session.commit() + else: + session.flush() diff --git a/querybook/server/logic/user.py b/querybook/server/logic/user.py index 7014bb7b0..4ca891bb2 100644 --- a/querybook/server/logic/user.py +++ b/querybook/server/logic/user.py @@ -1,18 +1,14 @@ from sqlalchemy import func from app.db import with_session +from const.elasticsearch import ElasticsearchItem +from const.user import UserGroup +from const.user_roles import UserRoleType from lib.config import get_config_value from lib.logger import get_logger -from const.user_roles import UserRoleType -from const.elasticsearch import ElasticsearchItem -from models.user import ( - User, - UserSetting, - UserRole, -) +from models.user import User, UserRole, UserSetting, UserGroupMember from tasks.sync_elasticsearch import sync_elasticsearch - LOG = get_logger(__file__) user_settings_config = get_config_value("user_setting") @@ -144,6 +140,66 @@ def delete_user(uid, session=None): pass +@with_session +def create_or_update_user_group(user_group: UserGroup, commit=True, session=None): + group = get_user_by_name(user_group.name, session=session) + fields = { + "username": user_group.name, + "fullname": user_group.display_name, + "email": user_group.email, + "is_group": True, + "properties": {"description": user_group.description}, + } + + if not group: + # create a new group + group = User.create( + fields=fields, + commit=commit, + session=session, + ) + else: + # update the group + group = User.update( + id=group.id, + fields=fields, + commit=commit, + session=session, + ) + + # get current existing member user ids + existing_group_members = ( + session.query(UserGroupMember).filter(UserGroupMember.gid == group.id).all() + ) + existing_group_member_ids = set([m.uid for m in existing_group_members]) + + # get the latest member user ids by name + group_members = ( + session.query(User).filter(User.username.in_(user_group.members)).all() + ) + group_member_ids = set([m.id for m in group_members]) + + members_to_delete = list(existing_group_member_ids - group_member_ids) + members_to_add = list(group_member_ids - existing_group_member_ids) + + # delete group members not in the group anymore + if members_to_delete: + session.query(UserGroupMember).filter(UserGroupMember.gid == group.id).filter( + UserGroupMember.uid.in_(members_to_delete) + ).delete() + + # add new group members + if members_to_add: + session.add_all( + [UserGroupMember(gid=group.id, uid=user_id) for user_id in members_to_add] + ) + + if commit: + session.commit() + else: + session.flush() + + """ ---------------------------------------------------------------------------------------------------------- USER SETTINGS diff --git a/querybook/server/models/admin.py b/querybook/server/models/admin.py index b3f89c26a..cbfe7ebea 100644 --- a/querybook/server/models/admin.py +++ b/querybook/server/models/admin.py @@ -164,6 +164,7 @@ def to_dict(self): "id": self.id, "name": self.name, "config": loader_class.loader_config.to_dict(), + "owner_types": loader_class.get_table_owner_types(), } def to_dict_admin(self): diff --git a/querybook/server/models/metastore.py b/querybook/server/models/metastore.py index 5b9bd6d3b..2c5b11cc8 100644 --- a/querybook/server/models/metastore.py +++ b/querybook/server/models/metastore.py @@ -172,6 +172,9 @@ class DataTable(CRUDMixin, TruncateString("name", "type", "location"), Base): name = sql.Column(sql.String(length=name_length), index=True) type = sql.Column(sql.String(length=name_length), index=True) + + # This field is no longer being used, keep it here for backward compatibility only. + # Table ownership will be fully managed by DataTableOwnership owner = sql.Column(sql.String(length=name_length)) table_created_at = sql.Column(sql.DateTime) @@ -261,6 +264,7 @@ class DataTableInformation( description = sql.Column(sql.Text(length=mediumtext_length)) hive_metastore_description = sql.Column(sql.Text(length=mediumtext_length)) column_info = sql.Column(sql.JSON) + custom_properties = sql.Column(sql.JSON) def to_dict(self): table_information = { @@ -269,6 +273,7 @@ def to_dict(self): "description": self.description, "hive_metastore_description": self.hive_metastore_description, "column_info": self.column_info, + "custom_properties": self.custom_properties, } return table_information @@ -322,6 +327,7 @@ class DataTableOwnership(Base): uid = sql.Column( sql.Integer, sql.ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) + type = sql.Column(sql.String(name_length)) def to_dict(self): item = { @@ -329,6 +335,7 @@ def to_dict(self): "data_table_id": self.data_table_id, "created_at": self.created_at, "uid": self.uid, + "type": self.type, } return item diff --git a/querybook/server/models/tag.py b/querybook/server/models/tag.py index fe0698b8a..c7ecd40da 100644 --- a/querybook/server/models/tag.py +++ b/querybook/server/models/tag.py @@ -40,6 +40,11 @@ class TagItem(CRUDMixin, Base): table_id = sql.Column( sql.Integer, sql.ForeignKey("data_table.id", ondelete="CASCADE"), nullable=True ) + column_id = sql.Column( + sql.Integer, + sql.ForeignKey("data_table_column.id", ondelete="CASCADE"), + nullable=True, + ) uid = sql.Column( sql.Integer, sql.ForeignKey("user.id", ondelete="SET NULL"), nullable=True @@ -55,3 +60,8 @@ class TagItem(CRUDMixin, Base): backref=backref("tags", cascade="all, delete", passive_deletes=True), foreign_keys=[table_id], ) + column = relationship( + "DataTableColumn", + backref=backref("tags", cascade="all, delete", passive_deletes=True), + foreign_keys=[column_id], + ) diff --git a/querybook/server/models/usergroup.py b/querybook/server/models/usergroup.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py b/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py index 390ce696b..61881c460 100644 --- a/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py +++ b/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py @@ -1,11 +1,10 @@ import unittest +from datetime import datetime from unittest import TestCase import boto3 -from datetime import datetime - +from const.metastore import DataColumn, DataTable from lib.metastore.loaders.glue_data_catalog_loader import GlueDataCatalogLoader -from lib.metastore.base_metastore_loader import DataColumn, DataTable moto_import_failed = False try: diff --git a/querybook/webapp/components/DataTableTags/DataTableTags.scss b/querybook/webapp/components/DataTableTags/DataTableTags.scss index 135dc893b..328d8bb03 100644 --- a/querybook/webapp/components/DataTableTags/DataTableTags.scss +++ b/querybook/webapp/components/DataTableTags/DataTableTags.scss @@ -1,7 +1,7 @@ .DataTableTags { flex-wrap: wrap; - .Tag { + .TableTag { margin: 4px 12px 4px 0px; cursor: pointer; } diff --git a/querybook/webapp/components/DataTableTags/DataTableTags.tsx b/querybook/webapp/components/DataTableTags/DataTableTags.tsx index b39a86cc5..13e9e0d0a 100644 --- a/querybook/webapp/components/DataTableTags/DataTableTags.tsx +++ b/querybook/webapp/components/DataTableTags/DataTableTags.tsx @@ -124,7 +124,7 @@ export const TableTag: React.FC<{ ); return ( - <> +
{canUserUpdate && ( setShowConfigModal(false)} /> )} - - {tagMeta.icon && ( - - )} - {tag.name} - - + /> +
); }; diff --git a/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx b/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx index 8604cd6eb..1ac4804bd 100644 --- a/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx +++ b/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx @@ -88,6 +88,11 @@ export const TableTagConfigModal: React.FC<{ {({ submitForm }) => (
+ ( handleTagRemove(tag)} - > - {tag} - + /> ))}
) : null; diff --git a/querybook/webapp/components/DataTableView/DataTableHeader.scss b/querybook/webapp/components/DataTableView/DataTableHeader.scss index afb8cf51a..9f1284e44 100644 --- a/querybook/webapp/components/DataTableView/DataTableHeader.scss +++ b/querybook/webapp/components/DataTableView/DataTableHeader.scss @@ -6,22 +6,13 @@ .table-title { font-family: var(--font-accent); } - .header-subtitle { - align-self: flex-start; - } .DataTableHeader-owner-list { .owner-badges { flex-wrap: wrap; min-height: 30px; - .UserBadge + .IconButton { - margin-left: 12px; - } } } .DataTableHeader-tags { - .header-subtitle { - margin-top: 6px; - } .DataTableTags { min-height: 34px; } diff --git a/querybook/webapp/components/DataTableView/DataTableHeader.tsx b/querybook/webapp/components/DataTableView/DataTableHeader.tsx index 0f908dbab..fb1d523f6 100644 --- a/querybook/webapp/components/DataTableView/DataTableHeader.tsx +++ b/querybook/webapp/components/DataTableView/DataTableHeader.tsx @@ -1,13 +1,19 @@ import { last } from 'lodash'; import React from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch } from 'react-redux'; import { BoardItemAddButton } from 'components/BoardItemAddButton/BoardItemAddButton'; import { DataTableTags } from 'components/DataTableTags/DataTableTags'; import { ImpressionWidget } from 'components/ImpressionWidget/ImpressionWidget'; import { UserBadge } from 'components/UserBadge/UserBadge'; -import { IDataTable } from 'const/metastore'; +import { + IDataTable, + IQueryMetastore, + MetadataMode, + MetadataType, +} from 'const/metastore'; import { IMyUserInfo } from 'const/user'; +import { useShallowSelector } from 'hooks/redux/useShallowSelector'; import * as Utils from 'lib/utils'; import { createDataTableOwnership, @@ -26,6 +32,7 @@ export interface IDataTableHeader { table: IDataTable; userInfo: IMyUserInfo; tableName: string; + metastore: IQueryMetastore; updateDataTableGolden: (golden: boolean) => any; } @@ -33,6 +40,7 @@ export const DataTableHeader: React.FunctionComponent = ({ table, tableName, userInfo, + metastore, updateDataTableGolden, }) => { @@ -50,11 +58,13 @@ export const DataTableHeader: React.FunctionComponent = ({ [table.id, userInfo.uid] ); - const { username, tableOwnerships } = useSelector((state: IStoreState) => ({ - username: state.user.userInfoById[userInfo.uid].username, - tableOwnerships: - state.dataSources.dataTableOwnershipByTableId[table.id], - })); + const { username, tableOwnerships = [] } = useShallowSelector( + (state: IStoreState) => ({ + username: state.user.userInfoById[userInfo.uid].username, + tableOwnerships: + state.dataSources.dataTableOwnershipByTableId[table.id], + }) + ); const dbTableOwner = (table.owner || '').split('@')[0]; const isDBTableOwner = dbTableOwner === username; const isTableOwner = (tableOwnerships || []).find( @@ -124,6 +134,42 @@ export const DataTableHeader: React.FunctionComponent = ({ /> )); + const metastoreOwnerDOM = (metastore.owner_types || []).map((ownerType) => { + const ownerships = tableOwnerships.filter( + (ownership) => ownership.type === ownerType.name + ); + + if (ownerships.length === 0) { + return null; + } + + return ( +
+
+ + {ownerType.display_name} + +
+ {ownerships.map((ownership) => ( + + ))} +
+
+
+ ); + }); + // Ownership cannot be removed if owner in db const ownerDOM = (
@@ -135,7 +181,7 @@ export const DataTableHeader: React.FunctionComponent = ({ > Owners -
+
{dbTableOwner && ( )} @@ -177,7 +223,13 @@ export const DataTableHeader: React.FunctionComponent = ({ > Tags - +
); @@ -185,7 +237,9 @@ export const DataTableHeader: React.FunctionComponent = ({
{topDOM} {titleDOM} - {ownerDOM} + {metastore.config[MetadataType.OWNER] !== MetadataMode.WRITE_LOCAL + ? metastoreOwnerDOM + : ownerDOM} {tagDOM}
); diff --git a/querybook/webapp/components/DataTableView/DataTableView.tsx b/querybook/webapp/components/DataTableView/DataTableView.tsx index 29aa13e33..022d58207 100644 --- a/querybook/webapp/components/DataTableView/DataTableView.tsx +++ b/querybook/webapp/components/DataTableView/DataTableView.tsx @@ -363,6 +363,7 @@ export const DataTableView: React.FC = ({ tableId }) => { table={table} tableName={tableName} userInfo={userInfo} + metastore={metastore} updateDataTableGolden={updateDataTableGolden} /> = ({ onEditColumnDescriptionRedirect, updateDataColumnDescription, }) => { + const { data: columnTags } = useResource( + React.useCallback( + () => TableColumnResource.getTags(column.id), + [column.id] + ) + ); const [expanded, , toggleExpanded] = useToggleState(false); const parsedType = useMemo(() => parseType('', column.type), [column.type]); + const tagsDOM = (columnTags || []).map((tag) => ( + + )); + const userCommentsContent = ( = ({ /> )} + {tagsDOM.length > 0 && ( + +
+ {tagsDOM} +
+
+ )} {column.comment && ( {column.comment} diff --git a/querybook/webapp/components/DataTableViewOverview/DataTableViewOverview.tsx b/querybook/webapp/components/DataTableViewOverview/DataTableViewOverview.tsx index d4365fcf1..15df1c0cc 100644 --- a/querybook/webapp/components/DataTableViewOverview/DataTableViewOverview.tsx +++ b/querybook/webapp/components/DataTableViewOverview/DataTableViewOverview.tsx @@ -150,6 +150,14 @@ export const DataTableViewOverview: React.FC< ); }); + const customPropertiesDOM = Object.entries( + table.custom_properties ?? {} + ).map(([key, value]) => ( + + {value} + + )); + const rawMetastoreInfoDOM = table.hive_metastore_description ? (
             
             {detailsDOM}
+            {customPropertiesDOM}
         
     );
 
diff --git a/querybook/webapp/components/Search/TableSelect.tsx b/querybook/webapp/components/Search/TableSelect.tsx
index b1edbf65c..5f2c41d51 100644
--- a/querybook/webapp/components/Search/TableSelect.tsx
+++ b/querybook/webapp/components/Search/TableSelect.tsx
@@ -126,6 +126,7 @@ export const TableSelect: React.FunctionComponent = ({
                     {tableNames.map((tableName) => (
                          {
                                 const newTableNames = tableNames.filter(
@@ -133,9 +134,7 @@ export const TableSelect: React.FunctionComponent = ({
                                 );
                                 onTableNamesChange(newTableNames);
                             }}
-                        >
-                            {tableName}
-                        
+                        />
                     ))}
                 
) : null} diff --git a/querybook/webapp/components/UserBadge/UserBadge.scss b/querybook/webapp/components/UserBadge/UserBadge.scss index bc5a55768..9b0c592f1 100644 --- a/querybook/webapp/components/UserBadge/UserBadge.scss +++ b/querybook/webapp/components/UserBadge/UserBadge.scss @@ -45,10 +45,6 @@ } } - + .UserBadge { - margin-left: 12px; - } - &:not(.mini) { &.card-style { .UserBadge-icon { diff --git a/querybook/webapp/const/metastore.ts b/querybook/webapp/const/metastore.ts index d2b913fa2..2826abb11 100644 --- a/querybook/webapp/const/metastore.ts +++ b/querybook/webapp/const/metastore.ts @@ -15,10 +15,19 @@ export enum MetadataMode { WRITE_LOCAL = 'write_local', WRITE_BACK = 'write_back', } + +// Keep it in sync with DataOwnerType in server/lib/metastore/metastore_data_types.py +export interface IDataOwnerType { + name: string; + display_name: string; + description?: string; +} + export interface IQueryMetastore { id: number; name: string; config: Record; + owner_types: [IDataOwnerType]; } export interface IDataSchema { @@ -58,6 +67,7 @@ export interface IDataTable { column_info?: { partition_keys?: string[]; }; + custom_properties?: Record; schema: number; schema_id: number; @@ -175,6 +185,7 @@ export interface IDataTableOwnership { data_table_id: number; uid: number; created_at: number; + type: string; } export type TableStatValue = number | string | Array; diff --git a/querybook/webapp/const/tag.ts b/querybook/webapp/const/tag.ts index 4c136350c..19caba59b 100644 --- a/querybook/webapp/const/tag.ts +++ b/querybook/webapp/const/tag.ts @@ -1,4 +1,5 @@ export interface ITagMeta { + type?: string; admin?: boolean; color?: string; icon?: string; diff --git a/querybook/webapp/resource/table.ts b/querybook/webapp/resource/table.ts index 2aa24c34e..bab6c3ecb 100644 --- a/querybook/webapp/resource/table.ts +++ b/querybook/webapp/resource/table.ts @@ -177,6 +177,7 @@ export const TableColumnResource = { return ds.update(`/column/${columnId}/`, params); }, + getTags: (columnId: number) => ds.fetch(`/column/${columnId}/tag/`), }; export const TableLineageResource = { diff --git a/querybook/webapp/stylesheets/_utilities.scss b/querybook/webapp/stylesheets/_utilities.scss index 9993cb2eb..594935618 100644 --- a/querybook/webapp/stylesheets/_utilities.scss +++ b/querybook/webapp/stylesheets/_utilities.scss @@ -109,6 +109,10 @@ text-overflow: ellipsis; } +.cursor-pointer { + cursor: pointer; +} + @mixin padding-classes { $padding-size-options: 0, 2, 4, 8, 12, 16, 20, 24; @each $size in $padding-size-options { @@ -190,3 +194,16 @@ } @include flex-classes; + +@mixin gap-classes { + $gap-size-options: 0, 2, 4, 8, 12, 16, 20, 24, 36, 48; + @each $size in $gap-size-options { + $gap: $size * 1px; + + .gap#{$size} { + gap: $gap; + } + } +} + +@include gap-classes; diff --git a/querybook/webapp/ui/Tag/HoverIconTag.tsx b/querybook/webapp/ui/Tag/HoverIconTag.tsx index 8b1124fc0..75788664f 100644 --- a/querybook/webapp/ui/Tag/HoverIconTag.tsx +++ b/querybook/webapp/ui/Tag/HoverIconTag.tsx @@ -4,16 +4,19 @@ import React from 'react'; import { Icon } from 'ui/Icon/Icon'; import type { AllLucideIconNames } from 'ui/Icon/LucideIcons'; -import { ITagProps, Tag } from './Tag'; +import { ITagProps, Tag, TagGroup } from './Tag'; -export interface IHoverIconTagProps extends ITagProps { +export interface IHoverIconTagProps extends Omit { + name: string; + type?: string; + icon?: string; iconOnHover?: AllLucideIconNames; onIconHoverClick?: (e?: React.MouseEvent) => any; } export const HoverIconTag = React.forwardRef< HTMLSpanElement, IHoverIconTagProps ->(({ iconOnHover, onIconHoverClick, children, ...tagProps }, ref) => { +>(({ name, type, icon, iconOnHover, onIconHoverClick, ...tagProps }, ref) => { const hoverDOM = iconOnHover ? (
@@ -22,9 +25,37 @@ export const HoverIconTag = React.forwardRef< const className = clsx(tagProps['className'], 'HoverIconTag'); + const iconDOM = icon && ( + + ); + + if (type) { + const { tooltip, tooltipPos, color, mini, onClick, ...extraProps } = + tagProps; + return ( + + + {iconDOM} + {type} + + + {name} + {hoverDOM} + + + ); + } + return ( - {children} + {iconDOM} + {name} {hoverDOM} ); diff --git a/querybook/webapp/ui/Tag/Tag.scss b/querybook/webapp/ui/Tag/Tag.scss index 1314e5670..68def6334 100644 --- a/querybook/webapp/ui/Tag/Tag.scss +++ b/querybook/webapp/ui/Tag/Tag.scss @@ -10,7 +10,7 @@ display: inline-flex; margin: 2px 0px; border-radius: var(--border-radius-sm); - cursor: default; + cursor: inherit; } .Tag + .Tag { margin-left: 6px; @@ -61,8 +61,6 @@ } .TagGroup { - overflow: hidden; - .Tag { margin: 0; border-radius: 0; @@ -72,6 +70,14 @@ color: var(--color-accent-lightest-0); } } + .Tag:first-child { + border-top-left-radius: var(--border-radius-sm); + border-bottom-left-radius: var(--border-radius-sm); + } + .Tag:last-child { + border-top-right-radius: var(--border-radius-sm); + border-bottom-right-radius: var(--border-radius-sm); + } &.small .Tag { @include small-tag(); diff --git a/querybook/webapp/ui/Tag/Tag.tsx b/querybook/webapp/ui/Tag/Tag.tsx index de5186318..ee4019204 100644 --- a/querybook/webapp/ui/Tag/Tag.tsx +++ b/querybook/webapp/ui/Tag/Tag.tsx @@ -8,8 +8,9 @@ import { TooltipDirection } from 'const/tooltip'; import './Tag.scss'; export interface ITagGroupProps { + tooltip?: string; + tooltipPos?: TooltipDirection; className?: string; - children: React.ReactNode; } export interface ITagProps { @@ -21,7 +22,7 @@ export interface ITagProps { withBorder?: boolean; color?: string; - tooltip?: React.ReactNode; + tooltip?: string; tooltipPos?: TooltipDirection; onClick?: () => any; @@ -29,10 +30,15 @@ export interface ITagProps { className?: string; } -export const TagGroup: React.FunctionComponent = ({ - className, - children, -}) =>
{children}
; +export const TagGroup = styled.div.attrs<{ + tooltip?: string; + tooltipPos?: TooltipDirection; + className?: string; +}>(({ tooltip, tooltipPos, className }) => ({ + 'aria-label': tooltip, + 'data-balloon-pos': tooltipPos, + className: `${className} TagGroup`, +}))``; const StyledColorTag = styled.span.attrs<{ highlighted?: boolean;