From 1308ea7c378c0783316998f4b5cde1bdd52e37be Mon Sep 17 00:00:00 2001 From: "J.C. Zhong" Date: Wed, 15 Feb 2023 14:32:23 -0800 Subject: [PATCH] feat: load tag from metastore (#1164) * feat: load tag from metastore * comments * comments --- querybook/server/const/metastore.py | 8 +++ .../lib/metastore/base_metastore_loader.py | 61 ++++--------------- .../loaders/glue_data_catalog_loader.py | 4 +- .../loaders/hive_metastore_loader.py | 4 +- .../loaders/mysql_metastore_loader.py | 5 +- .../loaders/sqlalchemy_metastore_loader.py | 4 +- .../loaders/thrifthive_metastore_loader.py | 5 +- .../lib/metastore/metastore_data_types.py | 60 ++++++++++++++++++ querybook/server/logic/tag.py | 45 +++++++++++++- .../test_glue_data_catalog_loader.py | 5 +- .../DataTableTags/DataTableTags.scss | 2 +- .../DataTableTags/DataTableTags.tsx | 20 ++---- .../DataTableTags/TableTagConfigModal.tsx | 5 ++ .../DataTableTags/TableTagGroupSelect.tsx | 5 +- .../DataTableView/DataTableHeader.scss | 6 -- .../DataTableView/DataTableHeader.tsx | 17 +++++- .../DataTableView/DataTableView.tsx | 1 + .../webapp/components/Search/TableSelect.tsx | 5 +- querybook/webapp/const/tag.ts | 1 + querybook/webapp/stylesheets/_utilities.scss | 4 ++ querybook/webapp/ui/Tag/HoverIconTag.tsx | 39 ++++++++++-- querybook/webapp/ui/Tag/Tag.scss | 12 +++- querybook/webapp/ui/Tag/Tag.tsx | 18 ++++-- 23 files changed, 233 insertions(+), 103 deletions(-) create mode 100644 querybook/server/lib/metastore/metastore_data_types.py diff --git a/querybook/server/const/metastore.py b/querybook/server/const/metastore.py index edcf02d4d..18b408660 100644 --- a/querybook/server/const/metastore.py +++ b/querybook/server/const/metastore.py @@ -18,6 +18,7 @@ class MetadataMode(Enum): READ_ONLY = "read_only" # On saving, metadata will only be written to querybook db. This is the default mode if not specified. + # It also indicates that it will not load this metadata from the metastore. WRITE_LOCAL = "write_local" # On saving, metadata will be written back to metastore, as well as querybook db @@ -37,5 +38,12 @@ class MetastoreLoaderConfig: def __init__(self, config: dict[MetadataType, MetadataMode]): self._config = {**self._default_config, **config} + def can_load_external_metadata(self, metadataType: MetadataType) -> bool: + """Check if the given metadata type will be loaded from metastore""" + return self._config.get(metadataType, MetadataMode.WRITE_LOCAL) in ( + MetadataMode.READ_ONLY, + MetadataMode.WRITE_BACK, + ) + def to_dict(self): return {key.value: value.value for (key, value) in self._config.items()} diff --git a/querybook/server/lib/metastore/base_metastore_loader.py b/querybook/server/lib/metastore/base_metastore_loader.py index 702e55496..d34c94ec9 100644 --- a/querybook/server/lib/metastore/base_metastore_loader.py +++ b/querybook/server/lib/metastore/base_metastore_loader.py @@ -1,7 +1,7 @@ from abc import ABCMeta, abstractmethod, abstractclassmethod import gevent import math -from typing import NamedTuple, List, Dict, Tuple, Optional +from typing import List, Dict, Tuple, Optional import traceback from app.db import DBSession, with_session @@ -26,60 +26,14 @@ get_schema_by_name, get_table_by_schema_id_and_name, ) +from logic.tag import create_table_tags +from .metastore_data_types import DataTable, DataColumn from .utils import MetastoreTableACLChecker LOG = get_logger(__name__) -class DataSchema(NamedTuple): - name: str - - -class DataTable(NamedTuple): - name: str - - # The type of table, it can be an arbitrary string - type: str = None - owner: str = None - - # description from metastore, expect HTML format - description: str = None - - # Expected in UTC seconds - table_created_at: int = None - table_updated_at: int = None - table_updated_by: str = None - - # size of table - data_size_bytes: int = None - # Location of the raw file - location: str = None - - # Json arrays of partitions - partitions: List = [] - - # Store the raw info here - raw_description: str = None - - # Arrays of partition keys - partition_keys: List[str] = [] - - # Custom properties - custom_properties: dict[str, str] = None - - -class DataColumn(NamedTuple): - name: str - type: str - - # column comment from sql query when creating the table - comment: str = None - - # user edited description from metastore, expect HTML format - description: str = None - - class BaseMetastoreLoader(metaclass=ABCMeta): loader_config: MetastoreLoaderConfig = MetastoreLoaderConfig({}) @@ -384,6 +338,15 @@ def _create_table_table( commit=False, session=session, ) + + # create tags if the metastore is configured to sync tags + if self.loader_config.can_load_external_metadata(MetadataType.TAG): + create_table_tags( + table_id=table_id, + tags=table.tags, + commit=False, + session=session, + ) session.commit() update_table_by_id(table_id, session=session) return table_id diff --git a/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py b/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py index 513e28e6e..cf5bc5530 100644 --- a/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py +++ b/querybook/server/lib/metastore/loaders/glue_data_catalog_loader.py @@ -3,8 +3,8 @@ from clients.glue_client import GlueDataCatalogClient from lib.form import StructFormField, FormField -from lib.metastore.base_metastore_loader import ( - BaseMetastoreLoader, +from lib.metastore.base_metastore_loader import BaseMetastoreLoader +from lib.metastore.metastore_data_types import ( DataTable, DataColumn, ) diff --git a/querybook/server/lib/metastore/loaders/hive_metastore_loader.py b/querybook/server/lib/metastore/loaders/hive_metastore_loader.py index bb4ac0705..5a48d84cd 100644 --- a/querybook/server/lib/metastore/loaders/hive_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/hive_metastore_loader.py @@ -3,8 +3,8 @@ from hmsclient.genthrift.hive_metastore.ttypes import NoSuchObjectException from clients.hms_client import HiveMetastoreClient -from lib.metastore.base_metastore_loader import ( - BaseMetastoreLoader, +from lib.metastore.base_metastore_loader import BaseMetastoreLoader +from lib.metastore.metastore_data_types import ( DataTable, DataColumn, ) diff --git a/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py b/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py index f634dc721..25c37f16e 100644 --- a/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/mysql_metastore_loader.py @@ -2,7 +2,10 @@ from lib.utils.utils import DATETIME_TO_UTC from lib.utils import json as ujson -from lib.metastore.base_metastore_loader import DataTable, DataColumn +from lib.metastore.metastore_data_types import ( + DataTable, + DataColumn, +) from .sqlalchemy_metastore_loader import SqlAlchemyMetastoreLoader diff --git a/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py b/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py index 485b933ac..d44cdf756 100644 --- a/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/sqlalchemy_metastore_loader.py @@ -1,7 +1,7 @@ from typing import Dict, List, Tuple -from lib.metastore.base_metastore_loader import ( - BaseMetastoreLoader, +from lib.metastore.base_metastore_loader import BaseMetastoreLoader +from lib.metastore.metastore_data_types import ( DataTable, DataColumn, ) diff --git a/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py b/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py index 1655d3c80..710cc018a 100644 --- a/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py +++ b/querybook/server/lib/metastore/loaders/thrifthive_metastore_loader.py @@ -6,7 +6,10 @@ from clients.hms_client import HiveMetastoreClient from lib.metastore.loaders.hive_metastore_loader import HMSMetastoreLoader from lib.metastore.loaders.form_fileds import load_partitions_field -from lib.metastore.base_metastore_loader import DataTable, DataColumn +from lib.metastore.metastore_data_types import ( + DataTable, + DataColumn, +) class HMSThriftMetastoreLoader(HMSMetastoreLoader): diff --git a/querybook/server/lib/metastore/metastore_data_types.py b/querybook/server/lib/metastore/metastore_data_types.py new file mode 100644 index 000000000..2b56970f0 --- /dev/null +++ b/querybook/server/lib/metastore/metastore_data_types.py @@ -0,0 +1,60 @@ +from typing import NamedTuple, List + + +class DataSchema(NamedTuple): + name: str + + +class DataTag(NamedTuple): + name: str + # below properties will be stored in tag.meta + type: str = None + description: str = None + color: str = None + + +class DataTable(NamedTuple): + name: str + + # The type of table, it can be an arbitrary string + type: str = None + owner: str = None + + # description from metastore, expect HTML format + description: str = None + + # list of tags + tags: List[DataTag] = [] + + # Expected in UTC seconds + table_created_at: int = None + table_updated_at: int = None + table_updated_by: str = None + + # size of table + data_size_bytes: int = None + # Location of the raw file + location: str = None + + # Json arrays of partitions + partitions: List = [] + + # Store the raw info here + raw_description: str = None + + # Arrays of partition keys + partition_keys: List[str] = [] + + # Custom properties + custom_properties: dict[str, str] = None + + +class DataColumn(NamedTuple): + name: str + type: str + + # column comment from sql query when creating the table + comment: str = None + + # user edited description from metastore, expect HTML format + description: str = None diff --git a/querybook/server/logic/tag.py b/querybook/server/logic/tag.py index 947778a8b..f0ce5663d 100644 --- a/querybook/server/logic/tag.py +++ b/querybook/server/logic/tag.py @@ -2,6 +2,7 @@ from app.db import with_session from models.tag import Tag, TagItem from logic.metastore import update_es_tables_by_id +from lib.metastore.metastore_data_types import DataTag @with_session @@ -28,19 +29,19 @@ def get_tags_by_keyword(keyword, limit=10, session=None): @with_session -def create_or_update_tag(tag_name, commit=True, session=None): +def create_or_update_tag(tag_name, meta={}, commit=True, session=None): tag = Tag.get(name=tag_name, session=session) if not tag: tag = Tag.create( - {"name": tag_name, "count": 1, "meta": {}}, + {"name": tag_name, "count": 1, "meta": meta}, commit=commit, session=session, ) else: tag = Tag.update( id=tag.id, - fields={"count": tag.count + 1}, + fields={"count": tag.count + 1, "meta": meta}, skip_if_value_none=True, commit=commit, session=session, @@ -89,3 +90,41 @@ def delete_tag_from_table( update_es_tables_by_id(tag_item.table_id) else: session.flush() + + +@with_session +def create_table_tags( + table_id: int = None, + tags: list[DataTag] = [], + commit=True, + session=None, +): + """This function is used for loading tags from metastore.""" + # delete all tags from the table + session.query(TagItem).filter_by(table_id=table_id).delete() + + for tag in tags: + meta = { + "type": tag.type, + "tooltip": tag.description, + "color": tag.color, + "admin": True, + } + # filter out properties with none values + meta = {k: v for k, v in meta.items() if v is not None} + + # update or create a new tag if not exist + create_or_update_tag( + tag_name=tag.name, meta=meta, commit=commit, session=session + ) + + # add a new tag_item to associate with the table + TagItem.create( + {"tag_name": tag.name, "table_id": table_id, "uid": None}, + session=session, + ) + + if commit: + session.commit() + else: + session.flush() diff --git a/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py b/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py index 390ce696b..3446bd8f1 100644 --- a/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py +++ b/querybook/tests/test_lib/test_metastore/test_loaders/test_glue_data_catalog_loader.py @@ -5,7 +5,10 @@ from datetime import datetime from lib.metastore.loaders.glue_data_catalog_loader import GlueDataCatalogLoader -from lib.metastore.base_metastore_loader import DataColumn, DataTable +from lib.metastore.metastore_data_types import ( + DataTable, + DataColumn, +) moto_import_failed = False try: diff --git a/querybook/webapp/components/DataTableTags/DataTableTags.scss b/querybook/webapp/components/DataTableTags/DataTableTags.scss index 135dc893b..328d8bb03 100644 --- a/querybook/webapp/components/DataTableTags/DataTableTags.scss +++ b/querybook/webapp/components/DataTableTags/DataTableTags.scss @@ -1,7 +1,7 @@ .DataTableTags { flex-wrap: wrap; - .Tag { + .TableTag { margin: 4px 12px 4px 0px; cursor: pointer; } diff --git a/querybook/webapp/components/DataTableTags/DataTableTags.tsx b/querybook/webapp/components/DataTableTags/DataTableTags.tsx index b39a86cc5..13e9e0d0a 100644 --- a/querybook/webapp/components/DataTableTags/DataTableTags.tsx +++ b/querybook/webapp/components/DataTableTags/DataTableTags.tsx @@ -124,7 +124,7 @@ export const TableTag: React.FC<{ ); return ( - <> +
{canUserUpdate && ( setShowConfigModal(false)} /> )} - - {tagMeta.icon && ( - - )} - {tag.name} - - + /> +
); }; diff --git a/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx b/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx index 8604cd6eb..1ac4804bd 100644 --- a/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx +++ b/querybook/webapp/components/DataTableTags/TableTagConfigModal.tsx @@ -88,6 +88,11 @@ export const TableTagConfigModal: React.FC<{ {({ submitForm }) => (
+ ( handleTagRemove(tag)} - > - {tag} - + /> ))}
) : null; diff --git a/querybook/webapp/components/DataTableView/DataTableHeader.scss b/querybook/webapp/components/DataTableView/DataTableHeader.scss index afb8cf51a..bf8802a59 100644 --- a/querybook/webapp/components/DataTableView/DataTableHeader.scss +++ b/querybook/webapp/components/DataTableView/DataTableHeader.scss @@ -6,9 +6,6 @@ .table-title { font-family: var(--font-accent); } - .header-subtitle { - align-self: flex-start; - } .DataTableHeader-owner-list { .owner-badges { flex-wrap: wrap; @@ -19,9 +16,6 @@ } } .DataTableHeader-tags { - .header-subtitle { - margin-top: 6px; - } .DataTableTags { min-height: 34px; } diff --git a/querybook/webapp/components/DataTableView/DataTableHeader.tsx b/querybook/webapp/components/DataTableView/DataTableHeader.tsx index 0f908dbab..4045b10ce 100644 --- a/querybook/webapp/components/DataTableView/DataTableHeader.tsx +++ b/querybook/webapp/components/DataTableView/DataTableHeader.tsx @@ -6,7 +6,12 @@ import { BoardItemAddButton } from 'components/BoardItemAddButton/BoardItemAddBu import { DataTableTags } from 'components/DataTableTags/DataTableTags'; import { ImpressionWidget } from 'components/ImpressionWidget/ImpressionWidget'; import { UserBadge } from 'components/UserBadge/UserBadge'; -import { IDataTable } from 'const/metastore'; +import { + IDataTable, + IQueryMetastore, + MetadataMode, + MetadataType, +} from 'const/metastore'; import { IMyUserInfo } from 'const/user'; import * as Utils from 'lib/utils'; import { @@ -26,6 +31,7 @@ export interface IDataTableHeader { table: IDataTable; userInfo: IMyUserInfo; tableName: string; + metastore: IQueryMetastore; updateDataTableGolden: (golden: boolean) => any; } @@ -33,6 +39,7 @@ export const DataTableHeader: React.FunctionComponent = ({ table, tableName, userInfo, + metastore, updateDataTableGolden, }) => { @@ -177,7 +184,13 @@ export const DataTableHeader: React.FunctionComponent = ({ > Tags - + ); diff --git a/querybook/webapp/components/DataTableView/DataTableView.tsx b/querybook/webapp/components/DataTableView/DataTableView.tsx index 29aa13e33..022d58207 100644 --- a/querybook/webapp/components/DataTableView/DataTableView.tsx +++ b/querybook/webapp/components/DataTableView/DataTableView.tsx @@ -363,6 +363,7 @@ export const DataTableView: React.FC = ({ tableId }) => { table={table} tableName={tableName} userInfo={userInfo} + metastore={metastore} updateDataTableGolden={updateDataTableGolden} /> = ({ {tableNames.map((tableName) => ( { const newTableNames = tableNames.filter( @@ -133,9 +134,7 @@ export const TableSelect: React.FunctionComponent = ({ ); onTableNamesChange(newTableNames); }} - > - {tableName} - + /> ))} ) : null} diff --git a/querybook/webapp/const/tag.ts b/querybook/webapp/const/tag.ts index 4c136350c..19caba59b 100644 --- a/querybook/webapp/const/tag.ts +++ b/querybook/webapp/const/tag.ts @@ -1,4 +1,5 @@ export interface ITagMeta { + type?: string; admin?: boolean; color?: string; icon?: string; diff --git a/querybook/webapp/stylesheets/_utilities.scss b/querybook/webapp/stylesheets/_utilities.scss index 9993cb2eb..e4ca6f046 100644 --- a/querybook/webapp/stylesheets/_utilities.scss +++ b/querybook/webapp/stylesheets/_utilities.scss @@ -109,6 +109,10 @@ text-overflow: ellipsis; } +.cursor-pointer { + cursor: pointer; +} + @mixin padding-classes { $padding-size-options: 0, 2, 4, 8, 12, 16, 20, 24; @each $size in $padding-size-options { diff --git a/querybook/webapp/ui/Tag/HoverIconTag.tsx b/querybook/webapp/ui/Tag/HoverIconTag.tsx index 8b1124fc0..75788664f 100644 --- a/querybook/webapp/ui/Tag/HoverIconTag.tsx +++ b/querybook/webapp/ui/Tag/HoverIconTag.tsx @@ -4,16 +4,19 @@ import React from 'react'; import { Icon } from 'ui/Icon/Icon'; import type { AllLucideIconNames } from 'ui/Icon/LucideIcons'; -import { ITagProps, Tag } from './Tag'; +import { ITagProps, Tag, TagGroup } from './Tag'; -export interface IHoverIconTagProps extends ITagProps { +export interface IHoverIconTagProps extends Omit { + name: string; + type?: string; + icon?: string; iconOnHover?: AllLucideIconNames; onIconHoverClick?: (e?: React.MouseEvent) => any; } export const HoverIconTag = React.forwardRef< HTMLSpanElement, IHoverIconTagProps ->(({ iconOnHover, onIconHoverClick, children, ...tagProps }, ref) => { +>(({ name, type, icon, iconOnHover, onIconHoverClick, ...tagProps }, ref) => { const hoverDOM = iconOnHover ? (
@@ -22,9 +25,37 @@ export const HoverIconTag = React.forwardRef< const className = clsx(tagProps['className'], 'HoverIconTag'); + const iconDOM = icon && ( + + ); + + if (type) { + const { tooltip, tooltipPos, color, mini, onClick, ...extraProps } = + tagProps; + return ( + + + {iconDOM} + {type} + + + {name} + {hoverDOM} + + + ); + } + return ( - {children} + {iconDOM} + {name} {hoverDOM} ); diff --git a/querybook/webapp/ui/Tag/Tag.scss b/querybook/webapp/ui/Tag/Tag.scss index 1314e5670..68def6334 100644 --- a/querybook/webapp/ui/Tag/Tag.scss +++ b/querybook/webapp/ui/Tag/Tag.scss @@ -10,7 +10,7 @@ display: inline-flex; margin: 2px 0px; border-radius: var(--border-radius-sm); - cursor: default; + cursor: inherit; } .Tag + .Tag { margin-left: 6px; @@ -61,8 +61,6 @@ } .TagGroup { - overflow: hidden; - .Tag { margin: 0; border-radius: 0; @@ -72,6 +70,14 @@ color: var(--color-accent-lightest-0); } } + .Tag:first-child { + border-top-left-radius: var(--border-radius-sm); + border-bottom-left-radius: var(--border-radius-sm); + } + .Tag:last-child { + border-top-right-radius: var(--border-radius-sm); + border-bottom-right-radius: var(--border-radius-sm); + } &.small .Tag { @include small-tag(); diff --git a/querybook/webapp/ui/Tag/Tag.tsx b/querybook/webapp/ui/Tag/Tag.tsx index de5186318..ee4019204 100644 --- a/querybook/webapp/ui/Tag/Tag.tsx +++ b/querybook/webapp/ui/Tag/Tag.tsx @@ -8,8 +8,9 @@ import { TooltipDirection } from 'const/tooltip'; import './Tag.scss'; export interface ITagGroupProps { + tooltip?: string; + tooltipPos?: TooltipDirection; className?: string; - children: React.ReactNode; } export interface ITagProps { @@ -21,7 +22,7 @@ export interface ITagProps { withBorder?: boolean; color?: string; - tooltip?: React.ReactNode; + tooltip?: string; tooltipPos?: TooltipDirection; onClick?: () => any; @@ -29,10 +30,15 @@ export interface ITagProps { className?: string; } -export const TagGroup: React.FunctionComponent = ({ - className, - children, -}) =>
{children}
; +export const TagGroup = styled.div.attrs<{ + tooltip?: string; + tooltipPos?: TooltipDirection; + className?: string; +}>(({ tooltip, tooltipPos, className }) => ({ + 'aria-label': tooltip, + 'data-balloon-pos': tooltipPos, + className: `${className} TagGroup`, +}))``; const StyledColorTag = styled.span.attrs<{ highlighted?: boolean;