diff --git a/.github/actions/prepare-for-docker-build/action.yml b/.github/actions/prepare-for-docker-build/action.yml new file mode 100644 index 000000000000..d14cdca7dfd8 --- /dev/null +++ b/.github/actions/prepare-for-docker-build/action.yml @@ -0,0 +1,59 @@ +name: Prepare for Docker Build +description: Set up Docker Build dependencies (without pushing) and run Maven build + +inputs: + image: + description: Image name + required: true + tag: + description: Docker tag to use + required: true + is_ingestion: + description: true if we are building an Ingestion image, false otherwise + required: true + default: "false" + release_version: + description: OpenMetadata Release Version + +outputs: + tags: + description: Generated Docker Tags + value: ${{ steps.meta.outputs.tags }} + +runs: + using: composite + steps: + - name: Install Ubuntu dependencies + shell: bash + run: | + # stop relying on apt cache of GitHub runners + sudo apt-get update + sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ + librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev jq + - name: Set up JDK 17 + if: inputs.is_ingestion == 'false' + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: 'temurin' + + - name: Install antlr cli + shell: bash + run: | + sudo make install_antlr_cli + + - name: Build OpenMetadata Server Application + if: inputs.is_ingestion == 'false' + shell: bash + run: | + mvn -DskipTests clean package + + - name: Install OpenMetadata Ingestion Dependencies + if: inputs.is_ingestion == 'true' + shell: bash + run: | + python3 -m venv env + source env/bin/activate + pip install --upgrade pip + sudo make install_antlr_cli + make install_dev generate \ No newline at end of file diff --git a/.github/trivy/templates/github.tpl b/.github/trivy/templates/github.tpl new file mode 100644 index 000000000000..08c8660c1e06 --- /dev/null +++ b/.github/trivy/templates/github.tpl @@ -0,0 +1,35 @@ +{{- range . }} +

🛡️ TRIVY SCAN RESULT 🛡️

+

Target: {{ .Target }}

+ {{- if .Vulnerabilities }} +

Vulnerabilities ({{ len .Vulnerabilities }})

+ + + + + + + + + + + + {{- range .Vulnerabilities }} + + + + + + + + {{- end }} + +
PackageVulnerability IDSeverityInstalled VersionFixed Version
{{ .PkgName }}{{ .VulnerabilityID }} + {{- if eq .Severity "CRITICAL" }} 🔥 CRITICAL + {{- else if eq .Severity "HIGH" }} 🚨 HIGH + {{- else }} {{ .Severity }} {{- end }} + {{ .InstalledVersion }}{{ if .FixedVersion }}{{ .FixedVersion }}{{ else }}N/A{{ end }}
+ {{- else }} +

No Vulnerabilities Found

+ {{- end }} +{{- end }} diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 9acbfd797753..138f4095e06e 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -65,7 +65,7 @@ jobs: - name: configure aws credentials if: contains('quicksight', matrix.e2e-test) || contains('datalake_s3', matrix.e2e-test) || contains('athena', matrix.e2e-test) - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.E2E_AWS_IAM_ROLE_ARN }} role-session-name: github-ci-aws-e2e-tests diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index 586489b012d8..7ca4a5f620d4 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -55,7 +55,7 @@ jobs: docker-images: false - name: Wait for the labeler - uses: lewagon/wait-on-check-action@v1.3.3 + uses: lewagon/wait-on-check-action@v1.3.4 if: ${{ github.event_name == 'pull_request_target' }} with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/trivy-scan-ingestion-base-slim-image.yml b/.github/workflows/trivy-scan-ingestion-base-slim-image.yml new file mode 100644 index 000000000000..3805100da053 --- /dev/null +++ b/.github/workflows/trivy-scan-ingestion-base-slim-image.yml @@ -0,0 +1,81 @@ +name: Trivy Scan For OpenMetadata Ingestion Base Slim Docker Image + +on: + pull_request_target: + types: [labeled, opened, synchronize, reopened] + paths: + - "ingestion/**" + - "openmetadata-service/**" + - "openmetadata-spec/src/main/resources/json/schema/**" + - "pom.xml" + - "Makefile" + +concurrency: + group: trivy-ingestion-base-slim-scan-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-and-scan: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Wait for the labeler + uses: lewagon/wait-on-check-action@v1.3.3 + if: ${{ github.event_name == 'pull_request_target' }} + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: Team Label + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 90 + + - name: Verify PR labels + uses: jesusvasquez333/verify-pr-label-action@v1.4.0 + if: ${{ github.event_name == 'pull_request_target' }} + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + valid-labels: 'safe to test' + pull-request-number: '${{ github.event.pull_request.number }}' + disable-reviews: true # To not auto approve changes + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Prepare for Docker Build + id: prepare + uses: ./.github/actions/prepare-for-docker-build + with: + image: openmetadata-ingestion-base-slim + tag: trivy + is_ingestion: true + + + - name: Build Docker Image + run: | + docker build -t openmetadata-ingestion-base-slim:trivy -f ingestion/operators/docker/Dockerfile.ci . + + + - name: Run Trivy Image Scan + id: trivy_scan + uses: aquasecurity/trivy-action@master + with: + scan-type: "image" + image-ref: openmetadata-ingestion-base-slim:trivy + hide-progress: false + ignore-unfixed: true + severity: "HIGH,CRITICAL" + skip-dirs: "/opt/airflow/dags,/home/airflow/ingestion/pipelines" + scan-ref: . + format: 'template' + template: "@.github/trivy/templates/github.tpl" + output: "trivy-result-ingestion-base-slim.md" + env: + TRIVY_DISABLE_VEX_NOTICE: "true" + + - name: Comment Trivy Scan Results on PR + uses: marocchino/sticky-pull-request-comment@v2 + with: + path: trivy-result-ingestion-base-slim.md + header: "trivy-scan-${{ github.workflow }}" diff --git a/.github/workflows/trivy-scan-ingestion-image.yml b/.github/workflows/trivy-scan-ingestion-image.yml new file mode 100644 index 000000000000..5ac8156bf1a1 --- /dev/null +++ b/.github/workflows/trivy-scan-ingestion-image.yml @@ -0,0 +1,81 @@ +name: Trivy Scan For OpenMetadata Ingestion Docker Image + +on: + pull_request_target: + types: [labeled, opened, synchronize, reopened] + paths: + - "ingestion/**" + - "openmetadata-service/**" + - "openmetadata-spec/src/main/resources/json/schema/**" + - "pom.xml" + - "Makefile" + +concurrency: + group: trivy-ingestion-scan-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-and-scan: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Wait for the labeler + uses: lewagon/wait-on-check-action@v1.3.3 + if: ${{ github.event_name == 'pull_request_target' }} + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: Team Label + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 90 + + - name: Verify PR labels + uses: jesusvasquez333/verify-pr-label-action@v1.4.0 + if: ${{ github.event_name == 'pull_request_target' }} + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + valid-labels: 'safe to test' + pull-request-number: '${{ github.event.pull_request.number }}' + disable-reviews: true # To not auto approve changes + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Prepare for Docker Build + id: prepare + uses: ./.github/actions/prepare-for-docker-build + with: + image: openmetadata-ingestion + tag: trivy + is_ingestion: true + + + - name: Build Docker Image + run: | + docker build -t openmetadata-ingestion:trivy -f ingestion/Dockerfile.ci . + + + - name: Run Trivy Image Scan + id: trivy_scan + uses: aquasecurity/trivy-action@master + with: + scan-type: "image" + image-ref: openmetadata-ingestion:trivy + hide-progress: false + ignore-unfixed: true + severity: "HIGH,CRITICAL" + skip-dirs: "/opt/airflow/dags,/home/airflow/ingestion/pipelines" + scan-ref: . + format: 'template' + template: "@.github/trivy/templates/github.tpl" + output: "trivy-results-ingestion.md" + env: + TRIVY_DISABLE_VEX_NOTICE: "true" + + - name: Comment Trivy Scan Results on PR + uses: marocchino/sticky-pull-request-comment@v2 + with: + path: trivy-results-ingestion.md + header: "trivy-scan-${{ github.workflow }}" diff --git a/.github/workflows/trivy-scan-openmetadta-server.yml b/.github/workflows/trivy-scan-openmetadta-server.yml new file mode 100644 index 000000000000..615a1229555a --- /dev/null +++ b/.github/workflows/trivy-scan-openmetadta-server.yml @@ -0,0 +1,80 @@ +name: Trivy Scan For OpenMetadata Server Docker Image +on: + pull_request_target: + types: [labeled, opened, synchronize, reopened] + paths: + - "openmetadata-service/**" + - "openmetadata-spec/src/main/resources/json/schema/**" + - "openmetadata-dist/**" + - "openmetadata-clients/**" + - "common/**" + - "pom.xml" + - "yarn.lock" + - "Makefile" + - "bootstrap/**" +concurrency: + group: trivy-server-scan-${{ github.head_ref || github.run_id }} + cancel-in-progress: true +jobs: + build-and-scan: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Wait for the labeler + uses: lewagon/wait-on-check-action@v1.3.3 + if: ${{ github.event_name == 'pull_request_target' }} + with: + ref: ${{ github.event.pull_request.head.sha }} + check-name: Team Label + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 90 + + - name: Verify PR labels + uses: jesusvasquez333/verify-pr-label-action@v1.4.0 + if: ${{ github.event_name == 'pull_request_target' }} + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + valid-labels: 'safe to test' + pull-request-number: '${{ github.event.pull_request.number }}' + disable-reviews: true # To not auto approve changes + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Prepare for Docker Build + id: prepare + uses: ./.github/actions/prepare-for-docker-build + with: + image: openmetadata-server + tag: trivy + is_ingestion: false + + - name: Build Docker Image + run: | + docker build -t openmetadata-server:trivy -f docker/development/Dockerfile . + + - name: Run Trivy Image Scan + id: trivy_scan + uses: aquasecurity/trivy-action@master + with: + scan-type: "image" + image-ref: openmetadata-server:trivy + hide-progress: false + ignore-unfixed: true + severity: "HIGH,CRITICAL" + scan-ref: . + format: 'template' + template: "@.github/trivy/templates/github.tpl" + output: trivy-result-openmetadata-server.md + env: + TRIVY_DISABLE_VEX_NOTICE: "true" + + - name: Comment Trivy Scan Results on PR + uses: marocchino/sticky-pull-request-comment@v2 + with: + path: trivy-result-openmetadata-server.md + header: "trivy-scan-${{ github.workflow }}" + diff --git a/README.md b/README.md index 54f9d2547669..3df44614a9b0 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Contents: - [Features](#key-features-of-openmetadata) - [Try our Sandbox](#try-our-sandbox) - [Install & Run](#install-and-run-openmetadata) -- [Roadmap](https://docs.open-metadata.org/v1.3.x/roadmap) +- [Roadmap](https://docs.open-metadata.org/latest/roadmap) - [Documentation and Support](#documentation-and-support) - [Contributors](#contributors) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index b5509137a521..ceaef3cececc 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -492,7 +492,7 @@ services: DB_HOST: ${AIRFLOW_DB_HOST:-mysql} DB_PORT: ${AIRFLOW_DB_PORT:-3306} AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} - DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} diff --git a/docker/docker-compose-ingestion/docker-compose-ingestion.yml b/docker/docker-compose-ingestion/docker-compose-ingestion.yml index b6adb2319e31..00749041d98c 100644 --- a/docker/docker-compose-ingestion/docker-compose-ingestion.yml +++ b/docker/docker-compose-ingestion/docker-compose-ingestion.yml @@ -26,7 +26,7 @@ services: DB_HOST: ${AIRFLOW_DB_HOST:-mysql} DB_PORT: ${AIRFLOW_DB_PORT:-3306} AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} - DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} # extra connection-string properties for the database diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml index a66b57ddfb63..0bec2001ba72 100644 --- a/docker/docker-compose-quickstart/docker-compose.yml +++ b/docker/docker-compose-quickstart/docker-compose.yml @@ -497,7 +497,7 @@ services: DB_HOST: ${AIRFLOW_DB_HOST:-mysql} DB_PORT: ${AIRFLOW_DB_PORT:-3306} AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db} - DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql} + DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb} DB_USER: ${AIRFLOW_DB_USER:-airflow_user} DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass} # extra connection-string properties for the database diff --git a/docker/images/minimal-ubuntu/Dockerfile b/docker/images/minimal-ubuntu/Dockerfile index 5457d209599c..5316f7f75979 100644 --- a/docker/images/minimal-ubuntu/Dockerfile +++ b/docker/images/minimal-ubuntu/Dockerfile @@ -15,7 +15,7 @@ FROM ubuntu:xenial-20210416 # environment variables -ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_FRONTEND=noninteractive # update RUN apt update -y && \ diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index cc0df5ad259c..2be62d4c75e3 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -1,6 +1,6 @@ -FROM mysql:8.3 as mysql +FROM mysql:8.3 AS mysql -FROM apache/airflow:2.9.1-python3.10 +FROM apache/airflow:2.9.3-python3.10 USER root RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - RUN curl -sS https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list @@ -78,7 +78,7 @@ ENV PIP_NO_CACHE_DIR=1 ENV PIP_QUIET=1 ARG RI_VERSION="1.6.0.0.dev0" RUN pip install --upgrade pip -RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.10.txt" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index f8433f776ca3..3e6f0346a220 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -1,6 +1,6 @@ -FROM mysql:8.3 as mysql +FROM mysql:8.3 AS mysql -FROM apache/airflow:2.9.1-python3.10 +FROM apache/airflow:2.9.3-python3.10 USER root RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - RUN curl -sS https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list @@ -73,7 +73,7 @@ COPY --chown=airflow:0 openmetadata-airflow-apis /home/airflow/openmetadata-airf COPY --chown=airflow:0 ingestion/examples/airflow/dags /opt/airflow/dags USER airflow -ARG AIRFLOW_CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/constraints-2.9.1/constraints-3.10.txt" +ARG AIRFLOW_CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt" # Disable pip cache dir # https://pip.pypa.io/en/stable/topics/caching/#avoiding-caching diff --git a/ingestion/ingestion_dependency.sh b/ingestion/ingestion_dependency.sh index ee54d6f6ac95..2b8372f852f4 100755 --- a/ingestion/ingestion_dependency.sh +++ b/ingestion/ingestion_dependency.sh @@ -15,7 +15,7 @@ DB_PORT=${DB_PORT:-3306} AIRFLOW_DB=${AIRFLOW_DB:-airflow_db} DB_USER=${DB_USER:-airflow_user} -DB_SCHEME=${DB_SCHEME:-mysql+pymysql} +DB_SCHEME=${DB_SCHEME:-mysql+mysqldb} DB_PASSWORD=${DB_PASSWORD:-airflow_pass} DB_PROPERTIES=${DB_PROPERTIES:-""} diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index d5d57088d256..c44d33106a3a 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -41,6 +41,7 @@ RUN dpkg --configure -a \ && rm -rf /var/lib/apt/lists/* # Add updated postgres/redshift dependencies based on libq +ENV DEBIAN_FRONTEND=noninteractive RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \ apt-get -qq update; \ diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index 27752f954383..a12409b92956 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -41,6 +41,7 @@ RUN apt-get -qq update \ && rm -rf /var/lib/apt/lists/* # Add updated postgres/redshift dependencies based on libq +ENV DEBIAN_FRONTEND=noninteractive RUN curl -sS https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo "deb https://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list; \ apt-get -qq update; \ diff --git a/ingestion/setup.py b/ingestion/setup.py index 96c426aef8f7..fa0ec7614286 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -19,7 +19,7 @@ # Add here versions required for multiple plugins VERSIONS = { - "airflow": "apache-airflow==2.9.1", + "airflow": "apache-airflow==2.9.3", "adlfs": "adlfs>=2023.1.0", "avro": "avro>=1.11.3,<1.12", "boto3": "boto3>=1.20,<2.0", # No need to add botocore separately. It's a dep from boto3 @@ -56,9 +56,13 @@ "elasticsearch8": "elasticsearch8~=8.9.0", "giturlparse": "giturlparse", "validators": "validators~=0.22.0", - "teradata": "teradatasqlalchemy>=20.0.0.0", + "teradata": "teradatasqlalchemy==20.0.0.2", "cockroach": "sqlalchemy-cockroachdb~=2.0", "cassandra": "cassandra-driver>=3.28.0", + "pydoris": "pydoris==1.0.2", + "pyiceberg": "pyiceberg==0.5.1", + "google-cloud-bigtable": "google-cloud-bigtable>=2.0.0", + "pyathena": "pyathena~=3.0", } COMMONS = { @@ -98,7 +102,7 @@ DATA_DIFF = { driver: f"collate-data-diff[{driver}]" # data-diff uses different drivers out-of-the-box than OpenMetadata - # the exrtas are described here: + # the extras are described here: # https://github.com/open-metadata/collate-data-diff/blob/main/pyproject.toml#L68 # install all data diffs with "pip install collate-data-diff[all-dbs]" for driver in [ @@ -143,8 +147,13 @@ "tabulate==0.9.0", "typing-inspect", "packaging", # For version parsing + "setuptools~=70.0", "shapely", "collate-data-diff", + # TODO: Remove one once we have updated datadiff version + "snowflake-connector-python>=3.13.1,<4.0.0", + "mysql-connector-python>=8.0.29;python_version<'3.9'", + "mysql-connector-python>=9.1;python_version>='3.9'", } plugins: Dict[str, Set[str]] = { @@ -155,7 +164,7 @@ VERSIONS["airflow"], }, # Same as ingestion container. For development. "amundsen": {VERSIONS["neo4j"]}, - "athena": {"pyathena~=3.0"}, + "athena": {VERSIONS["pyathena"]}, "atlas": {}, "azuresql": {VERSIONS["pyodbc"]}, "azure-sso": {VERSIONS["msal"]}, @@ -168,7 +177,11 @@ VERSIONS["numpy"], "sqlalchemy-bigquery>=1.2.2", }, - "bigtable": {"google-cloud-bigtable>=2.0.0", VERSIONS["pandas"], VERSIONS["numpy"]}, + "bigtable": { + VERSIONS["google-cloud-bigtable"], + VERSIONS["pandas"], + VERSIONS["numpy"], + }, "clickhouse": { "clickhouse-driver~=0.2", "clickhouse-sqlalchemy~=0.2", @@ -247,7 +260,7 @@ "impyla~=0.18.0", }, "iceberg": { - "pyiceberg==0.5.1", + VERSIONS["pyiceberg"], # Forcing the version of a few packages so it plays nicely with other requirements. VERSIONS["pydantic"], VERSIONS["adlfs"], @@ -313,7 +326,7 @@ VERSIONS["geoalchemy2"], }, "sagemaker": {VERSIONS["boto3"]}, - "salesforce": {"simple_salesforce~=1.11"}, + "salesforce": {"simple_salesforce~=1.11", "authlib>=1.3.1"}, "sample-data": {VERSIONS["avro"], VERSIONS["grpc-tools"]}, "sap-hana": {"hdbcli", "sqlalchemy-hana"}, "sas": {}, @@ -386,6 +399,8 @@ VERSIONS["grpc-tools"], VERSIONS["neo4j"], VERSIONS["cockroach"], + VERSIONS["pydoris"], + VERSIONS["pyiceberg"], "testcontainers==3.7.1;python_version<'3.9'", "testcontainers~=4.8.0;python_version>='3.9'", "minio==7.2.5", @@ -404,6 +419,13 @@ *plugins["dagster"], *plugins["oracle"], *plugins["mssql"], + VERSIONS["validators"], + VERSIONS["pyathena"], + VERSIONS["pyiceberg"], + VERSIONS["pydoris"], + "python-liquid", + VERSIONS["google-cloud-bigtable"], + *plugins["bigquery"], } e2e_test = { diff --git a/ingestion/src/metadata/ingestion/lineage/masker.py b/ingestion/src/metadata/ingestion/lineage/masker.py index 69aab2d7ba01..e55783934052 100644 --- a/ingestion/src/metadata/ingestion/lineage/masker.py +++ b/ingestion/src/metadata/ingestion/lineage/masker.py @@ -127,4 +127,4 @@ def mask_query( except Exception as exc: logger.debug(f"Failed to mask query with sqlfluff: {exc}") logger.debug(traceback.format_exc()) - return query + return None diff --git a/ingestion/src/metadata/ingestion/lineage/parser.py b/ingestion/src/metadata/ingestion/lineage/parser.py index 93bae226d74f..b9925fb1e35e 100644 --- a/ingestion/src/metadata/ingestion/lineage/parser.py +++ b/ingestion/src/metadata/ingestion/lineage/parser.py @@ -338,7 +338,7 @@ def stateful_add_joins_from_statement( logger.debug( f"Can't extract table names when parsing JOIN information from {comparison}" ) - logger.debug(f"Query: {self.masked_query}") + logger.debug(f"Query: {self.masked_query or self.query}") continue left_table_column = TableColumn(table=table_left, column=column_left) @@ -463,7 +463,7 @@ def get_sqlfluff_lineage_runner(qry: str, dlct: str) -> LineageRunner: self.masked_query = mask_query(self._clean_query, parser=lr_sqlparser) logger.debug( - f"Using sqlparse for lineage parsing for query: {self.masked_query}" + f"Using sqlparse for lineage parsing for query: {self.masked_query or self.query}" ) return lr_sqlparser diff --git a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py index f7bd21265fbd..577e6a87498a 100644 --- a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py +++ b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py @@ -625,8 +625,8 @@ def get_lineage_by_query( try: lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) - masked_query = lineage_parser.masked_query or query - logger.debug(f"Running lineage with query: {masked_query}") + masked_query = lineage_parser.masked_query + logger.debug(f"Running lineage with query: {masked_query or query}") raw_column_lineage = lineage_parser.column_lineage column_lineage.update(populate_column_lineage_map(raw_column_lineage)) @@ -697,7 +697,7 @@ def get_lineage_by_query( if not lineage_parser.query_parsing_success: query_parsing_failures.add( QueryParsingError( - query=masked_query, + query=masked_query or query, error=lineage_parser.query_parsing_failure_reason, ) ) @@ -729,8 +729,10 @@ def get_lineage_via_table_entity( try: lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) - masked_query = lineage_parser.masked_query or query - logger.debug(f"Getting lineage via table entity using query: {masked_query}") + masked_query = lineage_parser.masked_query + logger.debug( + f"Getting lineage via table entity using query: {masked_query or query}" + ) to_table_name = table_entity.name.root for from_table_name in lineage_parser.source_tables: diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py index be2de4fbf4a0..c0b217a804e8 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/query_mixin.py @@ -42,6 +42,8 @@ def _get_query_hash(self, query: str) -> str: return str(result.hexdigest()) def _get_or_create_query(self, query: CreateQueryRequest) -> Optional[Query]: + if query.query.root is None: + return None query_hash = self._get_query_hash(query=query.query.root) query_entity = self.get_by_name(entity=Query, fqn=query_hash) if query_entity is None: diff --git a/ingestion/src/metadata/ingestion/ometa/routes.py b/ingestion/src/metadata/ingestion/ometa/routes.py index 8759c663059b..b8538923110a 100644 --- a/ingestion/src/metadata/ingestion/ometa/routes.py +++ b/ingestion/src/metadata/ingestion/ometa/routes.py @@ -92,6 +92,7 @@ from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import ( CreateIngestionPipelineRequest, ) +from metadata.generated.schema.api.teams.createPersona import CreatePersonaRequest from metadata.generated.schema.api.teams.createRole import CreateRoleRequest from metadata.generated.schema.api.teams.createTeam import CreateTeamRequest from metadata.generated.schema.api.teams.createUser import CreateUserRequest @@ -157,6 +158,7 @@ from metadata.generated.schema.entity.services.pipelineService import PipelineService from metadata.generated.schema.entity.services.searchService import SearchService from metadata.generated.schema.entity.services.storageService import StorageService +from metadata.generated.schema.entity.teams.persona import Persona from metadata.generated.schema.entity.teams.role import Role from metadata.generated.schema.entity.teams.team import Team from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User @@ -217,6 +219,8 @@ CreateTeamRequest.__name__: "/teams", User.__name__: "/users", CreateUserRequest.__name__: "/users", + Persona.__name__: "/personas", + CreatePersonaRequest.__name__: "/personas", AuthenticationMechanism.__name__: "/users/auth-mechanism", Bot.__name__: "/bots", CreateBot.__name__: "/bots", diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py index 598ca5d591bf..fafc59429bb5 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py @@ -21,6 +21,7 @@ """ import copy import os +import re import traceback from datetime import datetime from pathlib import Path @@ -39,6 +40,7 @@ import giturlparse import lkml +import networkx as nx from liquid import Template from looker_sdk.sdk.api40.methods import Looker40SDK from looker_sdk.sdk.api40.models import Dashboard as LookerDashboard @@ -134,6 +136,10 @@ LIST_DASHBOARD_FIELDS = ["id", "title"] IMPORTED_PROJECTS_DIR = "imported_projects" +# we need to find the derived references in the SQL query using regex +# https://cloud.google.com/looker/docs/derived-tables#referencing_derived_tables_in_other_derived_tables +DERIVED_REFERENCES = r"\${([\w\s\d_.]+)\.SQL_TABLE_NAME}" + # Here we can update the fields to get further information, such as: # created_at, updated_at, last_updater_id, deleted_at, deleter_id, favorite_count, last_viewed_at GET_DASHBOARD_FIELDS = [ @@ -165,6 +171,13 @@ def build_datamodel_name(model_name: str, explore_name: str) -> str: return clean_dashboard_name(model_name + "_" + explore_name) +def find_derived_references(sql_query: str) -> List[str]: + if sql_query is None: + return [] + matches = re.findall(DERIVED_REFERENCES, sql_query) + return matches + + class LookerSource(DashboardServiceSource): """ Looker Source Class. @@ -172,6 +185,8 @@ class LookerSource(DashboardServiceSource): Its client uses Looker 40 from the SDK: client = looker_sdk.init40() """ + # pylint: disable=too-many-instance-attributes + config: WorkflowSource metadata: OpenMetadata client: Looker40SDK @@ -192,6 +207,10 @@ def __init__( self._main__lookml_manifest: Optional[LookMLManifest] = None self._view_data_model: Optional[DashboardDataModel] = None + self._parsed_views: Optional[Dict[str, str]] = {} + self._unparsed_views: Optional[Dict[str, str]] = {} + self._derived_dependencies = nx.DiGraph() + self._added_lineage: Optional[Dict] = {} @classmethod @@ -557,6 +576,68 @@ def _process_view( ) ) + def replace_derived_references(self, sql_query): + """ + Replace all derived references with the parsed views sql query + will replace the derived references in the SQL query using regex + for e.g. It will replace ${view_name.SQL_TABLE_NAME} with the parsed view query for view_name + https://cloud.google.com/looker/docs/derived-tables#referencing_derived_tables_in_other_derived_tables + """ + try: + sql_query = re.sub( + DERIVED_REFERENCES, + # from `${view_name.SQL_TABLE_NAME}` we want the `view_name`. + # match.group(1) will give us the `view_name` + lambda match: f"({self._parsed_views.get(match.group(1), match.group(0))})", + sql_query, + ) + except Exception as e: + logger.warning( + f"Something went wrong while replacing derived view references: {e}" + ) + return sql_query + + def build_lineage_for_unparsed_views(self) -> Iterable[Either[AddLineageRequest]]: + """ + build lineage by parsing the unparsed views containing derived references + """ + try: + # Doing a reversed topological sort to process the views in the right order + for view_name in reversed( + list(nx.topological_sort(self._derived_dependencies)) + ): + if view_name in self._parsed_views: + # Skip if already processed + continue + sql_query = self.replace_derived_references( + self._unparsed_views[view_name] + ) + if view_references := find_derived_references(sql_query): + # There are still derived references in the view query + logger.debug( + f"Views {view_references} not found for {view_name}. Skipping." + ) + continue + self._parsed_views[view_name] = sql_query + del self._unparsed_views[view_name] + yield from self._build_lineage_for_view(view_name, sql_query) + + except Exception as err: + yield Either( + left=StackTraceError( + name="parse_unparsed_views", + error=f"Error parsing unparsed views: {err}", + stackTrace=traceback.format_exc(), + ) + ) + + def _add_dependency_edge(self, view_name: str, view_references: List[str]): + """ + Add a dependency edge between the view and the derived reference + """ + for dependent_view_name in view_references: + self._derived_dependencies.add_edge(view_name, dependent_view_name) + def add_view_lineage( self, view: LookMlView, explore: LookmlModelExplore ) -> Iterable[Either[AddLineageRequest]]: @@ -589,6 +670,7 @@ def add_view_lineage( for db_service_name in db_service_names or []: dialect = self._get_db_dialect(db_service_name) source_table_name = self._clean_table_name(sql_table_name, dialect) + self._parsed_views[view.name] = source_table_name # View to the source is only there if we are informing the dbServiceNames yield self.build_lineage_request( @@ -601,20 +683,19 @@ def add_view_lineage( sql_query = view.derived_table.sql if not sql_query: return + if find_derived_references(sql_query): + sql_query = self.replace_derived_references(sql_query) + # If we still have derived references, we cannot process the view + if view_references := find_derived_references(sql_query): + self._add_dependency_edge(view.name, view_references) + logger.warning( + f"Not all references are replaced for view [{view.name}]. Parsing it later." + ) + return logger.debug(f"Processing view [{view.name}] with SQL: \n[{sql_query}]") - for db_service_name in db_service_names or []: - lineage_parser = LineageParser( - sql_query, - self._get_db_dialect(db_service_name), - timeout_seconds=30, - ) - if lineage_parser.source_tables: - for from_table_name in lineage_parser.source_tables: - yield self.build_lineage_request( - source=str(from_table_name), - db_service_name=db_service_name, - to_entity=self._view_data_model, - ) + yield from self._build_lineage_for_view(view.name, sql_query) + if self._unparsed_views: + self.build_lineage_for_unparsed_views() except Exception as err: yield Either( @@ -625,6 +706,27 @@ def add_view_lineage( ) ) + def _build_lineage_for_view( + self, view_name: str, sql_query: str + ) -> Iterable[Either[AddLineageRequest]]: + """ + Parse the SQL query and build lineage for the view. + """ + for db_service_name in self.get_db_service_names() or []: + lineage_parser = LineageParser( + sql_query, + self._get_db_dialect(db_service_name), + timeout_seconds=30, + ) + if lineage_parser.source_tables: + self._parsed_views[view_name] = sql_query + for from_table_name in lineage_parser.source_tables: + yield self.build_lineage_request( + source=str(from_table_name), + db_service_name=db_service_name, + to_entity=self._view_data_model, + ) + def _get_db_dialect(self, db_service_name) -> Dialect: db_service = self.metadata.get_by_name(DatabaseService, db_service_name) return ConnectionTypeDialectMapper.dialect_of( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py index 381094e18010..cfaa47379ef0 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py @@ -47,7 +47,10 @@ logger = utils_logger() - +GETGROUPS_DEFAULT_PARAMS = {"$top": "1", "$skip": "0"} +API_RESPONSE_MESSAGE_KEY = "message" +AUTH_TOKEN_MAX_RETRIES = 5 +AUTH_TOKEN_RETRY_WAIT = 120 # Similar inner methods with mode client. That's fine. # pylint: disable=duplicate-code class PowerBiApiClient: @@ -59,6 +62,9 @@ class PowerBiApiClient: def __init__(self, config: PowerBIConnection): self.config = config + self.pagination_entity_per_page = min( + 100, self.config.pagination_entity_per_page + ) self.msal_client = msal.ConfidentialClientApplication( client_id=self.config.clientId, client_credential=self.config.clientSecret.get_secret_value(), @@ -82,42 +88,84 @@ def get_auth_token(self) -> Tuple[str, str]: """ logger.info("Generating PowerBi access token") - response_data = self.msal_client.acquire_token_silent( - scopes=self.config.scope, account=None - ) - + response_data = self.get_auth_token_from_cache() if not response_data: logger.info("Token does not exist in the cache. Getting a new token.") - response_data = self.msal_client.acquire_token_for_client( - scopes=self.config.scope - ) + response_data = self.generate_new_auth_token() + response_data = response_data or {} auth_response = PowerBiToken(**response_data) if not auth_response.access_token: raise InvalidSourceException( - "Failed to generate the PowerBi access token. Please check provided config" + f"Failed to generate the PowerBi access token. Please check provided config {response_data}" ) logger.info("PowerBi Access Token generated successfully") return auth_response.access_token, auth_response.expires_in + def generate_new_auth_token(self) -> Optional[dict]: + """generate new auth token""" + retry = AUTH_TOKEN_MAX_RETRIES + while retry: + try: + response_data = self.msal_client.acquire_token_for_client( + scopes=self.config.scope + ) + return response_data + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error generating new auth token: {exc}") + # wait for time and retry + retry -= 1 + if retry: + logger.warning( + f"Error generating new token: {exc}, " + f"sleep {AUTH_TOKEN_RETRY_WAIT} seconds retrying {retry} more times.." + ) + sleep(AUTH_TOKEN_RETRY_WAIT) + else: + logger.warning( + "Could not generate new token after maximum retries, " + "Please check provided configs" + ) + return None + + def get_auth_token_from_cache(self) -> Optional[dict]: + """fetch auth token from cache""" + retry = AUTH_TOKEN_MAX_RETRIES + while retry: + try: + response_data = self.msal_client.acquire_token_silent( + scopes=self.config.scope, account=None + ) + return response_data + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error getting token from cache: {exc}") + retry -= 1 + if retry: + logger.warning( + f"Error getting token from cache: {exc}, " + f"sleep {AUTH_TOKEN_RETRY_WAIT} seconds retrying {retry} more times.." + ) + sleep(AUTH_TOKEN_RETRY_WAIT) + else: + logger.warning( + "Could not get token from cache after maximum retries, " + "Please check provided configs" + ) + return None + def fetch_dashboards(self) -> Optional[List[PowerBIDashboard]]: """Get dashboards method Returns: List[PowerBIDashboard] """ - try: - if self.config.useAdminApis: - response_data = self.client.get("/myorg/admin/dashboards") - response = DashboardsResponse(**response_data) - return response.value - group = self.fetch_all_workspaces()[0] - return self.fetch_all_org_dashboards(group_id=group.id) - - except Exception as exc: # pylint: disable=broad-except - logger.debug(traceback.format_exc()) - logger.warning(f"Error fetching dashboards: {exc}") - - return None + if self.config.useAdminApis: + response_data = self.client.get("/myorg/admin/dashboards") + response = DashboardsResponse(**response_data) + return response.value + group = self.fetch_all_workspaces()[0] + return self.fetch_all_org_dashboards(group_id=group.id) def fetch_all_org_dashboards( self, group_id: str @@ -205,6 +253,7 @@ def fetch_dataset_tables( return None + # pylint: disable=too-many-branches,too-many-statements def fetch_all_workspaces(self) -> Optional[List[Group]]: """Method to fetch all powerbi workspace details Returns: @@ -213,28 +262,94 @@ def fetch_all_workspaces(self) -> Optional[List[Group]]: try: admin = "admin/" if self.config.useAdminApis else "" api_url = f"/myorg/{admin}groups" - entities_per_page = self.config.pagination_entity_per_page - params_data = {"$top": "1"} - response_data = self.client.get(api_url, data=params_data) - response = GroupsResponse(**response_data) - count = response.odata_count + entities_per_page = self.pagination_entity_per_page + failed_indexes = [] + params_data = GETGROUPS_DEFAULT_PARAMS + response = self.client.get(api_url, data=params_data) + if ( + not response + or API_RESPONSE_MESSAGE_KEY in response + or len(response) != len(GroupsResponse.__annotations__) + ): + logger.warning("Error fetching workspaces between results: (0, 1)") + if response and response.get(API_RESPONSE_MESSAGE_KEY): + logger.warning( + "Error message from API response: " + f"{str(response.get(API_RESPONSE_MESSAGE_KEY))}" + ) + failed_indexes.append(params_data) + count = 0 + else: + try: + response = GroupsResponse(**response) + count = response.odata_count + except Exception as exc: + logger.warning(f"Error processing GetGroups response: {exc}") + count = 0 indexes = math.ceil(count / entities_per_page) - workspaces = [] for index in range(indexes): params_data = { "$top": str(entities_per_page), "$skip": str(index * entities_per_page), } - response_data = self.client.get(api_url, data=params_data) - if not response_data: - logger.error( - "Error fetching workspaces between results: " - f"{str(index * entities_per_page)} - {str(entities_per_page)}" + response = self.client.get(api_url, data=params_data) + if ( + not response + or API_RESPONSE_MESSAGE_KEY in response + or len(response) != len(GroupsResponse.__annotations__) + ): + index_range = ( + int(params_data.get("$skip")), + int(params_data.get("$skip")) + int(params_data.get("$top")), + ) + logger.warning( + f"Error fetching workspaces between results: {str(index_range)}" ) + if response and response.get(API_RESPONSE_MESSAGE_KEY): + logger.warning( + "Error message from API response: " + f"{str(response.get(API_RESPONSE_MESSAGE_KEY))}" + ) + failed_indexes.append(params_data) continue - response = GroupsResponse(**response_data) - workspaces.extend(response.value) + try: + response = GroupsResponse(**response) + workspaces.extend(response.value) + except Exception as exc: + logger.warning(f"Error processing GetGroups response: {exc}") + + if failed_indexes: + logger.info( + "Retrying one more time on failed indexes to get workspaces" + ) + for params_data in failed_indexes: + response = self.client.get(api_url, data=params_data) + if ( + not response + or API_RESPONSE_MESSAGE_KEY in response + or len(response) != len(GroupsResponse.__annotations__) + ): + index_range = ( + int(params_data.get("$skip")), + int(params_data.get("$skip")) + + int(params_data.get("$top")), + ) + logger.warning( + f"Workspaces between results {str(index_range)} " + "could not be fetched on multiple attempts" + ) + if response and response.get(API_RESPONSE_MESSAGE_KEY): + logger.warning( + "Error message from API response: " + f"{str(response.get(API_RESPONSE_MESSAGE_KEY))}" + ) + continue + try: + response = GroupsResponse(**response) + workspaces.extend(response.value) + except Exception as exc: + logger.warning(f"Error processing GetGroups response: {exc}") return workspaces except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py index ff69eea373f6..7896ab23a631 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py @@ -212,6 +212,7 @@ def _get_database_name( if sqa_str: sqa_url = make_url(sqa_str) default_db_name = sqa_url.database if sqa_url else None + return get_database_name_for_lineage(db_service_entity, default_db_name) def _get_datasource_fqn( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index 718cffb0b833..8c5b0687cc17 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -13,7 +13,10 @@ """ import json import traceback -from typing import Iterable, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Tuple, Union + +from collate_sqllineage.core.models import Column as LineageColumn +from collate_sqllineage.core.models import Table as LineageTable from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel @@ -34,9 +37,12 @@ from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) +from metadata.generated.schema.type.entityLineage import ColumnLineage from metadata.generated.schema.type.entityReferenceList import EntityReferenceList from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.lineage.parser import LineageParser +from metadata.ingestion.lineage.sql_lineage import get_column_fqn from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.ingestion.source.dashboard.superset.models import ( @@ -47,6 +53,9 @@ FetchDashboard, SupersetDatasource, ) +from metadata.ingestion.source.dashboard.superset.utils import ( + get_dashboard_data_model_column_fqn, +) from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser from metadata.utils import fqn from metadata.utils.logger import ingestion_logger @@ -146,6 +155,156 @@ def _get_charts_of_dashboard( ) return [] + def _is_table_to_table_lineage(self, columns: tuple, table: LineageTable) -> bool: + from_column: LineageColumn = columns[0] + to_column: LineageColumn = columns[-1] + + if not isinstance(from_column.parent, LineageTable): + return False + + if not isinstance(to_column.parent, LineageTable): + return False + + if from_column.parent.schema.raw_name != table.schema.raw_name: + return False + + if from_column.parent.raw_name != table.raw_name: + return False + + return True + + def _append_value_to_dict_list( + self, input_dict: Dict[str, List[str]], dict_key: str, list_value: str + ) -> None: + if input_dict.get(dict_key): + input_dict[dict_key].append(list_value) + else: + input_dict[dict_key] = [list_value] + + def _get_table_schema(self, table: LineageTable, chart: FetchChart) -> str: + if table.schema.raw_name == table.schema.unknown: + return chart.table_schema + + return table.schema.raw_name + + def _create_column_lineage_mapping( + self, parser: LineageParser, table: LineageTable, chart: FetchChart + ) -> Dict[str, List[str]]: + result = {} + table_to_table_lineage = [ + _columns + for _columns in parser.column_lineage + if self._is_table_to_table_lineage(_columns, table) + ] + + for columns in table_to_table_lineage: + from_column_name = columns[0].raw_name + to_column_name = columns[-1].raw_name + + if from_column_name != "*" and to_column_name != "*": + self._append_value_to_dict_list( + result, to_column_name, from_column_name + ) + + if from_column_name == "*" and to_column_name == "*": + for col_name in self._get_columns_list_for_lineage(chart): + self._append_value_to_dict_list(result, col_name, col_name) + + return result + + def _parse_lineage_from_dataset_sql( + self, chart_json: FetchChart + ) -> List[Tuple[FetchChart, Dict[str, List[str]]]]: + # Every SQL query in tables is a SQL statement SELECTING data. + # To get lineage we 'simulate' INSERT INTO query into dummy table. + result = [] + parser = LineageParser(f"INSERT INTO dummy_table {chart_json.sql}") + + for table in parser.source_tables: + table_name = table.raw_name + table_schema = self._get_table_schema(table, chart_json) + + column_mapping: Dict[str, List[str]] = self._create_column_lineage_mapping( + parser, table, chart_json + ) + + result.append( + ( + FetchChart( + table_name=table_name, + schema=table_schema, + sqlalchemy_uri=chart_json.sqlalchemy_uri, + ), + column_mapping, + ) + ) + + return result + + def _enrich_raw_input_tables( + self, + from_entities: List[Tuple[FetchChart, Dict[str, List[str]]]], + to_entity: DashboardDataModel, + db_service_entity: DatabaseService, + ): + result = [] + + for from_entity in from_entities: + input_table, _column_lineage = from_entity + datasource_fqn = self._get_datasource_fqn_for_lineage( + input_table, db_service_entity + ) + from_entity = self.metadata.get_by_name( + entity=Table, + fqn=datasource_fqn, + ) + + column_lineage: List[ColumnLineage] = [] + for to_column, from_columns in _column_lineage.items(): + _from_columns = [ + get_column_fqn(from_entity, from_column) + for from_column in from_columns + if get_column_fqn(from_entity, from_column) + ] + + _to_column = get_dashboard_data_model_column_fqn(to_entity, to_column) + + if _from_columns and _to_column: + column_lineage.append( + ColumnLineage( + fromColumns=_from_columns, + toColumn=_to_column, + ) + ) + + result.append((from_entity, column_lineage)) + + return result + + def _get_input_tables(self, chart: FetchChart): + if chart.sql: + result = self._parse_lineage_from_dataset_sql(chart) + else: + result = [ + (chart, {c: [c] for c in self._get_columns_list_for_lineage(chart)}) + ] + + return result + + def _get_dashboard_data_model_entity( + self, chart: FetchChart + ) -> Optional[DashboardDataModel]: + datamodel_fqn = fqn.build( + self.metadata, + entity_type=DashboardDataModel, + service_name=self.config.serviceName, + data_model_name=str(chart.datasource_id), + ) + return self.metadata.get_by_name( + entity=DashboardDataModel, + fqn=datamodel_fqn, + ) + def yield_dashboard_lineage_details( self, dashboard_details: Union[FetchDashboard, DashboardResult], @@ -158,51 +317,40 @@ def yield_dashboard_lineage_details( entity=DatabaseService, fqn=db_service_name ) if db_service_entity: - for chart_id in self._get_charts_of_dashboard(dashboard_details): - chart_json = self.all_charts.get(chart_id) - if chart_json: - try: - datasource_fqn = self._get_datasource_fqn_for_lineage( - chart_json, db_service_entity - ) - if not datasource_fqn: - continue - from_entity = self.metadata.get_by_name( - entity=Table, - fqn=datasource_fqn, - ) - datamodel_fqn = fqn.build( - self.metadata, - entity_type=DashboardDataModel, - service_name=self.config.serviceName, - data_model_name=str(chart_json.datasource_id), - ) - to_entity = self.metadata.get_by_name( - entity=DashboardDataModel, - fqn=datamodel_fqn, - ) + for chart_json in filter( + None, + [ + self.all_charts.get(chart_id) + for chart_id in self._get_charts_of_dashboard(dashboard_details) + ], + ): + try: + to_entity = self._get_dashboard_data_model_entity(chart_json) - columns_list = self._get_columns_list_for_lineage(chart_json) - column_lineage = self._get_column_lineage( - from_entity, to_entity, columns_list + if to_entity: + _input_tables = self._get_input_tables(chart_json) + input_tables = self._enrich_raw_input_tables( + _input_tables, to_entity, db_service_entity ) - if from_entity and to_entity: + for input_table in input_tables: + from_entity_table, column_lineage = input_table + yield self._get_add_lineage_request( to_entity=to_entity, - from_entity=from_entity, + from_entity=from_entity_table, column_lineage=column_lineage, ) - except Exception as exc: - yield Either( - left=StackTraceError( - name=db_service_name, - error=( - "Error to yield dashboard lineage details for DB " - f"service name [{db_service_name}]: {exc}" - ), - stackTrace=traceback.format_exc(), - ) + except Exception as exc: + yield Either( + left=StackTraceError( + name=db_service_name, + error=( + "Error to yield dashboard lineage details for DB " + f"service name [{db_service_name}]: {exc}" + ), + stackTrace=traceback.format_exc(), ) + ) def _get_datamodel( self, datamodel: Union[SupersetDatasource, FetchChart] @@ -238,6 +386,18 @@ def parse_array_data_type(self, col_parse: dict) -> Optional[str]: return DataType(col_parse["arrayDataType"]) return None + def parse_row_data_type(self, col_parse: dict) -> List[Column]: + """ + Set children to single UNKNOWN column for Trino row columns + to prevent validation error requiring non empty list of children. + """ + if col_parse["dataType"] == "ROW" and not col_parse.get("children"): + return [Column(name="unknown", dataType=DataType.UNKNOWN)] + + if col_parse.get("children"): + return col_parse["children"] + return [] + def get_column_info( self, data_source: List[Union[DataSourceResult, FetchColumn]] ) -> Optional[List[Column]]: @@ -259,9 +419,7 @@ def get_column_info( dataTypeDisplay=field.type, dataType=col_parse["dataType"], arrayDataType=self.parse_array_data_type(col_parse), - children=list(col_parse["children"]) - if col_parse.get("children") - else None, + children=self.parse_row_data_type(col_parse), name=str(field.id), displayName=field.column_name, description=field.description, diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py index d9d40e1214ad..15bae1c75d12 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/models.py @@ -151,6 +151,7 @@ class FetchChart(BaseModel): sqlalchemy_uri: Optional[str] = None viz_type: Optional[str] = None datasource_id: Optional[int] = None + sql: Optional[str] = None class FetchColumn(BaseModel): diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/queries.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/queries.py index 144c5322f749..b333f2b946ab 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/queries.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/queries.py @@ -23,6 +23,7 @@ t.id AS table_id, t.table_name, t.schema, + t.sql, db.database_name, db.sqlalchemy_uri from @@ -92,10 +93,10 @@ FETCH_COLUMN = """ select tc.id, - table_name , - column_name, - table_id, - type, + tc.table_name , + tc.column_name, + tc.table_id, + tc.type, tc.description from table_columns tc @@ -104,5 +105,5 @@ on t.id=tc.table_id where - table_id=%(table_id)s + tc.table_id=%(table_id)s """ diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/utils.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/utils.py new file mode 100644 index 000000000000..3c1eb910b6a4 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/utils.py @@ -0,0 +1,36 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Superset utils module +""" + +from typing import Optional + +from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel + + +def get_dashboard_data_model_column_fqn( + dashboard_data_model_entity: DashboardDataModel, column: str +) -> Optional[str]: + """ + Get fqn of column if exist in dashboard data model entity. + + This is Superset implementation specific as table name is stored within displayName (table name contains + numerical id), which is not consistent with implementations of dashboard data model columns of + other dashboard sources. + """ + if not dashboard_data_model_entity: + return None + for dashboard_data_model_column in dashboard_data_model_entity.columns: + if column.lower() == dashboard_data_model_column.displayName.lower(): + return dashboard_data_model_column.fullyQualifiedName.root + + return None diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py index 98480f6a4d46..6fdfd2de4733 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py @@ -34,9 +34,11 @@ from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode from metadata.generated.schema.entity.data.table import ( + ConstraintType, PartitionColumnDetails, PartitionIntervalTypes, Table, + TableConstraint, TablePartition, TableType, ) @@ -96,6 +98,7 @@ from metadata.ingestion.source.database.multi_db_source import MultiDBSource from metadata.utils import fqn from metadata.utils.credentials import GOOGLE_CREDENTIALS +from metadata.utils.execution_time_tracker import calculate_execution_time from metadata.utils.filters import filter_by_database, filter_by_schema from metadata.utils.helpers import retry_with_docker_host from metadata.utils.logger import ingestion_logger @@ -661,6 +664,42 @@ def _get_partition_column_name( ) return None + @calculate_execution_time() + def update_table_constraints( + self, + table_name, + schema_name, + db_name, + table_constraints, + foreign_columns, + columns, + ) -> List[TableConstraint]: + """ + From topology. + process the table constraints of all tables + """ + table_constraints = super().update_table_constraints( + table_name, + schema_name, + db_name, + table_constraints, + foreign_columns, + columns, + ) + try: + table = self.client.get_table(fqn._build(db_name, schema_name, table_name)) + if hasattr(table, "clustering_fields") and table.clustering_fields: + table_constraints.append( + TableConstraint( + constraintType=ConstraintType.CLUSTER_KEY, + columns=table.clustering_fields, + ) + ) + except Exception as exc: + logger.warning(f"Error getting clustering fields for {table_name}: {exc}") + logger.debug(traceback.format_exc()) + return table_constraints + def get_table_partition_details( self, table_name: str, schema_name: str, inspector: Inspector ) -> Tuple[bool, Optional[TablePartition]]: @@ -671,8 +710,10 @@ def get_table_partition_details( database = self.context.get().database table = self.client.get_table(fqn._build(database, schema_name, table_name)) columns = inspector.get_columns(table_name, schema_name, db_name=database) - if hasattr(table, "external_data_configuration") and hasattr( - table.external_data_configuration, "hive_partitioning" + if ( + hasattr(table, "external_data_configuration") + and hasattr(table.external_data_configuration, "hive_partitioning") + and table.external_data_configuration.hive_partitioning ): # Ingesting External Hive Partitioned Tables from google.cloud.bigquery.external_config import ( # pylint: disable=import-outside-toplevel @@ -739,6 +780,30 @@ def get_table_partition_details( table_partition.interval = table.range_partitioning.range_.interval table_partition.columnName = table.range_partitioning.field return True, TablePartition(columns=[table_partition]) + if ( + hasattr(table, "_properties") + and table._properties.get("partitionDefinition") + and table._properties.get("partitionDefinition").get( + "partitionedColumn" + ) + ): + + return True, TablePartition( + columns=[ + PartitionColumnDetails( + columnName=self._get_partition_column_name( + columns=columns, + partition_field_name=field.get("field"), + ), + intervalType=PartitionIntervalTypes.OTHER, + ) + for field in table._properties.get("partitionDefinition").get( + "partitionedColumn" + ) + if field and field.get("field") + ] + ) + except Exception as exc: logger.debug(traceback.format_exc()) logger.warning( diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index 1bc9ba40cb25..2ee49a4a9ccd 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -195,6 +195,12 @@ def get_schema_description(self, schema_name: str) -> Optional[str]: by default there will be no schema description """ + def get_stored_procedure_description(self, stored_procedure: str) -> Optional[str]: + """ + Method to fetch the stored procedure description + by default there will be no stored procedure description + """ + @calculate_execution_time_generator() def yield_database( self, database_name: str diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py index 14d3d3e392e9..7bae5d4b7f8a 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py @@ -38,9 +38,9 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.source.database.databricks.client import DatabricksClient from metadata.ingestion.source.database.databricks.queries import ( DATABRICKS_GET_CATALOGS, + DATABRICKS_SQL_STATEMENT_TEST, ) from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger @@ -81,7 +81,6 @@ def test_connection( Test connection. This can be executed either as part of a metadata workflow or during an Automation Workflow """ - client = DatabricksClient(service_connection) def test_database_query(engine: Engine, statement: str): """ @@ -106,7 +105,13 @@ def test_database_query(engine: Engine, statement: str): engine=connection, statement=DATABRICKS_GET_CATALOGS, ), - "GetQueries": client.test_query_api_access, + "GetQueries": partial( + test_database_query, + engine=connection, + statement=DATABRICKS_SQL_STATEMENT_TEST.format( + query_history=service_connection.queryHistoryTable + ), + ), } return test_connection_steps( diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py b/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py index a77cb780e56d..eb4b74b5d495 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/lineage.py @@ -11,12 +11,10 @@ """ Databricks lineage module """ -import traceback -from datetime import datetime -from typing import Iterator -from metadata.generated.schema.type.basic import DateTime -from metadata.generated.schema.type.tableQuery import TableQuery +from metadata.ingestion.source.database.databricks.queries import ( + DATABRICKS_SQL_STATEMENT, +) from metadata.ingestion.source.database.databricks.query_parser import ( DatabricksQueryParserSource, ) @@ -31,23 +29,13 @@ class DatabricksLineageSource(DatabricksQueryParserSource, LineageSource): Databricks Lineage Legacy Source """ - def yield_table_query(self) -> Iterator[TableQuery]: - data = self.client.list_query_history( - start_date=self.start, - end_date=self.end, + sql_stmt = DATABRICKS_SQL_STATEMENT + + filters = """ + AND ( + lower(statement_text) LIKE '%%create%%select%%' + OR lower(statement_text) LIKE '%%insert%%into%%select%%' + OR lower(statement_text) LIKE '%%update%%' + OR lower(statement_text) LIKE '%%merge%%' ) - for row in data or []: - try: - if self.client.is_query_valid(row): - yield TableQuery( - dialect=self.dialect.value, - query=row.get("query_text"), - userName=row.get("user_name"), - startTime=str(row.get("query_start_time_ms")), - endTime=str(row.get("execution_end_time_ms")), - analysisDate=DateTime(datetime.now()), - serviceName=self.config.serviceName, - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning(f"Error processing query_dict {row}: {exc}") + """ diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/queries.py b/ingestion/src/metadata/ingestion/source/database/databricks/queries.py index 732dc79ad685..25cdcedfc105 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/queries.py @@ -14,6 +14,30 @@ import textwrap +DATABRICKS_SQL_STATEMENT = textwrap.dedent( + """ + SELECT + statement_type AS query_type, + statement_text AS query_text, + executed_by AS user_name, + start_time AS start_time, + null AS database_name, + null AS schema_name, + end_time AS end_time, + total_duration_ms/1000 AS duration + from {query_history} + WHERE statement_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND statement_text NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND start_time between to_timestamp('{start_time}') and to_timestamp('{end_time}') + {filters} + LIMIT {result_limit} + """ +) + +DATABRICKS_SQL_STATEMENT_TEST = """ + SELECT statement_text from {query_history} LIMIT 1 +""" + DATABRICKS_VIEW_DEFINITIONS = textwrap.dedent( """ select diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py b/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py index 00628bfbddaa..c67b06aa30ed 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py @@ -22,7 +22,6 @@ ) from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.ingestion.source.database.databricks.client import DatabricksClient from metadata.ingestion.source.database.query_parser_source import QueryParserSource from metadata.utils.logger import ingestion_logger @@ -36,18 +35,6 @@ class DatabricksQueryParserSource(QueryParserSource, ABC): filters: str - def _init_super( - self, - config: WorkflowSource, - metadata: OpenMetadata, - ): - super().__init__(config, metadata, False) - - # pylint: disable=super-init-not-called - def __init__(self, config: WorkflowSource, metadata: OpenMetadata): - self._init_super(config=config, metadata=metadata) - self.client = DatabricksClient(self.service_connection) - @classmethod def create( cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None @@ -61,7 +48,16 @@ def create( ) return cls(config, metadata) - def prepare(self): + def get_sql_statement(self, start_time, end_time): """ - By default, there's nothing to prepare + returns sql statement to fetch query logs. + + Override if we have specific parameters """ + return self.sql_stmt.format( + start_time=start_time, + end_time=end_time, + filters=self.get_filters(), + result_limit=self.source_config.resultLimit, + query_history=self.service_connection.queryHistoryTable, + ) diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/usage.py b/ingestion/src/metadata/ingestion/source/database/databricks/usage.py index 0e5364a465d8..fedbab2da486 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/usage.py @@ -11,12 +11,10 @@ """ Databricks usage module """ -import traceback -from datetime import datetime -from typing import Iterable -from metadata.generated.schema.type.basic import DateTime -from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery +from metadata.ingestion.source.database.databricks.queries import ( + DATABRICKS_SQL_STATEMENT, +) from metadata.ingestion.source.database.databricks.query_parser import ( DatabricksQueryParserSource, ) @@ -31,36 +29,8 @@ class DatabricksUsageSource(DatabricksQueryParserSource, UsageSource): Databricks Usage Source """ - def yield_table_queries(self) -> Iterable[TableQuery]: - """ - Method to yield TableQueries - """ - queries = [] - data = self.client.list_query_history( - start_date=self.start, - end_date=self.end, - ) - for row in data or []: - try: - if self.client.is_query_valid(row): - queries.append( - TableQuery( - dialect=self.dialect.value, - query=row.get("query_text"), - userName=row.get("user_name"), - startTime=str(row.get("query_start_time_ms")), - endTime=str(row.get("execution_end_time_ms")), - analysisDate=DateTime(datetime.now()), - serviceName=self.config.serviceName, - duration=row.get("duration") - if row.get("duration") - else None, - ) - ) - except Exception as err: - logger.debug(traceback.format_exc()) - logger.warning( - f"Failed to process query {row.get('query_text')} due to: {err}" - ) + sql_stmt = DATABRICKS_SQL_STATEMENT - yield TableQueries(queries=queries) + filters = """ + AND statement_type NOT IN ('SHOW', 'DESCRIBE', 'USE') + """ diff --git a/ingestion/src/metadata/ingestion/source/database/db2/connection.py b/ingestion/src/metadata/ingestion/source/database/db2/connection.py index b25ac3efc757..c9efc348f730 100644 --- a/ingestion/src/metadata/ingestion/source/database/db2/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/db2/connection.py @@ -50,7 +50,7 @@ def get_connection(connection: Db2Connection) -> Engine: "w", encoding=UTF_8, ) as file: - file.write(connection.license) + file.write(connection.license.encode(UTF_8).decode("unicode-escape")) return create_generic_db_connection( connection=connection, diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py index adb8c689278a..a422fd6e4902 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py @@ -1093,7 +1093,9 @@ def add_dbt_test_result(self, dbt_test: dict): # Create the test case result object test_case_result = TestCaseResult( - timestamp=Timestamp(datetime_to_timestamp(dbt_timestamp)), + timestamp=Timestamp( + datetime_to_timestamp(dbt_timestamp, milliseconds=True) + ), testCaseStatus=test_case_status, testResultValue=[ TestResultValue( diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/metadata.py b/ingestion/src/metadata/ingestion/source/database/mssql/metadata.py index f3f9a91f26e9..1639363f7d50 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql/metadata.py @@ -30,7 +30,7 @@ from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.generated.schema.type.basic import EntityName +from metadata.generated.schema.type.basic import EntityName, Markdown from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException from metadata.ingestion.ometa.ometa_api import OpenMetadata @@ -41,6 +41,7 @@ ) from metadata.ingestion.source.database.mssql.queries import ( MSSQL_GET_DATABASE, + MSSQL_GET_STORED_PROCEDURE_COMMENTS, MSSQL_GET_STORED_PROCEDURES, ) from metadata.ingestion.source.database.mssql.utils import ( @@ -94,6 +95,14 @@ class MssqlSource(CommonDbSourceService, MultiDBSource): Database metadata from MSSQL Source """ + def __init__( + self, + config, + metadata, + ): + super().__init__(config, metadata) + self.stored_procedure_desc_map = {} + @classmethod def create( cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None @@ -107,6 +116,27 @@ def create( ) return cls(config, metadata) + def get_stored_procedure_description(self, stored_procedure: str) -> Optional[str]: + """ + Method to fetch the stored procedure description + """ + description = self.stored_procedure_desc_map.get( + ( + self.context.get().database, + self.context.get().database_schema, + stored_procedure, + ) + ) + return Markdown(description) if description else None + + def set_stored_procedure_description_map(self) -> None: + self.stored_procedure_desc_map.clear() + results = self.engine.execute(MSSQL_GET_STORED_PROCEDURE_COMMENTS).all() + self.stored_procedure_desc_map = { + (row.DATABASE_NAME, row.SCHEMA_NAME, row.STORED_PROCEDURE): row.COMMENT + for row in results + } + def get_configured_database(self) -> Optional[str]: if not self.service_connection.ingestAllDatabases: return self.service_connection.database @@ -118,6 +148,7 @@ def get_database_names_raw(self) -> Iterable[str]: def get_database_names(self) -> Iterable[str]: if not self.config.serviceConnection.root.config.ingestAllDatabases: configured_db = self.config.serviceConnection.root.config.database + self.set_stored_procedure_description_map() self.set_inspector(database_name=configured_db) yield configured_db else: @@ -178,7 +209,9 @@ def yield_stored_procedure( try: stored_procedure_request = CreateStoredProcedureRequest( name=EntityName(stored_procedure.name), - description=None, + description=self.get_stored_procedure_description( + stored_procedure.name + ), storedProcedureCode=StoredProcedureCode( language=STORED_PROC_LANGUAGE_MAP.get(stored_procedure.language), code=stored_procedure.definition, diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/queries.py b/ingestion/src/metadata/ingestion/source/database/mssql/queries.py index 0ed3f06da060..c3f3c5b3a1d4 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql/queries.py @@ -56,6 +56,23 @@ """ ) +MSSQL_GET_STORED_PROCEDURE_COMMENTS = textwrap.dedent( + """ +SELECT + DB_NAME() AS DATABASE_NAME, + s.name AS SCHEMA_NAME, + p.name AS STORED_PROCEDURE, + ep.value AS COMMENT +FROM sys.procedures p +JOIN sys.schemas s ON p.schema_id = s.schema_id +LEFT JOIN sys.extended_properties ep + ON ep.major_id = p.object_id + AND ep.minor_id = 0 + AND ep.class = 1 + AND ep.name = 'MS_Description'; +""" +) + MSSQL_ALL_VIEW_DEFINITIONS = textwrap.dedent( """ SELECT diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/usage.py b/ingestion/src/metadata/ingestion/source/database/postgres/usage.py index 579590b4ec59..522a89a9e2ee 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/usage.py @@ -15,6 +15,11 @@ from datetime import datetime from typing import Iterable +from sqlalchemy.exc import OperationalError + +from metadata.generated.schema.entity.services.ingestionPipelines.status import ( + StackTraceError, +) from metadata.generated.schema.type.basic import DateTime from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery from metadata.ingestion.source.connections import get_connection @@ -67,6 +72,16 @@ def process_table_query(self) -> Iterable[TableQueries]: logger.error(str(err)) if queries: yield TableQueries(queries=queries) + + except OperationalError as err: + self.status.failed( + StackTraceError( + name="Usage", + error=f"Source Usage failed due to - {err}", + stackTrace=traceback.format_exc(), + ) + ) + except Exception as err: if query: logger.debug( diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py index d1c87bd07289..244d96acce2a 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py @@ -11,7 +11,6 @@ """ Redshift source ingestion """ - import re import traceback from typing import Iterable, List, Optional, Tuple @@ -57,6 +56,9 @@ CommonDbSourceService, TableNameAndType, ) +from metadata.ingestion.source.database.external_table_lineage_mixin import ( + ExternalTableLineageMixin, +) from metadata.ingestion.source.database.incremental_metadata_extraction import ( IncrementalConfig, ) @@ -69,6 +71,7 @@ ) from metadata.ingestion.source.database.redshift.models import RedshiftStoredProcedure from metadata.ingestion.source.database.redshift.queries import ( + REDSHIFT_EXTERNAL_TABLE_LOCATION, REDSHIFT_GET_ALL_RELATION_INFO, REDSHIFT_GET_DATABASE_NAMES, REDSHIFT_GET_STORED_PROCEDURES, @@ -121,12 +124,13 @@ RedshiftDialect._get_all_relation_info = ( # pylint: disable=protected-access _get_all_relation_info ) - Inspector.get_all_table_ddls = get_all_table_ddls Inspector.get_table_ddl = get_table_ddl -class RedshiftSource(LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource): +class RedshiftSource( + ExternalTableLineageMixin, LifeCycleQueryMixin, CommonDbSourceService, MultiDBSource +): """ Implements the necessary methods to extract Database metadata from Redshift Source @@ -146,6 +150,7 @@ def __init__( self.incremental_table_processor: Optional[ RedshiftIncrementalTableProcessor ] = None + self.external_location_map = {} if self.incremental.enabled: logger.info( @@ -168,6 +173,14 @@ def create( ) return cls(config, metadata, incremental_config) + def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]: + """ + Method to fetch the location path of the table + """ + return self.external_location_map.get( + (self.context.get().database, schema_name, table_name) + ) + def get_partition_details(self) -> None: """ Populate partition details @@ -275,15 +288,23 @@ def _set_incremental_table_processor(self, database: str): for schema_name, table_name in self.incremental_table_processor.get_deleted() ) + def set_external_location_map(self, database_name: str) -> None: + self.external_location_map.clear() + results = self.engine.execute( + REDSHIFT_EXTERNAL_TABLE_LOCATION.format(database_name=database_name) + ).all() + self.external_location_map = { + (database_name, row.schemaname, row.tablename): row.location + for row in results + } + def get_database_names(self) -> Iterable[str]: if not self.config.serviceConnection.root.config.ingestAllDatabases: + configured_db = self.config.serviceConnection.root.config.database self.get_partition_details() - - self._set_incremental_table_processor( - self.config.serviceConnection.root.config.database - ) - - yield self.config.serviceConnection.root.config.database + self._set_incremental_table_processor(configured_db) + self.set_external_location_map(configured_db) + yield configured_db else: for new_database in self.get_database_names_raw(): database_fqn = fqn.build( @@ -307,9 +328,8 @@ def get_database_names(self) -> Iterable[str]: try: self.set_inspector(database_name=new_database) self.get_partition_details() - self._set_incremental_table_processor(new_database) - + self.set_external_location_map(new_database) yield new_database except Exception as exc: logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/queries.py b/ingestion/src/metadata/ingestion/source/database/redshift/queries.py index bc840ff1b00d..114ba4e33dd5 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/queries.py @@ -203,6 +203,11 @@ """ ) +REDSHIFT_EXTERNAL_TABLE_LOCATION = """ + SELECT schemaname, tablename, location + FROM svv_external_tables + where redshift_database_name='{database_name}' +""" REDSHIFT_PARTITION_DETAILS = """ select "schema", "table", diststyle diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py index e32479fe3987..6c98a201b319 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py @@ -193,10 +193,18 @@ def test_connection( engine_wrapper=engine_wrapper, ), "GetQueries": partial( - test_query, statement=SNOWFLAKE_TEST_GET_QUERIES, engine=engine + test_query, + statement=SNOWFLAKE_TEST_GET_QUERIES.format( + account_usage=service_connection.accountUsageSchema + ), + engine=engine, ), "GetTags": partial( - test_query, statement=SNOWFLAKE_TEST_FETCH_TAG, engine=engine + test_query, + statement=SNOWFLAKE_TEST_FETCH_TAG.format( + account_usage=service_connection.accountUsageSchema + ), + engine=engine, ), } diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py b/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py index 3ef0232b790d..1337635fe14f 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py @@ -62,6 +62,7 @@ def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]] start, _ = get_start_and_end(self.source_config.queryLogDuration) query = self.stored_procedure_query.format( start_date=start, + account_usage=self.service_connection.accountUsageSchema, ) queries_dict = self.procedure_queries_dict( query=query, diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py index da5a7034a26d..f6f026d39137 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py @@ -418,6 +418,7 @@ def yield_tag( SNOWFLAKE_FETCH_ALL_TAGS.format( database_name=self.context.get().database, schema_name=schema_name, + account_usage=self.service_connection.accountUsageSchema, ) ) @@ -431,6 +432,7 @@ def yield_tag( SNOWFLAKE_FETCH_ALL_TAGS.format( database_name=f'"{self.context.get().database}"', schema_name=f'"{self.context.get().database_schema}"', + account_usage=self.service_connection.accountUsageSchema, ) ) except Exception as inner_exc: @@ -635,6 +637,7 @@ def _get_stored_procedures_internal( query.format( database_name=self.context.get().database, schema_name=self.context.get().database_schema, + account_usage=self.service_connection.accountUsageSchema, ) ).all() for row in results: diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py b/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py index a0a4b819c0c1..55b2bf909c4a 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/queries.py @@ -25,7 +25,7 @@ start_time "start_time", end_time "end_time", total_elapsed_time "duration" - from snowflake.account_usage.query_history + from {account_usage}.query_history WHERE query_text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND query_text NOT LIKE '/* {{"app": "dbt", %%}} */%%' AND start_time between to_timestamp_ltz('{start_time}') and to_timestamp_ltz('{end_time}') @@ -39,7 +39,7 @@ SNOWFLAKE_FETCH_ALL_TAGS = textwrap.dedent( """ select TAG_NAME, TAG_VALUE, OBJECT_DATABASE, OBJECT_SCHEMA, OBJECT_NAME, COLUMN_NAME - from snowflake.account_usage.tag_references + from {account_usage}.tag_references where OBJECT_DATABASE = '{database_name}' and OBJECT_SCHEMA = '{schema_name}' """ @@ -234,11 +234,11 @@ """ SNOWFLAKE_TEST_FETCH_TAG = """ -select TAG_NAME from snowflake.account_usage.tag_references limit 1 +select TAG_NAME from {account_usage}.tag_references limit 1 """ SNOWFLAKE_TEST_GET_QUERIES = """ -SELECT query_text from snowflake.account_usage.query_history limit 1 +SELECT query_text from {account_usage}.query_history limit 1 """ SNOWFLAKE_TEST_GET_TABLES = """ @@ -296,10 +296,10 @@ ARGUMENT_SIGNATURE AS signature, COMMENT as comment, 'StoredProcedure' as procedure_type -FROM SNOWFLAKE.ACCOUNT_USAGE.PROCEDURES +FROM {account_usage}.PROCEDURES WHERE PROCEDURE_CATALOG = '{database_name}' AND PROCEDURE_SCHEMA = '{schema_name}' - AND DELETED IS NOT NULL + AND DELETED IS NULL """ ) @@ -313,10 +313,10 @@ ARGUMENT_SIGNATURE AS signature, COMMENT as comment, 'UDF' as procedure_type -FROM SNOWFLAKE.ACCOUNT_USAGE.FUNCTIONS +FROM {account_usage}.FUNCTIONS WHERE FUNCTION_CATALOG = '{database_name}' AND FUNCTION_SCHEMA = '{schema_name}' - AND DELETED IS NOT NULL + AND DELETED IS NULL """ ) @@ -336,7 +336,7 @@ SESSION_ID, START_TIME, END_TIME - FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY SP + FROM {account_usage}.QUERY_HISTORY SP WHERE QUERY_TYPE = 'CALL' AND START_TIME >= '{start_date}' AND QUERY_TEXT <> '' @@ -353,7 +353,7 @@ USER_NAME, SCHEMA_NAME, DATABASE_NAME - FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY SP + FROM {account_usage}.QUERY_HISTORY SP WHERE QUERY_TYPE <> 'CALL' AND QUERY_TEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' AND QUERY_TEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%' diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py b/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py index bbc528fc4c44..363495bd0f87 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py @@ -60,6 +60,7 @@ def get_sql_statement(self, start_time: datetime, end_time: datetime) -> str: end_time=end_time, result_limit=self.config.sourceConfig.config.resultLimit, filters=self.get_filters(), + account_usage=self.service_connection.accountUsageSchema, ) def check_life_cycle_query( diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/utils.py b/ingestion/src/metadata/ingestion/source/database/snowflake/utils.py index 4f2e7f007f61..a1660f13758f 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/utils.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/utils.py @@ -12,10 +12,12 @@ """ Module to define overriden dialect methods """ - +import operator +from functools import reduce from typing import Dict, Optional import sqlalchemy.types as sqltypes +from snowflake.sqlalchemy.snowdialect import SnowflakeDialect from sqlalchemy import exc as sa_exc from sqlalchemy import util as sa_util from sqlalchemy.engine import reflection @@ -52,6 +54,7 @@ get_table_comment_wrapper, ) +dialect = SnowflakeDialect() Query = str QueryMap = Dict[str, Query] @@ -83,6 +86,20 @@ } +def _denormalize_quote_join(*idents): + ip = dialect.identifier_preparer + split_idents = reduce( + operator.add, + [ip._split_schema_by_dot(ids) for ids in idents if ids is not None], + ) + quoted_identifiers = ip._quote_free_identifiers(*split_idents) + normalized_identifiers = ( + item if item.startswith('"') and item.endswith('"') else f'"{item}"' + for item in quoted_identifiers + ) + return ".".join(normalized_identifiers) + + def _quoted_name(entity_name: Optional[str]) -> Optional[str]: if entity_name: return fqn.quote_name(entity_name) @@ -256,17 +273,16 @@ def get_schema_columns(self, connection, schema, **kw): None, as it is cacheable and is an unexpected return type for this function""" ans = {} current_database, _ = self._current_database_schema(connection, **kw) - full_schema_name = self._denormalize_quote_join( - current_database, fqn.quote_name(schema) - ) + full_schema_name = _denormalize_quote_join(current_database, fqn.quote_name(schema)) try: schema_primary_keys = self._get_schema_primary_keys( connection, full_schema_name, **kw ) + # removing " " from schema name because schema name is in the WHERE clause of a query + table_schema = self.denormalize_name(fqn.unquote_name(schema)) + table_schema = table_schema.lower() if schema.islower() else table_schema result = connection.execute( - text(SNOWFLAKE_GET_SCHEMA_COLUMNS), - {"table_schema": self.denormalize_name(fqn.unquote_name(schema))} - # removing " " from schema name because schema name is in the WHERE clause of a query + text(SNOWFLAKE_GET_SCHEMA_COLUMNS), {"table_schema": table_schema} ) except sa_exc.ProgrammingError as p_err: @@ -362,9 +378,10 @@ def get_pk_constraint(self, connection, table_name, schema=None, **kw): schema = schema or self.default_schema_name schema = _quoted_name(entity_name=schema) current_database, current_schema = self._current_database_schema(connection, **kw) - full_schema_name = self._denormalize_quote_join( + full_schema_name = _denormalize_quote_join( current_database, schema if schema else current_schema ) + return self._get_schema_primary_keys( connection, self.denormalize_name(full_schema_name), **kw ).get(table_name, {"constrained_columns": [], "name": None}) @@ -378,7 +395,7 @@ def get_foreign_keys(self, connection, table_name, schema=None, **kw): schema = schema or self.default_schema_name schema = _quoted_name(entity_name=schema) current_database, current_schema = self._current_database_schema(connection, **kw) - full_schema_name = self._denormalize_quote_join( + full_schema_name = _denormalize_quote_join( current_database, schema if schema else current_schema ) @@ -452,9 +469,10 @@ def get_unique_constraints(self, connection, table_name, schema, **kw): schema = schema or self.default_schema_name schema = _quoted_name(entity_name=schema) current_database, current_schema = self._current_database_schema(connection, **kw) - full_schema_name = self._denormalize_quote_join( + full_schema_name = _denormalize_quote_join( current_database, schema if schema else current_schema ) + return self._get_schema_unique_constraints( connection, self.denormalize_name(full_schema_name), **kw ).get(table_name, []) diff --git a/ingestion/src/metadata/ingestion/source/database/trino/connection.py b/ingestion/src/metadata/ingestion/source/database/trino/connection.py index 6dbae4ac9c39..a87dd792b6bd 100644 --- a/ingestion/src/metadata/ingestion/source/database/trino/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/trino/connection.py @@ -39,6 +39,7 @@ create_generic_db_connection, get_connection_args_common, init_empty_connection_arguments, + init_empty_connection_options, ) from metadata.ingestion.connections.secrets import connection_with_options_secrets from metadata.ingestion.connections.test_connections import ( @@ -135,6 +136,10 @@ def get_connection(connection: TrinoConnection) -> Engine: # here we are creating a copy of connection, because we need to dynamically # add auth params to connectionArguments, which we do no intend to store # in original connection object and in OpenMetadata database + from trino.sqlalchemy.dialect import TrinoDialect + + TrinoDialect.is_disconnect = _is_disconnect + connection_copy = deepcopy(connection) if connection_copy.verify: connection_copy.connectionArguments = ( @@ -183,3 +188,11 @@ def test_connection( queries=queries, timeout_seconds=timeout_seconds, ) + + +# pylint: disable=unused-argument +def _is_disconnect(self, e, connection, cursor): + """is_disconnect method for the Databricks dialect""" + if "JWT expired" in str(e): + return True + return False diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py index 8f36b033fc10..92813459af5d 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py @@ -11,6 +11,7 @@ """ Databricks Unity Catalog Lineage Source Module """ +import traceback from typing import Iterable, Optional from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest @@ -27,6 +28,7 @@ EntitiesEdge, LineageDetails, ) +from metadata.generated.schema.type.entityLineage import Source as LineageSource from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.models import Either from metadata.ingestion.api.steps import InvalidSourceException, Source @@ -111,9 +113,59 @@ def _get_lineage_details( ) ) if col_lineage: - return LineageDetails(columnsLineage=col_lineage) + return LineageDetails( + columnsLineage=col_lineage, source=LineageSource.QueryLineage + ) return None + def _handle_upstream_table( + self, + table_streams: LineageTableStreams, + table: Table, + databricks_table_fqn: str, + ) -> Iterable[Either[AddLineageRequest]]: + for upstream_table in table_streams.upstream_tables: + try: + if not upstream_table.name: + continue + from_entity_fqn = fqn.build( + metadata=self.metadata, + entity_type=Table, + database_name=upstream_table.catalog_name, + schema_name=upstream_table.schema_name, + table_name=upstream_table.name, + service_name=self.config.serviceName, + ) + + from_entity = self.metadata.get_by_name( + entity=Table, fqn=from_entity_fqn + ) + if from_entity: + lineage_details = self._get_lineage_details( + from_table=from_entity, + to_table=table, + databricks_table_fqn=databricks_table_fqn, + ) + yield Either( + left=None, + right=AddLineageRequest( + edge=EntitiesEdge( + toEntity=EntityReference(id=table.id, type="table"), + fromEntity=EntityReference( + id=from_entity.id, type="table" + ), + lineageDetails=lineage_details, + ) + ), + ) + except Exception: + logger.debug( + "Error while processing lineage for " + f"{upstream_table.catalog_name}.{upstream_table.schema_name}.{upstream_table.name}" + f" -> {databricks_table_fqn}" + ) + logger.debug(traceback.format_exc()) + def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]: """ Based on the query logs, prepare the lineage @@ -130,37 +182,9 @@ def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]: table_streams: LineageTableStreams = self.client.get_table_lineage( databricks_table_fqn ) - for upstream_table in table_streams.upstream_tables: - from_entity_fqn = fqn.build( - metadata=self.metadata, - entity_type=Table, - database_name=upstream_table.catalog_name, - schema_name=upstream_table.schema_name, - table_name=upstream_table.name, - service_name=self.config.serviceName, - ) - - from_entity = self.metadata.get_by_name( - entity=Table, fqn=from_entity_fqn - ) - if from_entity: - lineage_details = self._get_lineage_details( - from_table=from_entity, - to_table=table, - databricks_table_fqn=databricks_table_fqn, - ) - yield Either( - left=None, - right=AddLineageRequest( - edge=EntitiesEdge( - toEntity=EntityReference(id=table.id, type="table"), - fromEntity=EntityReference( - id=from_entity.id, type="table" - ), - lineageDetails=lineage_details, - ) - ), - ) + yield from self._handle_upstream_table( + table_streams, table, databricks_table_fqn + ) def test_connection(self) -> None: test_connection_fn = get_test_connection_fn(self.service_connection) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py index 5a6b7933a28d..f2ac03b99f98 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/query_parser.py @@ -44,6 +44,13 @@ class UnityCatalogQueryParserSource( filters: str + def _init_super( + self, + config: WorkflowSource, + metadata: OpenMetadata, + ): + super().__init__(config, metadata, False) + # pylint: disable=super-init-not-called def __init__(self, config: WorkflowSource, metadata: OpenMetadata): self._init_super(config=config, metadata=metadata) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py index ca9f17b254e5..892f88ef7b74 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/service_spec.py @@ -11,6 +11,9 @@ from metadata.profiler.interface.sqlalchemy.unity_catalog.profiler_interface import ( UnityCatalogProfilerInterface, ) +from metadata.profiler.interface.sqlalchemy.unity_catalog.sampler_interface import ( + UnityCatalogSamplerInterface, +) from metadata.utils.service_spec.default import DefaultDatabaseSpec ServiceSpec = DefaultDatabaseSpec( @@ -19,4 +22,5 @@ usage_source_class=UnitycatalogUsageSource, profiler_class=UnityCatalogProfilerInterface, test_suite_class=UnityCatalogTestSuiteInterface, + sampler_class=UnityCatalogSamplerInterface, ) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py index 7a310f736a3f..c533454be8cb 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/usage.py @@ -11,17 +11,22 @@ """ unity catalog usage module """ +import traceback +from datetime import datetime +from typing import Iterable -from metadata.ingestion.source.database.databricks.usage import DatabricksUsageSource +from metadata.generated.schema.type.basic import DateTime +from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery from metadata.ingestion.source.database.unitycatalog.query_parser import ( UnityCatalogQueryParserSource, ) +from metadata.ingestion.source.database.usage_source import UsageSource from metadata.utils.logger import ingestion_logger logger = ingestion_logger() -class UnitycatalogUsageSource(UnityCatalogQueryParserSource, DatabricksUsageSource): +class UnitycatalogUsageSource(UnityCatalogQueryParserSource, UsageSource): """ UnityCatalog Usage Source @@ -29,3 +34,37 @@ class UnitycatalogUsageSource(UnityCatalogQueryParserSource, DatabricksUsageSour DatabricksUsageSource as both the sources would call the same API for fetching Usage Queries """ + + def yield_table_queries(self) -> Iterable[TableQuery]: + """ + Method to yield TableQueries + """ + queries = [] + data = self.client.list_query_history( + start_date=self.start, + end_date=self.end, + ) + for row in data or []: + try: + if self.client.is_query_valid(row): + queries.append( + TableQuery( + dialect=self.dialect.value, + query=row.get("query_text"), + userName=row.get("user_name"), + startTime=str(row.get("query_start_time_ms")), + endTime=str(row.get("execution_end_time_ms")), + analysisDate=DateTime(datetime.now()), + serviceName=self.config.serviceName, + duration=row.get("duration") + if row.get("duration") + else None, + ) + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Failed to process query {row.get('query_text')} due to: {err}" + ) + + yield TableQueries(queries=queries) diff --git a/ingestion/src/metadata/ingestion/source/database/usage_source.py b/ingestion/src/metadata/ingestion/source/database/usage_source.py index 6209d601364f..65d5f2635876 100644 --- a/ingestion/src/metadata/ingestion/source/database/usage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/usage_source.py @@ -153,7 +153,7 @@ def yield_table_queries(self) -> Iterable[TableQuery]: if query: logger.debug( ( - f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value)}" + f"###### USAGE QUERY #######\n{mask_query(query, self.dialect.value) or query}" "\n##########################" ) ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py index 097b8cc1dbb4..680a8cff54c5 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py @@ -17,8 +17,10 @@ from metadata.generated.schema.entity.automations.workflow import ( Workflow as AutomationWorkflow, ) +from metadata.generated.schema.entity.services.connections.pipeline.nifi.basicAuth import ( + NifiBasicAuth, +) from metadata.generated.schema.entity.services.connections.pipeline.nifiConnection import ( - BasicAuthentication, NifiConnection, ) from metadata.generated.schema.entity.services.connections.testConnectionResult import ( @@ -34,7 +36,7 @@ def get_connection(connection: NifiConnection) -> NifiClient: """ Create connection """ - if isinstance(connection.nifiConfig, BasicAuthentication): + if isinstance(connection.nifiConfig, NifiBasicAuth): return NifiClient( host_port=connection.hostPort, username=connection.nifiConfig.username, diff --git a/ingestion/src/metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py b/ingestion/src/metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py new file mode 100644 index 000000000000..12a4ae3eaacb --- /dev/null +++ b/ingestion/src/metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py @@ -0,0 +1,29 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Interfaces with database for all database engine +supporting sqlalchemy abstraction layer +""" +from metadata.ingestion.source.database.databricks.connection import ( + get_connection as databricks_get_connection, +) +from metadata.sampler.sqlalchemy.sampler import SQASampler + + +class UnityCatalogSamplerInterface(SQASampler): + def get_client(self): + """client is the session for SQA""" + self.connection = databricks_get_connection(self.service_connection_config) + self.client = super().get_client() + self.set_catalog(self.client) + + return self.client diff --git a/ingestion/src/metadata/profiler/source/fetcher/config.py b/ingestion/src/metadata/profiler/source/fetcher/config.py index ef44e9eca1e2..006f1d5181c3 100644 --- a/ingestion/src/metadata/profiler/source/fetcher/config.py +++ b/ingestion/src/metadata/profiler/source/fetcher/config.py @@ -42,3 +42,7 @@ def tableFilterPattern(self) -> Optional[FilterPattern]: @property def useFqnForFiltering(self) -> Optional[bool]: ... + + @property + def includeViews(self) -> Optional[bool]: + ... diff --git a/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py b/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py index 0fd55f84ccdc..60c04e3a8b81 100644 --- a/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py +++ b/ingestion/src/metadata/profiler/source/fetcher/fetcher_strategy.py @@ -17,6 +17,7 @@ from typing import Iterable, Iterator, Optional, cast from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.table import TableType from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -115,13 +116,13 @@ def __init__( super().__init__(config, metadata, global_profiler_config, status) self.source_config = cast( EntityFilterConfigInterface, self.source_config - ) # Satisfy typchecker + ) # Satisfy typechecker def _filter_databases(self, databases: Iterable[Database]) -> Iterable[Database]: """Filter databases based on the filter pattern Args: - database (Database): Database to filter + databases (Database): Database to filter Returns: bool @@ -192,6 +193,21 @@ def _filter_tables(self, table: Table) -> bool: return False + def _filter_views(self, table: Table) -> bool: + """Filter the tables based on include views configuration""" + # If we include views, nothing to filter + if self.source_config.includeViews: + return False + + # Otherwise, filter out views + if table.tableType == TableType.View: + self.status.filter( + table.name.root, f"We are not including views {table.name.root}" + ) + return True + + return False + def _filter_column_metrics_computation(self): """Filter""" @@ -242,6 +258,7 @@ def _filter_entities(self, tables: Iterable[Table]) -> Iterable[Table]: not self.source_config.classificationFilterPattern or not self.filter_classifications(table) ) + and not self._filter_views(table) ] return tables diff --git a/ingestion/src/metadata/sampler/sampler_interface.py b/ingestion/src/metadata/sampler/sampler_interface.py index 970793cb2feb..fe363816d01c 100644 --- a/ingestion/src/metadata/sampler/sampler_interface.py +++ b/ingestion/src/metadata/sampler/sampler_interface.py @@ -76,9 +76,6 @@ def __init__( self._columns: Optional[List[SQALikeColumn]] = None self.sample_config = sample_config - if not self.sample_config.profileSample: - self.sample_config.profileSample = 100 - self.entity = entity self.include_columns = include_columns self.exclude_columns = exclude_columns diff --git a/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py index 1712ccd6cef8..cd82565506b6 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/bigquery/sampler.py @@ -54,7 +54,6 @@ def __init__( sample_query: Optional[str] = None, storage_config: DataStorageConfig = None, sample_data_count: Optional[int] = SAMPLE_DATA_DEFAULT_COUNT, - table_type: TableType = None, **kwargs, ): super().__init__( @@ -68,7 +67,7 @@ def __init__( sample_data_count=sample_data_count, **kwargs, ) - self.raw_dataset_type: TableType = table_type + self.raw_dataset_type: Optional[TableType] = entity.tableType def set_tablesample(self, selectable: SqaTable): """Set the TABLESAMPLE clause for BigQuery diff --git a/ingestion/src/metadata/sampler/sqlalchemy/sampler.py b/ingestion/src/metadata/sampler/sqlalchemy/sampler.py index cd87b79368bb..4d28f29f4a95 100644 --- a/ingestion/src/metadata/sampler/sqlalchemy/sampler.py +++ b/ingestion/src/metadata/sampler/sqlalchemy/sampler.py @@ -12,6 +12,7 @@ Helper module to handle data sampling for the profiler """ +import hashlib import traceback from typing import List, Optional, Union, cast @@ -32,6 +33,7 @@ from metadata.profiler.orm.functions.random_num import RandomNumFn from metadata.profiler.processor.handle_partition import build_partition_predicate from metadata.sampler.sampler_interface import SamplerInterface +from metadata.utils.constants import UTF_8 from metadata.utils.helpers import is_safe_sql_query from metadata.utils.logger import profiler_interface_registry_logger @@ -109,17 +111,28 @@ def _base_sample_query(self, column: Optional[Column], label=None): query = self.get_partitioned_query(query) return query + def get_sampler_table_name(self) -> str: + """Get the base name of the SQA table for sampling. + We use MD5 as a hashing algorithm to generate a unique name for the table + keeping its length controlled. Otherwise, we ended up having issues + with names getting truncated when we add the suffixes to the identifiers + such as _sample, or _rnd. + """ + encoded_name = self.raw_dataset.__tablename__.encode(UTF_8) + hash_object = hashlib.md5(encoded_name) + return hash_object.hexdigest() + def get_sample_query(self, *, column=None) -> Query: """get query for sample data""" if self.sample_config.profileSampleType == ProfileSampleType.PERCENTAGE: rnd = self._base_sample_query( column, (ModuloFn(RandomNumFn(), 100)).label(RANDOM_LABEL), - ).cte(f"{self.raw_dataset.__tablename__}_rnd") + ).cte(f"{self.get_sampler_table_name()}_rnd") session_query = self.client.query(rnd) return session_query.where( rnd.c.random <= self.sample_config.profileSample - ).cte(f"{self.raw_dataset.__tablename__}_sample") + ).cte(f"{self.get_sampler_table_name()}_sample") table_query = self.client.query(self.raw_dataset) session_query = self._base_sample_query( @@ -129,7 +142,7 @@ def get_sample_query(self, *, column=None) -> Query: return ( session_query.order_by(RANDOM_LABEL) .limit(self.sample_config.profileSample) - .cte(f"{self.raw_dataset.__tablename__}_rnd") + .cte(f"{self.get_sampler_table_name()}_rnd") ) def get_dataset(self, column=None, **__) -> Union[DeclarativeMeta, AliasedClass]: @@ -143,7 +156,7 @@ def get_dataset(self, column=None, **__) -> Union[DeclarativeMeta, AliasedClass] if not self.sample_config.profileSample: if self.partition_details: partitioned = self._partitioned_table() - return partitioned.cte(f"{self.raw_dataset.__tablename__}_partitioned") + return partitioned.cte(f"{self.get_sampler_table_name()}_partitioned") return self.raw_dataset @@ -162,23 +175,23 @@ def fetch_sample_data(self, columns: Optional[List[Column]] = None) -> TableData return self._fetch_sample_data_from_user_query() # Add new RandomNumFn column - rnd = self.get_sample_query() + ds = self.get_dataset() if not columns: - sqa_columns = [col for col in inspect(rnd).c if col.name != RANDOM_LABEL] + sqa_columns = [col for col in inspect(ds).c if col.name != RANDOM_LABEL] else: # we can't directly use columns as it is bound to self.raw_dataset and not the rnd table. # If we use it, it will result in a cross join between self.raw_dataset and rnd table names = [col.name for col in columns] sqa_columns = [ col - for col in inspect(rnd).c + for col in inspect(ds).c if col.name != RANDOM_LABEL and col.name in names ] try: sqa_sample = ( self.client.query(*sqa_columns) - .select_from(rnd) + .select_from(ds) .limit(self.sample_limit) .all() ) @@ -224,7 +237,7 @@ def _rdn_sample_from_user_query(self) -> Query: stmt = stmt.columns(*list(inspect(self.raw_dataset).c)) return self.client.query(stmt.subquery()).cte( - f"{self.raw_dataset.__tablename__}_user_sampled" + f"{self.get_sampler_table_name()}_user_sampled" ) def _partitioned_table(self) -> Query: diff --git a/ingestion/tests/cli_e2e/base/config_builders/builders.py b/ingestion/tests/cli_e2e/base/config_builders/builders.py index a1a213eba21d..0b0eb4a83d2e 100644 --- a/ingestion/tests/cli_e2e/base/config_builders/builders.py +++ b/ingestion/tests/cli_e2e/base/config_builders/builders.py @@ -90,8 +90,8 @@ def build(self) -> dict: "type": "DatabaseLineage", "queryLogDuration": 1, "resultLimit": 10000, - "processQueryLineage": True, - "processStoredProcedureLineage": True, + "processQueryLineage": False, + "processStoredProcedureLineage": False, } } return self.config diff --git a/ingestion/tests/cli_e2e/test_cli_snowflake.py b/ingestion/tests/cli_e2e/test_cli_snowflake.py index 77db0556bdfe..5a41378cfbf5 100644 --- a/ingestion/tests/cli_e2e/test_cli_snowflake.py +++ b/ingestion/tests/cli_e2e/test_cli_snowflake.py @@ -181,7 +181,7 @@ def view_column_lineage_count(self) -> int: return 2 def expected_lineage_node(self) -> str: - return "e2e_snowflake.E2E_DB.E2E_TEST.view_persons" + return "e2e_snowflake.E2E_DB.E2E_TEST.VIEW_PERSONS" @staticmethod def fqn_created_table() -> str: diff --git a/ingestion/tests/cli_e2e/test_cli_vertica.py b/ingestion/tests/cli_e2e/test_cli_vertica.py index 1591c9d160f3..f0e66ac784b2 100644 --- a/ingestion/tests/cli_e2e/test_cli_vertica.py +++ b/ingestion/tests/cli_e2e/test_cli_vertica.py @@ -74,7 +74,7 @@ def view_column_lineage_count(self) -> int: return 2 def expected_lineage_node(self) -> str: - return "e2e_vertica.VMart.public.vendor_dimension_v" + return "e2e_vertica.VMart.public.vendor_dimension" @staticmethod def fqn_created_table() -> str: diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index bed6db07c0e1..ab65a16089ae 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -162,7 +162,7 @@ "serviceConnection": {{ "config": {service_config} }}, - "sourceConfig": {{"config": {{"type":"Profiler"}}}} + "sourceConfig": {{"config": {{"type":"Profiler", "profileSample": 100}}}} }}, "processor": {{"type": "orm-profiler", "config": {{}}}}, "sink": {{"type": "metadata-rest", "config": {{}}}}, diff --git a/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py b/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py index 960a15adae92..cd8e7515fa6b 100644 --- a/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_custom_properties_api.py @@ -93,7 +93,7 @@ "description": "Rating of a table", "propertyType": {"name": "enum"}, "customPropertyConfig": { - "config": {"values": ["Good", "Average", "Bad"], "multiSelect": False}, + "config": {"values": ["Average", "Bad", "Good"], "multiSelect": False}, }, }, { diff --git a/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py b/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py index 2d1d976ec72e..d6ca5e8a7e0e 100644 --- a/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py +++ b/ingestion/tests/integration/orm_profiler/test_orm_profiler_e2e.py @@ -549,8 +549,7 @@ def test_workflow_values_partition(ingest, metadata, service_name): profile = metadata.get_latest_table_profile(table.fullyQualifiedName).profile assert profile.rowCount == 4.0 - # If we don't have any sample, default to 100 - assert profile.profileSample == 100.0 + assert profile.profileSample == None workflow_config["processor"] = { "type": "orm-profiler", diff --git a/ingestion/tests/integration/trino/hive/Dockerfile b/ingestion/tests/integration/trino/hive/Dockerfile index eb218fd1d48c..576c37d40a35 100644 --- a/ingestion/tests/integration/trino/hive/Dockerfile +++ b/ingestion/tests/integration/trino/hive/Dockerfile @@ -2,8 +2,8 @@ ARG BASE_IMAGE=bitsondatadev/hive-metastore:latest FROM ${BASE_IMAGE} COPY conf/metastore-site.xml /opt/apache-hive-metastore-3.0.0-bin/conf/metastore-site.xml COPY entrypoint.sh /entrypoint.sh -ENV JDBC_CONNECTION_URL "" -ENV MINIO_ENDPOINT "" +ENV JDBC_CONNECTION_URL="" +ENV MINIO_ENDPOINT="" USER root RUN chmod +x /entrypoint.sh USER hive \ No newline at end of file diff --git a/ingestion/tests/integration/trino/test_profiler.py b/ingestion/tests/integration/trino/test_profiler.py index 6e12ca7b7b0c..e8092c1ded7d 100644 --- a/ingestion/tests/integration/trino/test_profiler.py +++ b/ingestion/tests/integration/trino/test_profiler.py @@ -65,7 +65,7 @@ class ProfilerTestParameters: ColumnProfile( name="three", timestamp=Timestamp(0), - valuesCount=1, + valuesCount=2, nullCount=1, ) ], @@ -101,7 +101,7 @@ class ProfilerTestParameters: ColumnProfile( name="gender", timestamp=Timestamp(0), - valuesCount=932, + valuesCount=1000, nullCount=0, ) ], diff --git a/ingestion/tests/integration/trino/trino/Dockerfile b/ingestion/tests/integration/trino/trino/Dockerfile index 87dd3e47bdc8..98c0ab84592b 100644 --- a/ingestion/tests/integration/trino/trino/Dockerfile +++ b/ingestion/tests/integration/trino/trino/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -ENV MINIO_ENDPOINT "" -ENV HIVE_METASTORE_URI "" +ENV MINIO_ENDPOINT="" +ENV HIVE_METASTORE_URI="" COPY etc /etc/trino \ No newline at end of file diff --git a/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py b/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py index 4f8de1611375..5c719d7fc753 100644 --- a/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py +++ b/ingestion/tests/unit/profiler/sqlalchemy/bigquery/test_bigquery_sampling.py @@ -127,20 +127,31 @@ def test_sampling_for_views(self, sampler_mock): """ Test view sampling """ + view_entity = Table( + id=uuid4(), + name="user", + columns=[ + EntityColumn( + name=ColumnName("id"), + dataType=DataType.INT, + ), + ], + tableType=TableType.View, + ) + sampler = BigQuerySampler( service_connection_config=self.bq_conn, ometa_client=None, - entity=self.table_entity, + entity=view_entity, sample_config=SampleConfig( profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), - table_type=TableType.View, ) query: CTE = sampler.get_sample_query() expected_query = ( - "WITH users_rnd AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n" - "FROM users)\n SELECT users_rnd.id, users_rnd.random \n" - "FROM users_rnd \nWHERE users_rnd.random <= 50.0" + 'WITH "9bc65c2abec141778ffaa729489f3e87_rnd" AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n' + 'FROM users)\n SELECT "9bc65c2abec141778ffaa729489f3e87_rnd".id, "9bc65c2abec141778ffaa729489f3e87_rnd".random \n' + 'FROM "9bc65c2abec141778ffaa729489f3e87_rnd" \nWHERE "9bc65c2abec141778ffaa729489f3e87_rnd".random <= 50.0' ) assert ( expected_query.casefold() @@ -151,10 +162,22 @@ def test_sampling_view_with_partition(self, sampler_mock): """ Test view sampling with partition """ + view_entity = Table( + id=uuid4(), + name="user", + columns=[ + EntityColumn( + name=ColumnName("id"), + dataType=DataType.INT, + ), + ], + tableType=TableType.View, + ) + sampler = BigQuerySampler( service_connection_config=self.bq_conn, ometa_client=None, - entity=self.table_entity, + entity=view_entity, sample_config=SampleConfig( profileSampleType=ProfileSampleType.PERCENTAGE, profileSample=50.0 ), @@ -168,9 +191,9 @@ def test_sampling_view_with_partition(self, sampler_mock): ) query: CTE = sampler.get_sample_query() expected_query = ( - "WITH users_rnd AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n" - "FROM users \nWHERE id in ('1', '2'))\n SELECT users_rnd.id, users_rnd.random \n" - "FROM users_rnd \nWHERE users_rnd.random <= 50.0" + 'WITH "9bc65c2abec141778ffaa729489f3e87_rnd" AS \n(SELECT users.id AS id, ABS(RANDOM()) * 100 %% 100 AS random \n' + "FROM users \nWHERE id in ('1', '2'))\n SELECT \"9bc65c2abec141778ffaa729489f3e87_rnd\".id, \"9bc65c2abec141778ffaa729489f3e87_rnd\".random \n" + 'FROM "9bc65c2abec141778ffaa729489f3e87_rnd" \nWHERE "9bc65c2abec141778ffaa729489f3e87_rnd".random <= 50.0' ) assert ( expected_query.casefold() diff --git a/ingestion/tests/unit/profiler/test_entity_fetcher.py b/ingestion/tests/unit/profiler/test_entity_fetcher.py new file mode 100644 index 000000000000..9e1d475cd661 --- /dev/null +++ b/ingestion/tests/unit/profiler/test_entity_fetcher.py @@ -0,0 +1,88 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Validate entity fetcher filtering strategies +""" +import uuid + +from metadata.generated.schema.entity.data.table import Table, TableType +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.metadataIngestion.databaseServiceAutoClassificationPipeline import ( + DatabaseServiceAutoClassificationPipeline, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, + Source, + SourceConfig, + WorkflowConfig, +) +from metadata.ingestion.api.status import Status +from metadata.profiler.source.fetcher.fetcher_strategy import DatabaseFetcherStrategy + +VIEW = Table( + id=uuid.uuid4(), + name="view", + columns=[], + tableType=TableType.View, +) + +TABLE = Table( + id=uuid.uuid4(), + name="table", + columns=[], + tableType=TableType.Regular, +) + + +def get_db_fetcher(source_config): + """Fetch database""" + workflow_config = OpenMetadataWorkflowConfig( + source=Source( + type="mysql", + serviceName="mysql", + sourceConfig=SourceConfig( + config=source_config, + ), + ), + workflowConfig=WorkflowConfig( + openMetadataServerConfig=OpenMetadataConnection( + hostPort="localhost:8585/api", + ) + ), + ) + return DatabaseFetcherStrategy( + config=workflow_config, + metadata=..., + global_profiler_config=..., + status=Status(), + ) + + +def test_include_views(): + """Validate we can include/exclude views""" + config = DatabaseServiceAutoClassificationPipeline( + includeViews=False, + ) + fetcher = get_db_fetcher(config) + + assert fetcher._filter_views(VIEW) + assert not fetcher._filter_views(TABLE) + + config = DatabaseServiceAutoClassificationPipeline( + includeViews=True, + ) + fetcher = get_db_fetcher(config) + + assert not fetcher._filter_views(VIEW) + assert not fetcher._filter_views(TABLE) diff --git a/ingestion/tests/unit/test_databricks_lineage.py b/ingestion/tests/unit/test_databricks_lineage.py index be2d4feb4af4..8647f63f3603 100644 --- a/ingestion/tests/unit/test_databricks_lineage.py +++ b/ingestion/tests/unit/test_databricks_lineage.py @@ -135,19 +135,3 @@ def __init__(self, methodName) -> None: mock_databricks_config["source"], config.workflowConfig.openMetadataServerConfig, ) - - @patch( - "metadata.ingestion.source.database.databricks.client.DatabricksClient.list_query_history" - ) - def test_get_table_query(self, list_query_history): - list_query_history.return_value = mock_data - results = self.databricks.get_table_query() - query_list = [] - for result in results: - if isinstance(result, TableQuery): - query_list.append(result) - for _, (expected, original) in enumerate( - zip(EXPECTED_DATABRICKS_DETAILS, query_list) - ): - expected.analysisDate = original.analysisDate = datetime.now() - self.assertEqual(expected, original) diff --git a/ingestion/tests/unit/topology/dashboard/test_looker.py b/ingestion/tests/unit/topology/dashboard/test_looker.py index 9acdc55111f8..5f0ca6b485b4 100644 --- a/ingestion/tests/unit/topology/dashboard/test_looker.py +++ b/ingestion/tests/unit/topology/dashboard/test_looker.py @@ -125,6 +125,13 @@ serviceType=DashboardServiceType.Looker, ) +EXPECTED_PARSED_VIEWS = { + "v1": "table1", + "v2": "select * from v2", + "v3": "select * from (select * from v2)", + "v4": "select * from (select * from (select * from v2)) inner join (table1)", +} + class LookerUnitTest(TestCase): """ @@ -560,3 +567,33 @@ def test_yield_dashboard_usage(self): self.assertIsNotNone( list(self.looker.yield_dashboard_usage(MOCK_LOOKER_DASHBOARD))[0].left ) + + def test_derived_view_references(self): + """ + Validate if we can find derived references in a SQL query + and replace them with their actual values + """ + # pylint: disable=protected-access + self.looker._parsed_views.update( + { + "v1": "table1", + "v2": "select * from v2", + } + ) + self.looker._unparsed_views.update( + { + "v3": "select * from ${v2.SQL_TABLE_NAME}", + "v4": "select * from ${v3.SQL_TABLE_NAME} inner join ${v1.SQL_TABLE_NAME}", + } + ) + self.looker._derived_dependencies.add_edges_from( + [ + ("v3", "v2"), + ("v4", "v3"), + ("v4", "v1"), + ] + ) + list(self.looker.build_lineage_for_unparsed_views()) + + self.assertEqual(self.looker._parsed_views, EXPECTED_PARSED_VIEWS) + self.assertEqual(self.looker._unparsed_views, {}) diff --git a/ingestion/tests/unit/topology/dashboard/test_superset.py b/ingestion/tests/unit/topology/dashboard/test_superset.py index b229095f4675..b8ea2c77c82a 100644 --- a/ingestion/tests/unit/topology/dashboard/test_superset.py +++ b/ingestion/tests/unit/topology/dashboard/test_superset.py @@ -18,6 +18,7 @@ from unittest import TestCase import sqlalchemy +from collate_sqllineage.core.models import Column, Schema, SubQuery, Table from testcontainers.core.generic import DockerContainer from testcontainers.postgres import PostgresContainer @@ -56,6 +57,7 @@ from metadata.generated.schema.type.entityReference import EntityReference from metadata.generated.schema.type.entityReferenceList import EntityReferenceList from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.lineage.parser import LineageParser from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.superset.api_source import SupersetAPISource from metadata.ingestion.source.dashboard.superset.db_source import SupersetDBSource @@ -170,7 +172,6 @@ owners=EXPECTED_USER, ) - EXPECTED_API_DASHBOARD = CreateDashboardRequest( name=EntityName("10"), displayName="Unicode Test", @@ -279,13 +280,33 @@ def setup_sample_data(postgres_container): id INTEGER PRIMARY KEY, table_name VARCHAR(255), schema VARCHAR(255), - database_id INTEGER + database_id INTEGER, + sql VARCHAR(4000) ); """ INSERT_TABLES_DATA = """ INSERT INTO tables(id, table_name, schema, database_id) VALUES (99, 'sample_table', 'main', 5); """ + CREATE_TABLE_COLUMNS_TABLE = """ + CREATE TABLE table_columns ( + id INTEGER PRIMARY KEY, + table_name VARCHAR(255), + table_id INTEGER, + column_name VARCHAR(255), + type VARCHAR(255), + description VARCHAR(255) + ); + """ + CREATE_TABLE_COLUMNS_DATA = """ + INSERT INTO + table_columns(id, table_name, table_id, column_name, type, description) + VALUES + (1099, 'sample_table', 99, 'id', 'VARCHAR', 'dummy description'), + (1199, 'sample_table', 99, 'timestamp', 'VARCHAR', 'dummy description'), + (1299, 'sample_table', 99, 'price', 'VARCHAR', 'dummy description'); + """ + connection.execute(sqlalchemy.text(CREATE_TABLE_AB_USER)) connection.execute(sqlalchemy.text(INSERT_AB_USER_DATA)) connection.execute(sqlalchemy.text(CREATE_TABLE_DASHBOARDS)) @@ -296,6 +317,8 @@ def setup_sample_data(postgres_container): connection.execute(sqlalchemy.text(INSERT_DBS_DATA)) connection.execute(sqlalchemy.text(CREATE_TABLES_TABLE)) connection.execute(sqlalchemy.text(INSERT_TABLES_DATA)) + connection.execute(sqlalchemy.text(CREATE_TABLE_COLUMNS_TABLE)) + connection.execute(sqlalchemy.text(CREATE_TABLE_COLUMNS_DATA)) INITIAL_SETUP = True @@ -616,3 +639,135 @@ def test_broken_column_type_in_datamodel(self): self.superset_db.prepare() parsed_datasource = self.superset_db.get_column_info(MOCK_DATASOURCE) assert parsed_datasource[0].dataType.value == "INT" + + def test_is_table_to_table_lineage(self): + table = Table(name="table_name", schema=Schema(name="schema_name")) + + for test_case in [ + ( + ( + Column(name="col_name"), + Table(name="table_name", schema=Schema(name="schema_name")), + Column(name="col_name"), + Table(name="dataset_name", schema=Schema(name="schema_name")), + ), + True, + ), + ( + ( + Column(name="col_name"), + Table(name="table_name", schema=Schema(name=Schema.unknown)), + Column(name="col_name"), + Table(name="dataset_name", schema=Schema(name="schema_name")), + ), + False, + ), + ( + ( + Column(name="col_name"), + Table(name="other_table_name", schema=Schema(name="schema_name")), + Column(name="col_name"), + Table(name="dataset_name", schema=Schema(name="schema_name")), + ), + False, + ), + ( + ( + Column(name="col_name"), + Table(name="table_name", schema=Schema(name="schema_name")), + Column(name="col_name"), + SubQuery( + subquery="select * from 1", + subquery_raw="select * from 1", + alias="dummy_subquery", + ), + ), + False, + ), + ]: + _columns, expected = test_case + + column_from, column_from_parent, column_to, column_to_parent = _columns + + column_from._parent.add(column_from_parent) + column_to._parent.add(column_to_parent) + + columns = (column_from, column_to) + self.assertEqual( + self.superset_db._is_table_to_table_lineage(columns, table), expected + ) + + def test_append_value_to_dict_list(self): + init_dict = {1: [2]} + + self.superset_db._append_value_to_dict_list(init_dict, 1, 3) + self.assertListEqual(init_dict[1], [2, 3]) + + self.superset_db._append_value_to_dict_list(init_dict, 2, 1) + self.assertListEqual(init_dict[2], [1]) + + def test_get_table_schema(self): + for test_case in [ + ( + Table(name="test_table", schema=Schema(name=Schema.unknown)), + FetchChart(schema="chart_table_schema"), + "chart_table_schema", + ), + ( + Table(name="test_table", schema=Schema(name="test_schema")), + FetchChart(schema="chart_table_schema"), + "test_schema", + ), + ]: + table, chart, expected = test_case + + self.assertEqual(self.superset_db._get_table_schema(table, chart), expected) + + def test_create_column_lineage_mapping_no_wildcard(self): + sql = """ + INSERT INTO dummy_table SELECT id, timestamp FROM input_table; + """ + + parser = LineageParser(sql) + table = Table(name="input_table", schema=Schema(name=Schema.unknown)) + chart = FetchChart(table_name="sample_table", table_schema="main", table_id=99) + + expected = {"id": ["id"], "timestamp": ["timestamp"]} + + self.assertDictEqual( + self.superset_db._create_column_lineage_mapping(parser, table, chart), + expected, + ) + + def test_create_column_lineage_mapping_with_wildcard(self): + sql = """ + INSERT INTO dummy_table SELECT * FROM input_table; + """ + + parser = LineageParser(sql) + table = Table(name="input_table", schema=Schema(name=Schema.unknown)) + chart = FetchChart(table_name="sample_table", table_schema="main", table_id=99) + + expected = {"id": ["id"], "timestamp": ["timestamp"], "price": ["price"]} + + self.assertDictEqual( + self.superset_db._create_column_lineage_mapping(parser, table, chart), + expected, + ) + + def test_get_input_tables_from_dataset_sql(self): + sql = """SELECT id, timestamp FROM sample_table""" + chart = FetchChart( + sql=sql, table_name="sample_table", table_schema="main", table_id=99 + ) + + result = self.superset_db._get_input_tables(chart)[0] + + self.assertSetEqual({"id", "timestamp"}, set(result[1])) + + def test_get_input_tables_when_table_has_no_sql(self): + chart = FetchChart(table_name="sample_table", table_schema="main", table_id=99) + + result = self.superset_db._get_input_tables(chart)[0] + + self.assertSetEqual({"id", "timestamp", "price"}, set(result[1])) diff --git a/ingestion/tests/utils/sqa.py b/ingestion/tests/utils/sqa.py index 19770c3f2969..73e370b7173c 100644 --- a/ingestion/tests/utils/sqa.py +++ b/ingestion/tests/utils/sqa.py @@ -18,6 +18,15 @@ class User(Base): age = Column(Integer) +class UserWithLongName(Base): + __tablename__ = "u" * 63 # Keep a max length name of 63 chars (max for Postgres) + id = Column(Integer, primary_key=True) + name = Column(String(256)) + fullname = Column(String(256)) + nickname = Column(String(256)) + age = Column(Integer) + + class SQATestUtils: def __init__(self, connection_url: str): self.connection_url = connection_url @@ -34,14 +43,16 @@ def load_data(self, data: Sequence[DeclarativeMeta]): self.session.commit() def load_user_data(self): - data = [ - User(name="John", fullname="John Doe", nickname="johnny b goode", age=30), # type: ignore - User(name="Jane", fullname="Jone Doe", nickname=None, age=31), # type: ignore - ] * 20 - self.load_data(data) + for clz in (User, UserWithLongName): + data = [ + clz(name="John", fullname="John Doe", nickname="johnny b goode", age=30), # type: ignore + clz(name="Jane", fullname="Jone Doe", nickname=None, age=31), # type: ignore + ] * 20 + self.load_data(data) def create_user_table(self): User.__table__.create(bind=self.session.get_bind()) + UserWithLongName.__table__.create(bind=self.session.get_bind()) def close(self): self.session.close() diff --git a/openmetadata-docs/content/partials/v1.6/connectors/dashboard/connectors-list.md b/openmetadata-docs/content/partials/v1.6/connectors/dashboard/connectors-list.md index 56bcab394e64..d91486345463 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/dashboard/connectors-list.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/dashboard/connectors-list.md @@ -7,12 +7,12 @@ {% connectorInfoCard name="MicroStrategy" stage="PROD" href="/connectors/dashboard/microstrategy" platform="OpenMetadata" / %} {% connectorInfoCard name="Mode" stage="PROD" href="/connectors/dashboard/mode" platform="OpenMetadata" / %} {% connectorInfoCard name="PowerBI" stage="PROD" href="/connectors/dashboard/powerbi" platform="OpenMetadata" / %} +{% connectorInfoCard name="PowerBI Report Server" stage="PROD" href="/connectors/dashboard/powerbireportserver" platform="Collate" / %} {% connectorInfoCard name="Qlik Sense" stage="PROD" href="/connectors/dashboard/qliksense" platform="OpenMetadata" / %} {% connectorInfoCard name="QuickSight" stage="PROD" href="/connectors/dashboard/quicksight" platform="OpenMetadata" / %} {% connectorInfoCard name="Redash" stage="PROD" href="/connectors/dashboard/redash" platform="OpenMetadata" / %} {% connectorInfoCard name="Superset" stage="PROD" href="/connectors/dashboard/superset" platform="OpenMetadata" / %} {% connectorInfoCard name="Sigma" stage="PROD" href="/connectors/dashboard/sigma" platform="OpenMetadata" / %} -{% connectorInfoCard name="PowerBI" stage="PROD" href="/connectors/dashboard/powerbi" platform="OpenMetadata" / %} {% connectorInfoCard name="Tableau" stage="PROD" href="/connectors/dashboard/tableau" platform="OpenMetadata" / %} {% /connectorsListContainer %} \ No newline at end of file diff --git a/openmetadata-docs/content/partials/v1.6/connectors/database/connectors-list.md b/openmetadata-docs/content/partials/v1.6/connectors/database/connectors-list.md index 2a8002e8ba7c..4791916897cc 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/database/connectors-list.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/database/connectors-list.md @@ -38,6 +38,7 @@ {% connectorInfoCard name="SingleStore" stage="PROD" href="/connectors/database/singlestore" platform="OpenMetadata" / %} {% connectorInfoCard name="Snowflake" stage="PROD" href="/connectors/database/snowflake" platform="OpenMetadata" / %} {% connectorInfoCard name="SQLite" stage="PROD" href="/connectors/database/sqlite" platform="OpenMetadata" / %} +{% connectorInfoCard name="Synapse" stage="PROD" href="/connectors/database/synapse" platform="Collate" / %} {% connectorInfoCard name="S3 Datalake" stage="PROD" href="/connectors/database/s3-datalake" platform="OpenMetadata" / %} {% connectorInfoCard name="Teradata" stage="PROD" href="/connectors/database/teradata" platform="OpenMetadata" / %} {% connectorInfoCard name="Trino" stage="PROD" href="/connectors/database/trino" platform="OpenMetadata" / %} diff --git a/openmetadata-docs/content/partials/v1.6/connectors/yaml/auto-classification.md b/openmetadata-docs/content/partials/v1.6/connectors/yaml/auto-classification.md index 42d7263a154a..25727a1f3121 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/yaml/auto-classification.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/yaml/auto-classification.md @@ -151,4 +151,8 @@ After saving the YAML config, we will run the command the same way we did for th metadata classify -c ``` -Note now instead of running `ingest`, we are using the `classify` command to select the Auto Classification workflow. +{% note %} + +Now instead of running `ingest`, we are using the `classify` command to select the Auto Classification workflow. + +{% /note %} diff --git a/openmetadata-docs/content/partials/v1.6/connectors/yaml/lineage.md b/openmetadata-docs/content/partials/v1.6/connectors/yaml/lineage.md index d695495b6f02..bb5e97cbafcf 100644 --- a/openmetadata-docs/content/partials/v1.6/connectors/yaml/lineage.md +++ b/openmetadata-docs/content/partials/v1.6/connectors/yaml/lineage.md @@ -3,7 +3,6 @@ After running a Metadata Ingestion workflow, we can run Lineage workflow. While the `serviceName` will be the same to that was used in Metadata Ingestion, so the ingestion bot can get the `serviceConnection` details from the server. - ### 1. Define the YAML Config This is a sample config for BigQuery Lineage: @@ -67,35 +66,35 @@ You can find all the definitions and types for the `sourceConfig` [here](https: {% /codeInfo %} -{% codeInfo srNumber=49 %} +{% codeInfo srNumber=51 %} **overrideViewLineage**: Set the 'Override View Lineage' toggle to control whether to override the existing view lineage. {% /codeInfo %} -{% codeInfo srNumber=51 %} +{% codeInfo srNumber=52 %} **processViewLineage**: Set the 'Process View Lineage' toggle to control whether to process view lineage. {% /codeInfo %} -{% codeInfo srNumber=52 %} +{% codeInfo srNumber=53 %} **processQueryLineage**: Set the 'Process Query Lineage' toggle to control whether to process query lineage. {% /codeInfo %} -{% codeInfo srNumber=53 %} +{% codeInfo srNumber=54 %} **processStoredProcedureLineage**: Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored procedure lineage. {% /codeInfo %} -{% codeInfo srNumber=54 %} +{% codeInfo srNumber=55 %} **threads**: Number of Threads to use in order to parallelize lineage ingestion. @@ -107,6 +106,7 @@ You can find all the definitions and types for the `sourceConfig` [here](https: #### Sink Configuration To send the metadata to OpenMetadata, it needs to be specified as `type: metadata-rest`. + {% /codeInfo %} @@ -178,13 +178,6 @@ source: # - table3 # - table4 ``` - -```yaml {% srNumber=49 %} -sink: - type: metadata-rest - config: {} -``` - ```yaml {% srNumber=51 %} overrideViewLineage: false ``` @@ -205,6 +198,12 @@ sink: threads: 1 ``` +```yaml {% srNumber=49 %} +sink: + type: metadata-rest + config: {} +``` + {% partial file="/v1.6/connectors/yaml/workflow-config.md" /%} {% /codeBlock %} diff --git a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md index 78def9dcdd72..a35feb19c047 100644 --- a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md @@ -46,6 +46,23 @@ You can refer to the following guide to get more details about the backup and re {% /inlineCallout %} {% /inlineCalloutContainer %} +## Understanding the "Running" State in OpenMetadata + +In OpenMetadata, the **"Running"** state indicates that the OpenMetadata server has received a response from Airflow confirming that a workflow is in progress. However, if Airflow unexpectedly stops or crashes before it can send a failure status update through the **Failure Callback**, OpenMetadata remains unaware of the workflow’s actual state. As a result, the workflow may appear to be stuck in **"Running"** even though it is no longer executing. + +This situation can also occur during an OpenMetadata upgrade. If an ingestion pipeline was running at the time of the upgrade and the process caused Airflow to shut down, OpenMetadata would not receive any further updates from Airflow. Consequently, the pipeline status remains **"Running"** indefinitely. + +{% image + src="/images/v1.6/deployment/upgrade/running-state-in-openmetadata.png" + alt="Running State in OpenMetadata" + caption="Running State in OpenMetadata" /%} + +### Expected Steps to Resolve +To resolve this issue: +- Ensure that Airflow is restarted properly after an unexpected shutdown. +- Manually update the pipeline status if necessary. +- Check Airflow logs to verify if the DAG execution was interrupted. + ### Update `sort_buffer_size` (MySQL) or `work_mem` (Postgres) Before running the migrations, it is important to update these parameters to ensure there are no runtime errors. @@ -86,6 +103,18 @@ After the migration is finished, you can revert this changes. # Backward Incompatible Changes +## 1.6.4 + +### Airflow 2.9.3 + +We are upgrading the Ingestion Airflow version to 2.9.3. + +The upgrade from the existing 2.9.1 -> 2.9.3 should happen transparently. The only thing to note is that there's +an ongoing issue with Airflow migrations and the `pymysql` driver, which we used before. If you are specifying +on your end the `DB_SCHEME` environment variable in the ingestion image, make sure it now is set to `mysql+mysqldb`. + +We have updated the default values accordingly. + ## 1.6.2 ### Executable Logical Test Suites diff --git a/openmetadata-docs/content/partials/v1.7/connectors/dashboard/connectors-list.md b/openmetadata-docs/content/partials/v1.7/connectors/dashboard/connectors-list.md index 56bcab394e64..d91486345463 100644 --- a/openmetadata-docs/content/partials/v1.7/connectors/dashboard/connectors-list.md +++ b/openmetadata-docs/content/partials/v1.7/connectors/dashboard/connectors-list.md @@ -7,12 +7,12 @@ {% connectorInfoCard name="MicroStrategy" stage="PROD" href="/connectors/dashboard/microstrategy" platform="OpenMetadata" / %} {% connectorInfoCard name="Mode" stage="PROD" href="/connectors/dashboard/mode" platform="OpenMetadata" / %} {% connectorInfoCard name="PowerBI" stage="PROD" href="/connectors/dashboard/powerbi" platform="OpenMetadata" / %} +{% connectorInfoCard name="PowerBI Report Server" stage="PROD" href="/connectors/dashboard/powerbireportserver" platform="Collate" / %} {% connectorInfoCard name="Qlik Sense" stage="PROD" href="/connectors/dashboard/qliksense" platform="OpenMetadata" / %} {% connectorInfoCard name="QuickSight" stage="PROD" href="/connectors/dashboard/quicksight" platform="OpenMetadata" / %} {% connectorInfoCard name="Redash" stage="PROD" href="/connectors/dashboard/redash" platform="OpenMetadata" / %} {% connectorInfoCard name="Superset" stage="PROD" href="/connectors/dashboard/superset" platform="OpenMetadata" / %} {% connectorInfoCard name="Sigma" stage="PROD" href="/connectors/dashboard/sigma" platform="OpenMetadata" / %} -{% connectorInfoCard name="PowerBI" stage="PROD" href="/connectors/dashboard/powerbi" platform="OpenMetadata" / %} {% connectorInfoCard name="Tableau" stage="PROD" href="/connectors/dashboard/tableau" platform="OpenMetadata" / %} {% /connectorsListContainer %} \ No newline at end of file diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/connectors-list.md b/openmetadata-docs/content/partials/v1.7/connectors/database/connectors-list.md index 738373a226bd..af311463fc4f 100644 --- a/openmetadata-docs/content/partials/v1.7/connectors/database/connectors-list.md +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/connectors-list.md @@ -40,6 +40,7 @@ {% connectorInfoCard name="SingleStore" stage="PROD" href="/connectors/database/singlestore" platform="OpenMetadata" / %} {% connectorInfoCard name="Snowflake" stage="PROD" href="/connectors/database/snowflake" platform="OpenMetadata" / %} {% connectorInfoCard name="SQLite" stage="PROD" href="/connectors/database/sqlite" platform="OpenMetadata" / %} +{% connectorInfoCard name="Synapse" stage="PROD" href="/connectors/database/synapse" platform="Collate" / %} {% connectorInfoCard name="S3 Datalake" stage="PROD" href="/connectors/database/s3-datalake" platform="OpenMetadata" / %} {% connectorInfoCard name="Teradata" stage="PROD" href="/connectors/database/teradata" platform="OpenMetadata" / %} {% connectorInfoCard name="Trino" stage="PROD" href="/connectors/database/trino" platform="OpenMetadata" / %} diff --git a/openmetadata-docs/content/partials/v1.7/connectors/metadata-ingestion-ui.md b/openmetadata-docs/content/partials/v1.7/connectors/metadata-ingestion-ui.md index 167b5adcbdc6..3ef0542280ca 100644 --- a/openmetadata-docs/content/partials/v1.7/connectors/metadata-ingestion-ui.md +++ b/openmetadata-docs/content/partials/v1.7/connectors/metadata-ingestion-ui.md @@ -3,6 +3,7 @@ {% step srNumber=1 %} {% stepDescription title="1. Visit the Services Page" %} +Click `Settings` in the side navigation bar and then `Services`. The first step is to ingest the metadata from your sources. To do that, you first need to create a Service connection first. diff --git a/openmetadata-docs/content/partials/v1.7/connectors/yaml/auto-classification.md b/openmetadata-docs/content/partials/v1.7/connectors/yaml/auto-classification.md index 42d7263a154a..25727a1f3121 100644 --- a/openmetadata-docs/content/partials/v1.7/connectors/yaml/auto-classification.md +++ b/openmetadata-docs/content/partials/v1.7/connectors/yaml/auto-classification.md @@ -151,4 +151,8 @@ After saving the YAML config, we will run the command the same way we did for th metadata classify -c ``` -Note now instead of running `ingest`, we are using the `classify` command to select the Auto Classification workflow. +{% note %} + +Now instead of running `ingest`, we are using the `classify` command to select the Auto Classification workflow. + +{% /note %} diff --git a/openmetadata-docs/content/partials/v1.7/connectors/yaml/lineage.md b/openmetadata-docs/content/partials/v1.7/connectors/yaml/lineage.md index 95ef73ee9f1c..a940b3c93d8f 100644 --- a/openmetadata-docs/content/partials/v1.7/connectors/yaml/lineage.md +++ b/openmetadata-docs/content/partials/v1.7/connectors/yaml/lineage.md @@ -3,7 +3,6 @@ After running a Metadata Ingestion workflow, we can run Lineage workflow. While the `serviceName` will be the same to that was used in Metadata Ingestion, so the ingestion bot can get the `serviceConnection` details from the server. - ### 1. Define the YAML Config This is a sample config for BigQuery Lineage: @@ -67,35 +66,35 @@ You can find all the definitions and types for the `sourceConfig` [here](https: {% /codeInfo %} -{% codeInfo srNumber=49 %} +{% codeInfo srNumber=51 %} **overrideViewLineage**: Set the 'Override View Lineage' toggle to control whether to override the existing view lineage. {% /codeInfo %} -{% codeInfo srNumber=51 %} +{% codeInfo srNumber=52 %} **processViewLineage**: Set the 'Process View Lineage' toggle to control whether to process view lineage. {% /codeInfo %} -{% codeInfo srNumber=52 %} +{% codeInfo srNumber=53 %} **processQueryLineage**: Set the 'Process Query Lineage' toggle to control whether to process query lineage. {% /codeInfo %} -{% codeInfo srNumber=53 %} +{% codeInfo srNumber=54 %} **processStoredProcedureLineage**: Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored procedure lineage. {% /codeInfo %} -{% codeInfo srNumber=54 %} +{% codeInfo srNumber=55 %} **threads**: Number of Threads to use in order to parallelize lineage ingestion. @@ -180,12 +179,6 @@ source: # - table4 ``` -```yaml {% srNumber=49 %} -sink: - type: metadata-rest - config: {} -``` - ```yaml {% srNumber=51 %} overrideViewLineage: false ``` @@ -206,6 +199,12 @@ sink: threads: 1 ``` +```yaml {% srNumber=49 %} +sink: + type: metadata-rest + config: {} +``` + {% partial file="/v1.7/connectors/yaml/workflow-config.md" /%} {% /codeBlock %} diff --git a/openmetadata-docs/content/partials/v1.7/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.7/deployment/upgrade/upgrade-prerequisites.md index 3004c74156d7..bcda1577075d 100644 --- a/openmetadata-docs/content/partials/v1.7/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.7/deployment/upgrade/upgrade-prerequisites.md @@ -46,6 +46,23 @@ You can refer to the following guide to get more details about the backup and re {% /inlineCallout %} {% /inlineCalloutContainer %} +## Understanding the "Running" State in OpenMetadata + +In OpenMetadata, the **"Running"** state indicates that the OpenMetadata server has received a response from Airflow confirming that a workflow is in progress. However, if Airflow unexpectedly stops or crashes before it can send a failure status update through the **Failure Callback**, OpenMetadata remains unaware of the workflow’s actual state. As a result, the workflow may appear to be stuck in **"Running"** even though it is no longer executing. + +This situation can also occur during an OpenMetadata upgrade. If an ingestion pipeline was running at the time of the upgrade and the process caused Airflow to shut down, OpenMetadata would not receive any further updates from Airflow. Consequently, the pipeline status remains **"Running"** indefinitely. + +{% image + src="/images/v1.7/deployment/upgrade/running-state-in-openmetadata.png" + alt="Running State in OpenMetadata" + caption="Running State in OpenMetadata" /%} + +### Expected Steps to Resolve +To resolve this issue: +- Ensure that Airflow is restarted properly after an unexpected shutdown. +- Manually update the pipeline status if necessary. +- Check Airflow logs to verify if the DAG execution was interrupted. + ### Update `sort_buffer_size` (MySQL) or `work_mem` (Postgres) Before running the migrations, it is important to update these parameters to ensure there are no runtime errors. diff --git a/openmetadata-docs/content/v1.5.x/getting-started/day-1/index.md b/openmetadata-docs/content/v1.5.x/getting-started/day-1/index.md index 46f809d3bf1a..167c7b44ae59 100644 --- a/openmetadata-docs/content/v1.5.x/getting-started/day-1/index.md +++ b/openmetadata-docs/content/v1.5.x/getting-started/day-1/index.md @@ -24,7 +24,7 @@ with links to more detailed documentation. ## Step 1: Set up a Data Connector Once you’re able to login to your Collate instance, set up a data connector to start bringing metadata into Collate. -There are [80+ turnkey connectors](/connectors) to various services: data warehouses, data lakes, databases, dashboards, +There are [90+ turnkey connectors](/connectors) to various services: data warehouses, data lakes, databases, dashboards, messaging services, pipelines, ML models, storage services, and other Metadata Services. Connections to [custom data sources](/connectors/custom-connectors) can also be created via API. diff --git a/openmetadata-docs/content/v1.5.x/getting-started/index.md b/openmetadata-docs/content/v1.5.x/getting-started/index.md index 4de859fbebf7..b3a0c941ecb6 100644 --- a/openmetadata-docs/content/v1.5.x/getting-started/index.md +++ b/openmetadata-docs/content/v1.5.x/getting-started/index.md @@ -7,7 +7,7 @@ collate: true # Getting Started Welcome to Collate's unified platform for data discovery, observability, and governance! Our platform centralizes all -the context around your data to help you build high-quality and AI assets. This guide gives you all the information you +the context around your data to help you build high-quality data and AI assets. This guide gives you all the information you need to set up your Collate environment in 30 minutes. ## How Does Collate Work? @@ -15,14 +15,14 @@ need to set up your Collate environment in 30 minutes. Collate is designed for both technical and non-technical data practitioners to work together across a broad set of use cases, including data discovery, lineage, observability, quality, collaboration, governance, and insights. -A library of 80+ turnkey connectors is available to easily ingest metadata into Collate, such as data warehouses, data lakes, +A library of 90+ turnkey connectors is available to easily ingest metadata into Collate, such as data warehouses, data lakes, streaming, dashboards, ML models, and more. APIs are also available to easily ingest metadata from custom data sources. Metadata from these different sources is organized into a Unified Metadata Graph, which provides a single, comprehensive source of truth across your entire data estate. This centralized information is surfaced through a unified user interface for all your use cases so that different data practitioners no longer need to switch between different data catalogs, quality, or governance tools. Additionally, -Collate can be extended through the application ecosystem, such as with AI productivity applications like MetaPilot, +Collate can be extended through the application ecosystem, such as with AI productivity applications like Collate AI, or with customer-built workflows to integrate Collate with your existing systems. These capabilities are built around native collaboration capabilities for shared workflows across different teams so that every data practitioner can work together: data platform, data governance, data scientist/analyst, and business user. @@ -31,7 +31,7 @@ together: data platform, data governance, data scientist/analyst, and business u Before we get started, here’s a quick summary of some of Collate’s main features: -- **Discovery**: integrated catalog, quality, and glossary; natural language search, filtering, and faceting, 80+ turnkey data connectors, and MetaPilot AI Chatbot. +- **Discovery**: integrated catalog, quality, and glossary; natural language search, filtering, and faceting, 90+ turnkey data connectors, and Collate AI Chatbot. - **Lineage**: table and column-level lineage, automated data estate mapping and APIs, lineage layers and search, governance and PII automation and manual customization. - **Observability**: alerting and notifications, incident management, third-party notifications, pipeline monitoring, root cause analysis, anomaly detection, data profiler. - **Quality**: table and column test cases, no-code and SQL data quality tests, test suites, test case reporting, quality dashboards, widgets and data quality lineage maps. diff --git a/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md index 5a4af7e59086..1dd42a667d1f 100644 --- a/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md +++ b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md @@ -7,10 +7,10 @@ slug: /quick-start/getting-started/day-1 Get started with your OpenMetadata service in a few simple steps: -1. Set up a Data Connector: Connect your data sources to begin collecting metadata. -2. Ingest Metadata: Run the metadata ingestion process to gather and push data insights. -3. Invite Users: Add team members to collaborate and manage metadata together. -4. Explore the Features: Dive into OpenMetadata's extensive feature set to unlock the full potential of your data. +1. **Set up a Data Connector**: Connect your data sources to begin collecting metadata. +2. **Ingest Metadata**: Run the metadata ingestion process to gather and push data insights. +3. **Invite Users**: Add team members to collaborate and manage metadata together. +4. **Explore the Features**: Dive into OpenMetadata's extensive feature set to unlock the full potential of your data. **Ready to begin? Let's get started!** @@ -20,7 +20,7 @@ You should receive your initial OpenMetadata credentials from OpenMetadata suppo ## Step 1: Set up a Data Connector -Once you have logged into your OpenMetadata instance, set up a data connector to start ingesting metadata. OpenMetadata provides [80+ turnkey connectors](/connectors) for a wide range of services, including: +Once you have logged into your OpenMetadata instance, set up a data connector to start ingesting metadata. OpenMetadata provides [90+ turnkey connectors](/connectors) for a wide range of services, including: - Databases - Dashboards diff --git a/openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md index 778794a85fd3..7130d1987f6a 100644 --- a/openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md +++ b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md @@ -11,9 +11,9 @@ Welcome to OpenMetadata's unified platform for data discovery, observability, an OpenMetadata is designed to support both technical and non-technical data practitioners across various use cases, including data discovery, lineage, observability, quality, collaboration, governance, and insights. -The platform includes a library of 80+ turnkey connectors to easily ingest metadata from sources such as data warehouses, data lakes, streaming platforms, dashboards, and ML models. For custom data sources, APIs are available to streamline metadata ingestion. Metadata from these sources is organized into a Unified Metadata Graph, providing a single, comprehensive source of truth for your entire data estate. +The platform includes a library of 90+ turnkey connectors to easily ingest metadata from sources such as data warehouses, data lakes, streaming platforms, dashboards, and ML models. For custom data sources, APIs are available to streamline metadata ingestion. Metadata from these sources is organized into a Unified Metadata Graph, providing a single, comprehensive source of truth for your entire data estate. -This centralized metadata is accessible through a unified user interface, eliminating the need for practitioners to switch between multiple catalogs, quality, or governance tools. OpenMetadata can also be extended with applications, such as AI-driven productivity tools like MetaPilot, or through custom-built workflows that integrate the platform with existing systems. +This centralized metadata is accessible through a unified user interface, eliminating the need for practitioners to switch between multiple catalogs, quality, or governance tools. OpenMetadata can also be extended with applications, such as AI-driven productivity tools like Collate AI, or through custom-built workflows that integrate the platform with existing systems. The platform’s native collaboration features support shared workflows, enabling different teams—data platform engineers, governance professionals, data scientists/analysts, and business users—to collaborate effectively in a single environment. ## Key Features of OpenMetadata @@ -23,7 +23,7 @@ Before we get started, here’s a quick summary of some of OpenMetadata’s main ### Discovery - Integrated catalog, data quality, and glossary - Natural language search, filtering, and faceting -- 80+ turnkey data connectors +- 90+ turnkey data connectors ### Lineage - Table and column-level lineage diff --git a/openmetadata-docs/content/v1.5.x/releases/releases/index.md b/openmetadata-docs/content/v1.5.x/releases/releases/index.md index 3320ea48721d..f1f8b529900e 100644 --- a/openmetadata-docs/content/v1.5.x/releases/releases/index.md +++ b/openmetadata-docs/content/v1.5.x/releases/releases/index.md @@ -8,7 +8,7 @@ slug: /releases/all-releases {% note %} The OpenMetadata community is on a monthly release cadence. At every 4-5 weeks we will be releasing a new -version. To see what's coming in next releases, please check our [Roadmap](/releases/roadmap) section. +version. To see what's coming in next releases, please check our {% collateContent %}[Roadmap](https://www.getcollate.io/roadmap){% /collateContent %}{% ossContent %}[Roadmap](/roadmap){% /ossContent %} section. {% /note %} @@ -542,7 +542,7 @@ To continue pursuing this objective, the application was completely refactored t ## Ingestion Connectors -80+ connectors to help teams to centralize metadata. We continue to push the boundaries of this mission, in +90+ connectors to help teams to centralize metadata. We continue to push the boundaries of this mission, in - **Apache Flink** as a Pipeline Connector - **SAP ERP**, after a long and successful collaboration with our community and SAP experts diff --git a/openmetadata-docs/content/v1.6.x/collate-menu.md b/openmetadata-docs/content/v1.6.x/collate-menu.md index ff9212d46b90..29bddb700981 100644 --- a/openmetadata-docs/content/v1.6.x/collate-menu.md +++ b/openmetadata-docs/content/v1.6.x/collate-menu.md @@ -216,6 +216,10 @@ site_menu: url: /connectors/database/s3-datalake/yaml - category: Connectors / Database / S3 Datalake / Troubleshooting url: /connectors/database/s3-datalake/troubleshooting + - category: Connectors / Database / Teradata + url: /connectors/database/teradata + - category: Connectors / Database / Teradata / Run Externally + url: /connectors/database/teradata/yaml - category: Connectors / Database / Trino url: /connectors/database/trino - category: Connectors / Database / Trino / Run Externally @@ -626,6 +630,8 @@ site_menu: url: /how-to-guides/data-discovery/details - category: How-to Guides / Data Discovery / Add Complex Queries using Advanced Search url: /how-to-guides/data-discovery/advanced + - category: How-to Guides / Data Discovery / Troubleshooting + url: /how-to-guides/data-discovery/troubleshooting - category: How-to Guides / Data Discovery / Bulk Upload Data Assets url: /how-to-guides/data-discovery/bulk-upload - category: How-to Guides / Data Discovery / How to Bulk Import Data Asset diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/db2/index.md b/openmetadata-docs/content/v1.6.x/connectors/database/db2/index.md index ed64e43dd0c0..affc9b5ca3ea 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/database/db2/index.md +++ b/openmetadata-docs/content/v1.6.x/connectors/database/db2/index.md @@ -98,6 +98,21 @@ If you are using DB2 for IBM i: - In Host and Port you should not add the Port Number. {% /note %} +{% note %} +If you have a **db2jcc_license_cisuz.jar** file, it will not work with **ibm_db**. This file is a **Db2 Connect** license for the Java Driver. +For **non-Java drivers**, such as the Python Client used in OpenMetadata ingestion, a **Db2 Connect** client-side license is required, typically named **db2con*.lic**. + +The **db2jcc_license_cisuz.jar** is specifically for Java-based clients, whereas OpenMetadata ingestion operates with a Python Client, making the `.jar` file incompatible. + +For activating a **non-Java license** for Db2 Connect **Application Server Edition**, **Advanced Application Server Edition**, **Enterprise Edition**, or **Trial**, follow these steps: +- Download the **license activation kit** from IBM Passport Advantage: [IBM PPA](https://www.ibm.com/software/passportadvantage/pao_customer.html). +- Unzip the package and locate the **non-Java license file** (e.g., `db2consv_ee.lic`). +- Apply the `.lic` file to activate the license. + +For further reference, check this IBM post: [Everything About Db2 Connect Licensing](https://community.ibm.com/community/user/datamanagement/blogs/shilu-mathai2/2023/05/05/everything-about-db2-connect-licensing). + +{% /note %} + {% partial file="/v1.6/connectors/database/advanced-configuration.md" /%} {% /extraContent %} diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/mongodb/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/mongodb/yaml.md index 660fbc964fd6..05fa8a104147 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/database/mongodb/yaml.md +++ b/openmetadata-docs/content/v1.6.x/connectors/database/mongodb/yaml.md @@ -351,7 +351,7 @@ Here we are also importing all the basic requirements to parse YAMLs, handle dat import yaml from datetime import timedelta from airflow import DAG -from metadata.profiler.api.workflow import ProfilerWorkflow +from metadata.workflow.profiler import ProfilerWorkflow try: from airflow.operators.python import PythonOperator diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/mysql/yaml.md index d5225a37c150..eea7d6f4b08d 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/database/mysql/yaml.md +++ b/openmetadata-docs/content/v1.6.x/connectors/database/mysql/yaml.md @@ -191,8 +191,8 @@ For a simple, local installation using our docker containers, this looks like: ```yaml {% srNumber=40 %} source: - type: mssql-lineage - serviceName: local_mssql + type: mysql-lineage + serviceName: local_mysql sourceConfig: config: type: DatabaseLineage @@ -243,12 +243,6 @@ source: # - table4 ``` -```yaml {% srNumber=49 %} -sink: - type: metadata-rest - config: {} -``` - ```yaml {% srNumber=51 %} overrideViewLineage: false ``` @@ -269,6 +263,12 @@ sink: threads: 1 ``` +```yaml {% srNumber=49 %} +sink: + type: metadata-rest + config: {} +``` + {% partial file="/v1.6/connectors/yaml/workflow-config.md" /%} {% /codeBlock %} diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/index.md index c6f7ab738035..408b6b0b12ef 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/index.md @@ -114,6 +114,9 @@ You can find more information about the `account_usage` schema [here](https://do - **Include Temporary and Transient Tables**: Optional configuration for ingestion of `TRANSIENT` and `TEMPORARY` tables, By default, it will skip the `TRANSIENT` and `TEMPORARY` tables. - **Client Session Keep Alive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. +- **Account Usage Schema Name**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. {% partial file="/v1.6/connectors/database/advanced-configuration.md" /%} diff --git a/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/yaml.md b/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/yaml.md index 8a824e07928b..bd10edc7097b 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/yaml.md +++ b/openmetadata-docs/content/v1.6.x/connectors/database/snowflake/yaml.md @@ -150,6 +150,14 @@ This is a sample config for Snowflake: {% /codeInfo %} +{% codeInfo srNumber=40 %} + +**accountUsageSchema**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. + +{% /codeInfo %} + {% codeInfo srNumber=6 %} **includeTransientTables**: Optional configuration for ingestion of TRANSIENT and TEMPORARY tables, By default, it will skip the TRANSIENT and TEMPORARY tables. @@ -231,6 +239,9 @@ source: ```yaml {% srNumber=5 %} # database: ``` +```yaml {% srNumber=40 %} + # accountUsageSchema: SNOWFLAKE.ACCOUNT_USAGE +``` ```yaml {% srNumber=6 %} includeTransientTables: false ``` diff --git a/openmetadata-docs/content/v1.6.x/connectors/pipeline/matillion/index.md b/openmetadata-docs/content/v1.6.x/connectors/pipeline/matillion/index.md index 971b25269adf..9906fe618fa6 100644 --- a/openmetadata-docs/content/v1.6.x/connectors/pipeline/matillion/index.md +++ b/openmetadata-docs/content/v1.6.x/connectors/pipeline/matillion/index.md @@ -30,6 +30,8 @@ Configure and schedule Matillion metadata and profiler workflows from the OpenMe To extract metadata from Matillion, you need to create a user with the following permissions: - `API` Permission ( While Creating the User, from Admin -> User ) +- To retrieve lineage data, the user must be granted [Component-level permissions](https://docs.matillion.com/metl/docs/2932106/#component). +- To enable lineage tracking in Matillion, **Matillion Enterprise Mode** is required. For detailed setup instructions and further information, refer to the official documentation: [Matillion Lineage Documentation](https://docs.matillion.com/metl/docs/2881895/). ### Matillion Versions diff --git a/openmetadata-docs/content/v1.6.x/deployment/oss-security.md b/openmetadata-docs/content/v1.6.x/deployment/oss-security.md new file mode 100644 index 000000000000..d0bd56e5c4e5 --- /dev/null +++ b/openmetadata-docs/content/v1.6.x/deployment/oss-security.md @@ -0,0 +1,44 @@ +--- +title: OSS Security Best Practices +slug: /deployment/oss-security +collate: false +--- + +# OSS Security + +## Encryption of Connection Credentials + +OpenMetadata ensures that sensitive information, such as passwords and connection secrets, is securely stored. + +- **Encryption Algorithm**: OpenMetadata uses **Fernet encryption** to encrypt secrets and passwords before storing them in the database. +- **Fernet Encryption Details**: + - Uses **AES-128 in CBC mode** with a strong key-based approach. + - **Not based on hashing or salting**, but rather an encryption/decryption method with a symmetric key. +- **Secrets Manager Support**: + - Users can **avoid storing credentials** in OpenMetadata by configuring an external **Secrets Manager**. + - More details on setting up a Secrets Manager can be found here: + 🔗 [Secrets Manager Documentation](https://docs.open-metadata.org/latest/deployment/secrets-manager) + +## Secure Connections to Data Sources + +OpenMetadata supports **encrypted connections** to various databases and services. + +- **SSL/TLS Support**: + - OpenMetadata allows users to configure **SSL/TLS encryption** for secure data transmission. + - Users can specify **SSL modes** and provide **CA certificates** for SSL validation. +- **How to Enable SSL?** + - Each connector supports different SSL configurations. + - Follow the detailed guide for enabling SSL in OpenMetadata: + 🔗 [Enable SSL in OpenMetadata](https://docs.open-metadata.org/latest/deployment/security/enable-ssl) + +## **Additional Security Measures** + +- **Role-Based Access Control (RBAC)**: OpenMetadata allows administrators to define user roles and permissions. +- **Authentication & Authorization**: OpenMetadata supports integration with OAuth, SAML, and LDAP for secure authentication. +- **Data Access Control**: Users can restrict access to metadata based on policies and governance rules. + +{% note %} +- **Passwords and secrets are securely encrypted** using **Fernet encryption**. +- **Connections to data sources can be encrypted** using **SSL/TLS**. +- **Secrets Managers** can be used to manage credentials externally. +{% /note %} diff --git a/openmetadata-docs/content/v1.6.x/getting-started/day-1/index.md b/openmetadata-docs/content/v1.6.x/getting-started/day-1/index.md index d423c6889b15..441f6ebb025f 100644 --- a/openmetadata-docs/content/v1.6.x/getting-started/day-1/index.md +++ b/openmetadata-docs/content/v1.6.x/getting-started/day-1/index.md @@ -8,10 +8,10 @@ collate: true Get started with your Collate service in just few simple steps: -1. Set up a Data Connector: Connect your data sources to begin collecting metadata. -2. Ingest Metadata: Run the metadata ingestion to gather and push data insights. -3. Invite Users: Add team members to collaborate and manage metadata together. -4. Explore the Features: Dive into Collate's rich feature set to unlock the full potential of your data. +1. **Set up a Data Connector**: Connect your data sources to begin collecting metadata. +2. **Ingest Metadata**: Run the metadata ingestion to gather and push data insights. +3. **Invite Users**: Add team members to collaborate and manage metadata together. +4. **Explore the Features**: Dive into Collate's rich feature set to unlock the full potential of your data. **Ready to begin? Let's get started!** @@ -24,7 +24,7 @@ with links to more detailed documentation. ## Step 1: Set up a Data Connector Once you’re able to login to your Collate instance, set up a data connector to start bringing metadata into Collate. -There are [80+ turnkey connectors](/connectors) to various services: data warehouses, data lakes, databases, dashboards, +There are [90+ turnkey connectors](/connectors) to various services: data warehouses, data lakes, databases, dashboards, messaging services, pipelines, ML models, storage services, and other Metadata Services. Connections to [custom data sources](/connectors/custom-connectors) can also be created via API. diff --git a/openmetadata-docs/content/v1.6.x/getting-started/index.md b/openmetadata-docs/content/v1.6.x/getting-started/index.md index 4de859fbebf7..065744ff02ef 100644 --- a/openmetadata-docs/content/v1.6.x/getting-started/index.md +++ b/openmetadata-docs/content/v1.6.x/getting-started/index.md @@ -7,7 +7,7 @@ collate: true # Getting Started Welcome to Collate's unified platform for data discovery, observability, and governance! Our platform centralizes all -the context around your data to help you build high-quality and AI assets. This guide gives you all the information you +the context around your data to help you build high-quality data and AI assets. This guide gives you all the information you need to set up your Collate environment in 30 minutes. ## How Does Collate Work? @@ -15,7 +15,7 @@ need to set up your Collate environment in 30 minutes. Collate is designed for both technical and non-technical data practitioners to work together across a broad set of use cases, including data discovery, lineage, observability, quality, collaboration, governance, and insights. -A library of 80+ turnkey connectors is available to easily ingest metadata into Collate, such as data warehouses, data lakes, +A library of 90+ turnkey connectors is available to easily ingest metadata into Collate, such as data warehouses, data lakes, streaming, dashboards, ML models, and more. APIs are also available to easily ingest metadata from custom data sources. Metadata from these different sources is organized into a Unified Metadata Graph, which provides a single, comprehensive source of truth across your entire data estate. @@ -31,7 +31,7 @@ together: data platform, data governance, data scientist/analyst, and business u Before we get started, here’s a quick summary of some of Collate’s main features: -- **Discovery**: integrated catalog, quality, and glossary; natural language search, filtering, and faceting, 80+ turnkey data connectors, and MetaPilot AI Chatbot. +- **Discovery**: integrated catalog, quality, and glossary; natural language search, filtering, and faceting, 90+ turnkey data connectors, and MetaPilot AI Chatbot. - **Lineage**: table and column-level lineage, automated data estate mapping and APIs, lineage layers and search, governance and PII automation and manual customization. - **Observability**: alerting and notifications, incident management, third-party notifications, pipeline monitoring, root cause analysis, anomaly detection, data profiler. - **Quality**: table and column test cases, no-code and SQL data quality tests, test suites, test case reporting, quality dashboards, widgets and data quality lineage maps. diff --git a/openmetadata-docs/content/v1.6.x/how-to-guides/admin-guide/roles-policies/authorization.md b/openmetadata-docs/content/v1.6.x/how-to-guides/admin-guide/roles-policies/authorization.md index fd765e58c3dc..73a24e20e89e 100644 --- a/openmetadata-docs/content/v1.6.x/how-to-guides/admin-guide/roles-policies/authorization.md +++ b/openmetadata-docs/content/v1.6.x/how-to-guides/admin-guide/roles-policies/authorization.md @@ -34,6 +34,7 @@ Here are some examples of conditions. | **matchAllTags(tagFqn, [tagFqn…])** | Returns true if the resource has all the tags from the tag list. | | **matchAnyTag(tagFqn, [tagFqn…])** | Returns true if the resource has any of the tags from the tag list. | | **matchTeam()** | Returns true if the user belongs to the team that owns the resource. | +| **hasDomain()** | Returns true if the logged in user is the has domain access of the entity being accessed | Conditions are used to assess DataAsset like Tables/Topics/Dashboards etc.. for specific attributes. diff --git a/openmetadata-docs/content/v1.6.x/how-to-guides/data-discovery/troubleshooting.md b/openmetadata-docs/content/v1.6.x/how-to-guides/data-discovery/troubleshooting.md new file mode 100644 index 000000000000..5122d4e01d32 --- /dev/null +++ b/openmetadata-docs/content/v1.6.x/how-to-guides/data-discovery/troubleshooting.md @@ -0,0 +1,42 @@ +--- +title: Troubleshooting for Export issue +slug: /how-to-guides/data-discovery/troubleshooting +--- + +# Troubleshooting Export Issue +When attempting to export a **CSV file for a Glossary**, the process gets stuck on the message **"Export initiated successfully."** and never completes. The file is not downloaded, and the export button remains disabled. + +This issue may occur if **WebSockets are blocked** in your network setup due to a **proxy** or **load balancer** configuration. OpenMetadata relies on WebSockets for real-time communication, and if they are blocked, the export process cannot complete. + +## Troubleshooting Steps + +### Step 1: Check for Load Balancer or Proxy + +If your setup includes a **load balancer** or **proxy**, verify whether WebSockets are being blocked. + +1. Run the following API request to check the export status: + +```bash +curl -X GET "https:///api/v1/glossaries/name//exportAsync" +``` + +If the response does not return a file and remains in an active state indefinitely, WebSockets might be blocked. + +### Step 2: Verify WebSocket Connectivity + +1. Open the Developer Tools in your browser (F12 or Ctrl + Shift + I in Chrome). +2. Navigate to the Network tab. +3. Filter requests by WebSockets (WS). +4. Check if WebSocket requests to OpenMetadata (wss://) are blocked, failing, or not established. + +### Step 3: Adjust WebSocket Settings in Your Proxy + +If WebSockets are blocked, update your proxy configuration to allow WebSocket traffic. + +### Step 4: Restart Services and Verify + +1. Restart your proxy or load balancer after making the configuration changes. +2. Clear browser cache and cookies. +3. Retry the CSV export in OpenMetadata. + +Once WebSockets are enabled in the proxy settings, the glossary export should complete successfully, and the CSV file should be available for download. diff --git a/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/automation/index.md b/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/automation/index.md index 7409a360ff06..6f047c6242cc 100644 --- a/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/automation/index.md +++ b/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/automation/index.md @@ -23,7 +23,23 @@ Managing metadata manually can be challenging, particularly in dynamic environme ## Key Use Cases for Collate Automations -### 1. Bulk Ownership and Domain Assignment +### 1. Bulk Description + +{% image +src="/images/v1.6/how-to-guides/governance/automator-description.png" +alt="Getting started with Automation" +caption="Getting started with Automation" +/%} + +- **Problem**: Many datasets lack descriptions, making it difficult for users to understand the data's purpose and contents. Sometimes, the same column description needs to be added to multiple datasets. +- **Solution**: Automations can bulk-apply descriptions to tables and columns, ensuring that all data assets are consistently documented. +- **Benefit**: This use case improves data discoverability and understanding, making it easier for users to find and use the data effectively. + +For the Action Configuration: +- **Apply to Children**: Lets you apply the description to the selected child assets (e.g., columns) within an asset. +- **Overwrite Metadata**: Allows you to overwrite existing descriptions with the new description. Otherwise, we will only apply the description to empty tables or columns. + +### 2. Bulk Ownership and Domain Assignment {% image src="/images/v1.6/how-to-guides/governance/bulk-ownership-and.png" @@ -35,7 +51,10 @@ caption="Getting started with Automation" - **Solution**: Automations can bulk-assign ownership and domains to datasets, ensuring all data assets are correctly categorized and owned. This process can be applied to tables, schemas, or other assets within Collate. - **Benefit**: This use case ensures data assets have a designated owner and are organized under the appropriate domain, making data more discoverable and accountable. -### 2. Bulk Tagging and Glossary Term Assignment +For the Action Configuration: +- **Overwrite Metadata**: Allows you to overwrite existing owner or domain with the configured one. Otherwise, we will only apply the owner or domain to assets that do not have an existing owner or domain. + +### 3. Bulk Tagging and Glossary Term Assignment {% image src="/images/v1.6/how-to-guides/governance/bulk-tagging-glossary.png" @@ -47,7 +66,12 @@ caption="Getting started with Automation" - **Solution**: Automations allow users to bulk-apply tags (e.g., PII) or glossary terms (e.g., Customer ID) to specific datasets, ensuring uniformity across the platform. - **Benefit**: This automation reduces the risk of missing important tags like PII-sensitive and ensures that key metadata elements are applied consistently across datasets. -### 3. Metadata Propagation via Lineage +For the Action Configuration: +- **Apply to Children**: Lets you apply the Tags or Glossary Terms to the selected child assets (e.g., columns) within an asset. +- **Overwrite Metadata**: Allows you to overwrite existing Tags or Terms with the configured one. Otherwise, we will add the new Tags or Terms to the existing ones. + + +### 4. Metadata Propagation via Lineage {% image src="/images/v1.6/how-to-guides/governance/metadata-propogation.png" @@ -59,7 +83,19 @@ caption="Getting started with Automation" - **Solution**: Use automations to propagate metadata across related datasets, ensuring that all relevant data inherits the correct metadata properties from the source dataset. - **Benefit**: Metadata consistency is ensured across the entire data lineage, reducing the need for manual updates and maintaining a single source of truth. -### 4. Automatic PII Detection and Tagging +For the Action Configuration: +1. First, we can choose if we want the propagation to happen at the Parent level (e.g., Table), Column Level, or both. This can be configured by selecting **Propagate Parent** and/or **Propagate Column Level**. +2. Then, we can control which pieces of metadata we want to propagate via lineage: + - **Propagate Description**: Propagates the description from the source asset to the downstream assets. Works for both parent and column-level. + - **Propagate Tags**: Propagates the tags from the source asset to the downstream assets. Works for both parent and column-level. + - **Propagate Glossary Terms**: Propagates the glossary terms from the source asset to the downstream assets. Works for both parent and column-level. + - **Propagate Owners**: Only applicable for Parent assets. Propagates the owner information to downstream assets. + - **Propagate Tier**: Only applicable for Parent assets. Propagated the tier information to downstream assets. + +As with other actions, you can choose to **Overwrite Metadata** or keep the existing metadata and only apply the new metadata to assets that do not have the metadata already. + + +### 5. Automatic PII Detection and Tagging {% image src="/images/v1.6/how-to-guides/governance/automatic-detection.png" @@ -67,6 +103,15 @@ alt="Getting started with Automation" caption="Getting started with Automation" /%} +{% note noteType="Warning" %} + +Note that we recommend using the **Auto Classification** workflow instead, which allows you to discover PII data automatically, +even in cases where you don't want to ingest the Sample Data into Collate. + +Note that this automation, the ML Tagging, will be deprecated in future releases. + +{% /note %} + - **Problem**: Manually identifying and tagging Personally Identifiable Information (PII) across large datasets is labor-intensive and prone to errors. - **Solution**: Automations can automatically detect PII data (e.g., emails, usernames) and apply relevant tags to ensure that sensitive data is flagged appropriately for compliance. - **Benefit**: Ensures compliance with data protection regulations by consistently tagging sensitive data, reducing the risk of non-compliance. diff --git a/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md b/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md index 9bf30da68f38..c691a6062151 100644 --- a/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md +++ b/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md @@ -7,11 +7,6 @@ slug: /how-to-guides/data-governance/classification/auto/auto-pii-tagging Auto PII tagging for Sensitive/NonSensitive at the column level is performed based on the two approaches described below. -{% note %} -PII Tagging is only available during `Profiler Ingestion`. -{% /note %} - - ## Tagging logic 1. **Column Name Scanner**: We validate the column names of the table against a set of regex rules that help us identify diff --git a/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md b/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md index 2d82baac5abe..6c5c464f270b 100644 --- a/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md +++ b/openmetadata-docs/content/v1.6.x/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md @@ -42,6 +42,19 @@ The Auto Classification Workflow enables automatic tagging of sensitive informat - When set to `true`, filtering patterns will be applied to the Fully Qualified Name of a table (e.g., `service_name.db_name.schema_name.table_name`). - When set to `false`, filtering applies only to raw table names. +## Auto Classification Workflow Execution + +To execute the **Auto Classification Workflow**, follow the steps below: + +### 1. Install the Required Python Package +Ensure you have the correct OpenMetadata ingestion package installed, including the **PII Processor** module: + +```bash +pip install "openmetadata-ingestion[pii-processor]" +``` +## 2. Define and Execute the Python Workflow +Instead of using a YAML configuration, use the AutoClassificationWorkflow from OpenMetadata to trigger the ingestion process programmatically. + ## Sample Auto Classification Workflow yaml ```yaml @@ -103,6 +116,14 @@ workflowConfig: jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" ``` +### 3. Expected Outcome +- Automatically classifies and tags sensitive data based on predefined patterns and confidence levels. +- Improves metadata enrichment and enhances data governance practices. +- Provides visibility into sensitive data across databases. +This approach ensures that the Auto Classification Workflow is executed correctly using the appropriate OpenMetadata ingestion framework. + +{% partial file="/v1.6/connectors/yaml/auto-classification.md" variables={connector: "snowflake"} /%} + ## Workflow Execution ### To Execute the Auto Classification Workflow: diff --git a/openmetadata-docs/content/v1.6.x/menu.md b/openmetadata-docs/content/v1.6.x/menu.md index a13449f2717a..b020a1eae5dd 100644 --- a/openmetadata-docs/content/v1.6.x/menu.md +++ b/openmetadata-docs/content/v1.6.x/menu.md @@ -223,6 +223,9 @@ site_menu: - category: Deployment / Metrics url: /deployment/metrics + - category: Deployment / OSS Security + url: /deployment/oss-security + - category: Connectors url: /connectors @@ -799,6 +802,8 @@ site_menu: url: /how-to-guides/data-discovery/details - category: How-to Guides / Data Discovery / Add Complex Queries using Advanced Search url: /how-to-guides/data-discovery/advanced + - category: How-to Guides / Data Discovery / Troubleshooting + url: /how-to-guides/data-discovery/troubleshooting - category: How-to Guides / Data Discovery / OpenMetadata Chrome Extension url: /how-to-guides/data-discovery/openmetadata-extension - category: How-to Guides / Data Discovery / OpenMetadata Chrome Extension / Viewing Activity Feeds and Managing Tasks diff --git a/openmetadata-docs/content/v1.6.x/quick-start/getting-started/day-1/index.md b/openmetadata-docs/content/v1.6.x/quick-start/getting-started/day-1/index.md index 6bc58494e729..14e6212205cd 100644 --- a/openmetadata-docs/content/v1.6.x/quick-start/getting-started/day-1/index.md +++ b/openmetadata-docs/content/v1.6.x/quick-start/getting-started/day-1/index.md @@ -7,10 +7,10 @@ slug: /quick-start/getting-started/day-1 Get started with your OpenMetadata service in a few simple steps: -1. Set up a Data Connector: Connect your data sources to begin collecting metadata. -2. Ingest Metadata: Run the metadata ingestion process to gather and push data insights. -3. Invite Users: Add team members to collaborate and manage metadata together. -4. Explore the Features: Dive into OpenMetadata's extensive feature set to unlock the full potential of your data. +1. **Set up a Data Connector**: Connect your data sources to begin collecting metadata. +2. **Ingest Metadata**: Run the metadata ingestion process to gather and push data insights. +3. **Invite Users**: Add team members to collaborate and manage metadata together. +4. **Explore the Features**: Dive into OpenMetadata's extensive feature set to unlock the full potential of your data. **Ready to begin? Let's get started!** @@ -20,7 +20,7 @@ You should receive your initial OpenMetadata credentials from OpenMetadata suppo ## Step 1: Set up a Data Connector -Once you have logged into your OpenMetadata instance, set up a data connector to start ingesting metadata. OpenMetadata provides [80+ turnkey connectors](/connectors) for a wide range of services, including: +Once you have logged into your OpenMetadata instance, set up a data connector to start ingesting metadata. OpenMetadata provides [90+ turnkey connectors](/connectors) for a wide range of services, including: - Databases - Dashboards diff --git a/openmetadata-docs/content/v1.6.x/quick-start/getting-started/index.md b/openmetadata-docs/content/v1.6.x/quick-start/getting-started/index.md index 778794a85fd3..fb8b26990b6b 100644 --- a/openmetadata-docs/content/v1.6.x/quick-start/getting-started/index.md +++ b/openmetadata-docs/content/v1.6.x/quick-start/getting-started/index.md @@ -11,7 +11,7 @@ Welcome to OpenMetadata's unified platform for data discovery, observability, an OpenMetadata is designed to support both technical and non-technical data practitioners across various use cases, including data discovery, lineage, observability, quality, collaboration, governance, and insights. -The platform includes a library of 80+ turnkey connectors to easily ingest metadata from sources such as data warehouses, data lakes, streaming platforms, dashboards, and ML models. For custom data sources, APIs are available to streamline metadata ingestion. Metadata from these sources is organized into a Unified Metadata Graph, providing a single, comprehensive source of truth for your entire data estate. +The platform includes a library of 90+ turnkey connectors to easily ingest metadata from sources such as data warehouses, data lakes, streaming platforms, dashboards, and ML models. For custom data sources, APIs are available to streamline metadata ingestion. Metadata from these sources is organized into a Unified Metadata Graph, providing a single, comprehensive source of truth for your entire data estate. This centralized metadata is accessible through a unified user interface, eliminating the need for practitioners to switch between multiple catalogs, quality, or governance tools. OpenMetadata can also be extended with applications, such as AI-driven productivity tools like MetaPilot, or through custom-built workflows that integrate the platform with existing systems. The platform’s native collaboration features support shared workflows, enabling different teams—data platform engineers, governance professionals, data scientists/analysts, and business users—to collaborate effectively in a single environment. @@ -23,7 +23,7 @@ Before we get started, here’s a quick summary of some of OpenMetadata’s main ### Discovery - Integrated catalog, data quality, and glossary - Natural language search, filtering, and faceting -- 80+ turnkey data connectors +- 90+ turnkey data connectors ### Lineage - Table and column-level lineage diff --git a/openmetadata-docs/content/v1.6.x/quick-start/local-docker-deployment.md b/openmetadata-docs/content/v1.6.x/quick-start/local-docker-deployment.md index 9dc26222aaef..0003c3023b95 100644 --- a/openmetadata-docs/content/v1.6.x/quick-start/local-docker-deployment.md +++ b/openmetadata-docs/content/v1.6.x/quick-start/local-docker-deployment.md @@ -119,15 +119,15 @@ The latest version is at the top of the page You can use the curl or wget command as well to fetch the docker compose files from your terminal - ```commandline -curl -sL -o docker-compose.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.6.1-release/docker-compose.yml +curl -sL -o docker-compose.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.6.3-release/docker-compose.yml -curl -sL -o docker-compose-postgres.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.6.1-release/docker-compose-postgres.yml +curl -sL -o docker-compose-postgres.yml https://github.com/open-metadata/OpenMetadata/releases/download/1.6.3-release/docker-compose-postgres.yml ``` ```commandline -wget https://github.com/open-metadata/OpenMetadata/releases/download/1.6.1-release/docker-compose.yml +wget https://github.com/open-metadata/OpenMetadata/releases/download/1.6.3-release/docker-compose.yml -wget https://github.com/open-metadata/OpenMetadata/releases/download/1.6.1-release/docker-compose-postgres.yml +wget https://github.com/open-metadata/OpenMetadata/releases/download/1.6.3-release/docker-compose-postgres.yml ``` ### 3. Start the Docker Compose Services @@ -166,10 +166,10 @@ You can validate that all containers are up by running with command `docker ps`. ```commandline ❯ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -470cc8149826 openmetadata/server:1.6.1 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server -63578aacbff5 openmetadata/ingestion:1.6.1 "./ingestion_depende…" 45 seconds ago Up 43 seconds 0.0.0.0:8080->8080/tcp openmetadata_ingestion +470cc8149826 openmetadata/server:1.6.3 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server +63578aacbff5 openmetadata/ingestion:1.6.3 "./ingestion_depende…" 45 seconds ago Up 43 seconds 0.0.0.0:8080->8080/tcp openmetadata_ingestion 9f5ee8334f4b docker.elastic.co/elasticsearch/elasticsearch:7.16.3 "/tini -- /usr/local…" 45 seconds ago Up 44 seconds 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp openmetadata_elasticsearch -08947ab3424b openmetadata/db:1.6.1 "/entrypoint.sh mysq…" 45 seconds ago Up 44 seconds (healthy) 3306/tcp, 33060-33061/tcp openmetadata_mysql +08947ab3424b openmetadata/db:1.6.3 "/entrypoint.sh mysq…" 45 seconds ago Up 44 seconds (healthy) 3306/tcp, 33060-33061/tcp openmetadata_mysql ``` In a few seconds, you should be able to access the OpenMetadata UI at [http://localhost:8585](http://localhost:8585) diff --git a/openmetadata-docs/content/v1.6.x/releases/releases/index.md b/openmetadata-docs/content/v1.6.x/releases/releases/index.md index 056ea001a84f..71d7576eaef3 100644 --- a/openmetadata-docs/content/v1.6.x/releases/releases/index.md +++ b/openmetadata-docs/content/v1.6.x/releases/releases/index.md @@ -8,7 +8,7 @@ slug: /releases/all-releases {% note %} The OpenMetadata community is on a monthly release cadence. At every 4-5 weeks we will be releasing a new -version. To see what's coming in next releases, please check our [Roadmap](/releases/roadmap) section. +version. To see what's coming in next releases, please check our {% collateContent %}[Roadmap](https://www.getcollate.io/roadmap){% /collateContent %}{% ossContent %}[Roadmap](/roadmap){% /ossContent %} section. {% /note %} @@ -168,7 +168,7 @@ ER diagrams help you better understand and manage your data architecture by show Organizations often struggle with data governance due to rigid, pre-defined manual workflows. OpenMetadata 1.6 introduces a new, automated data governance framework designed to be customized to each organization's needs. -In Collate 1.6, the Glossary Approval Workflow has been migrated to this new framework. Now, you can create custom approval processes with specific conditions and rules and easily visualize them through intuitive workflow diagrams. You can also create smart approval processes for glossary terms with real-time state changes and task creation to save time and streamline work.  +In Collate 1.6, the Glossary Approval Workflow has been migrated to this new framework. Now, you can create custom approval processes with specific conditions and rules and easily visualize them through intuitive workflow diagrams. You can also create smart approval processes for glossary terms with real-time state changes and task creation to save time and streamline work. ## Data Certification Workflows for Automated Bronze, Silver, & Gold Data Standardization! (Collate) @@ -202,7 +202,7 @@ OpenMetadata 1.6 extends Role-Based Access Control (RBAC) to search functionalit ## Expanded Connector Ecosystem and Diversity -OpenMetadata's ingestion framework contains 80+ native connectors. These connectors are the foundation of the platform and bring in all the metadata your team needs: technical metadata, lineage, usage, profiling, etc. +OpenMetadata's ingestion framework contains 90+ native connectors. These connectors are the foundation of the platform and bring in all the metadata your team needs: technical metadata, lineage, usage, profiling, etc. We bring new connectors in each release, continuously expanding our coverage. This time, release 1.6 comes with seven new connectors: @@ -770,7 +770,7 @@ To continue pursuing this objective, the application was completely refactored t ## Ingestion Connectors -80+ connectors to help teams to centralize metadata. We continue to push the boundaries of this mission, in +90+ connectors to help teams to centralize metadata. We continue to push the boundaries of this mission, in - **Apache Flink** as a Pipeline Connector - **SAP ERP**, after a long and successful collaboration with our community and SAP experts diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md index bfc76e4c23ab..55adbb4271b5 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md @@ -224,6 +224,10 @@ site_menu: url: /connectors/database/s3-datalake/yaml - category: Connectors / Database / S3 Datalake / Troubleshooting url: /connectors/database/s3-datalake/troubleshooting + - category: Connectors / Database / Teradata + url: /connectors/database/teradata + - category: Connectors / Database / Teradata / Run Externally + url: /connectors/database/teradata/yaml - category: Connectors / Database / Trino url: /connectors/database/trino - category: Connectors / Database / Trino / Run Externally @@ -634,6 +638,8 @@ site_menu: url: /how-to-guides/data-discovery/details - category: How-to Guides / Data Discovery / Add Complex Queries using Advanced Search url: /how-to-guides/data-discovery/advanced + - category: How-to Guides / Data Discovery / Troubleshooting + url: /how-to-guides/data-discovery/troubleshooting - category: How-to Guides / Data Discovery / Bulk Upload Data Assets url: /how-to-guides/data-discovery/bulk-upload - category: How-to Guides / Data Discovery / How to Bulk Import Data Asset diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/db2/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/db2/index.md index d489b105ae95..4dd22e7392e7 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/db2/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/db2/index.md @@ -98,6 +98,20 @@ If you are using DB2 for IBM i: - In Host and Port you should not add the Port Number. {% /note %} +{% note %} +If you have a **db2jcc_license_cisuz.jar** file, it will not work with **ibm_db**. This file is a **Db2 Connect** license for the Java Driver. +For **non-Java drivers**, such as the Python Client used in OpenMetadata ingestion, a **Db2 Connect** client-side license is required, typically named **db2con*.lic**. + +The **db2jcc_license_cisuz.jar** is specifically for Java-based clients, whereas OpenMetadata ingestion operates with a Python Client, making the `.jar` file incompatible. + +For activating a **non-Java license** for Db2 Connect **Application Server Edition**, **Advanced Application Server Edition**, **Enterprise Edition**, or **Trial**, follow these steps: +- Download the **license activation kit** from IBM Passport Advantage: [IBM PPA](https://www.ibm.com/software/passportadvantage/pao_customer.html). +- Unzip the package and locate the **non-Java license file** (e.g., `db2consv_ee.lic`). +- Apply the `.lic` file to activate the license. + +For further reference, check this IBM post: [Everything About Db2 Connect Licensing](https://community.ibm.com/community/user/datamanagement/blogs/shilu-mathai2/2023/05/05/everything-about-db2-connect-licensing). +{% /note %} + {% partial file="/v1.7/connectors/database/advanced-configuration.md" /%} {% /extraContent %} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mongodb/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mongodb/yaml.md index 2fc2e94fc6cb..34b900f559dd 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mongodb/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mongodb/yaml.md @@ -351,7 +351,7 @@ Here we are also importing all the basic requirements to parse YAMLs, handle dat import yaml from datetime import timedelta from airflow import DAG -from metadata.profiler.api.workflow import ProfilerWorkflow +from metadata.workflow.profiler import ProfilerWorkflow try: from airflow.operators.python import PythonOperator diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md index 756a5f2b534f..87cc7afda4f0 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md @@ -191,8 +191,8 @@ For a simple, local installation using our docker containers, this looks like: ```yaml {% srNumber=40 %} source: - type: mssql-lineage - serviceName: local_mssql + type: mysql-lineage + serviceName: local_mysql sourceConfig: config: type: DatabaseLineage @@ -243,12 +243,6 @@ source: # - table4 ``` -```yaml {% srNumber=49 %} -sink: - type: metadata-rest - config: {} -``` - ```yaml {% srNumber=51 %} overrideViewLineage: false ``` @@ -269,6 +263,12 @@ sink: threads: 1 ``` +```yaml {% srNumber=49 %} +sink: + type: metadata-rest + config: {} +``` + {% partial file="/v1.6/connectors/yaml/workflow-config.md" /%} {% /codeBlock %} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md index 70bb0f68f8d6..d0649194a9c6 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md @@ -115,6 +115,9 @@ You can find more information about the `account_usage` schema [here](https://do - **Include Temporary and Transient Tables**: Optional configuration for ingestion of `TRANSIENT` and `TEMPORARY` tables, By default, it will skip the `TRANSIENT` and `TEMPORARY` tables. - **Client Session Keep Alive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. +- **Account Usage Schema Name**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. {% partial file="/v1.7/connectors/database/advanced-configuration.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md index e46e4d4f2c80..9ca018518eeb 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md @@ -150,6 +150,14 @@ This is a sample config for Snowflake: {% /codeInfo %} +{% codeInfo srNumber=40 %} + +**accountUsageSchema**: Full name of account usage schema, used in case your used do not have direct access to `SNOWFLAKE.ACCOUNT_USAGE` schema. In such case you can replicate tables `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS` to a custom schema let's say `CUSTOM_DB.CUSTOM_SCHEMA` and provide the same name in this field. + +When using this field make sure you have all these tables available within your custom schema `QUERY_HISTORY`, `TAG_REFERENCES`, `PROCEDURES`, `FUNCTIONS`. + +{% /codeInfo %} + {% codeInfo srNumber=6 %} **includeTransientTables**: Optional configuration for ingestion of TRANSIENT and TEMPORARY tables, By default, it will skip the TRANSIENT and TEMPORARY tables. @@ -231,6 +239,9 @@ source: ```yaml {% srNumber=5 %} # database: ``` +```yaml {% srNumber=40 %} + # accountUsageSchema: SNOWFLAKE.ACCOUNT_USAGE +``` ```yaml {% srNumber=6 %} includeTransientTables: false ``` diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/matillion/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/matillion/index.md index 824d1d28c601..f1c0de671585 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/matillion/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/pipeline/matillion/index.md @@ -30,6 +30,8 @@ Configure and schedule Matillion metadata and profiler workflows from the OpenMe To extract metadata from Matillion, you need to create a user with the following permissions: - `API` Permission ( While Creating the User, from Admin -> User ) +- To retrieve lineage data, the user must be granted [Component-level permissions](https://docs.matillion.com/metl/docs/2932106/#component). +- To enable lineage tracking in Matillion, **Matillion Enterprise Mode** is required. For detailed setup instructions and further information, refer to the official documentation: [Matillion Lineage Documentation](https://docs.matillion.com/metl/docs/2881895/). ### Matillion Versions diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/deployment/oss-security.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/deployment/oss-security.md new file mode 100644 index 000000000000..385e5f2fb8ee --- /dev/null +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/deployment/oss-security.md @@ -0,0 +1,44 @@ +--- +title: OSS Security Best Practices +slug: /deployment/oss-security +collate: false +--- + +# OSS Security + +## Encryption of Connection Credentials + +OpenMetadata ensures that sensitive information, such as passwords and connection secrets, is securely stored. + +- **Encryption Algorithm**: OpenMetadata uses **Fernet encryption** to encrypt secrets and passwords before storing them in the database. +- **Fernet Encryption Details**: + - Uses **AES-128 in CBC mode** with a strong key-based approach. + - **Not based on hashing or salting**, but rather an encryption/decryption method with a symmetric key. +- **Secrets Manager Support**: + - Users can **avoid storing credentials** in OpenMetadata by configuring an external **Secrets Manager**. + - More details on setting up a Secrets Manager can be found here: + 🔗 [Secrets Manager Documentation](https://docs.open-metadata.org/latest/deployment/secrets-manager) + +## Secure Connections to Data Sources + +OpenMetadata supports **encrypted connections** to various databases and services. + +- **SSL/TLS Support**: + - OpenMetadata allows users to configure **SSL/TLS encryption** for secure data transmission. + - Users can specify **SSL modes** and provide **CA certificates** for SSL validation. +- **How to Enable SSL?** + - Each connector supports different SSL configurations. + - Follow the detailed guide for enabling SSL in OpenMetadata: + 🔗 [Enable SSL in OpenMetadata](https://docs.open-metadata.org/latest/deployment/security/enable-ssl) + +## **Additional Security Measures** + +- **Role-Based Access Control (RBAC)**: OpenMetadata allows administrators to define user roles and permissions. +- **Authentication & Authorization**: OpenMetadata supports integration with OAuth, SAML, and LDAP for secure authentication. +- **Data Access Control**: Users can restrict access to metadata based on policies and governance rules. + +{% note %} +- **Passwords and secrets are securely encrypted** using **Fernet encryption**. +- **Connections to data sources can be encrypted** using **SSL/TLS**. +- **Secrets Managers** can be used to manage credentials externally. +{% /note %} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/day-1/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/day-1/index.md index af90c55cbc86..a92ab62dbbba 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/day-1/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/day-1/index.md @@ -8,10 +8,10 @@ collate: true Get started with your Collate service in just few simple steps: -1. Set up a Data Connector: Connect your data sources to begin collecting metadata. -2. Ingest Metadata: Run the metadata ingestion to gather and push data insights. -3. Invite Users: Add team members to collaborate and manage metadata together. -4. Explore the Features: Dive into Collate's rich feature set to unlock the full potential of your data. +1. **Set up a Data Connector**: Connect your data sources to begin collecting metadata. +2. **Ingest Metadata**: Run the metadata ingestion to gather and push data insights. +3. **Invite Users**: Add team members to collaborate and manage metadata together. +4. **Explore the Features**: Dive into Collate's rich feature set to unlock the full potential of your data. **Ready to begin? Let's get started!** @@ -24,7 +24,7 @@ with links to more detailed documentation. ## Step 1: Set up a Data Connector Once you’re able to login to your Collate instance, set up a data connector to start bringing metadata into Collate. -There are [80+ turnkey connectors](/connectors) to various services: data warehouses, data lakes, databases, dashboards, +There are [90+ turnkey connectors](/connectors) to various services: data warehouses, data lakes, databases, dashboards, messaging services, pipelines, ML models, storage services, and other Metadata Services. Connections to [custom data sources](/connectors/custom-connectors) can also be created via API. diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/index.md index 4de859fbebf7..b3a0c941ecb6 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/getting-started/index.md @@ -7,7 +7,7 @@ collate: true # Getting Started Welcome to Collate's unified platform for data discovery, observability, and governance! Our platform centralizes all -the context around your data to help you build high-quality and AI assets. This guide gives you all the information you +the context around your data to help you build high-quality data and AI assets. This guide gives you all the information you need to set up your Collate environment in 30 minutes. ## How Does Collate Work? @@ -15,14 +15,14 @@ need to set up your Collate environment in 30 minutes. Collate is designed for both technical and non-technical data practitioners to work together across a broad set of use cases, including data discovery, lineage, observability, quality, collaboration, governance, and insights. -A library of 80+ turnkey connectors is available to easily ingest metadata into Collate, such as data warehouses, data lakes, +A library of 90+ turnkey connectors is available to easily ingest metadata into Collate, such as data warehouses, data lakes, streaming, dashboards, ML models, and more. APIs are also available to easily ingest metadata from custom data sources. Metadata from these different sources is organized into a Unified Metadata Graph, which provides a single, comprehensive source of truth across your entire data estate. This centralized information is surfaced through a unified user interface for all your use cases so that different data practitioners no longer need to switch between different data catalogs, quality, or governance tools. Additionally, -Collate can be extended through the application ecosystem, such as with AI productivity applications like MetaPilot, +Collate can be extended through the application ecosystem, such as with AI productivity applications like Collate AI, or with customer-built workflows to integrate Collate with your existing systems. These capabilities are built around native collaboration capabilities for shared workflows across different teams so that every data practitioner can work together: data platform, data governance, data scientist/analyst, and business user. @@ -31,7 +31,7 @@ together: data platform, data governance, data scientist/analyst, and business u Before we get started, here’s a quick summary of some of Collate’s main features: -- **Discovery**: integrated catalog, quality, and glossary; natural language search, filtering, and faceting, 80+ turnkey data connectors, and MetaPilot AI Chatbot. +- **Discovery**: integrated catalog, quality, and glossary; natural language search, filtering, and faceting, 90+ turnkey data connectors, and Collate AI Chatbot. - **Lineage**: table and column-level lineage, automated data estate mapping and APIs, lineage layers and search, governance and PII automation and manual customization. - **Observability**: alerting and notifications, incident management, third-party notifications, pipeline monitoring, root cause analysis, anomaly detection, data profiler. - **Quality**: table and column test cases, no-code and SQL data quality tests, test suites, test case reporting, quality dashboards, widgets and data quality lineage maps. diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.md index 936756f1912f..15133572cda0 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.md @@ -34,6 +34,7 @@ Here are some examples of conditions. | **matchAllTags(tagFqn, [tagFqn…])** | Returns true if the resource has all the tags from the tag list. | | **matchAnyTag(tagFqn, [tagFqn…])** | Returns true if the resource has any of the tags from the tag list. | | **matchTeam()** | Returns true if the user belongs to the team that owns the resource. | +| **hasDomain()** | Returns true if the logged in user is the has domain access of the entity being accessed | Conditions are used to assess DataAsset like Tables/Topics/Dashboards etc.. for specific attributes. diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-discovery/troubleshooting.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-discovery/troubleshooting.md new file mode 100644 index 000000000000..5122d4e01d32 --- /dev/null +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-discovery/troubleshooting.md @@ -0,0 +1,42 @@ +--- +title: Troubleshooting for Export issue +slug: /how-to-guides/data-discovery/troubleshooting +--- + +# Troubleshooting Export Issue +When attempting to export a **CSV file for a Glossary**, the process gets stuck on the message **"Export initiated successfully."** and never completes. The file is not downloaded, and the export button remains disabled. + +This issue may occur if **WebSockets are blocked** in your network setup due to a **proxy** or **load balancer** configuration. OpenMetadata relies on WebSockets for real-time communication, and if they are blocked, the export process cannot complete. + +## Troubleshooting Steps + +### Step 1: Check for Load Balancer or Proxy + +If your setup includes a **load balancer** or **proxy**, verify whether WebSockets are being blocked. + +1. Run the following API request to check the export status: + +```bash +curl -X GET "https:///api/v1/glossaries/name//exportAsync" +``` + +If the response does not return a file and remains in an active state indefinitely, WebSockets might be blocked. + +### Step 2: Verify WebSocket Connectivity + +1. Open the Developer Tools in your browser (F12 or Ctrl + Shift + I in Chrome). +2. Navigate to the Network tab. +3. Filter requests by WebSockets (WS). +4. Check if WebSocket requests to OpenMetadata (wss://) are blocked, failing, or not established. + +### Step 3: Adjust WebSocket Settings in Your Proxy + +If WebSockets are blocked, update your proxy configuration to allow WebSocket traffic. + +### Step 4: Restart Services and Verify + +1. Restart your proxy or load balancer after making the configuration changes. +2. Clear browser cache and cookies. +3. Retry the CSV export in OpenMetadata. + +Once WebSockets are enabled in the proxy settings, the glossary export should complete successfully, and the CSV file should be available for download. diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md index fa90865335a0..85681472541f 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md @@ -23,10 +23,26 @@ Managing metadata manually can be challenging, particularly in dynamic environme ## Key Use Cases for Collate Automations -### 1. Bulk Ownership and Domain Assignment +### 1. Bulk Description {% image -src="/images/v1.7/how-to-guides/governance/bulk-ownership-and.png" +src="/images/v1.6/how-to-guides/governance/automator-description.png" +alt="Getting started with Automation" +caption="Getting started with Automation" +/%} + +- **Problem**: Many datasets lack descriptions, making it difficult for users to understand the data's purpose and contents. Sometimes, the same column description needs to be added to multiple datasets. +- **Solution**: Automations can bulk-apply descriptions to tables and columns, ensuring that all data assets are consistently documented. +- **Benefit**: This use case improves data discoverability and understanding, making it easier for users to find and use the data effectively. + +For the Action Configuration: +- **Apply to Children**: Lets you apply the description to the selected child assets (e.g., columns) within an asset. +- **Overwrite Metadata**: Allows you to overwrite existing descriptions with the new description. Otherwise, we will only apply the description to empty tables or columns. + +### 2. Bulk Ownership and Domain Assignment + +{% image +src="/images/v1.6/how-to-guides/governance/bulk-ownership-and.png" alt="Getting started with Automation" caption="Getting started with Automation" /%} @@ -35,10 +51,13 @@ caption="Getting started with Automation" - **Solution**: Automations can bulk-assign ownership and domains to datasets, ensuring all data assets are correctly categorized and owned. This process can be applied to tables, schemas, or other assets within Collate. - **Benefit**: This use case ensures data assets have a designated owner and are organized under the appropriate domain, making data more discoverable and accountable. -### 2. Bulk Tagging and Glossary Term Assignment +For the Action Configuration: +- **Overwrite Metadata**: Allows you to overwrite existing owner or domain with the configured one. Otherwise, we will only apply the owner or domain to assets that do not have an existing owner or domain. + +### 3. Bulk Tagging and Glossary Term Assignment {% image -src="/images/v1.7/how-to-guides/governance/bulk-tagging-glossary.png" +src="/images/v1.6/how-to-guides/governance/bulk-tagging-glossary.png" alt="Getting started with Automation" caption="Getting started with Automation" /%} @@ -47,10 +66,15 @@ caption="Getting started with Automation" - **Solution**: Automations allow users to bulk-apply tags (e.g., PII) or glossary terms (e.g., Customer ID) to specific datasets, ensuring uniformity across the platform. - **Benefit**: This automation reduces the risk of missing important tags like PII-sensitive and ensures that key metadata elements are applied consistently across datasets. -### 3. Metadata Propagation via Lineage +For the Action Configuration: +- **Apply to Children**: Lets you apply the Tags or Glossary Terms to the selected child assets (e.g., columns) within an asset. +- **Overwrite Metadata**: Allows you to overwrite existing Tags or Terms with the configured one. Otherwise, we will add the new Tags or Terms to the existing ones. + + +### 4. Metadata Propagation via Lineage {% image -src="/images/v1.7/how-to-guides/governance/metadata-propogation.png" +src="/images/v1.6/how-to-guides/governance/metadata-propogation.png" alt="Getting started with Automation" caption="Getting started with Automation" /%} @@ -59,14 +83,35 @@ caption="Getting started with Automation" - **Solution**: Use automations to propagate metadata across related datasets, ensuring that all relevant data inherits the correct metadata properties from the source dataset. - **Benefit**: Metadata consistency is ensured across the entire data lineage, reducing the need for manual updates and maintaining a single source of truth. -### 4. Automatic PII Detection and Tagging +For the Action Configuration: +1. First, we can choose if we want the propagation to happen at the Parent level (e.g., Table), Column Level, or both. This can be configured by selecting **Propagate Parent** and/or **Propagate Column Level**. +2. Then, we can control which pieces of metadata we want to propagate via lineage: + - **Propagate Description**: Propagates the description from the source asset to the downstream assets. Works for both parent and column-level. + - **Propagate Tags**: Propagates the tags from the source asset to the downstream assets. Works for both parent and column-level. + - **Propagate Glossary Terms**: Propagates the glossary terms from the source asset to the downstream assets. Works for both parent and column-level. + - **Propagate Owners**: Only applicable for Parent assets. Propagates the owner information to downstream assets. + - **Propagate Tier**: Only applicable for Parent assets. Propagated the tier information to downstream assets. + +As with other actions, you can choose to **Overwrite Metadata** or keep the existing metadata and only apply the new metadata to assets that do not have the metadata already. + + +### 5. Automatic PII Detection and Tagging {% image -src="/images/v1.7/how-to-guides/governance/automatic-detection.png" +src="/images/v1.6/how-to-guides/governance/automatic-detection.png" alt="Getting started with Automation" caption="Getting started with Automation" /%} +{% note noteType="Warning" %} + +Note that we recommend using the **Auto Classification** workflow instead, which allows you to discover PII data automatically, +even in cases where you don't want to ingest the Sample Data into Collate. + +Note that this automation, the ML Tagging, will be deprecated in future releases. + +{% /note %} + - **Problem**: Manually identifying and tagging Personally Identifiable Information (PII) across large datasets is labor-intensive and prone to errors. - **Solution**: Automations can automatically detect PII data (e.g., emails, usernames) and apply relevant tags to ensure that sensitive data is flagged appropriately for compliance. - **Benefit**: Ensures compliance with data protection regulations by consistently tagging sensitive data, reducing the risk of non-compliance. diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md index 9bf30da68f38..c691a6062151 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/auto-pii-tagging.md @@ -7,11 +7,6 @@ slug: /how-to-guides/data-governance/classification/auto/auto-pii-tagging Auto PII tagging for Sensitive/NonSensitive at the column level is performed based on the two approaches described below. -{% note %} -PII Tagging is only available during `Profiler Ingestion`. -{% /note %} - - ## Tagging logic 1. **Column Name Scanner**: We validate the column names of the table against a set of regex rules that help us identify diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md index 2d82baac5abe..7c57f81bf8d8 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/how-to-guides/data-governance/classification/Auto Classification/external-workflow.md @@ -42,6 +42,19 @@ The Auto Classification Workflow enables automatic tagging of sensitive informat - When set to `true`, filtering patterns will be applied to the Fully Qualified Name of a table (e.g., `service_name.db_name.schema_name.table_name`). - When set to `false`, filtering applies only to raw table names. +## Auto Classification Workflow Execution + +To execute the **Auto Classification Workflow**, follow the steps below: + +### 1. Install the Required Python Package +Ensure you have the correct OpenMetadata ingestion package installed, including the **PII Processor** module: + +```bash +pip install "openmetadata-ingestion[pii-processor]" +``` +## 2. Define and Execute the Python Workflow +Instead of using a YAML configuration, use the AutoClassificationWorkflow from OpenMetadata to trigger the ingestion process programmatically. + ## Sample Auto Classification Workflow yaml ```yaml @@ -103,6 +116,14 @@ workflowConfig: jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" ``` +### 3. Expected Outcome +- Automatically classifies and tags sensitive data based on predefined patterns and confidence levels. +- Improves metadata enrichment and enhances data governance practices. +- Provides visibility into sensitive data across databases. +This approach ensures that the Auto Classification Workflow is executed correctly using the appropriate OpenMetadata ingestion framework. + +{% partial file="/v1.7/connectors/yaml/auto-classification.md" variables={connector: "snowflake"} /%} + ## Workflow Execution ### To Execute the Auto Classification Workflow: diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md index f5833c56a26a..93f5de33997c 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md @@ -222,6 +222,9 @@ site_menu: - category: Deployment / Metrics url: /deployment/metrics + + - category: Deployment / OSS Security + url: /deployment/oss-security - category: Connectors url: /connectors @@ -807,6 +810,8 @@ site_menu: url: /how-to-guides/data-discovery/details - category: How-to Guides / Data Discovery / Add Complex Queries using Advanced Search url: /how-to-guides/data-discovery/advanced + - category: How-to Guides / Data Discovery / Troubleshooting + url: /how-to-guides/data-discovery/troubleshooting - category: How-to Guides / Data Discovery / OpenMetadata Chrome Extension url: /how-to-guides/data-discovery/openmetadata-extension - category: How-to Guides / Data Discovery / OpenMetadata Chrome Extension / Viewing Activity Feeds and Managing Tasks diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/day-1/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/day-1/index.md index 831688efda9b..dca07d407fdd 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/day-1/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/day-1/index.md @@ -7,10 +7,10 @@ slug: /quick-start/getting-started/day-1 Get started with your OpenMetadata service in a few simple steps: -1. Set up a Data Connector: Connect your data sources to begin collecting metadata. -2. Ingest Metadata: Run the metadata ingestion process to gather and push data insights. -3. Invite Users: Add team members to collaborate and manage metadata together. -4. Explore the Features: Dive into OpenMetadata's extensive feature set to unlock the full potential of your data. +1. **Set up a Data Connector**: Connect your data sources to begin collecting metadata. +2. **Ingest Metadata**: Run the metadata ingestion process to gather and push data insights. +3. **Invite Users**: Add team members to collaborate and manage metadata together. +4. **Explore the Features**: Dive into OpenMetadata's extensive feature set to unlock the full potential of your data. **Ready to begin? Let's get started!** @@ -20,7 +20,7 @@ You should receive your initial OpenMetadata credentials from OpenMetadata suppo ## Step 1: Set up a Data Connector -Once you have logged into your OpenMetadata instance, set up a data connector to start ingesting metadata. OpenMetadata provides [80+ turnkey connectors](/connectors) for a wide range of services, including: +Once you have logged into your OpenMetadata instance, set up a data connector to start ingesting metadata. OpenMetadata provides [90+ turnkey connectors](/connectors) for a wide range of services, including: - Databases - Dashboards diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/index.md index 778794a85fd3..7130d1987f6a 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/quick-start/getting-started/index.md @@ -11,9 +11,9 @@ Welcome to OpenMetadata's unified platform for data discovery, observability, an OpenMetadata is designed to support both technical and non-technical data practitioners across various use cases, including data discovery, lineage, observability, quality, collaboration, governance, and insights. -The platform includes a library of 80+ turnkey connectors to easily ingest metadata from sources such as data warehouses, data lakes, streaming platforms, dashboards, and ML models. For custom data sources, APIs are available to streamline metadata ingestion. Metadata from these sources is organized into a Unified Metadata Graph, providing a single, comprehensive source of truth for your entire data estate. +The platform includes a library of 90+ turnkey connectors to easily ingest metadata from sources such as data warehouses, data lakes, streaming platforms, dashboards, and ML models. For custom data sources, APIs are available to streamline metadata ingestion. Metadata from these sources is organized into a Unified Metadata Graph, providing a single, comprehensive source of truth for your entire data estate. -This centralized metadata is accessible through a unified user interface, eliminating the need for practitioners to switch between multiple catalogs, quality, or governance tools. OpenMetadata can also be extended with applications, such as AI-driven productivity tools like MetaPilot, or through custom-built workflows that integrate the platform with existing systems. +This centralized metadata is accessible through a unified user interface, eliminating the need for practitioners to switch between multiple catalogs, quality, or governance tools. OpenMetadata can also be extended with applications, such as AI-driven productivity tools like Collate AI, or through custom-built workflows that integrate the platform with existing systems. The platform’s native collaboration features support shared workflows, enabling different teams—data platform engineers, governance professionals, data scientists/analysts, and business users—to collaborate effectively in a single environment. ## Key Features of OpenMetadata @@ -23,7 +23,7 @@ Before we get started, here’s a quick summary of some of OpenMetadata’s main ### Discovery - Integrated catalog, data quality, and glossary - Natural language search, filtering, and faceting -- 80+ turnkey data connectors +- 90+ turnkey data connectors ### Lineage - Table and column-level lineage diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/releases/releases/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/releases/releases/index.md index 5636e4c0cd67..b69b062859da 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/releases/releases/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/releases/releases/index.md @@ -8,7 +8,7 @@ slug: /releases/all-releases {% note %} The OpenMetadata community is on a monthly release cadence. At every 4-5 weeks we will be releasing a new -version. To see what's coming in next releases, please check our [Roadmap](/releases/roadmap) section. +version. To see what's coming in next releases, please check our {% collateContent %}[Roadmap](https://www.getcollate.io/roadmap){% /collateContent %}{% ossContent %}[Roadmap](/roadmap){% /ossContent %} section. {% /note %} @@ -168,7 +168,7 @@ ER diagrams help you better understand and manage your data architecture by show Organizations often struggle with data governance due to rigid, pre-defined manual workflows. OpenMetadata 1.6 introduces a new, automated data governance framework designed to be customized to each organization's needs. -In Collate 1.6, the Glossary Approval Workflow has been migrated to this new framework. Now, you can create custom approval processes with specific conditions and rules and easily visualize them through intuitive workflow diagrams. You can also create smart approval processes for glossary terms with real-time state changes and task creation to save time and streamline work.  +In Collate 1.6, the Glossary Approval Workflow has been migrated to this new framework. Now, you can create custom approval processes with specific conditions and rules and easily visualize them through intuitive workflow diagrams. You can also create smart approval processes for glossary terms with real-time state changes and task creation to save time and streamline work. ## Data Certification Workflows for Automated Bronze, Silver, & Gold Data Standardization! (Collate) @@ -202,7 +202,7 @@ OpenMetadata 1.6 extends Role-Based Access Control (RBAC) to search functionalit ## Expanded Connector Ecosystem and Diversity -OpenMetadata's ingestion framework contains 80+ native connectors. These connectors are the foundation of the platform and bring in all the metadata your team needs: technical metadata, lineage, usage, profiling, etc. +OpenMetadata's ingestion framework contains 90+ native connectors. These connectors are the foundation of the platform and bring in all the metadata your team needs: technical metadata, lineage, usage, profiling, etc. We bring new connectors in each release, continuously expanding our coverage. This time, release 1.6 comes with seven new connectors: @@ -770,7 +770,7 @@ To continue pursuing this objective, the application was completely refactored t ## Ingestion Connectors -80+ connectors to help teams to centralize metadata. We continue to push the boundaries of this mission, in +90+ connectors to help teams to centralize metadata. We continue to push the boundaries of this mission, in - **Apache Flink** as a Pipeline Connector - **SAP ERP**, after a long and successful collaboration with our community and SAP experts diff --git a/openmetadata-docs/images/connectors/synapse.webp b/openmetadata-docs/images/connectors/synapse.webp new file mode 100644 index 000000000000..36197c9efe3a Binary files /dev/null and b/openmetadata-docs/images/connectors/synapse.webp differ diff --git a/openmetadata-docs/images/v1.6/connectors/adls/add-new-service.png b/openmetadata-docs/images/v1.6/connectors/adls/add-new-service.png new file mode 100644 index 000000000000..b9c8c2d7681a Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/adls/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/adls/select-service.png b/openmetadata-docs/images/v1.6/connectors/adls/select-service.png new file mode 100644 index 000000000000..7f29fdc0818e Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/adls/select-service.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/adls/service-connection.png b/openmetadata-docs/images/v1.6/connectors/adls/service-connection.png new file mode 100644 index 000000000000..9e175a777333 Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/adls/service-connection.png differ diff --git a/openmetadata-docs/images/v1.6/deployment/upgrade/running-state-in-openmetadata.png b/openmetadata-docs/images/v1.6/deployment/upgrade/running-state-in-openmetadata.png new file mode 100644 index 000000000000..cd808c4d92a8 Binary files /dev/null and b/openmetadata-docs/images/v1.6/deployment/upgrade/running-state-in-openmetadata.png differ diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automator-description.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automator-description.png new file mode 100644 index 000000000000..edd397fc9521 Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automator-description.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/adls/add-new-service.png b/openmetadata-docs/images/v1.7/connectors/adls/add-new-service.png new file mode 100644 index 000000000000..b9c8c2d7681a Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/adls/add-new-service.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/adls/select-service.png b/openmetadata-docs/images/v1.7/connectors/adls/select-service.png new file mode 100644 index 000000000000..7f29fdc0818e Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/adls/select-service.png differ diff --git a/openmetadata-docs/images/v1.7/connectors/adls/service-connection.png b/openmetadata-docs/images/v1.7/connectors/adls/service-connection.png new file mode 100644 index 000000000000..9e175a777333 Binary files /dev/null and b/openmetadata-docs/images/v1.7/connectors/adls/service-connection.png differ diff --git a/openmetadata-docs/images/v1.7/deployment/upgrade/running-state-in-openmetadata.png b/openmetadata-docs/images/v1.7/deployment/upgrade/running-state-in-openmetadata.png new file mode 100644 index 000000000000..cd808c4d92a8 Binary files /dev/null and b/openmetadata-docs/images/v1.7/deployment/upgrade/running-state-in-openmetadata.png differ diff --git a/openmetadata-docs/images/v1.7/how-to-guides/governance/automator-description.png b/openmetadata-docs/images/v1.7/how-to-guides/governance/automator-description.png new file mode 100644 index 000000000000..edd397fc9521 Binary files /dev/null and b/openmetadata-docs/images/v1.7/how-to-guides/governance/automator-description.png differ diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index 11b04594f47a..64c6aefcc52c 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -16,7 +16,7 @@ ${project.basedir}/target/site/jacoco-aggregate/jacoco.xml ${project.basedir}/src/test/java 1.20.3 - 2.29.15 + 2.30.19 1.14.0 4.9.0 1.0.0 @@ -28,6 +28,7 @@ 3.6.0 3.3.1 2.1.1 + 2.5.2 @@ -89,7 +90,7 @@ net.minidev json-smart - 2.5.1 + ${json-smart.version} org.open-metadata diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 0121dbdc6d07..f13ae7b53a5c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -50,6 +50,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; @@ -193,7 +194,11 @@ public final void addRecord(CsvFile csvFile, List recordList) { } /** Owner field is in entityType:entityName format */ - public List getOwners(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) + public List getOwners( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + Function invalidMessageCreator) throws IOException { if (!processRecord) { return null; @@ -207,7 +212,7 @@ public List getOwners(CSVPrinter printer, CSVRecord csvRecord, for (String owner : owners) { List ownerTypes = listOrEmpty(CsvUtil.fieldToEntities(owner)); if (ownerTypes.size() != 2) { - importFailure(printer, invalidOwner(fieldNumber), csvRecord); + importFailure(printer, invalidMessageCreator.apply(fieldNumber), csvRecord); return Collections.emptyList(); } EntityReference ownerRef = @@ -219,6 +224,16 @@ public List getOwners(CSVPrinter printer, CSVRecord csvRecord, return refs.isEmpty() ? null : refs; } + public List getOwners(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) + throws IOException { + return getOwners(printer, csvRecord, fieldNumber, EntityCsv::invalidOwner); + } + + public List getReviewers( + CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) throws IOException { + return getOwners(printer, csvRecord, fieldNumber, EntityCsv::invalidReviewer); + } + /** Owner field is in entityName format */ public EntityReference getOwnerAsUser(CSVPrinter printer, CSVRecord csvRecord, int fieldNumber) throws IOException { @@ -868,6 +883,11 @@ public static String invalidOwner(int field) { return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); } + public static String invalidReviewer(int field) { + String error = "Reviewer should be of format user:userName or team:teamName"; + return String.format(FIELD_ERROR_MSG, CsvErrorType.INVALID_FIELD, field + 1, error); + } + public static String invalidExtension(int field, String key, String value) { String error = "Invalid key-value pair in extension string: Key = " diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java index a294a79cc51b..78c26a5668c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/TypeRegistry.java @@ -14,15 +14,22 @@ package org.openmetadata.service; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.resources.types.TypeResource.PROPERTIES_FIELD; import com.networknt.schema.JsonSchema; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.Type; +import org.openmetadata.schema.entity.type.Category; import org.openmetadata.schema.entity.type.CustomProperty; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.TypeRepository; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @@ -49,6 +56,35 @@ public static TypeRegistry instance() { return INSTANCE; } + public final void initialize(TypeRepository repository) { + // Load types defined in OpenMetadata schemas + long now = System.currentTimeMillis(); + List types = JsonUtils.getTypes(); + types.forEach( + type -> { + type.withId(UUID.randomUUID()).withUpdatedBy(ADMIN_USER_NAME).withUpdatedAt(now); + LOG.debug("Loading type {}", type.getName()); + try { + EntityUtil.Fields fields = repository.getFields(PROPERTIES_FIELD); + try { + Type storedType = repository.getByName(null, type.getName(), fields); + type.setId(storedType.getId()); + // If entity type already exists, then carry forward custom properties + if (storedType.getCategory().equals(Category.Entity)) { + type.setCustomProperties(storedType.getCustomProperties()); + } + } catch (Exception e) { + LOG.debug( + "Type '{}' not found. Proceeding to add new type entity in database.", + type.getName()); + } + repository.addToRegistry(type); + } catch (Exception e) { + LOG.error("Error loading type {}", type.getName(), e); + } + }); + } + public void addType(Type type) { TYPES.put(type.getName(), type); @@ -111,34 +147,25 @@ public static String getPropertyName(String propertyFQN) { } public static String getCustomPropertyType(String entityType, String propertyName) { - Type type = TypeRegistry.TYPES.get(entityType); - if (type != null && type.getCustomProperties() != null) { - for (CustomProperty property : type.getCustomProperties()) { - if (property.getName().equals(propertyName)) { - return property.getPropertyType().getName(); - } - } + String fqn = getCustomPropertyFQN(entityType, propertyName); + CustomProperty property = CUSTOM_PROPERTIES.get(fqn); + if (property == null) { + throw EntityNotFoundException.byMessage( + CatalogExceptionMessage.entityNotFound(propertyName, entityType)); } - throw EntityNotFoundException.byMessage( - CatalogExceptionMessage.entityNotFound(Entity.TYPE, String.valueOf(type))); + return property.getPropertyType().getName(); } public static String getCustomPropertyConfig(String entityType, String propertyName) { - Type type = TypeRegistry.TYPES.get(entityType); - if (type != null && type.getCustomProperties() != null) { - for (CustomProperty property : type.getCustomProperties()) { - if (property.getName().equals(propertyName) - && property.getCustomPropertyConfig() != null - && property.getCustomPropertyConfig().getConfig() != null) { - Object config = property.getCustomPropertyConfig().getConfig(); - if (config instanceof String || config instanceof Integer) { - return config.toString(); // for simple type config return as string - } else { - return JsonUtils.pojoToJson( - config); // for complex object in config return as JSON string - } - } - } + String fqn = getCustomPropertyFQN(entityType, propertyName); + CustomProperty property = CUSTOM_PROPERTIES.get(fqn); + if (property != null + && property.getCustomPropertyConfig() != null + && property.getCustomPropertyConfig().getConfig() != null) { + Object config = property.getCustomPropertyConfig().getConfig(); + return (config instanceof String || config instanceof Integer) + ? config.toString() // for simple type config return as string + : JsonUtils.pojoToJson(config); // for complex object in config return as JSON string } return null; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java index a0a11f031205..7db4c4e4f89c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/AbstractEventConsumer.java @@ -32,10 +32,12 @@ import org.openmetadata.schema.entity.events.EventSubscriptionOffset; import org.openmetadata.schema.entity.events.FailedEvent; import org.openmetadata.schema.entity.events.SubscriptionDestination; +import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.service.Entity; import org.openmetadata.service.events.errors.EventPublisherException; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ResultList; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; import org.quartz.JobDetail; @@ -70,8 +72,9 @@ private void init(JobExecutionContext context) { (EventSubscription) context.getJobDetail().getJobDataMap().get(ALERT_INFO_KEY); this.jobDetail = context.getJobDetail(); this.eventSubscription = sub; - this.offset = loadInitialOffset(context).getCurrentOffset(); - this.startingOffset = loadInitialOffset(context).getStartingOffset(); + EventSubscriptionOffset eventSubscriptionOffset = loadInitialOffset(context); + this.offset = eventSubscriptionOffset.getCurrentOffset(); + this.startingOffset = eventSubscriptionOffset.getStartingOffset(); this.alertMetrics = loadInitialMetrics(); this.destinationMap = loadDestinationsMap(context); this.doInit(context); @@ -240,34 +243,46 @@ public void commit(JobExecutionContext jobExecutionContext) { } @Override - public List pollEvents(long offset, long batchSize) { - // Read from Change Event Table + public ResultList pollEvents(long offset, long batchSize) { List eventJson = Entity.getCollectionDAO().changeEventDAO().list(batchSize, offset); - List changeEvents = new ArrayList<>(); + List errorEvents = new ArrayList<>(); for (String json : eventJson) { - ChangeEvent event = JsonUtils.readValue(json, ChangeEvent.class); - changeEvents.add(event); + try { + ChangeEvent event = JsonUtils.readValue(json, ChangeEvent.class); + changeEvents.add(event); + } catch (Exception ex) { + errorEvents.add(new EntityError().withMessage(ex.getMessage()).withEntity(json)); + LOG.error("Error in Parsing Change Event : {} , Message: {} ", json, ex.getMessage(), ex); + } } - return changeEvents; + return new ResultList<>(changeEvents, errorEvents, null, null, eventJson.size()); } @Override public void execute(JobExecutionContext jobExecutionContext) { // Must Have , Before Execute the Init, Quartz Requires a Non-Arg Constructor this.init(jobExecutionContext); - // Poll Events from Change Event Table - List batch = pollEvents(offset, eventSubscription.getBatchSize()); - int batchSize = batch.size(); - Map> eventsWithReceivers = createEventsWithReceivers(batch); + long batchSize = 0; + Map> eventsWithReceivers = new HashMap<>(); try { + // Poll Events from Change Event Table + ResultList batch = pollEvents(offset, eventSubscription.getBatchSize()); + batchSize = batch.getPaging().getTotal(); + eventsWithReceivers.putAll(createEventsWithReceivers(batch.getData())); // Publish Events if (!eventsWithReceivers.isEmpty()) { alertMetrics.withTotalEvents(alertMetrics.getTotalEvents() + eventsWithReceivers.size()); publishEvents(eventsWithReceivers); } } catch (Exception e) { - LOG.error("Error in executing the Job : {} ", e.getMessage()); + LOG.error( + "Error in polling events for alert : {} , Offset : {} , Batch Size : {} ", + e.getMessage(), + offset, + batchSize, + e); + } finally { if (!eventsWithReceivers.isEmpty()) { // Commit the Offset diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java index 9f1c8482062a..0d1be623ce08 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/changeEvent/Consumer.java @@ -13,16 +13,16 @@ package org.openmetadata.service.apps.bundles.changeEvent; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.service.events.errors.EventPublisherException; +import org.openmetadata.service.util.ResultList; import org.quartz.JobExecutionContext; public interface Consumer { - List pollEvents(long offset, long batchSize); + ResultList pollEvents(long offset, long batchSize); void publishEvents(Map> events); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java index 840b7694fc5f..210aaf38bdbe 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/workflows/dataAssets/processors/DataInsightsEntityEnricherProcessor.java @@ -4,6 +4,7 @@ import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.END_TIMESTAMP_KEY; import static org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils.START_TIMESTAMP_KEY; import static org.openmetadata.service.apps.bundles.insights.workflows.dataAssets.DataAssetsWorkflow.ENTITY_TYPE_FIELDS_KEY; +import static org.openmetadata.service.search.SearchIndexUtils.parseFollowers; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_TYPE_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.TIMESTAMP_KEY; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.getUpdatedStats; @@ -231,10 +232,8 @@ private Map enrichEntity( oCustomProperties.ifPresent( o -> entityMap.put(String.format("%sCustomProperty", entityType), o)); - // Remove 'changeDescription' field - entityMap.remove("changeDescription"); - // Remove 'sampleData' - entityMap.remove("sampleData"); + // Parse Followers: + entityMap.put("followers", parseFollowers(entity.getFollowers())); return entityMap; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java index 8d256f4167fa..d110da423fca 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AbstractOmAppJobListener.java @@ -61,7 +61,8 @@ public void jobToBeExecuted(JobExecutionContext jobExecutionContext) { .withTimestamp(jobStartTime) .withRunType(runType) .withStatus(AppRunRecord.Status.RUNNING) - .withScheduleInfo(jobApp.getAppSchedule()); + .withScheduleInfo(jobApp.getAppSchedule()) + .withConfig(JsonUtils.getMap(jobApp.getAppConfiguration())); boolean update = false; if (jobExecutionContext.isRecovering()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java index c9db40ee835d..50c1fee40983 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/scheduler/AppScheduler.java @@ -12,6 +12,9 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; @@ -94,12 +97,17 @@ private AppScheduler( .getListenerManager() .addJobListener(new OmAppJobListener(dao), jobGroupEquals(APPS_JOB_GROUP)); - this.resetErrorTriggers(); + ScheduledExecutorService threadScheduler = Executors.newScheduledThreadPool(1); + threadScheduler.scheduleAtFixedRate(this::resetErrorTriggers, 0, 24, TimeUnit.HOURS); // Start Scheduler this.scheduler.start(); } + /* Quartz triggers can go into an "ERROR" state in some cases. Most notably when the jobs + constructor throws an error. I do not know why this happens and the issues seem to be transient. + This method resets all triggers in the ERROR state to the normal state. + */ private void resetErrorTriggers() { try { scheduler diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java index 6944de845734..caa0f6fa13d9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/EventSubscriptionJobException.java @@ -5,6 +5,10 @@ public EventSubscriptionJobException(String message) { super(message); } + public EventSubscriptionJobException(String message, Throwable throwable) { + super(message, throwable); + } + public EventSubscriptionJobException(Throwable throwable) { super(throwable); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java index 7b6bb3111590..d65fa9b503ca 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/Workflow.java @@ -7,15 +7,16 @@ @Getter public class Workflow { + public static final String INGESTION_PIPELINE_ID_VARIABLE = "ingestionPipelineId"; public static final String RELATED_ENTITY_VARIABLE = "relatedEntity"; - public static final String PAYLOAD = "payload"; public static final String RESULT_VARIABLE = "result"; - public static final String RESOLVED_BY_VARIABLE = "resolvedBy"; + public static final String UPDATED_BY_VARIABLE = "updatedBy"; public static final String STAGE_INSTANCE_STATE_ID_VARIABLE = "stageInstanceStateId"; public static final String WORKFLOW_INSTANCE_EXECUTION_ID_VARIABLE = "workflowInstanceExecutionId"; public static final String WORKFLOW_RUNTIME_EXCEPTION = "workflowRuntimeException"; public static final String EXCEPTION_VARIABLE = "exception"; + public static final String GLOBAL_NAMESPACE = "global"; private final TriggerWorkflow triggerWorkflow; private final MainWorkflow mainWorkflow; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java index 0167230c8330..eaa3cde064a6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowEventConsumer.java @@ -1,6 +1,9 @@ package org.openmetadata.service.governance.workflows; import static org.openmetadata.schema.entity.events.SubscriptionDestination.SubscriptionType.GOVERNANCE_WORKFLOW_CHANGE_EVENT; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; +import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; +import static org.openmetadata.service.governance.workflows.WorkflowVariableHandler.getNamespacedVariableName; import java.util.HashMap; import java.util.List; @@ -60,7 +63,9 @@ public void sendMessage(ChangeEvent event) throws EventPublisherException { Map variables = new HashMap<>(); - variables.put("relatedEntity", entityLink.getLinkString()); + variables.put( + getNamespacedVariableName(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE), + entityLink.getLinkString()); WorkflowHandler.getInstance().triggerWithSignal(signal, variables); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java index 16287cd21fda..305375a31e72 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java @@ -1,17 +1,20 @@ package org.openmetadata.service.governance.workflows; +import static org.openmetadata.service.governance.workflows.WorkflowVariableHandler.getNamespacedVariableName; import static org.openmetadata.service.governance.workflows.elements.TriggerFactory.getTriggerWorkflowId; import java.time.Duration; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.flowable.bpmn.converter.BpmnXMLConverter; -import org.flowable.common.engine.api.FlowableException; import org.flowable.common.engine.api.FlowableObjectNotFoundException; +import org.flowable.common.engine.impl.el.DefaultExpressionManager; import org.flowable.engine.HistoryService; import org.flowable.engine.ProcessEngine; import org.flowable.engine.ProcessEngineConfiguration; @@ -29,9 +32,11 @@ import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.exception.UnhandledServerException; import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.jdbi3.locator.ConnectionType; +import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineMapper; @Slf4j public class WorkflowHandler { @@ -40,8 +45,9 @@ public class WorkflowHandler { private RuntimeService runtimeService; private TaskService taskService; private HistoryService historyService; + private final Map expressionMap = new HashMap<>(); private static WorkflowHandler instance; - private static volatile boolean initialized = false; + @Getter private static volatile boolean initialized = false; private WorkflowHandler(OpenMetadataApplicationConfig config) { ProcessEngineConfiguration processEngineConfiguration = @@ -58,9 +64,18 @@ private WorkflowHandler(OpenMetadataApplicationConfig config) { processEngineConfiguration.setDatabaseType(ProcessEngineConfiguration.DATABASE_TYPE_POSTGRES); } + initializeExpressionMap(config); initializeNewProcessEngine(processEngineConfiguration); } + public void initializeExpressionMap(OpenMetadataApplicationConfig config) { + expressionMap.put("IngestionPipelineMapper", new IngestionPipelineMapper(config)); + expressionMap.put( + "PipelineServiceClient", + PipelineServiceClientFactory.createPipelineServiceClient( + config.getPipelineServiceClientConfiguration())); + } + public void initializeNewProcessEngine( ProcessEngineConfiguration currentProcessEngineConfiguration) { ProcessEngines.destroy(); @@ -86,6 +101,8 @@ public void initializeNewProcessEngine( .setAsyncExecutorMaxPoolSize(workflowSettings.getExecutorConfiguration().getMaxPoolSize()) .setAsyncExecutorThreadPoolQueueSize( workflowSettings.getExecutorConfiguration().getQueueSize()) + .setAsyncExecutorAsyncJobLockTimeInMillis( + workflowSettings.getExecutorConfiguration().getJobLockTimeInMillis()) .setAsyncExecutorMaxAsyncJobsDuePerAcquisition( workflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition()); @@ -96,6 +113,9 @@ public void initializeNewProcessEngine( Duration.ofDays( workflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays())); + // Add Expression Manager + processEngineConfiguration.setExpressionManager(new DefaultExpressionManager(expressionMap)); + // Add Global Failure Listener processEngineConfiguration.setEventListeners(List.of(new WorkflowFailureListener())); @@ -198,19 +218,57 @@ public void setCustomTaskId(String taskId, UUID customTaskId) { taskService.setVariable(taskId, "customTaskId", customTaskId.toString()); } + public String getParentActivityId(String executionId) { + String activityId = null; + + Execution execution = + runtimeService.createExecutionQuery().executionId(executionId).singleResult(); + + if (execution != null && execution.getParentId() != null) { + Execution parentExecution = + runtimeService.createExecutionQuery().executionId(execution.getParentId()).singleResult(); + + if (parentExecution != null) { + activityId = parentExecution.getActivityId(); + } + } + + return activityId; + } + + private Task getTaskFromCustomTaskId(UUID customTaskId) { + return taskService + .createTaskQuery() + .processVariableValueEquals("customTaskId", customTaskId.toString()) + .singleResult(); + } + + public Map transformToNodeVariables( + UUID customTaskId, Map variables) { + Map namespacedVariables = null; + Optional oTask = Optional.ofNullable(getTaskFromCustomTaskId(customTaskId)); + + if (oTask.isPresent()) { + Task task = oTask.get(); + String namespace = getParentActivityId(task.getExecutionId()); + namespacedVariables = new HashMap<>(); + for (Map.Entry entry : variables.entrySet()) { + namespacedVariables.put( + getNamespacedVariableName(namespace, entry.getKey()), entry.getValue()); + } + } else { + LOG.debug(String.format("Flowable Task for Task ID %s not found.", customTaskId)); + } + return namespacedVariables; + } + public void resolveTask(UUID taskId) { resolveTask(taskId, null); } public void resolveTask(UUID customTaskId, Map variables) { try { - Optional oTask = - Optional.ofNullable( - taskService - .createTaskQuery() - .processVariableValueEquals("customTaskId", customTaskId.toString()) - .singleResult()); - + Optional oTask = Optional.ofNullable(getTaskFromCustomTaskId(customTaskId)); if (oTask.isPresent()) { Task task = oTask.get(); Optional.ofNullable(variables) @@ -222,11 +280,6 @@ public void resolveTask(UUID customTaskId, Map variables) { } } catch (FlowableObjectNotFoundException ex) { LOG.debug(String.format("Flowable Task for Task ID %s not found.", customTaskId)); - } catch ( - FlowableException - ex) { // TODO: Remove this once we change the Task flow. Currently closeTask() is called - // twice. - LOG.debug(String.format("Flowable Exception: %s.", ex)); } } @@ -248,11 +301,6 @@ public void terminateTaskProcessInstance(UUID customTaskId, String reason) { } } catch (FlowableObjectNotFoundException ex) { LOG.debug(String.format("Flowable Task for Task ID %s not found.", customTaskId)); - } catch ( - FlowableException - ex) { // TODO: Remove this once we change the Task flow. Currently closeTask() is called - // twice. - LOG.debug(String.format("Flowable Exception: %s.", ex)); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowInstanceExecutionIdSetterListener.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowInstanceExecutionIdSetterListener.java index 9651615cd781..0106f2e276c3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowInstanceExecutionIdSetterListener.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowInstanceExecutionIdSetterListener.java @@ -1,5 +1,6 @@ package org.openmetadata.service.governance.workflows; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_INSTANCE_EXECUTION_ID_VARIABLE; import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; @@ -13,9 +14,11 @@ public class WorkflowInstanceExecutionIdSetterListener implements JavaDelegate { @Override public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); try { String workflowName = getProcessDefinitionKeyFromId(execution.getProcessDefinitionId()); - String relatedEntity = (String) execution.getVariable(RELATED_ENTITY_VARIABLE); + String relatedEntity = + (String) varHandler.getNamespacedVariable(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE); LOG.debug( String.format( "New Execution for Workflow '%s'. Related Entity: '%s'", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowVariableHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowVariableHandler.java new file mode 100644 index 000000000000..61e2f715a73b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowVariableHandler.java @@ -0,0 +1,65 @@ +package org.openmetadata.service.governance.workflows; + +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; + +import lombok.extern.slf4j.Slf4j; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.task.service.delegate.DelegateTask; +import org.flowable.variable.api.delegate.VariableScope; + +@Slf4j +public class WorkflowVariableHandler { + private final VariableScope varScope; + + public WorkflowVariableHandler(VariableScope varScope) { + this.varScope = varScope; + } + + public static String getNamespacedVariableName(String namespace, String varName) { + if (namespace != null) { + return String.format("%s_%s", namespace, varName); + } else { + return null; + } + } + + public Object getNamespacedVariable(String namespace, String varName) { + String namespacedVarName = getNamespacedVariableName(namespace, varName); + if (namespacedVarName != null) { + return varScope.getVariable(namespacedVarName); + } else { + return null; + } + } + + public void setNamespacedVariable(String namespace, String varName, Object varValue) { + String namespacedVarName = getNamespacedVariableName(namespace, varName); + if (namespacedVarName != null) { + varScope.setVariable(namespacedVarName, varValue); + LOG.debug(String.format("%s variable set to %s", namespacedVarName, varValue)); + } else { + throw new RuntimeException("Namespace can't be null when setting a namespaced variable."); + } + } + + public void setGlobalVariable(String varName, Object varValue) { + setNamespacedVariable(GLOBAL_NAMESPACE, varName, varValue); + } + + private String getNodeNamespace() { + if (varScope instanceof DelegateExecution) { + return ((DelegateExecution) varScope).getParent().getCurrentActivityId(); + } else if (varScope instanceof DelegateTask) { + return WorkflowHandler.getInstance() + .getParentActivityId(((DelegateTask) varScope).getExecutionId()); + } else { + throw new RuntimeException( + "varScope must be either an instance of 'DelegateExecution' or 'DelegateTask'."); + } + } + + public void setNodeVariable(String varName, Object varValue) { + String namespace = getNodeNamespace(); + setNamespacedVariable(namespace, varName, varValue); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/Edge.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/Edge.java index 0c2a579a3be5..06a8b1df7913 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/Edge.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/Edge.java @@ -1,6 +1,7 @@ package org.openmetadata.service.governance.workflows.elements; import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; +import static org.openmetadata.service.governance.workflows.WorkflowVariableHandler.getNamespacedVariableName; import org.flowable.bpmn.model.BpmnModel; import org.flowable.bpmn.model.Process; @@ -13,17 +14,15 @@ public class Edge { public Edge(org.openmetadata.schema.governance.workflows.elements.EdgeDefinition edgeDefinition) { SequenceFlow edge = new SequenceFlow(edgeDefinition.getFrom(), edgeDefinition.getTo()); if (!CommonUtil.nullOrEmpty(edgeDefinition.getCondition())) { - edge.setConditionExpression(getFlowableCondition(edgeDefinition.getCondition())); + edge.setConditionExpression( + getFlowableCondition(edgeDefinition.getFrom(), edgeDefinition.getCondition())); } this.edge = edge; } - private String getFlowableCondition(boolean condition) { - if (condition) { - return String.format("${%s}", RESULT_VARIABLE); - } else { - return String.format("${!%s}", RESULT_VARIABLE); - } + private String getFlowableCondition(String from, String condition) { + return String.format( + "${%s == '%s'}", getNamespacedVariableName(from, RESULT_VARIABLE), condition); } public void addToWorkflow(BpmnModel model, Process process) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java index 60e17fa138a3..c7265a5c91bf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/NodeFactory.java @@ -3,18 +3,23 @@ import org.openmetadata.schema.governance.workflows.elements.NodeSubType; import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; -import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CreateIngestionPipelineTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunAppTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunIngestionPipelineTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.endEvent.EndEventDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.gateway.ParallelGatewayDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.startEvent.StartEventDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.userTask.UserApprovalTaskDefinition; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTask; -import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.JsonLogicFilterTask; -import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.NoOpTask; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.CreateIngestionPipelineTask; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.RunIngestionPipelineTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTask; import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTask; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.runApp.RunAppTask; import org.openmetadata.service.governance.workflows.elements.nodes.endEvent.EndEvent; +import org.openmetadata.service.governance.workflows.elements.nodes.gateway.ParallelGateway; import org.openmetadata.service.governance.workflows.elements.nodes.startEvent.StartEvent; import org.openmetadata.service.governance.workflows.elements.nodes.userTask.UserApprovalTask; @@ -30,8 +35,12 @@ public static NodeInterface createNode(WorkflowNodeDefinitionInterface nodeDefin case SET_GLOSSARY_TERM_STATUS_TASK -> new SetGlossaryTermStatusTask( (SetGlossaryTermStatusTaskDefinition) nodeDefinition); case USER_APPROVAL_TASK -> new UserApprovalTask((UserApprovalTaskDefinition) nodeDefinition); - case PYTHON_WORKFLOW_AUTOMATION_TASK -> new NoOpTask(nodeDefinition); - case JSON_LOGIC_TASK -> new JsonLogicFilterTask((JsonLogicTaskDefinition) nodeDefinition); + case CREATE_INGESTION_PIPELINE_TASK -> new CreateIngestionPipelineTask( + (CreateIngestionPipelineTaskDefinition) nodeDefinition); + case RUN_INGESTION_PIPELINE_TASK -> new RunIngestionPipelineTask( + (RunIngestionPipelineTaskDefinition) nodeDefinition); + case RUN_APP_TASK -> new RunAppTask((RunAppTaskDefinition) nodeDefinition); + case PARALLEL_GATEWAY -> new ParallelGateway((ParallelGatewayDefinition) nodeDefinition); }; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java index 27ae31d267ec..6a295f4a715f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/TriggerFactory.java @@ -3,9 +3,7 @@ import org.openmetadata.schema.governance.workflows.TriggerType; import org.openmetadata.schema.governance.workflows.WorkflowDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.trigger.PeriodicBatchEntityTriggerDefinition; -import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; import org.openmetadata.schema.governance.workflows.elements.triggers.EventBasedEntityTriggerDefinition; -import org.openmetadata.service.governance.workflows.elements.triggers.CustomSignalTrigger; import org.openmetadata.service.governance.workflows.elements.triggers.EventBasedEntityTrigger; import org.openmetadata.service.governance.workflows.elements.triggers.PeriodicBatchEntityTrigger; @@ -18,10 +16,6 @@ public static TriggerInterface createTrigger(WorkflowDefinition workflow) { workflow.getName(), triggerWorkflowId, (EventBasedEntityTriggerDefinition) workflow.getTrigger()); - case CUSTOM_SIGNAL -> new CustomSignalTrigger( - workflow.getName(), - triggerWorkflowId, - (CustomSignalTriggerDefinition) workflow.getTrigger()); case PERIODIC_BATCH_ENTITY -> new PeriodicBatchEntityTrigger( workflow.getName(), triggerWorkflowId, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java index 808722afbc25..70337f7f9bf1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CheckEntityAttributesTask.java @@ -19,6 +19,7 @@ import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; +import org.openmetadata.service.util.JsonUtils; public class CheckEntityAttributesTask implements NodeInterface { private final SubProcess subProcess; @@ -33,7 +34,10 @@ public CheckEntityAttributesTask(CheckEntityAttributesTaskDefinition nodeDefinit new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); ServiceTask checkEntityAttributes = - getCheckEntityAttributesServiceTask(subProcessId, nodeDefinition.getConfig().getRules()); + getCheckEntityAttributesServiceTask( + subProcessId, + nodeDefinition.getConfig().getRules(), + JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())); EndEvent endEvent = new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); @@ -54,17 +58,22 @@ public BoundaryEvent getRuntimeExceptionBoundaryEvent() { return runtimeExceptionBoundaryEvent; } - private ServiceTask getCheckEntityAttributesServiceTask(String subProcessId, String rules) { + private ServiceTask getCheckEntityAttributesServiceTask( + String subProcessId, String rules, String inputNamespaceMap) { FieldExtension rulesExpr = new FieldExtensionBuilder().fieldName("rulesExpr").fieldValue(rules).build(); - - ServiceTask serviceTask = - new ServiceTaskBuilder() - .id(getFlowableElementId(subProcessId, "checkEntityAttributes")) - .implementation(CheckEntityAttributesImpl.class.getName()) + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(inputNamespaceMap) .build(); - serviceTask.getFieldExtensions().add(rulesExpr); - return serviceTask; + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "checkEntityAttributes")) + .implementation(CheckEntityAttributesImpl.class.getName()) + .addFieldExtension(rulesExpr) + .addFieldExtension(inputNamespaceMapExpr) + .build(); } public void addToWorkflow(BpmnModel model, Process process) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CreateIngestionPipelineTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CreateIngestionPipelineTask.java new file mode 100644 index 000000000000..aaaed47534ce --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/CreateIngestionPipelineTask.java @@ -0,0 +1,113 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask; + +import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; + +import org.flowable.bpmn.model.BoundaryEvent; +import org.flowable.bpmn.model.BpmnModel; +import org.flowable.bpmn.model.EndEvent; +import org.flowable.bpmn.model.FieldExtension; +import org.flowable.bpmn.model.Process; +import org.flowable.bpmn.model.SequenceFlow; +import org.flowable.bpmn.model.ServiceTask; +import org.flowable.bpmn.model.StartEvent; +import org.flowable.bpmn.model.SubProcess; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CreateIngestionPipelineTaskDefinition; +import org.openmetadata.service.governance.workflows.elements.NodeInterface; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl.CreateIngestionPipelineImpl; +import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.FieldExtensionBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; +import org.openmetadata.service.util.JsonUtils; + +public class CreateIngestionPipelineTask implements NodeInterface { + private final SubProcess subProcess; + private final BoundaryEvent runtimeExceptionBoundaryEvent; + + public CreateIngestionPipelineTask(CreateIngestionPipelineTaskDefinition nodeDefinition) { + String subProcessId = nodeDefinition.getName(); + + SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); + + StartEvent startEvent = + new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); + + ServiceTask createIngestionPipelineTask = + getCreateIngestionPipelineTask( + subProcessId, + nodeDefinition.getConfig().getPipelineType(), + nodeDefinition.getConfig().getDeploy(), + JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())); + + EndEvent endEvent = + new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); + + subProcess.addFlowElement(startEvent); + subProcess.addFlowElement(createIngestionPipelineTask); + subProcess.addFlowElement(endEvent); + + subProcess.addFlowElement( + new SequenceFlow(startEvent.getId(), createIngestionPipelineTask.getId())); + subProcess.addFlowElement( + new SequenceFlow(createIngestionPipelineTask.getId(), endEvent.getId())); + + this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.subProcess = subProcess; + } + + @Override + public BoundaryEvent getRuntimeExceptionBoundaryEvent() { + return runtimeExceptionBoundaryEvent; + } + + private ServiceTask getCreateIngestionPipelineTask( + String subProcessId, PipelineType pipelineType, boolean deploy, String inputNamespaceMap) { + FieldExtension pipelineTypeExpr = + new FieldExtensionBuilder() + .fieldName("pipelineTypeExpr") + .fieldValue(pipelineType.toString()) + .build(); + + FieldExtension deployExpr = + new FieldExtensionBuilder() + .fieldName("deployExpr") + .fieldValue(String.valueOf(deploy)) + .build(); + + FieldExtension ingestionPipelineMapperExpr = + new FieldExtensionBuilder() + .fieldName("ingestionPipelineMapperExpr") + .expression("${IngestionPipelineMapper}") + .build(); + + FieldExtension pipelineServiceClientExpr = + new FieldExtensionBuilder() + .fieldName("pipelineServiceClientExpr") + .expression("${PipelineServiceClient}") + .build(); + + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(inputNamespaceMap) + .build(); + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "checkIngestionPipelineSucceeded")) + .implementation(CreateIngestionPipelineImpl.class.getName()) + .addFieldExtension(pipelineTypeExpr) + .addFieldExtension(deployExpr) + .addFieldExtension(ingestionPipelineMapperExpr) + .addFieldExtension(pipelineServiceClientExpr) + .addFieldExtension(inputNamespaceMapExpr) + .setAsync(true) + .build(); + } + + public void addToWorkflow(BpmnModel model, Process process) { + process.addFlowElement(subProcess); + process.addFlowElement(runtimeExceptionBoundaryEvent); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java deleted file mode 100644 index b33d11d0cbe4..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/NoOpTask.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask; - -import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; - -import org.flowable.bpmn.model.BoundaryEvent; -import org.flowable.bpmn.model.BpmnModel; -import org.flowable.bpmn.model.EndEvent; -import org.flowable.bpmn.model.Process; -import org.flowable.bpmn.model.SequenceFlow; -import org.flowable.bpmn.model.ServiceTask; -import org.flowable.bpmn.model.StartEvent; -import org.flowable.bpmn.model.SubProcess; -import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; -import org.openmetadata.service.governance.workflows.elements.NodeInterface; -import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl.NoOpTaskImp; -import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; -import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; -import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; -import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; - -public class NoOpTask implements NodeInterface { - private final SubProcess subProcess; - private final BoundaryEvent runtimeExceptionBoundaryEvent; - - public NoOpTask(WorkflowNodeDefinitionInterface nodeDefinition) { - String subProcessId = nodeDefinition.getName(); - - SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); - - StartEvent startEvent = - new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); - - ServiceTask noOpTask = setNoOpTask(subProcessId); - - EndEvent endEvent = - new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); - - subProcess.addFlowElement(startEvent); - subProcess.addFlowElement(noOpTask); - subProcess.addFlowElement(endEvent); - - subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), noOpTask.getId())); - subProcess.addFlowElement(new SequenceFlow(noOpTask.getId(), endEvent.getId())); - - this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); - this.subProcess = subProcess; - } - - @Override - public BoundaryEvent getRuntimeExceptionBoundaryEvent() { - return runtimeExceptionBoundaryEvent; - } - - private ServiceTask setNoOpTask(String subProcessId) { - - return new ServiceTaskBuilder() - .id(getFlowableElementId(subProcessId, "printHelloTask")) - .implementation(NoOpTaskImp.class.getName()) - .build(); - } - - public void addToWorkflow(BpmnModel model, Process process) { - process.addFlowElement(subProcess); - process.addFlowElement(runtimeExceptionBoundaryEvent); - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/RunIngestionPipelineTask.java similarity index 51% rename from openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java rename to openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/RunIngestionPipelineTask.java index f73589a467c3..c2a9b466a494 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/JsonLogicFilterTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/RunIngestionPipelineTask.java @@ -11,20 +11,21 @@ import org.flowable.bpmn.model.ServiceTask; import org.flowable.bpmn.model.StartEvent; import org.flowable.bpmn.model.SubProcess; -import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunIngestionPipelineTaskDefinition; import org.openmetadata.service.governance.workflows.elements.NodeInterface; -import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl.JsonLogicFilterImpl; +import org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl.RunIngestionPipelineImpl; import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.FieldExtensionBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; +import org.openmetadata.service.util.JsonUtils; -public class JsonLogicFilterTask implements NodeInterface { +public class RunIngestionPipelineTask implements NodeInterface { private final SubProcess subProcess; private final BoundaryEvent runtimeExceptionBoundaryEvent; - public JsonLogicFilterTask(JsonLogicTaskDefinition nodeDefinition) { + public RunIngestionPipelineTask(RunIngestionPipelineTaskDefinition nodeDefinition) { String subProcessId = nodeDefinition.getName(); SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); @@ -32,18 +33,22 @@ public JsonLogicFilterTask(JsonLogicTaskDefinition nodeDefinition) { StartEvent startEvent = new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); - ServiceTask checkEntityAttributes = - getCheckEntityAttributesServiceTask(subProcessId, nodeDefinition.getConfig().getRules()); + ServiceTask runIngestionWorkflow = + getRunIngestionWorkflowServiceTask( + subProcessId, + nodeDefinition.getConfig().getWaitForCompletion(), + nodeDefinition.getConfig().getTimeoutSeconds(), + JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())); EndEvent endEvent = new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); subProcess.addFlowElement(startEvent); - subProcess.addFlowElement(checkEntityAttributes); + subProcess.addFlowElement(runIngestionWorkflow); subProcess.addFlowElement(endEvent); - subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), checkEntityAttributes.getId())); - subProcess.addFlowElement(new SequenceFlow(checkEntityAttributes.getId(), endEvent.getId())); + subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), runIngestionWorkflow.getId())); + subProcess.addFlowElement(new SequenceFlow(runIngestionWorkflow.getId(), endEvent.getId())); this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); this.subProcess = subProcess; @@ -54,17 +59,43 @@ public BoundaryEvent getRuntimeExceptionBoundaryEvent() { return runtimeExceptionBoundaryEvent; } - private ServiceTask getCheckEntityAttributesServiceTask(String subProcessId, String rules) { - FieldExtension rulesExpr = - new FieldExtensionBuilder().fieldName("rulesExpr").fieldValue(rules).build(); + private ServiceTask getRunIngestionWorkflowServiceTask( + String subProcessId, + boolean waitForCompletion, + long timeoutSeconds, + String inputNamespaceMap) { + FieldExtension waitExpr = + new FieldExtensionBuilder() + .fieldName("waitForCompletionExpr") + .fieldValue(String.valueOf(waitForCompletion)) + .build(); + FieldExtension timeoutSecondsExpr = + new FieldExtensionBuilder() + .fieldName("timeoutSecondsExpr") + .fieldValue(String.valueOf(timeoutSeconds)) + .build(); - ServiceTask serviceTask = - new ServiceTaskBuilder() - .id(getFlowableElementId(subProcessId, "jsonLogic")) - .implementation(JsonLogicFilterImpl.class.getName()) + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(inputNamespaceMap) .build(); - serviceTask.getFieldExtensions().add(rulesExpr); - return serviceTask; + + FieldExtension pipelineServiceClientExpr = + new FieldExtensionBuilder() + .fieldName("pipelineServiceClientExpr") + .expression("${PipelineServiceClient}") + .build(); + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "triggerIngestionWorkflow")) + .implementation(RunIngestionPipelineImpl.class.getName()) + .addFieldExtension(waitExpr) + .addFieldExtension(timeoutSecondsExpr) + .addFieldExtension(inputNamespaceMapExpr) + .addFieldExtension(pipelineServiceClientExpr) + .setAsync(true) + .build(); } public void addToWorkflow(BpmnModel model, Process process) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java index d17a544ce4de..d058f2b5d32d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetEntityCertificationTask.java @@ -21,6 +21,7 @@ import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; +import org.openmetadata.service.util.JsonUtils; public class SetEntityCertificationTask implements NodeInterface { private final SubProcess subProcess; @@ -38,7 +39,8 @@ public SetEntityCertificationTask(SetEntityCertificationTaskDefinition nodeDefin getSetEntityCertificationServiceTask( subProcessId, (CertificationConfiguration.CertificationEnum) - nodeDefinition.getConfig().getCertification()); + nodeDefinition.getConfig().getCertification(), + JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())); EndEvent endEvent = new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); @@ -60,7 +62,9 @@ public BoundaryEvent getRuntimeExceptionBoundaryEvent() { } private ServiceTask getSetEntityCertificationServiceTask( - String subProcessId, CertificationConfiguration.CertificationEnum certification) { + String subProcessId, + CertificationConfiguration.CertificationEnum certification, + String inputNamespaceMap) { FieldExtension certificationExpr = new FieldExtensionBuilder() .fieldName("certificationExpr") @@ -69,14 +73,18 @@ private ServiceTask getSetEntityCertificationServiceTask( .map(CertificationConfiguration.CertificationEnum::value) .orElse("")) .build(); - - ServiceTask serviceTask = - new ServiceTaskBuilder() - .id(getFlowableElementId(subProcessId, "setGlossaryTermStatus")) - .implementation(SetEntityCertificationImpl.class.getName()) + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(inputNamespaceMap) .build(); - serviceTask.getFieldExtensions().add(certificationExpr); - return serviceTask; + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "setGlossaryTermStatus")) + .implementation(SetEntityCertificationImpl.class.getName()) + .addFieldExtension(certificationExpr) + .addFieldExtension(inputNamespaceMapExpr) + .build(); } public void addToWorkflow(BpmnModel model, Process process) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java index ea3d4cd55ed9..f5cddda0b45e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/SetGlossaryTermStatusTask.java @@ -19,6 +19,7 @@ import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; +import org.openmetadata.service.util.JsonUtils; public class SetGlossaryTermStatusTask implements NodeInterface { private final SubProcess subProcess; @@ -34,7 +35,9 @@ public SetGlossaryTermStatusTask(SetGlossaryTermStatusTaskDefinition nodeDefinit ServiceTask setGlossaryTermStatus = getSetGlossaryTermStatusServiceTask( - subProcessId, nodeDefinition.getConfig().getGlossaryTermStatus().toString()); + subProcessId, + nodeDefinition.getConfig().getGlossaryTermStatus().toString(), + JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())); EndEvent endEvent = new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); @@ -55,17 +58,22 @@ public BoundaryEvent getRuntimeExceptionBoundaryEvent() { return runtimeExceptionBoundaryEvent; } - private ServiceTask getSetGlossaryTermStatusServiceTask(String subProcessId, String status) { + private ServiceTask getSetGlossaryTermStatusServiceTask( + String subProcessId, String status, String inputNamespaceMap) { FieldExtension statusExpr = new FieldExtensionBuilder().fieldName("statusExpr").fieldValue(status).build(); - - ServiceTask serviceTask = - new ServiceTaskBuilder() - .id(getFlowableElementId(subProcessId, "setGlossaryTermStatus")) - .implementation(SetGlossaryTermStatusImpl.class.getName()) + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(inputNamespaceMap) .build(); - serviceTask.getFieldExtensions().add(statusExpr); - return serviceTask; + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "setGlossaryTermStatus")) + .implementation(SetGlossaryTermStatusImpl.class.getName()) + .addFieldExtension(statusExpr) + .addFieldExtension(inputNamespaceMapExpr) + .build(); } public void addToWorkflow(BpmnModel model, Process process) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CheckEntityAttributesImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CheckEntityAttributesImpl.java index 7723107f45fd..43d4c9d7d6b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CheckEntityAttributesImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CheckEntityAttributesImpl.java @@ -8,6 +8,7 @@ import io.github.jamsesso.jsonlogic.JsonLogic; import io.github.jamsesso.jsonlogic.JsonLogicException; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.flowable.common.engine.api.delegate.Expression; import org.flowable.engine.delegate.BpmnError; @@ -16,26 +17,34 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.util.JsonUtils; @Slf4j public class CheckEntityAttributesImpl implements JavaDelegate { private Expression rulesExpr; + private Expression inputNamespaceMapExpr; @Override public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); String rules = (String) rulesExpr.getValue(execution); MessageParser.EntityLink entityLink = - MessageParser.EntityLink.parse((String) execution.getVariable(RELATED_ENTITY_VARIABLE)); - execution.setVariable(RESULT_VARIABLE, checkAttributes(entityLink, rules)); + MessageParser.EntityLink.parse( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); + varHandler.setNodeVariable(RESULT_VARIABLE, checkAttributes(entityLink, rules)); } catch (Exception exc) { LOG.error( String.format( "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), exc); - execution.setVariable(EXCEPTION_VARIABLE, exc.toString()); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CreateIngestionPipelineImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CreateIngestionPipelineImpl.java new file mode 100644 index 000000000000..cbf3858db4b0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/CreateIngestionPipelineImpl.java @@ -0,0 +1,183 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; + +import static org.openmetadata.service.Entity.API_SERVICE; +import static org.openmetadata.service.Entity.DASHBOARD_SERVICE; +import static org.openmetadata.service.Entity.DATABASE_SERVICE; +import static org.openmetadata.service.Entity.MESSAGING_SERVICE; +import static org.openmetadata.service.Entity.METADATA_SERVICE; +import static org.openmetadata.service.Entity.MLMODEL_SERVICE; +import static org.openmetadata.service.Entity.PIPELINE_SERVICE; +import static org.openmetadata.service.Entity.SEARCH_SERVICE; +import static org.openmetadata.service.Entity.STORAGE_SERVICE; +import static org.openmetadata.service.governance.workflows.Workflow.INGESTION_PIPELINE_ID_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.flowable.common.engine.api.delegate.Expression; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.engine.delegate.JavaDelegate; +import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType; +import org.openmetadata.schema.metadataIngestion.ApiServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.DashboardServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.DatabaseServiceAutoClassificationPipeline; +import org.openmetadata.schema.metadataIngestion.DatabaseServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.DatabaseServiceProfilerPipeline; +import org.openmetadata.schema.metadataIngestion.DatabaseServiceQueryLineagePipeline; +import org.openmetadata.schema.metadataIngestion.DatabaseServiceQueryUsagePipeline; +import org.openmetadata.schema.metadataIngestion.LogLevels; +import org.openmetadata.schema.metadataIngestion.MessagingServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.MlmodelServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.PipelineServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.SearchServiceMetadataPipeline; +import org.openmetadata.schema.metadataIngestion.SourceConfig; +import org.openmetadata.schema.metadataIngestion.StorageServiceMetadataPipeline; +import org.openmetadata.schema.type.Include; +import org.openmetadata.sdk.PipelineServiceClientInterface; +import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; +import org.openmetadata.service.jdbi3.IngestionPipelineRepository; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineMapper; +import org.openmetadata.service.util.JsonUtils; + +public class CreateIngestionPipelineImpl implements JavaDelegate { + private static final Map SUPPORT_FEATURE_MAP = new HashMap<>(); + + static { + SUPPORT_FEATURE_MAP.put(PipelineType.METADATA, "supportsMetadataExtraction"); + SUPPORT_FEATURE_MAP.put(PipelineType.USAGE, "supportsUsageExtraction"); + SUPPORT_FEATURE_MAP.put(PipelineType.LINEAGE, "supportsLineageExtraction"); + SUPPORT_FEATURE_MAP.put(PipelineType.PROFILER, "supportsProfiler"); + SUPPORT_FEATURE_MAP.put(PipelineType.AUTO_CLASSIFICATION, "supportsProfiler"); + } + + private static final Map DATABASE_PIPELINE_MAP = new HashMap<>(); + + static { + DATABASE_PIPELINE_MAP.put(PipelineType.METADATA, new DatabaseServiceMetadataPipeline()); + DATABASE_PIPELINE_MAP.put(PipelineType.USAGE, new DatabaseServiceQueryUsagePipeline()); + DATABASE_PIPELINE_MAP.put(PipelineType.LINEAGE, new DatabaseServiceQueryLineagePipeline()); + DATABASE_PIPELINE_MAP.put(PipelineType.PROFILER, new DatabaseServiceProfilerPipeline()); + DATABASE_PIPELINE_MAP.put( + PipelineType.AUTO_CLASSIFICATION, new DatabaseServiceAutoClassificationPipeline()); + } + + private static final Map SERVICE_TO_PIPELINE_MAP = new HashMap<>(); + + static { + SERVICE_TO_PIPELINE_MAP.put(MESSAGING_SERVICE, new MessagingServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(DASHBOARD_SERVICE, new DashboardServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(PIPELINE_SERVICE, new PipelineServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(MLMODEL_SERVICE, new MlmodelServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(METADATA_SERVICE, new DatabaseServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(STORAGE_SERVICE, new StorageServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(SEARCH_SERVICE, new SearchServiceMetadataPipeline()); + SERVICE_TO_PIPELINE_MAP.put(API_SERVICE, new ApiServiceMetadataPipeline()); + } + + private Expression pipelineTypeExpr; + private Expression deployExpr; + private Expression inputNamespaceMapExpr; + private Expression ingestionPipelineMapperExpr; + private Expression pipelineServiceClientExpr; + + @Override + public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); + PipelineType pipelineType = + PipelineType.fromValue((String) pipelineTypeExpr.getValue(execution)); + boolean deploy = Boolean.parseBoolean((String) deployExpr.getValue(execution)); + IngestionPipelineMapper mapper = + (IngestionPipelineMapper) ingestionPipelineMapperExpr.getValue(execution); + PipelineServiceClientInterface pipelineServiceClient = + (PipelineServiceClientInterface) pipelineServiceClientExpr.getValue(execution); + + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); + + ServiceEntityInterface service = Entity.getEntity(entityLink, "owners", Include.NON_DELETED); + + if (supportsPipelineType(pipelineType, JsonUtils.getMap(service.getConnection().getConfig()))) { + IngestionPipeline ingestionPipeline = createIngestionPipeline(mapper, pipelineType, service); + varHandler.setNodeVariable(INGESTION_PIPELINE_ID_VARIABLE, ingestionPipeline.getId()); + + boolean wasSuccessful = true; + + if (deploy) { + wasSuccessful = deployPipeline(pipelineServiceClient, ingestionPipeline, service); + } + // TODO: Use this variable to either continue the flow or send some kind of notification + varHandler.setNodeVariable(RESULT_VARIABLE, wasSuccessful); + } + } + + private boolean supportsPipelineType( + PipelineType pipelineType, Map connectionConfig) { + return Optional.ofNullable(connectionConfig.get(SUPPORT_FEATURE_MAP.get(pipelineType))) + .map(supports -> (boolean) supports) + .orElse(false); + } + + private boolean deployPipeline( + PipelineServiceClientInterface pipelineServiceClient, + IngestionPipeline ingestionPipeline, + ServiceEntityInterface service) { + PipelineServiceClientResponse response = + pipelineServiceClient.deployPipeline(ingestionPipeline, service); + return response.getCode() == 200; + } + + private IngestionPipeline createIngestionPipeline( + IngestionPipelineMapper mapper, PipelineType pipelineType, ServiceEntityInterface service) { + IngestionPipelineRepository repository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + + org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline create = + new org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline() + .withAirflowConfig(new AirflowConfig().withStartDate(getYesterdayDate())) + .withLoggerLevel(LogLevels.INFO) + .withName(UUID.randomUUID().toString()) + .withDisplayName(String.format("[%s] %s", service.getName(), pipelineType)) + .withOwners(service.getOwners()) + .withPipelineType(pipelineType) + .withService(service.getEntityReference()) + .withSourceConfig( + new SourceConfig().withConfig(getSourceConfig(pipelineType, service))); + IngestionPipeline ingestionPipeline = mapper.createToEntity(create, "governance-bot"); + + return repository.create(null, ingestionPipeline); + } + + private Object getSourceConfig(PipelineType pipelineType, ServiceEntityInterface service) { + String entityType = Entity.getEntityTypeFromObject(service); + if (entityType.equals(DATABASE_SERVICE)) { + return DATABASE_PIPELINE_MAP.get(pipelineType); + } else if (pipelineType.equals(PipelineType.METADATA)) { + return SERVICE_TO_PIPELINE_MAP.get(entityType); + } else { + return null; + } + } + + private Date getYesterdayDate() { + return Date.from( + LocalDate.now(ZoneOffset.UTC).minusDays(1).atStartOfDay(ZoneId.of("UTC")).toInstant()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java deleted file mode 100644 index 51e04b4b1d5e..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/JsonLogicFilterImpl.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; - -import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; -import static org.openmetadata.service.governance.workflows.Workflow.PAYLOAD; -import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; -import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; - -import io.github.jamsesso.jsonlogic.JsonLogic; -import io.github.jamsesso.jsonlogic.JsonLogicException; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.flowable.common.engine.api.delegate.Expression; -import org.flowable.engine.delegate.BpmnError; -import org.flowable.engine.delegate.DelegateExecution; -import org.flowable.engine.delegate.JavaDelegate; -import org.openmetadata.schema.EntityInterface; -import org.openmetadata.schema.type.ChangeEvent; -import org.openmetadata.schema.type.Include; -import org.openmetadata.service.Entity; -import org.openmetadata.service.util.JsonUtils; - -@Slf4j -public class JsonLogicFilterImpl implements JavaDelegate { - private Expression rulesExpr; - - @Override - public void execute(DelegateExecution execution) { - try { - // TODO why is 'rulesExpr' not passed as a variable? - String rules = (String) rulesExpr.getValue(execution); - String payload = (String) execution.getVariable("payload"); - List filtered = - JsonUtils.readObjects(payload, ChangeEvent.class).stream() - .filter( - ce -> { - EntityInterface entity = - Entity.getEntity(ce.getEntityType(), ce.getEntityId(), "*", Include.ALL); - return checkAttributes(rules, entity); - }) - .toList(); - - execution.setVariable(PAYLOAD, JsonUtils.pojoToJson(filtered)); - } catch (Exception exc) { - LOG.error( - "[{}] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId()), exc); - execution.setVariable(EXCEPTION_VARIABLE, exc.toString()); - throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); - } - } - - private Boolean checkAttributes(String rules, EntityInterface entity) { - JsonLogic jsonLogic = new JsonLogic(); - try { - return (boolean) jsonLogic.apply(rules, JsonUtils.getMap(entity)); - } catch (JsonLogicException e) { - throw new RuntimeException(e); - } - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java deleted file mode 100644 index 5480acc42320..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/NoOpTaskImp.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; - -import static org.openmetadata.service.governance.workflows.Workflow.PAYLOAD; - -import lombok.extern.slf4j.Slf4j; -import org.flowable.common.engine.api.delegate.Expression; -import org.flowable.engine.delegate.DelegateExecution; -import org.flowable.engine.delegate.JavaDelegate; - -@Slf4j -public class NoOpTaskImp implements JavaDelegate { - private Expression statusExpr; - - @Override - public void execute(DelegateExecution execution) { - String payload = (String) execution.getVariable(PAYLOAD); - System.out.println("NoOpTaskImp: " + payload); - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/RunIngestionPipelineImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/RunIngestionPipelineImpl.java new file mode 100644 index 000000000000..e5ba39ae467a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/RunIngestionPipelineImpl.java @@ -0,0 +1,123 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.impl; + +import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.INGESTION_PIPELINE_ID_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; +import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; +import static org.openmetadata.service.util.EntityUtil.Fields.EMPTY_FIELDS; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.flowable.common.engine.api.delegate.Expression; +import org.flowable.engine.delegate.BpmnError; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.engine.delegate.JavaDelegate; +import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineStatus; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineStatusType; +import org.openmetadata.schema.type.Include; +import org.openmetadata.sdk.PipelineServiceClientInterface; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; +import org.openmetadata.service.jdbi3.IngestionPipelineRepository; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; + +@Slf4j +public class RunIngestionPipelineImpl implements JavaDelegate { + private Expression inputNamespaceMapExpr; + private Expression pipelineServiceClientExpr; + private Expression waitForCompletionExpr; + private Expression timeoutSecondsExpr; + + @Override + public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); + try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); + + boolean waitForCompletion = + Boolean.parseBoolean((String) waitForCompletionExpr.getValue(execution)); + long timeoutSeconds = Long.parseLong((String) timeoutSecondsExpr.getValue(execution)); + + PipelineServiceClientInterface pipelineServiceClient = + (PipelineServiceClientInterface) pipelineServiceClientExpr.getValue(execution); + + UUID ingestionPipelineId = + (UUID) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(INGESTION_PIPELINE_ID_VARIABLE), + INGESTION_PIPELINE_ID_VARIABLE); + + IngestionPipelineRepository repository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + OpenMetadataApplicationConfig config = repository.getOpenMetadataApplicationConfig(); + + IngestionPipeline ingestionPipeline = repository.get(null, ingestionPipelineId, EMPTY_FIELDS); + ingestionPipeline.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(config).build()); + + ServiceEntityInterface service = + Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); + // TODO: Currently using this for v0. We should change to actually pooling the pipeline to + // check if it was deployed + Thread.sleep(60 * 1000); + pipelineServiceClient.runPipeline(ingestionPipeline, service); + + boolean success = true; + if (waitForCompletion) { + success = waitForIngestionPipeline(ingestionPipeline, repository, timeoutSeconds); + } + varHandler.setNodeVariable(RESULT_VARIABLE, success); + } catch (Exception exc) { + LOG.error( + String.format( + "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), + exc); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); + throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); + } + } + + private boolean waitForIngestionPipeline( + IngestionPipeline ingestionPipeline, + IngestionPipelineRepository repository, + long timeoutSeconds) { + + long startTimeMillis = System.currentTimeMillis(); + long timeoutMillis = timeoutSeconds * 1000; + while (true) { + if (System.currentTimeMillis() - startTimeMillis > timeoutMillis) { + return false; + } + + long FIVE_MINUTES_IN_MILLIS = 5 * 60 * 1000; + List statuses = + repository + .listPipelineStatus( + ingestionPipeline.getFullyQualifiedName(), + startTimeMillis - FIVE_MINUTES_IN_MILLIS, + startTimeMillis + timeoutMillis) + .getData(); + + if (statuses.isEmpty()) { + continue; + } + + PipelineStatus status = statuses.get(statuses.size() - 1); + + if (status.getPipelineState().equals(PipelineStatusType.FAILED)) { + return false; + } else if (status.getPipelineState().equals(PipelineStatusType.SUCCESS) + || status.getPipelineState().equals(PipelineStatusType.PARTIAL_SUCCESS)) { + return true; + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java index 122714e4d99a..fa873e8bbd32 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetEntityCertificationImpl.java @@ -2,10 +2,11 @@ import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; -import static org.openmetadata.service.governance.workflows.Workflow.RESOLVED_BY_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; +import java.util.Map; import java.util.Optional; import javax.json.JsonPatch; import lombok.extern.slf4j.Slf4j; @@ -18,6 +19,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.util.JsonUtils; @@ -25,12 +27,19 @@ @Slf4j public class SetEntityCertificationImpl implements JavaDelegate { private Expression certificationExpr; + private Expression inputNamespaceMapExpr; @Override public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); MessageParser.EntityLink entityLink = - MessageParser.EntityLink.parse((String) execution.getVariable(RELATED_ENTITY_VARIABLE)); + MessageParser.EntityLink.parse( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); String entityType = entityLink.getEntityType(); EntityInterface entity = Entity.getEntity(entityLink, "*", Include.ALL); @@ -39,7 +48,10 @@ public void execute(DelegateExecution execution) { .map(certificationExpr -> (String) certificationExpr.getValue(execution)) .orElse(null); String user = - Optional.ofNullable((String) execution.getVariable(RESOLVED_BY_VARIABLE)) + Optional.ofNullable( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(UPDATED_BY_VARIABLE), UPDATED_BY_VARIABLE)) .orElse("governance-bot"); setStatus(entity, entityType, user, certification); @@ -48,7 +60,7 @@ public void execute(DelegateExecution execution) { String.format( "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), exc); - execution.setVariable(EXCEPTION_VARIABLE, exc.toString()); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java index cfe38e652771..71827d466472 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/impl/SetGlossaryTermStatusImpl.java @@ -2,10 +2,11 @@ import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; -import static org.openmetadata.service.governance.workflows.Workflow.RESOLVED_BY_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; +import java.util.Map; import java.util.Objects; import java.util.Optional; import javax.json.JsonPatch; @@ -17,6 +18,7 @@ import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.jdbi3.GlossaryTermRepository; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.util.JsonUtils; @@ -24,17 +26,27 @@ @Slf4j public class SetGlossaryTermStatusImpl implements JavaDelegate { private Expression statusExpr; + private Expression inputNamespaceMapExpr; @Override public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); MessageParser.EntityLink entityLink = - MessageParser.EntityLink.parse((String) execution.getVariable(RELATED_ENTITY_VARIABLE)); + MessageParser.EntityLink.parse( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); GlossaryTerm glossaryTerm = Entity.getEntity(entityLink, "*", Include.ALL); String status = (String) statusExpr.getValue(execution); String user = - Optional.ofNullable((String) execution.getVariable(RESOLVED_BY_VARIABLE)) + Optional.ofNullable( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(UPDATED_BY_VARIABLE), UPDATED_BY_VARIABLE)) .orElse("governance-bot"); setStatus(glossaryTerm, user, status); @@ -43,7 +55,7 @@ public void execute(DelegateExecution execution) { String.format( "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), exc); - execution.setVariable(EXCEPTION_VARIABLE, exc.toString()); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppDelegate.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppDelegate.java new file mode 100644 index 000000000000..0e1fb26f18dc --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppDelegate.java @@ -0,0 +1,64 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.runApp; + +import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; +import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; + +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.flowable.common.engine.api.delegate.Expression; +import org.flowable.engine.delegate.BpmnError; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.engine.delegate.JavaDelegate; +import org.openmetadata.sdk.PipelineServiceClientInterface; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class RunAppDelegate implements JavaDelegate { + private Expression inputNamespaceMapExpr; + private Expression pipelineServiceClientExpr; + private Expression appNameExpr; + private Expression waitForCompletionExpr; + private Expression timeoutSecondsExpr; + + @Override + public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); + try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); + + String appName = (String) appNameExpr.getValue(execution); + boolean waitForCompletion = + Boolean.parseBoolean((String) waitForCompletionExpr.getValue(execution)); + long timeoutSeconds = Long.parseLong((String) timeoutSecondsExpr.getValue(execution)); + + PipelineServiceClientInterface pipelineServiceClient = + (PipelineServiceClientInterface) pipelineServiceClientExpr.getValue(execution); + + MessageParser.EntityLink entityLink = + MessageParser.EntityLink.parse( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); + + boolean success = + new RunAppImpl() + .execute( + pipelineServiceClient, appName, waitForCompletion, timeoutSeconds, entityLink); + + varHandler.setNodeVariable(RESULT_VARIABLE, success); + } catch (Exception exc) { + LOG.error( + String.format( + "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), + exc); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); + throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java new file mode 100644 index 000000000000..c3e9860194da --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppImpl.java @@ -0,0 +1,199 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.runApp; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.util.EntityUtil.Fields.EMPTY_FIELDS; + +import java.util.List; +import java.util.Set; +import javax.json.JsonPatch; +import lombok.SneakyThrows; +import org.openmetadata.schema.ServiceEntityInterface; +import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppRunRecord; +import org.openmetadata.schema.entity.app.AppType; +import org.openmetadata.schema.entity.app.external.CollateAIAppConfig; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineStatus; +import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineStatusType; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.sdk.PipelineServiceClientInterface; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.apps.ApplicationHandler; +import org.openmetadata.service.jdbi3.AppRepository; +import org.openmetadata.service.jdbi3.IngestionPipelineRepository; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.OpenMetadataConnectionBuilder; + +public class RunAppImpl { + public boolean execute( + PipelineServiceClientInterface pipelineServiceClient, + String appName, + boolean waitForCompletion, + long timeoutSeconds, + MessageParser.EntityLink entityLink) { + ServiceEntityInterface service = Entity.getEntity(entityLink, "owners", Include.NON_DELETED); + + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + App app = + appRepository.getByName(null, appName, new EntityUtil.Fields(Set.of("bot", "pipelines"))); + + if (!validateAppShouldRun(app, service)) { + return true; + } + + App updatedApp = getUpdatedApp(app, service); + + updateApp(appRepository, app, updatedApp); + + long startTime = System.currentTimeMillis(); + long timeoutMillis = timeoutSeconds * 1000; + boolean success = true; + + if (app.getAppType().equals(AppType.Internal)) { + success = runApp(appRepository, app, waitForCompletion, startTime, timeoutMillis); + } else { + success = runApp(pipelineServiceClient, app, waitForCompletion, startTime, timeoutMillis); + } + + updateApp(appRepository, updatedApp, app); + return success; + } + + private boolean validateAppShouldRun(App app, ServiceEntityInterface service) { + // We only want to run the CollateAIApplication for Databases + if (Entity.getEntityTypeFromObject(service).equals(Entity.DATABASE_SERVICE) + && app.getName().equals("CollateAIApplication")) { + return true; + } else { + return false; + } + } + + private App getUpdatedApp(App app, ServiceEntityInterface service) { + App updatedApp = JsonUtils.deepCopy(app, App.class); + Object updatedConfig = JsonUtils.deepCopy(app.getAppConfiguration(), Object.class); + + if (app.getName().equals("CollateAIApplication")) { + (JsonUtils.convertValue(updatedConfig, CollateAIAppConfig.class)) + .withFilter( + String.format( + "{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must\":[{\"term\":{\"Tier.TagFQN\":\"Tier.Tier1\"}}]}},{\"bool\":{\"must\":[{\"term\":{\"entityType\":\"table\"}}]}},{\"bool\":{\"must\":[{\"term\":{\"service.name.keyword\":\"%s\"}}]}}]}}}", + service.getName().toLowerCase())); + } + updatedApp.withAppConfiguration(updatedConfig); + return updatedApp; + } + + private void updateApp(AppRepository repository, App originalApp, App updatedApp) { + JsonPatch patch = JsonUtils.getJsonPatch(originalApp, updatedApp); + repository.patch(null, originalApp.getId(), "admin", patch); + } + + // Internal App Logic + @SneakyThrows + private boolean runApp( + AppRepository repository, + App app, + boolean waitForCompletion, + long startTime, + long timeoutMillis) { + ApplicationHandler.getInstance() + .triggerApplicationOnDemand(app, Entity.getCollectionDAO(), Entity.getSearchRepository()); + + if (waitForCompletion) { + return waitForCompletion(repository, app, startTime, timeoutMillis); + } else { + return true; + } + } + + private boolean waitForCompletion( + AppRepository repository, App app, long startTime, long timeoutMillis) { + AppRunRecord appRunRecord = null; + + do { + try { + if (System.currentTimeMillis() - startTime > timeoutMillis) { + return false; + } + appRunRecord = repository.getLatestAppRunsAfterStartTime(app, startTime); + } catch (Exception ignore) { + } + } while (!isRunCompleted(appRunRecord)); + + return appRunRecord.getStatus().equals(AppRunRecord.Status.SUCCESS) + || appRunRecord.getStatus().equals(AppRunRecord.Status.COMPLETED); + } + + private boolean isRunCompleted(AppRunRecord appRunRecord) { + if (appRunRecord == null) { + return false; + } + return !nullOrEmpty(appRunRecord.getExecutionTime()); + } + + // External App Logic + private boolean runApp( + PipelineServiceClientInterface pipelineServiceClient, + App app, + boolean waitForCompletion, + long startTime, + long timeoutMillis) { + EntityReference pipelineRef = app.getPipelines().get(0); + + IngestionPipelineRepository repository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + OpenMetadataApplicationConfig config = repository.getOpenMetadataApplicationConfig(); + + IngestionPipeline ingestionPipeline = repository.get(null, pipelineRef.getId(), EMPTY_FIELDS); + ingestionPipeline.setOpenMetadataServerConnection( + new OpenMetadataConnectionBuilder(config).build()); + + ServiceEntityInterface service = + Entity.getEntity(ingestionPipeline.getService(), "", Include.NON_DELETED); + + pipelineServiceClient.deployPipeline(ingestionPipeline, service); + pipelineServiceClient.runPipeline(ingestionPipeline, service); + + if (waitForCompletion) { + return waitForCompletion(repository, ingestionPipeline, startTime, timeoutMillis); + } else { + return true; + } + } + + private boolean waitForCompletion( + IngestionPipelineRepository repository, + IngestionPipeline ingestionPipeline, + long startTime, + long timeoutMillis) { + while (true) { + if (System.currentTimeMillis() - startTime > timeoutMillis) { + return false; + } + + List statuses = + repository + .listPipelineStatus( + ingestionPipeline.getFullyQualifiedName(), startTime, startTime + timeoutMillis) + .getData(); + + if (statuses.isEmpty()) { + continue; + } + + PipelineStatus status = statuses.get(statuses.size() - 1); + + if (status.getPipelineState().equals(PipelineStatusType.FAILED)) { + return false; + } else if (status.getPipelineState().equals(PipelineStatusType.SUCCESS) + || status.getPipelineState().equals(PipelineStatusType.PARTIAL_SUCCESS)) { + return true; + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java new file mode 100644 index 000000000000..1e31677752df --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/automatedTask/runApp/RunAppTask.java @@ -0,0 +1,110 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.automatedTask.runApp; + +import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; + +import org.flowable.bpmn.model.BoundaryEvent; +import org.flowable.bpmn.model.BpmnModel; +import org.flowable.bpmn.model.EndEvent; +import org.flowable.bpmn.model.FieldExtension; +import org.flowable.bpmn.model.Process; +import org.flowable.bpmn.model.SequenceFlow; +import org.flowable.bpmn.model.ServiceTask; +import org.flowable.bpmn.model.StartEvent; +import org.flowable.bpmn.model.SubProcess; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunAppTaskDefinition; +import org.openmetadata.service.governance.workflows.elements.NodeInterface; +import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.FieldExtensionBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.ServiceTaskBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; +import org.openmetadata.service.governance.workflows.flowable.builders.SubProcessBuilder; +import org.openmetadata.service.util.JsonUtils; + +public class RunAppTask implements NodeInterface { + private final SubProcess subProcess; + private final BoundaryEvent runtimeExceptionBoundaryEvent; + + public RunAppTask(RunAppTaskDefinition nodeDefinition) { + String subProcessId = nodeDefinition.getName(); + + SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); + + StartEvent startEvent = + new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); + + ServiceTask runApp = + getRunAppServiceTask( + subProcessId, + nodeDefinition.getConfig().getAppName(), + nodeDefinition.getConfig().getWaitForCompletion(), + nodeDefinition.getConfig().getTimeoutSeconds(), + JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())); + + EndEvent endEvent = + new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); + + subProcess.addFlowElement(startEvent); + subProcess.addFlowElement(runApp); + subProcess.addFlowElement(endEvent); + + subProcess.addFlowElement(new SequenceFlow(startEvent.getId(), runApp.getId())); + subProcess.addFlowElement(new SequenceFlow(runApp.getId(), endEvent.getId())); + + this.runtimeExceptionBoundaryEvent = getRuntimeExceptionBoundaryEvent(subProcess); + this.subProcess = subProcess; + } + + @Override + public BoundaryEvent getRuntimeExceptionBoundaryEvent() { + return runtimeExceptionBoundaryEvent; + } + + private ServiceTask getRunAppServiceTask( + String subProcessId, + String appName, + boolean waitForCompletion, + long timeoutSeconds, + String inputNamespaceMap) { + FieldExtension appNameExpr = + new FieldExtensionBuilder().fieldName("appNameExpr").fieldValue(appName).build(); + + FieldExtension waitExpr = + new FieldExtensionBuilder() + .fieldName("waitForCompletionExpr") + .fieldValue(String.valueOf(waitForCompletion)) + .build(); + FieldExtension timeoutSecondsExpr = + new FieldExtensionBuilder() + .fieldName("timeoutSecondsExpr") + .fieldValue(String.valueOf(timeoutSeconds)) + .build(); + + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(inputNamespaceMap) + .build(); + + FieldExtension pipelineServiceClientExpr = + new FieldExtensionBuilder() + .fieldName("pipelineServiceClientExpr") + .expression("${PipelineServiceClient}") + .build(); + + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "triggerIngestionWorkflow")) + .implementation(RunAppDelegate.class.getName()) + .addFieldExtension(appNameExpr) + .addFieldExtension(waitExpr) + .addFieldExtension(timeoutSecondsExpr) + .addFieldExtension(inputNamespaceMapExpr) + .addFieldExtension(pipelineServiceClientExpr) + .setAsync(true) + .build(); + } + + public void addToWorkflow(BpmnModel model, Process process) { + process.addFlowElement(subProcess); + process.addFlowElement(runtimeExceptionBoundaryEvent); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/gateway/ParallelGateway.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/gateway/ParallelGateway.java new file mode 100644 index 000000000000..153611d9758a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/gateway/ParallelGateway.java @@ -0,0 +1,19 @@ +package org.openmetadata.service.governance.workflows.elements.nodes.gateway; + +import org.flowable.bpmn.model.BpmnModel; +import org.flowable.bpmn.model.Process; +import org.openmetadata.schema.governance.workflows.elements.nodes.gateway.ParallelGatewayDefinition; +import org.openmetadata.service.governance.workflows.elements.NodeInterface; +import org.openmetadata.service.governance.workflows.flowable.builders.ParallelGatewayBuilder; + +public class ParallelGateway implements NodeInterface { + private final org.flowable.bpmn.model.ParallelGateway parallelGateway; + + public ParallelGateway(ParallelGatewayDefinition nodeDefinition) { + this.parallelGateway = new ParallelGatewayBuilder().id(nodeDefinition.getName()).build(); + } + + public void addToWorkflow(BpmnModel model, Process process) { + process.addFlowElement(parallelGateway); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java index b82dc8ab1ebc..6925e18cb4ae 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/UserApprovalTask.java @@ -53,15 +53,22 @@ public UserApprovalTask(UserApprovalTaskDefinition nodeDefinition) { .fieldValue(assigneesVarName) .build(); + FieldExtension inputNamespaceMapExpr = + new FieldExtensionBuilder() + .fieldName("inputNamespaceMapExpr") + .fieldValue(JsonUtils.pojoToJson(nodeDefinition.getInputNamespaceMap())) + .build(); + SubProcess subProcess = new SubProcessBuilder().id(subProcessId).build(); StartEvent startEvent = new StartEventBuilder().id(getFlowableElementId(subProcessId, "startEvent")).build(); ServiceTask setAssigneesVariable = - getSetAssigneesVariableServiceTask(subProcessId, assigneesExpr, assigneesVarNameExpr); + getSetAssigneesVariableServiceTask( + subProcessId, assigneesExpr, assigneesVarNameExpr, inputNamespaceMapExpr); - UserTask userTask = getUserTask(subProcessId, assigneesVarNameExpr); + UserTask userTask = getUserTask(subProcessId, assigneesVarNameExpr, inputNamespaceMapExpr); EndEvent endEvent = new EndEventBuilder().id(getFlowableElementId(subProcessId, "endEvent")).build(); @@ -101,37 +108,42 @@ public BoundaryEvent getRuntimeExceptionBoundaryEvent() { } private ServiceTask getSetAssigneesVariableServiceTask( - String subProcessId, FieldExtension assigneesExpr, FieldExtension assigneesVarNameExpr) { - ServiceTask serviceTask = - new ServiceTaskBuilder() - .id(getFlowableElementId(subProcessId, "setAssigneesVariable")) - .implementation(SetApprovalAssigneesImpl.class.getName()) - .build(); - serviceTask.getFieldExtensions().add(assigneesExpr); - serviceTask.getFieldExtensions().add(assigneesVarNameExpr); - return serviceTask; + String subProcessId, + FieldExtension assigneesExpr, + FieldExtension assigneesVarNameExpr, + FieldExtension inputNamespaceMapExpr) { + return new ServiceTaskBuilder() + .id(getFlowableElementId(subProcessId, "setAssigneesVariable")) + .implementation(SetApprovalAssigneesImpl.class.getName()) + .addFieldExtension(assigneesExpr) + .addFieldExtension(assigneesVarNameExpr) + .addFieldExtension(inputNamespaceMapExpr) + .build(); } - private UserTask getUserTask(String subProcessId, FieldExtension assigneesVarNameExpr) { + private UserTask getUserTask( + String subProcessId, + FieldExtension assigneesVarNameExpr, + FieldExtension inputNamespaceMapExpr) { FlowableListener setCandidateUsersListener = new FlowableListenerBuilder() .event("create") .implementation(SetCandidateUsersImpl.class.getName()) + .addFieldExtension(assigneesVarNameExpr) .build(); - setCandidateUsersListener.getFieldExtensions().add(assigneesVarNameExpr); FlowableListener createOpenMetadataTaskListener = new FlowableListenerBuilder() .event("create") .implementation(CreateApprovalTaskImpl.class.getName()) + .addFieldExtension(inputNamespaceMapExpr) .build(); - UserTask userTask = - new UserTaskBuilder().id(getFlowableElementId(subProcessId, "approvalTask")).build(); - userTask.getTaskListeners().add(setCandidateUsersListener); - userTask.getTaskListeners().add(createOpenMetadataTaskListener); - - return userTask; + return new UserTaskBuilder() + .id(getFlowableElementId(subProcessId, "approvalTask")) + .addListener(setCandidateUsersListener) + .addListener(createOpenMetadataTaskListener) + .build(); } private BoundaryEvent getTerminationEvent() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java index c20411d94689..9bad943b5384 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/CreateApprovalTaskImpl.java @@ -7,9 +7,11 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import lombok.extern.slf4j.Slf4j; +import org.flowable.common.engine.api.delegate.Expression; import org.flowable.engine.delegate.BpmnError; import org.flowable.engine.delegate.TaskListener; import org.flowable.identitylink.api.IdentityLink; @@ -25,20 +27,29 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.governance.workflows.WorkflowHandler; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.jdbi3.FeedRepository; import org.openmetadata.service.resources.feeds.FeedMapper; import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.util.JsonUtils; import org.openmetadata.service.util.WebsocketNotificationHandler; @Slf4j public class CreateApprovalTaskImpl implements TaskListener { + private Expression inputNamespaceMapExpr; + @Override public void notify(DelegateTask delegateTask) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(delegateTask); try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(delegateTask), Map.class); List assignees = getAssignees(delegateTask); MessageParser.EntityLink entityLink = MessageParser.EntityLink.parse( - (String) delegateTask.getVariable(RELATED_ENTITY_VARIABLE)); + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); GlossaryTerm entity = Entity.getEntity(entityLink, "*", Include.ALL); Thread task = createApprovalTask(entity, assignees); @@ -49,7 +60,7 @@ public void notify(DelegateTask delegateTask) { "[%s] Failure: ", getProcessDefinitionKeyFromId(delegateTask.getProcessDefinitionId())), exc); - delegateTask.setVariable(EXCEPTION_VARIABLE, exc.toString()); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetApprovalAssigneesImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetApprovalAssigneesImpl.java index f3a2ed0585d2..2660ce9b983f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetApprovalAssigneesImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetApprovalAssigneesImpl.java @@ -18,6 +18,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.util.JsonUtils; @@ -25,10 +26,14 @@ public class SetApprovalAssigneesImpl implements JavaDelegate { private Expression assigneesExpr; private Expression assigneesVarNameExpr; + private Expression inputNamespaceMapExpr; @Override public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); try { + Map inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); Map assigneesConfig = JsonUtils.readOrConvertValue(assigneesExpr.getValue(execution), Map.class); Boolean addReviewers = (Boolean) assigneesConfig.get("addReviewers"); @@ -40,7 +45,10 @@ public void execute(DelegateExecution execution) { if (addReviewers) { MessageParser.EntityLink entityLink = - MessageParser.EntityLink.parse((String) execution.getVariable(RELATED_ENTITY_VARIABLE)); + MessageParser.EntityLink.parse( + (String) + varHandler.getNamespacedVariable( + inputNamespaceMap.get(RELATED_ENTITY_VARIABLE), RELATED_ENTITY_VARIABLE)); EntityInterface entity = Entity.getEntity(entityLink, "*", Include.ALL); assignees.addAll(getEntityLinkStringFromEntityReference(entity.getReviewers())); } @@ -56,7 +64,7 @@ public void execute(DelegateExecution execution) { String.format( "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), exc); - execution.setVariable(EXCEPTION_VARIABLE, exc.toString()); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetCandidateUsersImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetCandidateUsersImpl.java index ee3234593f38..3aa41d608f49 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetCandidateUsersImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/nodes/userTask/impl/SetCandidateUsersImpl.java @@ -10,6 +10,7 @@ import org.flowable.engine.delegate.BpmnError; import org.flowable.engine.delegate.TaskListener; import org.flowable.task.service.delegate.DelegateTask; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.util.JsonUtils; @Slf4j @@ -18,6 +19,7 @@ public class SetCandidateUsersImpl implements TaskListener { @Override public void notify(DelegateTask delegateTask) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(delegateTask); try { List assignees = JsonUtils.readOrConvertValue( @@ -30,7 +32,7 @@ public void notify(DelegateTask delegateTask) { "[%s] Failure: ", getProcessDefinitionKeyFromId(delegateTask.getProcessDefinitionId())), exc); - delegateTask.setVariable(EXCEPTION_VARIABLE, exc.toString()); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java deleted file mode 100644 index 107fd2eb6377..000000000000 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/CustomSignalTrigger.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.openmetadata.service.governance.workflows.elements.triggers; - -import static org.openmetadata.service.governance.workflows.Workflow.PAYLOAD; -import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; -import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; - -import java.util.ArrayList; -import java.util.List; -import lombok.Getter; -import org.flowable.bpmn.model.BoundaryEvent; -import org.flowable.bpmn.model.BpmnModel; -import org.flowable.bpmn.model.CallActivity; -import org.flowable.bpmn.model.EndEvent; -import org.flowable.bpmn.model.ErrorEventDefinition; -import org.flowable.bpmn.model.IOParameter; -import org.flowable.bpmn.model.Process; -import org.flowable.bpmn.model.SequenceFlow; -import org.flowable.bpmn.model.Signal; -import org.flowable.bpmn.model.SignalEventDefinition; -import org.flowable.bpmn.model.StartEvent; -import org.jetbrains.annotations.NotNull; -import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; -import org.openmetadata.service.governance.workflows.elements.TriggerInterface; -import org.openmetadata.service.governance.workflows.flowable.builders.CallActivityBuilder; -import org.openmetadata.service.governance.workflows.flowable.builders.EndEventBuilder; -import org.openmetadata.service.governance.workflows.flowable.builders.SignalBuilder; -import org.openmetadata.service.governance.workflows.flowable.builders.StartEventBuilder; - -public class CustomSignalTrigger implements TriggerInterface { - private final Process process; - @Getter private final String triggerWorkflowId; - private StartEvent startEvent = null; - private final List signals = new ArrayList<>(); - - public CustomSignalTrigger( - String mainWorkflowName, - String triggerWorkflowId, - CustomSignalTriggerDefinition triggerDefinition) { - Process process = new Process(); - process.setId(triggerWorkflowId); - process.setName(triggerWorkflowId); - attachWorkflowInstanceListeners(process); - - setStartEvent(triggerWorkflowId, triggerDefinition); - - CallActivity workflowTrigger = getWorkflowTrigger(triggerWorkflowId, mainWorkflowName); - process.addFlowElement(workflowTrigger); - - BoundaryEvent runtimeExceptionBoundaryEvent = getBoundaryEvent(workflowTrigger); - process.addFlowElement(runtimeExceptionBoundaryEvent); - - EndEvent errorEndEvent = - new EndEventBuilder().id(getFlowableElementId(triggerWorkflowId, "errorEndEvent")).build(); - process.addFlowElement(errorEndEvent); - - EndEvent endEvent = - new EndEventBuilder().id(getFlowableElementId(triggerWorkflowId, "endEvent")).build(); - process.addFlowElement(endEvent); - - // Start Events -> FilterTask - process.addFlowElement(startEvent); - process.addFlowElement(new SequenceFlow(startEvent.getId(), workflowTrigger.getId())); - process.addFlowElement(new SequenceFlow(workflowTrigger.getId(), endEvent.getId())); - - // WorkflowTrigger -> End - process.addFlowElement( - new SequenceFlow(runtimeExceptionBoundaryEvent.getId(), errorEndEvent.getId())); - - this.process = process; - this.triggerWorkflowId = triggerWorkflowId; - } - - private static @NotNull BoundaryEvent getBoundaryEvent(CallActivity workflowTrigger) { - ErrorEventDefinition runtimeExceptionDefinition = new ErrorEventDefinition(); - runtimeExceptionDefinition.setErrorCode(WORKFLOW_RUNTIME_EXCEPTION); - - BoundaryEvent runtimeExceptionBoundaryEvent = new BoundaryEvent(); - runtimeExceptionBoundaryEvent.setId( - getFlowableElementId(workflowTrigger.getId(), "runtimeExceptionBoundaryEvent")); - runtimeExceptionBoundaryEvent.addEventDefinition(runtimeExceptionDefinition); - - runtimeExceptionBoundaryEvent.setAttachedToRef(workflowTrigger); - return runtimeExceptionBoundaryEvent; - } - - private void setStartEvent( - String workflowTriggerId, CustomSignalTriggerDefinition triggerDefinition) { - Signal signal = new SignalBuilder().id(triggerDefinition.getConfig().getSignal()).build(); - - SignalEventDefinition signalEventDefinition = new SignalEventDefinition(); - signalEventDefinition.setSignalRef(signal.getId()); - - StartEvent startEvent = - new StartEventBuilder().id(getFlowableElementId(workflowTriggerId, "customSignal")).build(); - startEvent.getEventDefinitions().add(signalEventDefinition); - - this.startEvent = startEvent; - this.signals.add(signal); - } - - private CallActivity getWorkflowTrigger(String triggerWorkflowId, String mainWorkflowName) { - CallActivity workflowTrigger = - new CallActivityBuilder() - .id(getFlowableElementId(triggerWorkflowId, "workflowTrigger")) - .calledElement(mainWorkflowName) - .inheritBusinessKey(true) - .build(); - - IOParameter inputParameter = new IOParameter(); - inputParameter.setSource(PAYLOAD); - inputParameter.setTarget(PAYLOAD); - - IOParameter outputParameter = new IOParameter(); - outputParameter.setSource(PAYLOAD); - outputParameter.setTarget(PAYLOAD); - - workflowTrigger.setInParameters(List.of(inputParameter)); - workflowTrigger.setOutParameters(List.of(outputParameter)); - - return workflowTrigger; - } - - @Override - public void addToWorkflow(BpmnModel model) { - model.addProcess(process); - for (Signal signal : signals) { - model.addSignal(signal); - } - } -} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/EventBasedEntityTrigger.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/EventBasedEntityTrigger.java index f8761bf96656..2076232e36de 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/EventBasedEntityTrigger.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/EventBasedEntityTrigger.java @@ -1,9 +1,11 @@ package org.openmetadata.service.governance.workflows.elements.triggers; import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; +import static org.openmetadata.service.governance.workflows.WorkflowVariableHandler.getNamespacedVariableName; import java.util.ArrayList; import java.util.List; @@ -144,12 +146,12 @@ private CallActivity getWorkflowTrigger(String triggerWorkflowId, String mainWor .build(); IOParameter inputParameter = new IOParameter(); - inputParameter.setSource(RELATED_ENTITY_VARIABLE); - inputParameter.setTarget(RELATED_ENTITY_VARIABLE); + inputParameter.setSource(getNamespacedVariableName(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE)); + inputParameter.setTarget(getNamespacedVariableName(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE)); IOParameter outputParameter = new IOParameter(); - outputParameter.setSource(EXCEPTION_VARIABLE); - outputParameter.setTarget(EXCEPTION_VARIABLE); + outputParameter.setSource(getNamespacedVariableName(GLOBAL_NAMESPACE, EXCEPTION_VARIABLE)); + outputParameter.setTarget(getNamespacedVariableName(GLOBAL_NAMESPACE, EXCEPTION_VARIABLE)); workflowTrigger.setInParameters(List.of(inputParameter)); workflowTrigger.setOutParameters(List.of(outputParameter)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/PeriodicBatchEntityTrigger.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/PeriodicBatchEntityTrigger.java index 800a614b3c6e..11ae73977233 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/PeriodicBatchEntityTrigger.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/PeriodicBatchEntityTrigger.java @@ -1,8 +1,10 @@ package org.openmetadata.service.governance.workflows.elements.triggers; import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.getFlowableElementId; +import static org.openmetadata.service.governance.workflows.WorkflowVariableHandler.getNamespacedVariableName; import java.util.List; import java.util.Optional; @@ -120,10 +122,10 @@ private CallActivity getWorkflowTriggerCallActivity( IOParameter inputParameter = new IOParameter(); inputParameter.setSource(RELATED_ENTITY_VARIABLE); - inputParameter.setTarget(RELATED_ENTITY_VARIABLE); + inputParameter.setTarget(getNamespacedVariableName(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE)); IOParameter outputParameter = new IOParameter(); - outputParameter.setSource(EXCEPTION_VARIABLE); + outputParameter.setSource(getNamespacedVariableName(GLOBAL_NAMESPACE, EXCEPTION_VARIABLE)); outputParameter.setTarget(EXCEPTION_VARIABLE); workflowTrigger.setInParameters(List.of(inputParameter)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/impl/FilterEntityImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/impl/FilterEntityImpl.java index 3d8bd581415f..834e9462e0b6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/impl/FilterEntityImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/elements/triggers/impl/FilterEntityImpl.java @@ -1,5 +1,6 @@ package org.openmetadata.service.governance.workflows.elements.triggers.impl; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; import static org.openmetadata.service.governance.workflows.elements.triggers.EventBasedEntityTrigger.PASSES_FILTER_VARIABLE; @@ -13,6 +14,7 @@ import org.openmetadata.schema.type.FieldChange; import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.util.JsonUtils; @@ -21,10 +23,12 @@ public class FilterEntityImpl implements JavaDelegate { @Override public void execute(DelegateExecution execution) { + WorkflowVariableHandler varHandler = new WorkflowVariableHandler(execution); List excludedFilter = JsonUtils.readOrConvertValue(excludedFilterExpr.getValue(execution), List.class); - String entityLinkStr = (String) execution.getVariable(RELATED_ENTITY_VARIABLE); + String entityLinkStr = + (String) varHandler.getNamespacedVariable(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE); execution.setVariable( PASSES_FILTER_VARIABLE, passesExcludedFilter(entityLinkStr, excludedFilter)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/BaseDelegate.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/BaseDelegate.java new file mode 100644 index 000000000000..01fea92c5e49 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/BaseDelegate.java @@ -0,0 +1,44 @@ +package org.openmetadata.service.governance.workflows.flowable; + +import static org.openmetadata.service.governance.workflows.Workflow.EXCEPTION_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.WORKFLOW_RUNTIME_EXCEPTION; +import static org.openmetadata.service.governance.workflows.WorkflowHandler.getProcessDefinitionKeyFromId; + +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.flowable.common.engine.api.delegate.Expression; +import org.flowable.engine.delegate.BpmnError; +import org.flowable.engine.delegate.DelegateExecution; +import org.flowable.engine.delegate.JavaDelegate; +import org.openmetadata.service.governance.workflows.WorkflowVariableHandler; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public abstract class BaseDelegate implements JavaDelegate { + private Expression inputNamespaceMapExpr; + private Expression configMapExpr; + + protected WorkflowVariableHandler varHandler; + protected Map inputNamespaceMap; + protected Map configMap; + + protected abstract void innerExecute(DelegateExecution execution); + + @Override + public void execute(DelegateExecution execution) { + varHandler = new WorkflowVariableHandler(execution); + try { + inputNamespaceMap = + JsonUtils.readOrConvertValue(inputNamespaceMapExpr.getValue(execution), Map.class); + configMap = JsonUtils.readOrConvertValue(configMapExpr.getValue(execution), Map.class); + innerExecute(execution); + } catch (Exception exc) { + LOG.error( + String.format( + "[%s] Failure: ", getProcessDefinitionKeyFromId(execution.getProcessDefinitionId())), + exc); + varHandler.setGlobalVariable(EXCEPTION_VARIABLE, exc.toString()); + throw new BpmnError(WORKFLOW_RUNTIME_EXCEPTION, exc.getMessage()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java index 53acaa0d8e0e..6fcafdacc5de 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/MainWorkflow.java @@ -1,8 +1,13 @@ package org.openmetadata.service.governance.workflows.flowable; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; + import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import lombok.Getter; import org.flowable.bpmn.model.BoundaryEvent; import org.flowable.bpmn.model.BpmnModel; @@ -15,6 +20,7 @@ import org.openmetadata.service.governance.workflows.elements.NodeFactory; import org.openmetadata.service.governance.workflows.elements.NodeInterface; import org.openmetadata.service.governance.workflows.elements.nodes.endEvent.EndEvent; +import org.openmetadata.service.util.JsonUtils; @Getter public class MainWorkflow { @@ -23,6 +29,8 @@ public class MainWorkflow { private final List runtimeExceptionBoundaryEvents = new ArrayList<>(); public MainWorkflow(WorkflowDefinition workflowDefinition) { + new WorkflowGraph(workflowDefinition).validate(); + BpmnModel model = new BpmnModel(); model.setTargetNamespace(""); String workflowName = workflowDefinition.getFullyQualifiedName(); @@ -63,4 +71,97 @@ private void configureRuntimeExceptionFlow(Process process) { process.addFlowElement(new SequenceFlow(event.getId(), errorEndEvent.getEndEvent().getId())); } } + + @Getter + public static class WorkflowGraph { + private final Map nodeMap; + private final Map> incomingEdgesMap; + private final Set globalVariables; + + public WorkflowGraph(WorkflowDefinition workflowDefinition) { + Map nodeMap = new HashMap<>(); + Map> incomingEdgesMap = new HashMap<>(); + + for (WorkflowNodeDefinitionInterface nodeDefinitionObj : workflowDefinition.getNodes()) { + nodeMap.put(nodeDefinitionObj.getName(), nodeDefinitionObj); + } + + for (EdgeDefinition edgeDefinition : workflowDefinition.getEdges()) { + incomingEdgesMap + .computeIfAbsent(edgeDefinition.getTo(), k -> new ArrayList<>()) + .add(edgeDefinition.getFrom()); + } + + this.nodeMap = nodeMap; + this.incomingEdgesMap = incomingEdgesMap; + this.globalVariables = workflowDefinition.getTrigger().getOutput(); + } + + private void validateNode(WorkflowNodeDefinitionInterface nodeDefinition) { + Map inputNamespaceMap = + (Map) + JsonUtils.readOrConvertValue(nodeDefinition.getInputNamespaceMap(), Map.class); + + if (inputNamespaceMap == null) { + return; + } + + for (Map.Entry entry : inputNamespaceMap.entrySet()) { + String variable = entry.getKey(); + String namespace = entry.getValue(); + + if (namespace.equals(GLOBAL_NAMESPACE)) { + if (!validateGlobalContainsVariable(variable)) { + throw new RuntimeException( + String.format( + "Invalid Workflow: [%s] is expecting '%s' to be a global variable, but it is not present.", + nodeDefinition.getName(), variable)); + } + } else { + if (!validateNodeOutputsVariable(namespace, variable)) { + throw new RuntimeException( + String.format( + "Invalid Workflow: [%s] is expecting '%s' to be an output from [%s], which it is not.", + nodeDefinition.getName(), variable, namespace)); + } + if (!validateNodeHasInput(nodeDefinition.getName(), namespace)) { + throw new RuntimeException( + String.format( + "Invalid Workflow: [%s] is expecting [%s] to be an input node, which it is not.", + nodeDefinition.getName(), namespace)); + } + } + } + } + + private boolean validateGlobalContainsVariable(String variable) { + return globalVariables.contains(variable); + } + + private boolean validateNodeOutputsVariable(String nodeName, String variable) { + WorkflowNodeDefinitionInterface nodeDefinition = nodeMap.get(nodeName); + + if (nodeDefinition == null) { + return false; + } + + List nodeOutput = nodeDefinition.getOutput(); + + if (nodeOutput == null) { + return false; + } + + return nodeOutput.contains(variable); + } + + private boolean validateNodeHasInput(String nodeName, String inputNodeName) { + return incomingEdgesMap.get(nodeName).contains(inputNodeName); + } + + public void validate() { + for (WorkflowNodeDefinitionInterface nodeDefinition : nodeMap.values()) { + validateNode(nodeDefinition); + } + } + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FieldExtensionBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FieldExtensionBuilder.java index fcbce5b6a095..e73048e1b49f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FieldExtensionBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FieldExtensionBuilder.java @@ -4,7 +4,9 @@ public class FieldExtensionBuilder { private String fieldName; - private String fieldValue; + private String fieldValue = null; + + private String expression = null; public FieldExtensionBuilder fieldName(String fieldName) { this.fieldName = fieldName; @@ -16,10 +18,22 @@ public FieldExtensionBuilder fieldValue(String fieldValue) { return this; } + public FieldExtensionBuilder expression(String expression) { + this.expression = expression; + return this; + } + public FieldExtension build() { FieldExtension fieldExtension = new FieldExtension(); fieldExtension.setFieldName(fieldName); - fieldExtension.setStringValue(fieldValue); + if (fieldValue != null) { + fieldExtension.setStringValue(fieldValue); + } else if (expression != null) { + fieldExtension.setExpression(expression); + } else { + throw new RuntimeException( + "FieldExtension must have either a 'fieldValue' or an 'expression'"); + } return fieldExtension; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FlowableListenerBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FlowableListenerBuilder.java index 0385587b043b..259a91d2b3b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FlowableListenerBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/FlowableListenerBuilder.java @@ -1,10 +1,15 @@ package org.openmetadata.service.governance.workflows.flowable.builders; +import java.util.ArrayList; +import java.util.List; +import org.flowable.bpmn.model.FieldExtension; import org.flowable.bpmn.model.FlowableListener; public class FlowableListenerBuilder { private String event; private String implementation; + private String implementationType = "class"; + private final List fieldExtensions = new ArrayList<>(); public FlowableListenerBuilder event(String event) { this.event = event; @@ -16,11 +21,25 @@ public FlowableListenerBuilder implementation(String implementation) { return this; } + public FlowableListenerBuilder implementationType(String implementationType) { + this.implementationType = implementationType; + return this; + } + + public FlowableListenerBuilder addFieldExtension(FieldExtension fieldExtension) { + this.fieldExtensions.add(fieldExtension); + return this; + } + public FlowableListener build() { FlowableListener listener = new FlowableListener(); listener.setEvent(event); - listener.setImplementationType("class"); + listener.setImplementationType(implementationType); listener.setImplementation(implementation); + + for (FieldExtension fieldExtension : fieldExtensions) { + listener.getFieldExtensions().add(fieldExtension); + } return listener; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ParallelGatewayBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ParallelGatewayBuilder.java new file mode 100644 index 000000000000..8c0fb1fa4eda --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ParallelGatewayBuilder.java @@ -0,0 +1,13 @@ +package org.openmetadata.service.governance.workflows.flowable.builders; + +import org.flowable.bpmn.model.ParallelGateway; + +public class ParallelGatewayBuilder extends FlowableElementBuilder { + @Override + public ParallelGateway build() { + ParallelGateway gateway = new ParallelGateway(); + gateway.setId(id); + gateway.setName(id); + return gateway; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ServiceTaskBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ServiceTaskBuilder.java index 6fc7472380eb..33927f4ceb68 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ServiceTaskBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/ServiceTaskBuilder.java @@ -1,22 +1,49 @@ package org.openmetadata.service.governance.workflows.flowable.builders; +import java.util.ArrayList; +import java.util.List; +import org.flowable.bpmn.model.FieldExtension; import org.flowable.bpmn.model.ServiceTask; public class ServiceTaskBuilder extends FlowableElementBuilder { private String implementation; + private String implementationType = "class"; + private boolean async = false; + private final List fieldExtensions = new ArrayList<>(); public ServiceTaskBuilder implementation(String implementation) { this.implementation = implementation; return this; } + public ServiceTaskBuilder implementationType(String implementationType) { + this.implementationType = implementationType; + return this; + } + + public ServiceTaskBuilder addFieldExtension(FieldExtension fieldExtension) { + this.fieldExtensions.add(fieldExtension); + return this; + } + + public ServiceTaskBuilder setAsync(boolean async) { + this.async = async; + return this; + } + @Override public ServiceTask build() { ServiceTask serviceTask = new ServiceTask(); serviceTask.setId(id); serviceTask.setName(id); - serviceTask.setImplementationType("class"); + serviceTask.setImplementationType(implementationType); serviceTask.setImplementation(implementation); + serviceTask.setAsynchronous(async); + + for (FieldExtension fieldExtension : fieldExtensions) { + serviceTask.getFieldExtensions().add(fieldExtension); + } + return serviceTask; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/UserTaskBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/UserTaskBuilder.java index 1b019df6961d..4851f96c9c99 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/UserTaskBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/flowable/builders/UserTaskBuilder.java @@ -1,13 +1,28 @@ package org.openmetadata.service.governance.workflows.flowable.builders; +import java.util.ArrayList; +import java.util.List; +import org.flowable.bpmn.model.FlowableListener; import org.flowable.bpmn.model.UserTask; public class UserTaskBuilder extends FlowableElementBuilder { + private final List listeners = new ArrayList<>(); + + public UserTaskBuilder addListener(FlowableListener listener) { + this.listeners.add(listener); + return this; + } + @Override public UserTask build() { UserTask userTask = new UserTask(); userTask.setId(id); userTask.setName(id); + + for (FlowableListener listener : listeners) { + userTask.getTaskListeners().add(listener); + } + return userTask; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 05f04a292d06..76d5c9e9e597 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4395,6 +4395,9 @@ List listWithoutEntityFilter( value = "SELECT MAX(\"offset\") FROM change_event", connectionType = POSTGRES) long getLatestOffset(); + + @SqlQuery("SELECT count(*) FROM change_event") + long listCount(); } class FailedEventResponseMapper implements RowMapper { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 5958b8eba064..cbb7bc7a885e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -111,6 +111,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import javax.json.JsonPatch; import javax.validation.ConstraintViolationException; import javax.validation.constraints.NotNull; @@ -1350,7 +1351,7 @@ private void deleteChildren(UUID id, boolean recursive, boolean hardDelete, Stri List.of(Relationship.CONTAINS.ordinal(), Relationship.PARENT_OF.ordinal())); if (childrenRecords.isEmpty()) { - LOG.info("No children to delete"); + LOG.debug("No children to delete"); return; } // Entity being deleted contains children entities @@ -1663,7 +1664,16 @@ private void validateAndUpdateExtensionBasedOnPropertyType( jsonNode.put(fieldName, formattedValue); } case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName); - case "enum" -> validateEnumKeys(fieldName, fieldValue, propertyConfig); + case "enum" -> { + validateEnumKeys(fieldName, fieldValue, propertyConfig); + List enumValues = + StreamSupport.stream(fieldValue.spliterator(), false) + .map(JsonNode::asText) + .sorted() + .collect(Collectors.toList()); + jsonNode.set(fieldName, JsonUtils.valueToTree(enumValues)); + entity.setExtension(jsonNode); + } default -> {} } } @@ -1832,8 +1842,19 @@ public final Object getExtension(T entity) { } ObjectNode objectNode = JsonUtils.getObjectNode(); for (ExtensionRecord extensionRecord : records) { - String fieldName = TypeRegistry.getPropertyName(extensionRecord.extensionName()); - objectNode.set(fieldName, JsonUtils.readTree(extensionRecord.extensionJson())); + String fieldName = extensionRecord.extensionName().substring(fieldFQNPrefix.length() + 1); + JsonNode fieldValue = JsonUtils.readTree(extensionRecord.extensionJson()); + String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName); + if ("enum".equals(customPropertyType) && fieldValue.isArray() && fieldValue.size() > 1) { + List sortedEnumValues = + StreamSupport.stream(fieldValue.spliterator(), false) + .map(JsonNode::asText) + .sorted() + .collect(Collectors.toList()); + fieldValue = JsonUtils.valueToTree(sortedEnumValues); + } + + objectNode.set(fieldName, fieldValue); } return objectNode; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java index 2eefbef79458..52ac95ba0e42 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionRepository.java @@ -15,6 +15,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.apps.bundles.changeEvent.AbstractEventConsumer.OFFSET_EXTENSION; import static org.openmetadata.service.events.subscription.AlertUtil.validateAndBuildFilteringConditions; import static org.openmetadata.service.fernet.Fernet.encryptWebhookSecretKey; import static org.openmetadata.service.util.EntityUtil.objectMatch; @@ -28,12 +29,14 @@ import org.openmetadata.schema.entity.events.ArgumentsInput; import org.openmetadata.schema.entity.events.EventFilterRule; import org.openmetadata.schema.entity.events.EventSubscription; +import org.openmetadata.schema.entity.events.EventSubscriptionOffset; import org.openmetadata.schema.entity.events.SubscriptionDestination; import org.openmetadata.service.Entity; import org.openmetadata.service.events.scheduled.EventSubscriptionScheduler; import org.openmetadata.service.events.subscription.AlertUtil; import org.openmetadata.service.resources.events.subscription.EventSubscriptionResource; import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.JsonUtils; @Slf4j public class EventSubscriptionRepository extends EntityRepository { @@ -111,6 +114,29 @@ private void validateFilterRules(EventSubscription entity) { } } + public EventSubscriptionOffset syncEventSubscriptionOffset(String eventSubscriptionName) { + EventSubscription eventSubscription = getByName(null, eventSubscriptionName, getFields("*")); + long latestOffset = daoCollection.changeEventDAO().getLatestOffset(); + long currentTime = System.currentTimeMillis(); + // Upsert Offset + EventSubscriptionOffset eventSubscriptionOffset = + new EventSubscriptionOffset() + .withCurrentOffset(latestOffset) + .withStartingOffset(latestOffset) + .withTimestamp(currentTime); + + Entity.getCollectionDAO() + .eventSubscriptionDAO() + .upsertSubscriberExtension( + eventSubscription.getId().toString(), + OFFSET_EXTENSION, + "eventSubscriptionOffset", + JsonUtils.pojoToJson(eventSubscriptionOffset)); + + EventSubscriptionScheduler.getInstance().updateEventSubscription(eventSubscription); + return eventSubscriptionOffset; + } + @Override public void storeEntity(EventSubscription entity, boolean update) { store(entity, update); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index b790bf8ac9c3..c0b7bb64805f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -310,8 +310,13 @@ public Thread getTask(EntityLink about, TaskType taskType, TaskStatus taskStatus for (Triple task : tasks) { if (task.getMiddle().equals(Entity.THREAD)) { UUID threadId = UUID.fromString(task.getLeft()); - Thread thread = - EntityUtil.validate(threadId, dao.feedDAO().findById(threadId), Thread.class); + Thread thread; + try { + thread = EntityUtil.validate(threadId, dao.feedDAO().findById(threadId), Thread.class); + } catch (EntityNotFoundException exc) { + LOG.debug(String.format("Thread '%s' not found.", threadId)); + continue; + } if (Optional.ofNullable(taskStatus).isPresent()) { if (thread.getTask() != null && thread.getTask().getType() == taskType diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 5192f1afe27b..d6519748d334 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -200,7 +200,7 @@ protected void createEntity(CSVPrinter printer, List csvRecords) thro .withTags( getTagLabels( printer, csvRecord, List.of(Pair.of(7, TagLabel.TagSource.CLASSIFICATION)))) - .withReviewers(getOwners(printer, csvRecord, 8)) + .withReviewers(getReviewers(printer, csvRecord, 8)) .withOwners(getOwners(printer, csvRecord, 9)) .withStatus(getTermStatus(printer, csvRecord)) .withExtension(getExtension(printer, csvRecord, 11)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 3ed9baf66d90..057aa04fd1d3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -24,8 +24,8 @@ import static org.openmetadata.service.Entity.TEAM; import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidGlossaryTermMove; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; -import static org.openmetadata.service.governance.workflows.Workflow.RESOLVED_BY_VARIABLE; import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusiveForParentAndSubField; import static org.openmetadata.service.resources.tags.TagLabelUtil.getUniqueTags; @@ -604,8 +604,10 @@ public EntityInterface performTask(String user, ResolveTask resolveTask) { UUID taskId = threadContext.getThread().getId(); Map variables = new HashMap<>(); variables.put(RESULT_VARIABLE, resolveTask.getNewValue().equalsIgnoreCase("approved")); - variables.put(RESOLVED_BY_VARIABLE, user); - WorkflowHandler.getInstance().resolveTask(taskId, variables); + variables.put(UPDATED_BY_VARIABLE, user); + WorkflowHandler workflowHandler = WorkflowHandler.getInstance(); + workflowHandler.resolveTask( + taskId, workflowHandler.transformToNodeVariables(taskId, variables)); // --- // TODO: performTask returns the updated Entity and the flow applies the new value. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java index a3f3b5262ab1..993456bc5f73 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java @@ -39,6 +39,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.FieldChange; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; import org.openmetadata.sdk.PipelineServiceClientInterface; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; @@ -53,6 +54,7 @@ import org.openmetadata.service.util.ResultList; public class IngestionPipelineRepository extends EntityRepository { + private static final String UPDATE_FIELDS = "sourceConfig,airflowConfig,loggerLevel,enabled,deployed"; private static final String PATCH_FIELDS = @@ -151,6 +153,14 @@ public void storeEntity(IngestionPipeline ingestionPipeline, boolean update) { @Override public void storeRelationships(IngestionPipeline ingestionPipeline) { addServiceRelationship(ingestionPipeline, ingestionPipeline.getService()); + if (ingestionPipeline.getIngestionAgent() != null) { + addRelationship( + ingestionPipeline.getId(), + ingestionPipeline.getIngestionAgent().getId(), + entityType, + ingestionPipeline.getIngestionAgent().getType(), + Relationship.HAS); + } } @Override @@ -291,8 +301,11 @@ public PipelineStatus getPipelineStatus(String ingestionPipelineFQN, UUID pipeli PipelineStatus.class); } - /** Handles entity updated from PUT and POST operation. */ + /** + * Handles entity updated from PUT and POST operation. + */ public class IngestionPipelineUpdater extends EntityUpdater { + public IngestionPipelineUpdater( IngestionPipeline original, IngestionPipeline updated, Operation operation) { super(buildIngestionPipelineDecrypted(original), updated, operation); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index edf54543dd63..859a7c848fca 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -127,7 +127,7 @@ public void setFields(TestSuite entity, EntityUtil.Fields fields) { @Override public void setInheritedFields(TestSuite testSuite, EntityUtil.Fields fields) { - if (Boolean.TRUE.equals(testSuite.getBasic())) { + if (Boolean.TRUE.equals(testSuite.getBasic()) && testSuite.getBasicEntityReference() != null) { Table table = Entity.getEntity( TABLE, testSuite.getBasicEntityReference().getId(), "owners,domain", ALL); @@ -154,6 +154,14 @@ public void setFullyQualifiedName(TestSuite testSuite) { } } + @Override + public EntityInterface getParentEntity(TestSuite entity, String fields) { + if (entity.getBasic() && entity.getBasicEntityReference() != null) { + return Entity.getEntity(entity.getBasicEntityReference(), fields, ALL); + } + return null; + } + private TestSummary getTestCasesExecutionSummary(JsonObject aggregation) { // Initialize the test summary with 0 values TestSummary testSummary = @@ -432,6 +440,7 @@ public RestUtil.DeleteResponse deleteLogicalTestSuite( String updatedBy = securityContext.getUserPrincipal().getName(); preDelete(original, updatedBy); setFieldsInternal(original, putFields); + deleteChildIngestionPipelines(original.getId(), hardDelete, updatedBy); EventType changeType; TestSuite updated = JsonUtils.readValue(JsonUtils.pojoToJson(original), TestSuite.class); @@ -452,6 +461,24 @@ public RestUtil.DeleteResponse deleteLogicalTestSuite( return new RestUtil.DeleteResponse<>(updated, changeType); } + /** + * Always delete as if it was marked recursive. Deleting a Logical Suite should + * just go ahead and clean the Ingestion Pipelines + */ + private void deleteChildIngestionPipelines(UUID id, boolean hardDelete, String updatedBy) { + List childrenRecords = + daoCollection + .relationshipDAO() + .findTo(id, entityType, Relationship.CONTAINS.ordinal(), Entity.INGESTION_PIPELINE); + + if (childrenRecords.isEmpty()) { + LOG.debug("No children to delete"); + return; + } + // Delete all the contained entities + deleteChildren(childrenRecords, hardDelete, updatedBy); + } + private void updateTestSummaryFromBucket(JsonObject bucket, TestSummary testSummary) { String key = bucket.getString("key"); Integer count = bucket.getJsonNumber("doc_count").intValue(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java index 8bde739c0b4c..9c73d1a5502c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -174,6 +175,11 @@ private List getCustomProperties(Type type) { for (Triple result : results) { CustomProperty property = JsonUtils.readValue(result.getRight(), CustomProperty.class); property.setPropertyType(this.getReferenceByName(result.getMiddle(), NON_DELETED)); + + if ("enum".equals(property.getPropertyType().getName())) { + sortEnumKeys(property); + } + customProperties.add(property); } customProperties.sort(EntityUtil.compareCustomProperty); @@ -270,6 +276,19 @@ private void validateTableTypeConfig(CustomPropertyConfig config) { } } + @SuppressWarnings("unchecked") + private void sortEnumKeys(CustomProperty property) { + Object enumConfig = property.getCustomPropertyConfig().getConfig(); + if (enumConfig instanceof Map) { + Map configMap = (Map) enumConfig; + if (configMap.get("values") instanceof List) { + List values = (List) configMap.get("values"); + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + configMap.put("values", sortedValues); + } + } + } + /** Handles entity updated from PUT and POST operation. */ public class TypeUpdater extends EntityUpdater { public TypeUpdater(Type original, Type updated, Operation operation) { @@ -460,6 +479,7 @@ private void postUpdateCustomPropertyConfig( Type entity, CustomProperty origProperty, CustomProperty updatedProperty) { String updatedBy = entity.getUpdatedBy(); if (origProperty.getPropertyType().getName().equals("enum")) { + sortEnumKeys(updatedProperty); EnumConfig origConfig = JsonUtils.convertValue( origProperty.getCustomPropertyConfig().getConfig(), EnumConfig.class); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java index 0e7e89fb5353..c728341e2281 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java @@ -52,7 +52,11 @@ protected void postDelete(WorkflowDefinition entity) { @Override protected void setFields(WorkflowDefinition entity, EntityUtil.Fields fields) { - entity.withDeployed(WorkflowHandler.getInstance().isDeployed(entity)); + if (WorkflowHandler.isInitialized()) { + entity.withDeployed(WorkflowHandler.getInstance().isDeployed(entity)); + } else { + LOG.debug("Can't get `deploy` status since WorkflowHandler is not initialized."); + } } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java new file mode 100644 index 000000000000..2ca9a36906bf --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v170/Migration.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.migration.mysql.v170; + +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateGovernanceWorkflowDefinitions; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + updateGovernanceWorkflowDefinitions(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java new file mode 100644 index 000000000000..93ed51876685 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v170/Migration.java @@ -0,0 +1,20 @@ +package org.openmetadata.service.migration.postgres.v170; + +import static org.openmetadata.service.migration.utils.v170.MigrationUtil.updateGovernanceWorkflowDefinitions; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + updateGovernanceWorkflowDefinitions(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java new file mode 100644 index 000000000000..153f09eea5d0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v170/MigrationUtil.java @@ -0,0 +1,110 @@ +package org.openmetadata.service.migration.utils.v170; + +import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.governance.workflows.WorkflowDefinition; +import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; +import org.openmetadata.service.Entity; +import org.openmetadata.service.governance.workflows.flowable.MainWorkflow; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.WorkflowDefinitionRepository; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.JsonUtils; + +@Slf4j +public class MigrationUtil { + + @SneakyThrows + private static void setDefaultInputNamespaceMap(WorkflowNodeDefinitionInterface nodeDefinition) { + try { + Class clazz = nodeDefinition.getClass(); + var field = clazz.getDeclaredField("inputNamespaceMap"); + + field.setAccessible(true); + + Object fieldValue = field.get(nodeDefinition); + + if (fieldValue == null) { + Class fieldType = field.getType(); + + Object newValue = fieldType.getDeclaredConstructor().newInstance(); + + field.set(nodeDefinition, newValue); + } + } catch (NoSuchFieldException ignored) { + + } + } + + @SneakyThrows + private static void updateInputNamespaceMap( + WorkflowNodeDefinitionInterface nodeDefinition, Map inputNamespaceMap) { + try { + Class clazz = nodeDefinition.getClass(); + var field = clazz.getDeclaredField("inputNamespaceMap"); + field.setAccessible(true); + Object inputNamespaceMapObj = field.get(nodeDefinition); + + if (inputNamespaceMapObj != null) { + Class fieldType = field.getType(); + + Field[] inputNamespaceMapFields = fieldType.getDeclaredFields(); + + for (Field inputNamespaceMapField : inputNamespaceMapFields) { + inputNamespaceMapField.setAccessible(true); + String fieldName = inputNamespaceMapField.getName(); + + if (inputNamespaceMap.containsKey(fieldName)) { + inputNamespaceMapField.set(inputNamespaceMapObj, inputNamespaceMap.get(fieldName)); + } + } + } + } catch (NoSuchFieldException ignored) { + + } + } + + public static void updateGovernanceWorkflowDefinitions() { + WorkflowDefinitionRepository repository = + (WorkflowDefinitionRepository) Entity.getEntityRepository(Entity.WORKFLOW_DEFINITION); + List workflowDefinitions = + repository.listAll(EntityUtil.Fields.EMPTY_FIELDS, new ListFilter()); + + for (WorkflowDefinition workflowDefinition : workflowDefinitions) { + MainWorkflow.WorkflowGraph graph = new MainWorkflow.WorkflowGraph(workflowDefinition); + + for (WorkflowNodeDefinitionInterface nodeDefinition : workflowDefinition.getNodes()) { + setDefaultInputNamespaceMap(nodeDefinition); + + Map nodeInputNamespaceMap = + (Map) + JsonUtils.readOrConvertValue(nodeDefinition.getInputNamespaceMap(), Map.class); + + if (nodeInputNamespaceMap == null) { + continue; + } + + if (nodeDefinition.getInput().contains(UPDATED_BY_VARIABLE) + && nodeInputNamespaceMap.get(UPDATED_BY_VARIABLE) == null) { + if (graph.getIncomingEdgesMap().containsKey(nodeDefinition.getName())) { + for (String incomeNodeName : + graph.getIncomingEdgesMap().get(nodeDefinition.getName())) { + List incomeNodeOutput = graph.getNodeMap().get(incomeNodeName).getOutput(); + if (incomeNodeOutput != null && incomeNodeOutput.contains(UPDATED_BY_VARIABLE)) { + nodeInputNamespaceMap.put(UPDATED_BY_VARIABLE, incomeNodeName); + updateInputNamespaceMap(nodeDefinition, nodeInputNamespaceMap); + break; + } + } + } + } + } + repository.createOrUpdate(null, workflowDefinition); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index 98412e76a3d0..774effc1b4ea 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -56,7 +56,9 @@ import org.openmetadata.service.limits.Limits; import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchSortFilter; +import org.openmetadata.service.security.AuthRequest; import org.openmetadata.service.security.AuthorizationException; +import org.openmetadata.service.security.AuthorizationLogic; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.CreateResourceContext; import org.openmetadata.service.security.policyevaluator.OperationContext; @@ -302,6 +304,21 @@ public Response create(UriInfo uriInfo, SecurityContext securityContext, T entit return Response.created(entity.getHref()).entity(entity).build(); } + public Response create( + UriInfo uriInfo, + SecurityContext securityContext, + List authRequests, + AuthorizationLogic authorizationLogic, + T entity) { + OperationContext operationContext = new OperationContext(entityType, CREATE); + CreateResourceContext createResourceContext = + new CreateResourceContext<>(entityType, entity); + limits.enforceLimits(securityContext, createResourceContext, operationContext); + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + entity = addHref(uriInfo, repository.create(uriInfo, entity)); + return Response.created(entity.getHref()).entity(entity).build(); + } + public Response createOrUpdate(UriInfo uriInfo, SecurityContext securityContext, T entity) { repository.prepareInternal(entity, true); // If entity does not exist, this is a create operation, else update operation @@ -325,6 +342,31 @@ public Response createOrUpdate(UriInfo uriInfo, SecurityContext securityContext, return response.toResponse(); } + public Response createOrUpdate( + UriInfo uriInfo, + SecurityContext securityContext, + List authRequests, + AuthorizationLogic authorizationLogic, + T entity) { + repository.prepareInternal(entity, true); + // If entity does not exist, this is a create operation, else update operation + ResourceContext resourceContext = getResourceContextByName(entity.getFullyQualifiedName()); + MetadataOperation operation = createOrUpdateOperation(resourceContext); + OperationContext operationContext = new OperationContext(entityType, operation); + if (operation == CREATE) { + CreateResourceContext createResourceContext = + new CreateResourceContext<>(entityType, entity); + limits.enforceLimits(securityContext, createResourceContext, operationContext); + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + entity = addHref(uriInfo, repository.create(uriInfo, entity)); + return new PutResponse<>(Response.Status.CREATED, entity, ENTITY_CREATED).toResponse(); + } + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + PutResponse response = repository.createOrUpdate(uriInfo, entity); + addHref(uriInfo, response.getEntity()); + return response.toResponse(); + } + public Response patchInternal( UriInfo uriInfo, SecurityContext securityContext, UUID id, JsonPatch patch) { OperationContext operationContext = new OperationContext(entityType, patch); @@ -338,6 +380,20 @@ public Response patchInternal( return response.toResponse(); } + public Response patchInternal( + UriInfo uriInfo, + SecurityContext securityContext, + List authRequests, + AuthorizationLogic authorizationLogic, + UUID id, + JsonPatch patch) { + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + PatchResponse response = + repository.patch(uriInfo, id, securityContext.getUserPrincipal().getName(), patch); + addHref(uriInfo, response.entity()); + return response.toResponse(); + } + public Response patchInternal( UriInfo uriInfo, SecurityContext securityContext, String fqn, JsonPatch patch) { OperationContext operationContext = new OperationContext(entityType, patch); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityTimeSeriesResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityTimeSeriesResource.java index 6b7896e8dbbf..f772cab7708e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityTimeSeriesResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityTimeSeriesResource.java @@ -1,6 +1,7 @@ package org.openmetadata.service.resources; import java.io.IOException; +import java.util.List; import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import lombok.Getter; @@ -10,6 +11,8 @@ import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository; import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchSortFilter; +import org.openmetadata.service.security.AuthRequest; +import org.openmetadata.service.security.AuthorizationLogic; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; @@ -61,6 +64,22 @@ protected ResultList listInternalFromSearch( fields, searchListFilter, limit, offset, searchSortFilter, q); } + protected ResultList listInternalFromSearch( + SecurityContext securityContext, + EntityUtil.Fields fields, + SearchListFilter searchListFilter, + int limit, + int offset, + SearchSortFilter searchSortFilter, + String q, + List authRequests, + AuthorizationLogic authorizationLogic) + throws IOException { + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + return repository.listFromSearchWithOffset( + fields, searchListFilter, limit, offset, searchSortFilter, q); + } + public ResultList listLatestFromSearch( SecurityContext securityContext, EntityUtil.Fields fields, @@ -74,6 +93,19 @@ public ResultList listLatestFromSearch( return repository.listLatestFromSearch(fields, searchListFilter, groupBy, q); } + public ResultList listLatestFromSearch( + SecurityContext securityContext, + EntityUtil.Fields fields, + SearchListFilter searchListFilter, + String groupBy, + String q, + List authRequests, + AuthorizationLogic authorizationLogic) + throws IOException { + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + return repository.listLatestFromSearch(fields, searchListFilter, groupBy, q); + } + protected T latestInternalFromSearch( SecurityContext securityContext, EntityUtil.Fields fields, @@ -85,4 +117,16 @@ protected T latestInternalFromSearch( authorizer.authorize(securityContext, operationContext, resourceContext); return repository.latestFromSearch(fields, searchListFilter, q); } + + protected T latestInternalFromSearch( + SecurityContext securityContext, + EntityUtil.Fields fields, + SearchListFilter searchListFilter, + String q, + List authRequests, + AuthorizationLogic authorizationLogic) + throws IOException { + authorizer.authorizeRequests(securityContext, authRequests, authorizationLogic); + return repository.latestFromSearch(fields, searchListFilter, q); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java index 374ab5ba719a..0400e2dc9f21 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java @@ -1,8 +1,5 @@ package org.openmetadata.service.resources.apps; -import static org.openmetadata.service.Entity.APPLICATION; -import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; - import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -12,7 +9,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.List; import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; @@ -52,6 +48,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.AppMarketPlaceUtil; import org.openmetadata.service.util.ResultList; @Path("/v1/apps/marketplace") @@ -65,30 +62,17 @@ public class AppMarketPlaceResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/apps/marketplace/"; - private PipelineServiceClientInterface pipelineServiceClient; private AppMarketPlaceMapper mapper; static final String FIELDS = "owners,tags"; @Override public void initialize(OpenMetadataApplicationConfig config) { try { - this.pipelineServiceClient = + PipelineServiceClientInterface pipelineServiceClient = PipelineServiceClientFactory.createPipelineServiceClient( config.getPipelineServiceClientConfiguration()); - mapper = new AppMarketPlaceMapper(pipelineServiceClient); - // Initialize Default Installed Applications - List createAppMarketPlaceDefinitionReqs = - getEntitiesFromSeedData( - APPLICATION, - String.format(".*json/data/%s/.*\\.json$", entityType), - CreateAppMarketPlaceDefinitionReq.class); - for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { - AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, "admin"); - // Update Fully Qualified Name - repository.setFullyQualifiedName(definition); - this.repository.createOrUpdate(null, definition); - } + AppMarketPlaceUtil.createAppMarketPlaceDefinitions(repository, mapper); } catch (Exception ex) { LOG.error("Failed in initializing App MarketPlace Resource", ex); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 0bf712ce9f65..dda74168f6a2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -251,7 +251,7 @@ public ResultList list( mediaType = "application/json", schema = @Schema(implementation = AppRunList.class))) }) - public Response listAppRuns( + public ResultList listAppRuns( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter(description = "Name of the App", schema = @Schema(type = "string")) @@ -281,9 +281,7 @@ public Response listAppRuns( Long endTs) { App installation = repository.getByName(uriInfo, name, repository.getFields("id,pipelines")); if (installation.getAppType().equals(AppType.Internal)) { - return Response.status(Response.Status.OK) - .entity(repository.listAppRuns(installation, limitParam, offset)) - .build(); + return repository.listAppRuns(installation, limitParam, offset); } if (!installation.getPipelines().isEmpty()) { EntityReference pipelineRef = installation.getPipelines().get(0); @@ -292,13 +290,27 @@ public Response listAppRuns( IngestionPipeline ingestionPipeline = ingestionPipelineRepository.get( uriInfo, pipelineRef.getId(), ingestionPipelineRepository.getFields(FIELD_OWNERS)); - return Response.ok( - ingestionPipelineRepository.listPipelineStatus( - ingestionPipeline.getFullyQualifiedName(), startTs, endTs), - MediaType.APPLICATION_JSON_TYPE) - .build(); + return ingestionPipelineRepository + .listPipelineStatus(ingestionPipeline.getFullyQualifiedName(), startTs, endTs) + .map(pipelineStatus -> convertPipelineStatus(installation, pipelineStatus)); } - throw new IllegalArgumentException("App does not have an associated pipeline."); + throw new IllegalArgumentException("App does not have a scheduled deployment"); + } + + private static AppRunRecord convertPipelineStatus(App app, PipelineStatus pipelineStatus) { + return new AppRunRecord() + .withAppId(app.getId()) + .withAppName(app.getName()) + .withExecutionTime(pipelineStatus.getStartDate()) + .withEndTime(pipelineStatus.getEndDate()) + .withStatus( + switch (pipelineStatus.getPipelineState()) { + case QUEUED -> AppRunRecord.Status.PENDING; + case SUCCESS -> AppRunRecord.Status.SUCCESS; + case FAILED, PARTIAL_SUCCESS -> AppRunRecord.Status.FAILED; + case RUNNING -> AppRunRecord.Status.RUNNING; + }) + .withConfig(pipelineStatus.getConfig()); } @GET @@ -617,7 +629,7 @@ public Response create( limits.enforceLimits( securityContext, getResourceContext(), - new OperationContext(Entity.APPLICATION, MetadataOperation.CREATE)); + new OperationContext(APPLICATION, MetadataOperation.CREATE)); if (SCHEDULED_TYPES.contains(app.getScheduleType())) { ApplicationHandler.getInstance() .installApplication( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java index 1fa08546cd17..f916c3285354 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResolutionStatusResource.java @@ -11,6 +11,8 @@ import io.swagger.v3.oas.annotations.tags.Tag; import java.beans.IntrospectionException; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; @@ -41,10 +43,14 @@ import org.openmetadata.service.jdbi3.TestCaseResolutionStatusRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityTimeSeriesResource; +import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.security.AuthRequest; +import org.openmetadata.service.security.AuthorizationLogic; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; -import org.openmetadata.service.security.policyevaluator.ReportDataContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; +import org.openmetadata.service.security.policyevaluator.TestCaseResourceContext; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; @@ -139,10 +145,19 @@ public ResultList list( schema = @Schema(type = "String")) @QueryParam("originEntityFQN") String originEntityFQN) { - OperationContext operationContext = + List requests = new ArrayList<>(); + OperationContext testCaseOperationContext = new OperationContext(Entity.TEST_CASE, MetadataOperation.VIEW_ALL); - ResourceContextInterface resourceContext = ReportDataContext.builder().build(); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface testCaseResourceContext = getTestCaseResourceContext(testCaseFQN); + requests.add(new AuthRequest(testCaseOperationContext, testCaseResourceContext)); + if (originEntityFQN != null) { + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); + ResourceContextInterface entityResourceContext = + new ResourceContext<>(Entity.TABLE, null, originEntityFQN); + requests.add(new AuthRequest(entityOperationContext, entityResourceContext)); + } + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); ListFilter filter = new ListFilter(null); filter.addQueryParam("testCaseResolutionStatusType", testCaseResolutionStatusType); @@ -172,10 +187,10 @@ public ResultList listForStateId( @Context SecurityContext securityContext, @Parameter(description = "Sequence ID", schema = @Schema(type = "UUID")) @PathParam("stateId") UUID stateId) { - OperationContext operationContext = + OperationContext testCaseOperationContext = new OperationContext(Entity.TEST_CASE, MetadataOperation.VIEW_ALL); - ResourceContextInterface resourceContext = ReportDataContext.builder().build(); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface testCaseResourceContext = TestCaseResourceContext.builder().build(); + authorizer.authorize(securityContext, testCaseOperationContext, testCaseResourceContext); return repository.listTestCaseResolutionStatusesForStateId(stateId); } @@ -200,10 +215,10 @@ public TestCaseResolutionStatus get( @Parameter(description = "Test Case Failure Status ID", schema = @Schema(type = "UUID")) @PathParam("id") UUID testCaseResolutionStatusId) { - OperationContext operationContext = + OperationContext testCaseOperationContext = new OperationContext(Entity.TEST_CASE, MetadataOperation.VIEW_ALL); - ResourceContextInterface resourceContext = ReportDataContext.builder().build(); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface testCaseResourceContext = TestCaseResourceContext.builder().build(); + authorizer.authorize(securityContext, testCaseOperationContext, testCaseResourceContext); return repository.getById(testCaseResolutionStatusId); } @@ -226,10 +241,19 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTestCaseResolutionStatus createTestCaseResolutionStatus) { - OperationContext operationContext = + OperationContext testCaseOperationContext = new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_TESTS); - ResourceContextInterface resourceContext = ReportDataContext.builder().build(); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface testCaseResourceContext = TestCaseResourceContext.builder().build(); + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); + ResourceContextInterface entityResourceContext = TestCaseResourceContext.builder().build(); + + List requests = + List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(testCaseOperationContext, testCaseResourceContext)); + + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); TestCaseResolutionStatus testCaseResolutionStatus = mapper.createToEntity( createTestCaseResolutionStatus, securityContext.getUserPrincipal().getName()); @@ -266,12 +290,34 @@ public Response patch( })) JsonPatch patch) throws IntrospectionException, InvocationTargetException, IllegalAccessException { - OperationContext operationContext = + OperationContext testCaseOperationContext = new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_TESTS); - ResourceContextInterface resourceContext = ReportDataContext.builder().build(); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface testCaseResourceContext = TestCaseResourceContext.builder().build(); + authorizer.authorize(securityContext, testCaseOperationContext, testCaseResourceContext); RestUtil.PatchResponse response = repository.patch(id, patch, securityContext.getUserPrincipal().getName()); return response.toResponse(); } + + protected static ResourceContextInterface getEntityResourceContext( + String fqn, String entityType) { + ResourceContextInterface resourceContext; + if (fqn != null) { + MessageParser.EntityLink entityLinkParsed = new MessageParser.EntityLink(entityType, fqn); + resourceContext = TestCaseResourceContext.builder().entityLink(entityLinkParsed).build(); + } else { + resourceContext = TestCaseResourceContext.builder().build(); + } + return resourceContext; + } + + protected static ResourceContextInterface getTestCaseResourceContext(String name) { + ResourceContextInterface resourceContext; + if (name != null) { + resourceContext = TestCaseResourceContext.builder().name(name).build(); + } else { + resourceContext = TestCaseResourceContext.builder().build(); + } + return resourceContext; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index 748abfd6c3ae..e467f45b49fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -65,6 +65,8 @@ import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchSortFilter; +import org.openmetadata.service.security.AuthRequest; +import org.openmetadata.service.security.AuthorizationLogic; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.mask.PIIMasker; import org.openmetadata.service.security.policyevaluator.CreateResourceContext; @@ -73,6 +75,7 @@ import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; import org.openmetadata.service.security.policyevaluator.TestCaseResourceContext; import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.RestUtil.DeleteResponse; import org.openmetadata.service.util.RestUtil.PatchResponse; @@ -645,19 +648,29 @@ public Response create( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTestCase create) { - // Override OperationContext to change the entity to table and operation from CREATE to - // EDIT_TESTS + EntityLink entityLink = EntityLink.parse(create.getEntityLink()); TestCase test = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - ResourceContextInterface resourceContext = - TestCaseResourceContext.builder().entityLink(entityLink).build(); limits.enforceLimits( securityContext, new CreateResourceContext<>(entityType, test), new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_TESTS)); - authorizer.authorize(securityContext, operationContext, resourceContext); + + OperationContext tableOpContext = + new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); + ResourceContextInterface tableResourceContext = + TestCaseResourceContext.builder().entityLink(entityLink).build(); + OperationContext testCaseOpContext = + new OperationContext(Entity.TEST_CASE, MetadataOperation.CREATE); + ResourceContextInterface testCaseResourceContext = + new CreateResourceContext<>(entityType, test); + TestCaseResourceContext.builder().name(test.getName()).build(); + + List requests = + List.of( + new AuthRequest(tableOpContext, tableResourceContext), + new AuthRequest(testCaseOpContext, testCaseResourceContext)); + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); repository.isTestSuiteBasic(create.getTestSuite()); test = addHref(uriInfo, repository.create(uriInfo, test)); return Response.created(test.getHref()).entity(test).build(); @@ -745,12 +758,19 @@ public Response patch( @ExampleObject("[{op:remove, path:/a},{op:add, path: /b, value: val}]") })) JsonPatch patch) { - // Override OperationContext to change the entity to table and operation from UPDATE to - // EDIT_TESTS - ResourceContextInterface resourceContext = TestCaseResourceContext.builder().id(id).build(); - OperationContext operationContext = + OperationContext tableOpContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface tableRC = TestCaseResourceContext.builder().id(id).build(); + + OperationContext testCaseOpContext = + new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_ALL); + ResourceContextInterface testCaseRC = TestCaseResourceContext.builder().id(id).build(); + + List requests = + List.of( + new AuthRequest(tableOpContext, tableRC), + new AuthRequest(testCaseOpContext, testCaseRC)); + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); PatchResponse response = repository.patch(uriInfo, id, securityContext.getUserPrincipal().getName(), patch); if (response.entity().getTestCaseResult() != null @@ -790,12 +810,19 @@ public Response patchTestCaseResult( @ExampleObject("[{op:remove, path:/a},{op:add, path: /b, value: val}]") })) JsonPatch patch) { - // Override OperationContext to change the entity to table and operation from UPDATE to - // EDIT_TESTS - ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); - OperationContext operationContext = + OperationContext tableOpContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface tableRC = TestCaseResourceContext.builder().name(fqn).build(); + + OperationContext testCaseOpContext = + new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_ALL); + ResourceContextInterface testCaseRC = TestCaseResourceContext.builder().name(fqn).build(); + + List requests = + List.of( + new AuthRequest(tableOpContext, tableRC), + new AuthRequest(testCaseOpContext, testCaseRC)); + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); PatchResponse patchResponse = repository.patchTestCaseResults( fqn, timestamp, patch, securityContext.getUserPrincipal().toString()); @@ -820,14 +847,26 @@ public Response createOrUpdate( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTestCase create) { - // Override OperationContext to change the entity to table and operation from CREATE/UPDATE to - // EDIT_TESTS EntityLink entityLink = EntityLink.parse(create.getEntityLink()); - ResourceContextInterface resourceContext = - TestCaseResourceContext.builder().entityLink(entityLink).build(); - OperationContext operationContext = + OperationContext tableOpContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface tableResourceContext = + TestCaseResourceContext.builder().entityLink(entityLink).build(); + OperationContext testCaseOpCreate = + new OperationContext(Entity.TEST_CASE, MetadataOperation.CREATE); + ResourceContextInterface testCaseRC = + TestCaseResourceContext.builder() + .name(FullyQualifiedName.add(entityLink.getEntityFQN(), create.getName())) + .build(); + OperationContext testCaseOpUpdate = + new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_ALL); + + List requests = + List.of( + new AuthRequest(tableOpContext, tableResourceContext), + new AuthRequest(testCaseOpCreate, testCaseRC), + new AuthRequest(testCaseOpUpdate, testCaseRC)); + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); TestCase test = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); repository.isTestSuiteBasic(create.getTestSuite()); repository.prepareInternal(test, true); @@ -861,12 +900,6 @@ public Response delete( @Parameter(description = "Id of the test case", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { - // Override OperationContext to change the entity to table and operation from DELETE to - // EDIT_TESTS - ResourceContextInterface resourceContext = TestCaseResourceContext.builder().id(id).build(); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); return delete(uriInfo, securityContext, id, recursive, hardDelete); } @@ -899,7 +932,7 @@ public Response delete( String fqn) { ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); + new OperationContext(Entity.TEST_CASE, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, resourceContext); return deleteByName(uriInfo, securityContext, fqn, recursive, hardDelete); } @@ -923,7 +956,7 @@ public Response deleteLogicalTestCase( @PathParam("id") UUID id) { ResourceContextInterface resourceContext = TestCaseResourceContext.builder().id(id).build(); OperationContext operationContext = - new OperationContext(Entity.TEST_SUITE, MetadataOperation.EDIT_TESTS); + new OperationContext(Entity.TEST_CASE, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, resourceContext); DeleteResponse response = repository.deleteTestCaseFromLogicalTestSuite(testSuiteId, id); @@ -976,10 +1009,19 @@ public Response addTestCaseResult( @PathParam("fqn") String fqn, @Valid TestCaseResult testCaseResult) { - ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); - OperationContext operationContext = + OperationContext tableOpContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + ResourceContextInterface tableRC = TestCaseResourceContext.builder().name(fqn).build(); + + OperationContext testCaseOpContext = + new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_ALL); + ResourceContextInterface testCaseRC = TestCaseResourceContext.builder().name(fqn).build(); + + List requests = + List.of( + new AuthRequest(tableOpContext, tableRC), + new AuthRequest(testCaseOpContext, testCaseRC)); + authorizer.authorizeRequests(securityContext, requests, AuthorizationLogic.ANY); if (testCaseResult.getTestCaseStatus() == TestCaseStatus.Success) { TestCase testCase = repository.findByName(fqn, Include.ALL); repository.deleteTestCaseFailedRowsSample(testCase.getId()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java index 080b7ddf1d1f..20f6bc7d70d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResultResource.java @@ -1,6 +1,8 @@ package org.openmetadata.service.resources.dqtests; +import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.Entity.TEST_CASE; +import static org.openmetadata.service.Entity.TEST_SUITE; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; @@ -12,8 +14,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -47,8 +52,11 @@ import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchSortFilter; +import org.openmetadata.service.security.AuthRequest; +import org.openmetadata.service.security.AuthorizationLogic; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; import org.openmetadata.service.security.policyevaluator.TestCaseResourceContext; import org.openmetadata.service.util.EntityUtil; @@ -104,10 +112,21 @@ public Response addTestCaseResult( @Valid CreateTestCaseResult createTestCaseResults) { // Needed in further validation to check if the testCase exists createTestCaseResults.withFqn(fqn); + TestCase testCase = getTestCase(fqn); ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); - OperationContext operationContext = + OperationContext operationContext = new OperationContext(TEST_CASE, MetadataOperation.EDIT_ALL); + ResourceContextInterface entityResourceContext = + TestCaseResourceContext.builder() + .entityLink(MessageParser.EntityLink.parse(testCase.getEntityLink())) + .build(); + OperationContext entityOperationContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + + List authRequests = + List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(operationContext, resourceContext)); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); return repository.addTestCaseResult( securityContext.getUserPrincipal().getName(), uriInfo, @@ -150,10 +169,23 @@ public ResultList list( schema = @Schema(type = "number")) @QueryParam("endTs") Long endTs) { - ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + TestCase testCase = getTestCase(fqn); + ResourceContextInterface testCaseResourceContext = + TestCaseResourceContext.builder().name(testCase.getFullyQualifiedName()).build(); + OperationContext testCaseOperationContext = + new OperationContext(TEST_CASE, MetadataOperation.VIEW_ALL); + ResourceContextInterface entityResourceContext = + TestCaseResourceContext.builder() + .entityLink(MessageParser.EntityLink.parse(testCase.getEntityLink())) + .build(); + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); + + List authRequests = + List.of( + new AuthRequest(testCaseOperationContext, testCaseResourceContext), + new AuthRequest(entityOperationContext, entityResourceContext)); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); return repository.getTestCaseResults(fqn, startTs, endTs); } @@ -294,12 +326,7 @@ public ResultList listTestCaseResultsFromSearch( Optional.ofNullable(dataQualityDimension) .ifPresent(dqd -> searchListFilter.addQueryParam("dataQualityDimension", dqd)); - ResourceContextInterface resourceContextInterface = getResourceContext(testCaseFQN); - // Override OperationContext to change the entity to table - // and operation from VIEW_ALL to VIEW_TESTS - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); - + List authRequests = getAuthRequestsForListOps(testCaseFQN, testSuiteId); if (latest.equals("true")) { return listLatestFromSearch( securityContext, @@ -307,8 +334,8 @@ public ResultList listTestCaseResultsFromSearch( searchListFilter, "testCaseFQN.keyword", q, - operationContext, - resourceContextInterface); + authRequests, + AuthorizationLogic.ANY); } return listInternalFromSearch( securityContext, @@ -318,8 +345,8 @@ public ResultList listTestCaseResultsFromSearch( offset, new SearchSortFilter("timestamp", "desc", null, null), q, - operationContext, - resourceContextInterface); + authRequests, + AuthorizationLogic.ANY); } @GET @@ -379,14 +406,10 @@ public TestCaseResult latestTestCaseResultFromSearch( Optional.ofNullable(testSuiteId) .ifPresent(tsi -> searchListFilter.addQueryParam("testSuiteId", tsi)); - ResourceContextInterface resourceContextInterface = getResourceContext(testCaseFQN); - // Override OperationContext to change the entity to table - // and operation from VIEW_ALL to VIEW_TESTS - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); + List authRequests = getAuthRequestsForListOps(testCaseFQN, testSuiteId); return super.latestInternalFromSearch( - securityContext, fields, searchListFilter, q, operationContext, resourceContextInterface); + securityContext, fields, searchListFilter, q, authRequests, AuthorizationLogic.ANY); } @PATCH @@ -418,10 +441,23 @@ public Response patchTestCaseResult( @ExampleObject("[{op:remove, path:/a},{op:add, path: /b, value: val}]") })) JsonPatch patch) { - ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); - OperationContext operationContext = + TestCase testCase = getTestCase(fqn); + ResourceContextInterface testCaseResourceContext = + TestCaseResourceContext.builder().name(testCase.getFullyQualifiedName()).build(); + OperationContext testCaseOperationContext = + new OperationContext(TEST_CASE, MetadataOperation.EDIT_ALL); + ResourceContextInterface entityResourceContext = + TestCaseResourceContext.builder() + .entityLink(MessageParser.EntityLink.parse(testCase.getEntityLink())) + .build(); + OperationContext entityOperationContext = new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + + List authRequests = + List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(testCaseOperationContext, testCaseResourceContext)); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); RestUtil.PatchResponse patchResponse = repository.patchTestCaseResults( fqn, timestamp, patch, securityContext.getUserPrincipal().getName()); @@ -454,10 +490,19 @@ public Response deleteTestCaseResult( @Parameter(description = "Timestamp of the testCase result", schema = @Schema(type = "long")) @PathParam("timestamp") Long timestamp) { + TestCase testCase = getTestCase(fqn); ResourceContextInterface resourceContext = TestCaseResourceContext.builder().name(fqn).build(); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + OperationContext operationContext = new OperationContext(TEST_CASE, MetadataOperation.DELETE); + ResourceContextInterface entityResourceContext = + TestCaseResourceContext.builder() + .entityLink(MessageParser.EntityLink.parse(testCase.getEntityLink())) + .build(); + OperationContext entityOperationContext = new OperationContext(TABLE, MetadataOperation.DELETE); + List authRequests = + List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(operationContext, resourceContext)); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ALL); return repository.deleteTestCaseResult(fqn, timestamp).toResponse(); } @@ -478,4 +523,42 @@ private ResourceContextInterface getResourceContext(String testCaseFQN) { } return resourceContext; } + + private List getAuthRequestsForListOps(String testCaseFQN, String testSuiteId) { + List authRequests = new ArrayList<>(); + if (testCaseFQN != null) { + TestCase testCase = getTestCase(testCaseFQN); + ResourceContextInterface testCaseResourceContext = + TestCaseResourceContext.builder().name(testCaseFQN).build(); + OperationContext testCaseOperationContext = + new OperationContext(TEST_CASE, MetadataOperation.VIEW_ALL); + ResourceContextInterface entityResourceContext = + TestCaseResourceContext.builder() + .entityLink(MessageParser.EntityLink.parse(testCase.getEntityLink())) + .build(); + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); + authRequests.add(new AuthRequest(entityOperationContext, entityResourceContext)); + authRequests.add(new AuthRequest(testCaseOperationContext, testCaseResourceContext)); + } else { + ResourceContextInterface resourceContext = getResourceContext(null); + OperationContext operationContext = + new OperationContext(TEST_CASE, MetadataOperation.VIEW_ALL); + authRequests.add(new AuthRequest(operationContext, resourceContext)); + } + + if (testSuiteId != null) { + ResourceContextInterface testSuiteResourceContext = + new ResourceContext<>(Entity.TEST_SUITE, UUID.fromString(testSuiteId), null); + OperationContext testSuiteOperationContext = + new OperationContext(TEST_SUITE, MetadataOperation.VIEW_ALL); + authRequests.add(new AuthRequest(testSuiteOperationContext, testSuiteResourceContext)); + } + + return authRequests; + } + + private TestCase getTestCase(String fqn) { + return Entity.getEntityByName(TEST_CASE, fqn, "", Include.ALL); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java index 05acefd8ac53..46ed7f312342 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java @@ -44,6 +44,7 @@ import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestSummary; import org.openmetadata.schema.type.EntityHistory; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.service.Entity; @@ -54,10 +55,14 @@ import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.search.SearchListFilter; import org.openmetadata.service.search.SearchSortFilter; +import org.openmetadata.service.security.AuthRequest; +import org.openmetadata.service.security.AuthorizationLogic; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.ResourceContext; +import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.ResultList; @@ -76,6 +81,8 @@ public class TestSuiteResource extends EntityResource"; public static final String NON_BASIC_TEST_SUITE_DELETION_ERROR = "Cannot delete executable test suite. To delete executable test suite, use DELETE /v1/dataQuality/testSuites/basic/<...>"; + public static final String BASIC_TEST_SUITE_WITHOUT_REF_ERROR = + "Cannot create a basic test suite without the BasicEntityReference field informed."; static final String FIELDS = "owners,tests,summary"; static final String SEARCH_FIELDS_EXCLUDE = "table,database,databaseSchema,service"; @@ -160,20 +167,11 @@ public ResultList list( filter.addQueryParam("includeEmptyTestSuites", includeEmptyTestSuites); EntityUtil.Fields fields = getFields(fieldsParam); - ResourceContext resourceContext = getResourceContext(); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); + List authRequests = getAuthRequestsForListOps(); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); return super.listInternal( - uriInfo, - securityContext, - fields, - filter, - limitParam, - before, - after, - operationContext, - resourceContext); + uriInfo, securityContext, fieldsParam, filter, limitParam, before, after); } @GET @@ -299,21 +297,10 @@ public ResultList listFromSearch( EntityUtil.Fields fields = getFields(fieldsParam); - ResourceContext resourceContext = getResourceContext(); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); - - return super.listInternalFromSearch( - uriInfo, - securityContext, - fields, - searchListFilter, - limit, - offset, - searchSortFilter, - q, - operationContext, - resourceContext); + List authRequests = getAuthRequestsForListOps(); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); + return repository.listFromSearchWithOffset( + uriInfo, fields, searchListFilter, limit, offset, searchSortFilter, q); } @GET @@ -471,10 +458,8 @@ public TestSummary getTestsExecutionSummary( schema = @Schema(type = "String", format = "uuid")) @QueryParam("testSuiteId") UUID testSuiteId) { - ResourceContext resourceContext = getResourceContext(); - OperationContext operationContext = - new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + List authRequests = getAuthRequestsForListOps(); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); // Set the deprecation header based on draft specification from IETF // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 response.setHeader("Deprecation", "Monday, October 30, 2024"); @@ -523,10 +508,8 @@ public DataQualityReport getDataQualityReport( @QueryParam("index") String index) throws IOException { - ResourceContext resourceContext = getResourceContext(); - OperationContext operationContext = - new OperationContext(entityType, MetadataOperation.VIEW_TESTS); - authorizer.authorize(securityContext, operationContext, resourceContext); + List authRequests = getAuthRequestsForListOps(); + authorizer.authorizeRequests(securityContext, authRequests, AuthorizationLogic.ANY); if (nullOrEmpty(aggregationQuery) || nullOrEmpty(index)) { throw new IllegalArgumentException("aggregationQuery and index are required parameters"); } @@ -558,7 +541,8 @@ public Response create( TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); testSuite.setBasic(false); - return create(uriInfo, securityContext, testSuite); + List authRequests = getAuthRequestsForPost(testSuite); + return create(uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, testSuite); } @POST @@ -584,12 +568,16 @@ public Response createExecutable( @Valid CreateTestSuite create) { TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + if (testSuite.getBasicEntityReference() == null) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_WITHOUT_REF_ERROR); + } testSuite.setBasic(true); // Set the deprecation header based on draft specification from IETF // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 response.setHeader("Deprecation", "Monday, March 24, 2025"); response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); - return create(uriInfo, securityContext, testSuite); + List authRequests = getAuthRequestsForPost(testSuite); + return create(uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, testSuite); } @POST @@ -615,8 +603,12 @@ public Response createBasic( @Valid CreateTestSuite create) { TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + if (testSuite.getBasicEntityReference() == null) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_WITHOUT_REF_ERROR); + } testSuite.setBasic(true); - return create(uriInfo, securityContext, testSuite); + List authRequests = getAuthRequestsForPost(testSuite); + return create(uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, testSuite); } @PATCH @@ -645,7 +637,10 @@ public Response updateDescription( @ExampleObject("[{op:remove, path:/a},{op:add, path: /b, value: val}]") })) JsonPatch patch) { - return patchInternal(uriInfo, securityContext, id, patch); + TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "", ALL); + List authRequests = + getAuthRequestsForUpdate(testSuite, ResourceContextInterface.Operation.PATCH, patch); + return patchInternal(uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, id, patch); } @PUT @@ -673,7 +668,12 @@ public Response createOrUpdate( TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); testSuite.setBasic(false); - return createOrUpdate(uriInfo, securityContext, testSuite); + List authRequests = + new java.util.ArrayList<>( + getAuthRequestsForUpdate(testSuite, ResourceContextInterface.Operation.PUT, null)); + authRequests.addAll(getAuthRequestsForPost(testSuite)); + return createOrUpdate( + uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, testSuite); } @PUT @@ -704,7 +704,12 @@ public Response createOrUpdateExecutable( // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 response.setHeader("Deprecation", "Monday, March 24, 2025"); response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); - return createOrUpdate(uriInfo, securityContext, testSuite); + List authRequests = + new java.util.ArrayList<>( + getAuthRequestsForUpdate(testSuite, ResourceContextInterface.Operation.PUT, null)); + authRequests.addAll(getAuthRequestsForPost(testSuite)); + return createOrUpdate( + uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, testSuite); } @PUT @@ -729,7 +734,12 @@ public Response createOrUpdateBasic( TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); testSuite.setBasic(true); - return createOrUpdate(uriInfo, securityContext, testSuite); + List authRequests = + new java.util.ArrayList<>( + getAuthRequestsForUpdate(testSuite, ResourceContextInterface.Operation.PUT, null)); + authRequests.addAll(getAuthRequestsForPost(testSuite)); + return createOrUpdate( + uriInfo, securityContext, authRequests, AuthorizationLogic.ANY, testSuite); } @DELETE @@ -754,7 +764,8 @@ public Response delete( @Parameter(description = "Id of the logical test suite", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { - OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + OperationContext operationContext = + new OperationContext(Entity.TEST_SUITE, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); if (Boolean.TRUE.equals(testSuite.getBasic())) { @@ -789,7 +800,8 @@ public Response delete( @Parameter(description = "FQN of the logical test suite", schema = @Schema(type = "String")) @PathParam("name") String name) { - OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + OperationContext operationContext = + new OperationContext(Entity.TEST_SUITE, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); if (Boolean.TRUE.equals(testSuite.getBasic())) { @@ -976,4 +988,65 @@ public Response restoreTestSuite( @Valid RestoreEntity restore) { return restoreEntity(uriInfo, securityContext, restore.getId()); } + + private List getAuthRequestsForListOps() { + ResourceContext entityResourceContext = new ResourceContext<>(Entity.TABLE); + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.VIEW_TESTS); + ResourceContext testSuiteResourceContext = getResourceContext(); + OperationContext testSuiteOperationContext = + new OperationContext(entityType, MetadataOperation.VIEW_ALL); + ResourceContext testCaseResourceContext = new ResourceContext<>(Entity.TEST_CASE); + OperationContext testCaseOperationContext = + new OperationContext(Entity.TEST_CASE, MetadataOperation.VIEW_ALL); + + return List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(testSuiteOperationContext, testSuiteResourceContext), + new AuthRequest(testCaseOperationContext, testCaseResourceContext)); + } + + private List getAuthRequestsForPost(TestSuite testSuite) { + ResourceContext entityResourceContext; + EntityReference entityReference = testSuite.getBasicEntityReference(); + if (entityReference != null) { + entityResourceContext = new ResourceContext<>(Entity.TABLE, entityReference.getId(), null); + } else { + entityResourceContext = new ResourceContext<>(Entity.TABLE); + } + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); + ResourceContext testSuiteResourceContext = getResourceContext(); + OperationContext testSuiteOperationContext = + new OperationContext(entityType, MetadataOperation.CREATE); + + return List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(testSuiteOperationContext, testSuiteResourceContext)); + } + + private List getAuthRequestsForUpdate( + TestSuite testSuite, ResourceContextInterface.Operation operation, JsonPatch patch) { + EntityReference entityReference = testSuite.getBasicEntityReference(); + ResourceContext entityResourceContext; + OperationContext testSuiteOperationContext; + if (entityReference != null) { + entityResourceContext = new ResourceContext<>(Entity.TABLE, entityReference.getId(), null); + } else { + entityResourceContext = new ResourceContext<>(Entity.TABLE); + } + OperationContext entityOperationContext = + new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); + ResourceContext testSuiteResourceContext = + getResourceContextByName(FullyQualifiedName.quoteName(testSuite.getName()), operation); + if (patch != null) { + testSuiteOperationContext = new OperationContext(entityType, patch); + } else { + testSuiteOperationContext = new OperationContext(entityType, MetadataOperation.EDIT_ALL); + } + + return List.of( + new AuthRequest(entityOperationContext, entityResourceContext), + new AuthRequest(testSuiteOperationContext, testSuiteResourceContext)); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java index d575fbf01c3d..99fee621e9a8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java @@ -538,8 +538,6 @@ public SubscriptionStatus getEventSubscriptionStatusByName( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_ALL); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); - - authorizer.authorizeAdmin(securityContext); EventSubscription sub = repository.getByName(null, name, repository.getFields("name")); return EventSubscriptionScheduler.getInstance() .getStatusForEventSubscription(sub.getId(), destinationId); @@ -1318,6 +1316,39 @@ public List getAllDestinationStatusesForSubscriptionByN return EventSubscriptionScheduler.getInstance().listAlertDestinations(sub.getId()); } + @PUT + @Path("name/{eventSubscriptionName}/syncOffset") + @Valid + @Operation( + operationId = "syncOffsetForEventSubscriptionByName", + summary = "Sync Offset for a specific Event Subscription by its name", + description = "Sync Offset for a specific Event Subscription by its name", + responses = { + @ApiResponse( + responseCode = "200", + description = "Returns the destinations for the Event Subscription", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = SubscriptionDestination.class))), + @ApiResponse( + responseCode = "404", + description = "Event Subscription with the name {fqn} is not found") + }) + public Response syncOffsetForEventSubscription( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Name of the Event Subscription", schema = @Schema(type = "string")) + @PathParam("eventSubscriptionName") + String name) { + OperationContext operationContext = + new OperationContext(entityType, MetadataOperation.EDIT_ALL); + authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); + return Response.status(Response.Status.OK) + .entity(repository.syncEventSubscriptionOffset(name)) + .build(); + } + @POST @Path("/testDestination") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java index e2134d9b11f7..f668a3acf65c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineMapper.java @@ -26,6 +26,7 @@ public IngestionPipeline createToEntity(CreateIngestionPipeline create, String u .withOpenMetadataServerConnection(openMetadataServerConnection) .withSourceConfig(create.getSourceConfig()) .withLoggerLevel(create.getLoggerLevel()) - .withService(create.getService()); + .withService(create.getService()) + .withIngestionAgent(create.getIngestionAgent()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java index 839dd1ca1655..70e74cad6ba9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java @@ -49,7 +49,6 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.SystemRepository; import org.openmetadata.service.util.JsonUtils; @Slf4j @@ -60,7 +59,6 @@ public class SettingsCache { .maximumSize(1000) .expireAfterWrite(3, TimeUnit.MINUTES) .build(new SettingsLoader()); - protected static SystemRepository systemRepository; private SettingsCache() { // Private constructor for singleton @@ -69,7 +67,6 @@ private SettingsCache() { // Expected to be called only once from the DefaultAuthorizer public static void initialize(OpenMetadataApplicationConfig config) { if (!initialized) { - systemRepository = Entity.getSystemRepository(); initialized = true; createDefaultConfiguration(config); } @@ -77,26 +74,21 @@ public static void initialize(OpenMetadataApplicationConfig config) { private static void createDefaultConfiguration(OpenMetadataApplicationConfig applicationConfig) { // Initialise Email Setting - Settings storedSettings = systemRepository.getConfigWithKey(EMAIL_CONFIGURATION.toString()); + Settings storedSettings = + Entity.getSystemRepository().getConfigWithKey(EMAIL_CONFIGURATION.toString()); if (storedSettings == null) { // Only in case a config doesn't exist in DB we insert it - SmtpSettings emailConfig = - new SmtpSettings() - .withPassword(StringUtils.EMPTY) - .withEmailingEntity("OpenMetadata") - .withSupportUrl("https://slack.open-metadata.org") - .withEnableSmtpServer(Boolean.FALSE) - .withTransportationStrategy(SmtpSettings.TransportationStrategy.SMTP_TLS) - .withTemplates(SmtpSettings.Templates.OPENMETADATA); + SmtpSettings emailConfig = getDefaultSmtpSettings(); Settings setting = new Settings().withConfigType(EMAIL_CONFIGURATION).withConfigValue(emailConfig); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise OM base url setting Settings storedOpenMetadataBaseUrlConfiguration = - systemRepository.getConfigWithKey(OPEN_METADATA_BASE_URL_CONFIGURATION.toString()); + Entity.getSystemRepository() + .getConfigWithKey(OPEN_METADATA_BASE_URL_CONFIGURATION.toString()); if (storedOpenMetadataBaseUrlConfiguration == null) { String url = new HttpUrl.Builder().scheme("http").host("localhost").port(8585).build().toString(); @@ -106,12 +98,12 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app new Settings() .withConfigType(OPEN_METADATA_BASE_URL_CONFIGURATION) .withConfigValue(new OpenMetadataBaseUrlConfiguration().withOpenMetadataUrl(baseUrl)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Theme Setting Settings storedCustomUiThemeConf = - systemRepository.getConfigWithKey(CUSTOM_UI_THEME_PREFERENCE.toString()); + Entity.getSystemRepository().getConfigWithKey(CUSTOM_UI_THEME_PREFERENCE.toString()); if (storedCustomUiThemeConf == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = @@ -131,12 +123,13 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app .withErrorColor("") .withWarningColor("") .withInfoColor(""))); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Login Configuration // Initialise Logo Setting - Settings storedLoginConf = systemRepository.getConfigWithKey(LOGIN_CONFIGURATION.toString()); + Settings storedLoginConf = + Entity.getSystemRepository().getConfigWithKey(LOGIN_CONFIGURATION.toString()); if (storedLoginConf == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = @@ -147,23 +140,24 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app .withMaxLoginFailAttempts(3) .withAccessBlockTime(30) .withJwtTokenExpiryTime(3600)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Rbac Settings - Settings storedRbacSettings = systemRepository.getConfigWithKey(SEARCH_SETTINGS.toString()); + Settings storedRbacSettings = + Entity.getSystemRepository().getConfigWithKey(SEARCH_SETTINGS.toString()); if (storedRbacSettings == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = new Settings() .withConfigType(SEARCH_SETTINGS) .withConfigValue(new SearchSettings().withEnableAccessControl(false)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Certification Settings Settings certificationSettings = - systemRepository.getConfigWithKey(ASSET_CERTIFICATION_SETTINGS.toString()); + Entity.getSystemRepository().getConfigWithKey(ASSET_CERTIFICATION_SETTINGS.toString()); if (certificationSettings == null) { Settings setting = new Settings() @@ -172,11 +166,12 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app new AssetCertificationSettings() .withAllowedClassification("Certification") .withValidityPeriod("P30D")); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } // Initialise Workflow Settings - Settings workflowSettings = systemRepository.getConfigWithKey(WORKFLOW_SETTINGS.toString()); + Settings workflowSettings = + Entity.getSystemRepository().getConfigWithKey(WORKFLOW_SETTINGS.toString()); if (workflowSettings == null) { Settings setting = new Settings() @@ -185,10 +180,11 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app new WorkflowSettings() .withExecutorConfiguration(new ExecutorConfiguration()) .withHistoryCleanUpConfiguration(new HistoryCleanUpConfiguration())); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } - Settings lineageSettings = systemRepository.getConfigWithKey(LINEAGE_SETTINGS.toString()); + Settings lineageSettings = + Entity.getSystemRepository().getConfigWithKey(LINEAGE_SETTINGS.toString()); if (lineageSettings == null) { // Only in case a config doesn't exist in DB we insert it Settings setting = @@ -199,7 +195,7 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app .withDownstreamDepth(2) .withUpstreamDepth(2) .withLineageLayer(LineageLayer.ENTITY_LINEAGE)); - systemRepository.createNewSetting(setting); + Entity.getSystemRepository().createNewSetting(setting); } } @@ -226,40 +222,55 @@ public static void invalidateSettings(String settingsName) { } } + private static SmtpSettings getDefaultSmtpSettings() { + return new SmtpSettings() + .withPassword(StringUtils.EMPTY) + .withEmailingEntity("OpenMetadata") + .withSupportUrl("https://slack.open-metadata.org") + .withEnableSmtpServer(Boolean.FALSE) + .withTransportationStrategy(SmtpSettings.TransportationStrategy.SMTP_TLS) + .withTemplates(SmtpSettings.Templates.OPENMETADATA); + } + static class SettingsLoader extends CacheLoader { @Override public @NonNull Settings load(@CheckForNull String settingsName) { Settings fetchedSettings; switch (SettingsType.fromValue(settingsName)) { case EMAIL_CONFIGURATION -> { - fetchedSettings = systemRepository.getEmailConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getEmailConfigInternal(); + if (fetchedSettings == null) { + return new Settings() + .withConfigType(EMAIL_CONFIGURATION) + .withConfigValue(getDefaultSmtpSettings()); + } LOG.info("Loaded Email Setting"); } case OPEN_METADATA_BASE_URL_CONFIGURATION -> { - fetchedSettings = systemRepository.getOMBaseUrlConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getOMBaseUrlConfigInternal(); } case SLACK_APP_CONFIGURATION -> { // Only if available - fetchedSettings = systemRepository.getSlackApplicationConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackApplicationConfigInternal(); LOG.info("Loaded Slack Application Configuration"); } case SLACK_BOT -> { // Only if available - fetchedSettings = systemRepository.getSlackbotConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackbotConfigInternal(); LOG.info("Loaded Slack Bot Configuration"); } case SLACK_INSTALLER -> { // Only if available - fetchedSettings = systemRepository.getSlackInstallerConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackInstallerConfigInternal(); LOG.info("Loaded Slack Installer Configuration"); } case SLACK_STATE -> { // Only if available - fetchedSettings = systemRepository.getSlackStateConfigInternal(); + fetchedSettings = Entity.getSystemRepository().getSlackStateConfigInternal(); LOG.info("Loaded Slack state Configuration"); } default -> { - fetchedSettings = systemRepository.getConfigWithKey(settingsName); + fetchedSettings = Entity.getSystemRepository().getConfigWithKey(settingsName); LOG.info("Loaded Setting {}", fetchedSettings.getConfigType()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java index dc90db9aea29..99e8330e6016 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java @@ -118,7 +118,9 @@ public void initialize(OpenMetadataApplicationConfig config) { type.setCustomProperties(storedType.getCustomProperties()); } } catch (Exception e) { - LOG.debug("Creating entity that does not exist ", e); + LOG.debug( + "Type '{}' not found. Proceeding to add new type entity in database.", + type.getName()); } this.repository.createOrUpdate(null, type); this.repository.addToRegistry(type); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 69ebeafd86e5..1cfe293ec49e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -1125,7 +1125,7 @@ private void getLineage( searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); - if (CommonUtil.nullOrEmpty(deleted)) { + if (!CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn))) @@ -1309,6 +1309,7 @@ private Map searchPipelineLineage( Object[] searchAfter = null; long processedRecords = 0; long totalRecords = -1; + // Process pipeline as edge while (totalRecords != processedRecords) { es.org.elasticsearch.action.search.SearchRequest searchRequest = new es.org.elasticsearch.action.search.SearchRequest( @@ -1320,13 +1321,14 @@ private Map searchPipelineLineage( .must(QueryBuilders.termQuery("lineage.pipeline.fullyQualifiedName.keyword", fqn))); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); + searchSourceBuilder.size(1000); FieldSortBuilder sortBuilder = SortBuilders.fieldSort("fullyQualifiedName"); searchSourceBuilder.sort(sortBuilder); searchSourceBuilder.query(boolQueryBuilder); if (searchAfter != null) { searchSourceBuilder.searchAfter(searchAfter); } - if (CommonUtil.nullOrEmpty(deleted)) { + if (!CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(boolQueryBuilder) @@ -1342,27 +1344,9 @@ private Map searchPipelineLineage( HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); nodes.add(tempMap); for (Map lin : lineage) { - HashMap fromEntity = (HashMap) lin.get("fromEntity"); - HashMap toEntity = (HashMap) lin.get("toEntity"); HashMap pipeline = (HashMap) lin.get("pipeline"); if (pipeline != null && pipeline.get("fullyQualifiedName").equalsIgnoreCase(fqn)) { edges.add(lin); - getLineage( - fromEntity.get("fqn"), - upstreamDepth, - edges, - nodes, - queryFilter, - "lineage.toEntity.fqn.keyword", - deleted); - getLineage( - toEntity.get("fqn"), - downstreamDepth, - edges, - nodes, - queryFilter, - "lineage.fromEntity.fqn.keyword", - deleted); } } } @@ -1372,13 +1356,22 @@ private Map searchPipelineLineage( if (currentHits > 0) { searchAfter = searchResponse.getHits().getHits()[currentHits - 1].getSortValues(); } else { - searchAfter = null; + // when current records are 0 break the loop + break; } } + + // Process pipeline as node getLineage( - fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + fqn, + downstreamDepth, + edges, + nodes, + queryFilter, + "lineage.fromEntity.fqnHash.keyword", + deleted); getLineage( - fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); + fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqnHash.keyword", deleted); // TODO: Fix this , this is hack if (edges.isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java index c712f5c74922..fd40d921ce03 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ColumnIndex.java @@ -50,6 +50,6 @@ default String getColumnDescriptionStatus(EntityInterface entity) { } } } - return CommonUtil.nullOrEmpty(entity.getDescription()) ? "INCOMPLETE" : "COMPLETE"; + return "COMPLETE"; } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java index 46d67218d746..b08894a05741 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java @@ -70,7 +70,9 @@ private void setParentRelationships(Map doc, TestCase testCase) } TestSuite testSuite = Entity.getEntityOrNull(testSuiteEntityReference, "", Include.ALL); EntityReference entityReference = testSuite.getBasicEntityReference(); - TestSuiteIndex.addTestSuiteParentEntityRelations(entityReference, doc); + if (entityReference != null) { + TestSuiteIndex.addTestSuiteParentEntityRelations(entityReference, doc); + } } public static Map getFields() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java index b63d3940db8d..56de9830049e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java @@ -65,7 +65,9 @@ private void setParentRelationships(Map doc) { TestSuite testSuite = Entity.getEntityOrNull(testCase.getTestSuite(), "", Include.ALL); if (testSuite == null) return; doc.put("testSuite", testSuite.getEntityReference()); - TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getBasicEntityReference(), doc); + if (testSuite.getBasicEntityReference() != null) { + TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getBasicEntityReference(), doc); + } } public static Map getFields() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 50a90e724866..787bee189581 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -1120,7 +1120,7 @@ private void getLineage( searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn)))); - if (CommonUtil.nullOrEmpty(deleted)) { + if (!CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(direction, FullyQualifiedName.buildHash(fqn))) @@ -1303,6 +1303,7 @@ private Map searchPipelineLineage( Object[] searchAfter = null; long processedRecords = 0; long totalRecords = -1; + // Process pipeline as edge while (totalRecords != processedRecords) { os.org.opensearch.action.search.SearchRequest searchRequest = new os.org.opensearch.action.search.SearchRequest( @@ -1315,11 +1316,12 @@ private Map searchPipelineLineage( searchSourceBuilder.fetchSource(null, SOURCE_FIELDS_TO_EXCLUDE.toArray(String[]::new)); FieldSortBuilder sortBuilder = SortBuilders.fieldSort("fullyQualifiedName"); searchSourceBuilder.sort(sortBuilder); + searchSourceBuilder.size(1000); searchSourceBuilder.query(boolQueryBuilder); if (searchAfter != null) { searchSourceBuilder.searchAfter(searchAfter); } - if (CommonUtil.nullOrEmpty(deleted)) { + if (!CommonUtil.nullOrEmpty(deleted)) { searchSourceBuilder.query( QueryBuilders.boolQuery() .must(boolQueryBuilder) @@ -1335,27 +1337,9 @@ private Map searchPipelineLineage( HashMap tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap())); nodes.add(tempMap); for (Map lin : lineage) { - HashMap fromEntity = (HashMap) lin.get("fromEntity"); - HashMap toEntity = (HashMap) lin.get("toEntity"); HashMap pipeline = (HashMap) lin.get("pipeline"); if (pipeline != null && pipeline.get("fullyQualifiedName").equalsIgnoreCase(fqn)) { edges.add(lin); - getLineage( - fromEntity.get("fqn"), - upstreamDepth, - edges, - nodes, - queryFilter, - "lineage.toEntity.fqn.keyword", - deleted); - getLineage( - toEntity.get("fqn"), - downstreamDepth, - edges, - nodes, - queryFilter, - "lineage.fromEntity.fqn.keyword", - deleted); } } } @@ -1365,13 +1349,22 @@ private Map searchPipelineLineage( if (currentHits > 0) { searchAfter = searchResponse.getHits().getHits()[currentHits - 1].getSortValues(); } else { - searchAfter = null; + // when current records are 0 break the loop + break; } } + + // Process pipeline as node getLineage( - fqn, downstreamDepth, edges, nodes, queryFilter, "lineage.fromEntity.fqn.keyword", deleted); + fqn, + downstreamDepth, + edges, + nodes, + queryFilter, + "lineage.fromEntity.fqnHash.keyword", + deleted); getLineage( - fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqn.keyword", deleted); + fqn, upstreamDepth, edges, nodes, queryFilter, "lineage.toEntity.fqnHash.keyword", deleted); if (edges.isEmpty()) { os.org.opensearch.action.search.SearchRequest searchRequestForEntity = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index a48ebd67c47d..137bd4cb0a6a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -42,7 +42,10 @@ import org.openmetadata.schema.services.connections.database.datalake.GCSConfig; import org.openmetadata.schema.services.connections.database.deltalake.StorageConfig; import org.openmetadata.schema.services.connections.database.iceberg.IcebergFileSystem; +import org.openmetadata.schema.services.connections.mlmodel.VertexAIConnection; import org.openmetadata.schema.services.connections.pipeline.AirflowConnection; +import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; +import org.openmetadata.schema.services.connections.pipeline.NifiConnection; import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; import org.openmetadata.schema.services.connections.storage.GCSConnection; @@ -87,7 +90,10 @@ private ClassConverterFactory() { new TestServiceConnectionRequestClassConverter()), Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), Map.entry(Workflow.class, new WorkflowClassConverter()), - Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter())); + Map.entry(CockroachConnection.class, new CockroachConnectionClassConverter()), + Map.entry(NifiConnection.class, new NifiConnectionClassConverter()), + Map.entry(MatillionConnection.class, new MatillionConnectionClassConverter()), + Map.entry(VertexAIConnection.class, new VertexAIConnectionClassConverter())); Map.entry(Workflow.class, new WorkflowClassConverter()); Map.entry(CassandraConnection.class, new CassandraConnectionClassConverter()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java new file mode 100644 index 000000000000..90b4a572fc30 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java @@ -0,0 +1,38 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; +import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `MatillionConnection` object. */ +public class MatillionConnectionClassConverter extends ClassConverter { + + public MatillionConnectionClassConverter() { + super(MatillionConnection.class); + } + + @Override + public Object convert(Object object) { + MatillionConnection matillionConnection = + (MatillionConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(matillionConnection.getConnection(), List.of(MatillionETLAuth.class)) + .ifPresent(matillionConnection::setConnection); + + return matillionConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NifiConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NifiConnectionClassConverter.java new file mode 100644 index 000000000000..234456b39632 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/NifiConnectionClassConverter.java @@ -0,0 +1,39 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.services.connections.pipeline.NifiConnection; +import org.openmetadata.schema.services.connections.pipeline.nifi.BasicAuth; +import org.openmetadata.schema.services.connections.pipeline.nifi.ClientCertificateAuth; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `NifiConnection` object. */ +public class NifiConnectionClassConverter extends ClassConverter { + + public NifiConnectionClassConverter() { + super(NifiConnection.class); + } + + @Override + public Object convert(Object object) { + NifiConnection nifiConnection = (NifiConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail( + nifiConnection.getNifiConfig(), List.of(BasicAuth.class, ClientCertificateAuth.class)) + .ifPresent(nifiConnection::setNifiConfig); + + return nifiConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/VertexAIConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/VertexAIConnectionClassConverter.java new file mode 100644 index 000000000000..ecdd571f9121 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/VertexAIConnectionClassConverter.java @@ -0,0 +1,37 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.security.credentials.GCPCredentials; +import org.openmetadata.schema.services.connections.mlmodel.VertexAIConnection; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `VertexAIConnection` object. */ +public class VertexAIConnectionClassConverter extends ClassConverter { + + public VertexAIConnectionClassConverter() { + super(VertexAIConnection.class); + } + + @Override + public Object convert(Object object) { + VertexAIConnection vertexAIConnection = + (VertexAIConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(vertexAIConnection.getCredentials(), List.of(GCPCredentials.class)) + .ifPresent(obj -> vertexAIConnection.setCredentials((GCPCredentials) obj)); + + return vertexAIConnection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthRequest.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthRequest.java new file mode 100644 index 000000000000..8e65c7b65e24 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthRequest.java @@ -0,0 +1,7 @@ +package org.openmetadata.service.security; + +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; + +public record AuthRequest( + OperationContext operationContext, ResourceContextInterface resourceContext) {} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthorizationLogic.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthorizationLogic.java new file mode 100644 index 000000000000..da0183864ff0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthorizationLogic.java @@ -0,0 +1,6 @@ +package org.openmetadata.service.security; + +public enum AuthorizationLogic { + ANY, + ALL +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java index f94193c0d817..21c2bc01a4ff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java @@ -41,6 +41,9 @@ void authorize( OperationContext operationContext, ResourceContextInterface resourceContext); + void authorizeRequests( + SecurityContext securityContext, List requests, AuthorizationLogic logic); + void authorizeAdmin(SecurityContext securityContext); void authorizeAdmin(String adminName); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java index a8636b53ad44..f79acc0a9551 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java @@ -92,6 +92,36 @@ public void authorize( PolicyEvaluator.hasPermission(subjectContext, resourceContext, operationContext); } + public void authorizeRequests( + SecurityContext securityContext, List requests, AuthorizationLogic logic) { + SubjectContext subjectContext = getSubjectContext(securityContext); + + if (subjectContext.isAdmin()) { + return; + } + + if (logic == AuthorizationLogic.ANY) { + boolean anySuccess = false; + for (AuthRequest req : requests) { + try { + PolicyEvaluator.hasPermission( + subjectContext, req.resourceContext(), req.operationContext()); + anySuccess = true; + break; + } catch (AuthorizationException ignored) { + } + } + if (!anySuccess) { + throw new AuthorizationException("User does not have ANY of the required permissions."); + } + } else { // ALL + for (AuthRequest req : requests) { + PolicyEvaluator.hasPermission( + subjectContext, req.resourceContext(), req.operationContext()); + } + } + } + @Override public void authorizeAdmin(SecurityContext securityContext) { SubjectContext subjectContext = getSubjectContext(securityContext); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java index 355bfe809905..8f0a438c2953 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java @@ -65,6 +65,12 @@ public void authorize( /* Always authorize */ } + @Override + public void authorizeRequests( + SecurityContext securityContext, List requests, AuthorizationLogic logic) { + /* Always authorize */ + } + private void addAnonymousUser() { String username = "anonymous"; try { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java index c1d189b2d189..60d1260c378d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java @@ -43,7 +43,7 @@ public class TestCaseResourceContext implements ResourceContextInterface { @Override public String getResource() { - return entityLink.getEntityType(); + return entity != null ? entity.getEntityReference().getType() : Entity.TEST_CASE; } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java new file mode 100644 index 000000000000..28a5c5657649 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/AppMarketPlaceUtil.java @@ -0,0 +1,59 @@ +package org.openmetadata.service.util; + +import static org.openmetadata.service.Entity.APPLICATION; +import static org.openmetadata.service.jdbi3.AppRepository.APP_BOT_ROLE; +import static org.openmetadata.service.jdbi3.EntityRepository.getEntitiesFromSeedData; + +import java.io.IOException; +import java.util.List; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.app.CreateAppMarketPlaceDefinitionReq; +import org.openmetadata.schema.entity.teams.Role; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.AppMarketPlaceRepository; +import org.openmetadata.service.jdbi3.PolicyRepository; +import org.openmetadata.service.jdbi3.RoleRepository; +import org.openmetadata.service.jdbi3.TeamRepository; +import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; + +public class AppMarketPlaceUtil { + public static void createAppMarketPlaceDefinitions( + AppMarketPlaceRepository appMarketRepository, AppMarketPlaceMapper mapper) + throws IOException { + PolicyRepository policyRepository = Entity.getPolicyRepository(); + RoleRepository roleRepository = Entity.getRoleRepository(); + + try { + roleRepository.findByName(APP_BOT_ROLE, Include.NON_DELETED); + } catch (EntityNotFoundException e) { + policyRepository.initSeedDataFromResources(); + List roles = roleRepository.getEntitiesFromSeedData(); + for (Role role : roles) { + role.setFullyQualifiedName(role.getName()); + List policies = role.getPolicies(); + for (EntityReference policy : policies) { + EntityReference ref = + Entity.getEntityReferenceByName(Entity.POLICY, policy.getName(), Include.NON_DELETED); + policy.setId(ref.getId()); + } + roleRepository.initializeEntity(role); + } + TeamRepository teamRepository = (TeamRepository) Entity.getEntityRepository(Entity.TEAM); + teamRepository.initOrganization(); + } + + List createAppMarketPlaceDefinitionReqs = + getEntitiesFromSeedData( + APPLICATION, + String.format(".*json/data/%s/.*\\.json$", Entity.APP_MARKET_PLACE_DEF), + CreateAppMarketPlaceDefinitionReq.class); + for (CreateAppMarketPlaceDefinitionReq definitionReq : createAppMarketPlaceDefinitionReqs) { + AppMarketPlaceDefinition definition = mapper.createToEntity(definitionReq, "admin"); + appMarketRepository.setFullyQualifiedName(definition); + appMarketRepository.createOrUpdate(null, definition); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 9196cff53120..46109b2eaeba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -65,6 +65,7 @@ import org.openmetadata.sdk.PipelineServiceClientInterface; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.apps.ApplicationHandler; import org.openmetadata.service.apps.scheduler.AppScheduler; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; @@ -74,15 +75,18 @@ import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EventSubscriptionRepository; import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.SystemRepository; +import org.openmetadata.service.jdbi3.TypeRepository; import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.apps.AppMapper; +import org.openmetadata.service.resources.apps.AppMarketPlaceMapper; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.secrets.SecretsManager; @@ -328,45 +332,108 @@ public Integer configureEmailSettings( public Integer installApp( @Option( names = {"-n", "--name"}, - description = "Number of records to process in each batch.", + description = "The name of the application to install.", required = true) - String appName) { + String appName, + @Option( + names = {"--force"}, + description = "Forces migrations to be run again, even if they have ran previously", + defaultValue = "false") + boolean force) { try { parseConfig(); - CollectionRegistry.initialize(); - ApplicationHandler.initialize(config); - CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); - ApplicationHandler.initialize(config); - AppScheduler.initialize(config, collectionDAO, searchRepository); - // Instantiate JWT Token Generator - JWTTokenGenerator.getInstance() - .init( - config.getAuthenticationConfiguration().getTokenValidationAlgorithm(), - config.getJwtTokenConfiguration()); - AppMarketPlaceRepository marketPlaceRepository = - (AppMarketPlaceRepository) Entity.getEntityRepository(Entity.APP_MARKET_PLACE_DEF); - AppMarketPlaceDefinition definition = - marketPlaceRepository.getByName(null, appName, marketPlaceRepository.getFields("id")); AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); - CreateApp createApp = - new CreateApp() - .withName(definition.getName()) - .withDescription(definition.getDescription()) - .withDisplayName(definition.getDisplayName()) - .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.NONE)) - .withAppConfiguration(new AppConfiguration()); - AppMapper appMapper = new AppMapper(); - App entity = appMapper.createToEntity(createApp, ADMIN_USER_NAME); - appRepository.prepareInternal(entity, true); - appRepository.createOrUpdate(null, entity); + + if (!force && isAppInstalled(appRepository, appName)) { + LOG.info("App already installed."); + return 0; + } + + if (force && deleteApplication(appRepository, appName)) { + LOG.info("App deleted."); + } + + LOG.info("App not installed. Installing..."); + installApplication(appName, appRepository); LOG.info("App Installed."); return 0; } catch (Exception e) { - LOG.error("Install Application Failed ", e); + LOG.error("Install Application Failed", e); return 1; } } + @Command(name = "delete-app", description = "Delete the installed application.") + public Integer deleteApp( + @Option( + names = {"-n", "--name"}, + description = "The name of the application to install.", + required = true) + String appName) { + try { + parseConfig(); + AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); + if (deleteApplication(appRepository, appName)) { + LOG.info("App deleted."); + } + return 0; + } catch (Exception e) { + LOG.error("Delete Application Failed", e); + return 1; + } + } + + private boolean isAppInstalled(AppRepository appRepository, String appName) { + try { + appRepository.findByName(appName, Include.NON_DELETED); + return true; + } catch (EntityNotFoundException e) { + return false; + } + } + + private boolean deleteApplication(AppRepository appRepository, String appName) { + try { + appRepository.deleteByName(ADMIN_USER_NAME, appName, true, true); + return true; + } catch (EntityNotFoundException e) { + return false; + } + } + + private void installApplication(String appName, AppRepository appRepository) throws Exception { + PipelineServiceClientInterface pipelineServiceClient = + PipelineServiceClientFactory.createPipelineServiceClient( + config.getPipelineServiceClientConfiguration()); + + JWTTokenGenerator.getInstance() + .init( + config.getAuthenticationConfiguration().getTokenValidationAlgorithm(), + config.getJwtTokenConfiguration()); + + AppMarketPlaceMapper mapper = new AppMarketPlaceMapper(pipelineServiceClient); + AppMarketPlaceRepository appMarketRepository = + (AppMarketPlaceRepository) Entity.getEntityRepository(Entity.APP_MARKET_PLACE_DEF); + + AppMarketPlaceUtil.createAppMarketPlaceDefinitions(appMarketRepository, mapper); + + AppMarketPlaceDefinition definition = + appMarketRepository.getByName(null, appName, appMarketRepository.getFields("id")); + + CreateApp createApp = + new CreateApp() + .withName(definition.getName()) + .withDescription(definition.getDescription()) + .withDisplayName(definition.getDisplayName()) + .withAppSchedule(new AppSchedule().withScheduleTimeline(ScheduleTimeline.NONE)) + .withAppConfiguration(new AppConfiguration()); + + AppMapper appMapper = new AppMapper(); + App entity = appMapper.createToEntity(createApp, ADMIN_USER_NAME); + appRepository.prepareInternal(entity, true); + appRepository.createOrUpdate(null, entity); + } + @Command( name = "check-connection", description = @@ -576,6 +643,8 @@ public Integer reIndex( ApplicationHandler.initialize(config); CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); ApplicationHandler.initialize(config); + TypeRepository typeRepository = (TypeRepository) Entity.getEntityRepository(Entity.TYPE); + TypeRegistry.instance().initialize(typeRepository); AppScheduler.initialize(config, collectionDAO, searchRepository); String appName = "SearchIndexingApplication"; Set entities = @@ -599,6 +668,26 @@ public Integer reIndex( } } + @Command(name = "syncAlertOffset", description = "Sync the Alert Offset.") + public Integer reIndex( + @Option( + names = {"-n", "--name"}, + description = "Name of the alerts.", + required = true) + String alertName) { + try { + parseConfig(); + CollectionRegistry.initialize(); + EventSubscriptionRepository repository = + (EventSubscriptionRepository) Entity.getEntityRepository(Entity.EVENT_SUBSCRIPTION); + repository.syncEventSubscriptionOffset(alertName); + return 0; + } catch (Exception e) { + LOG.error("Failed to sync alert offset due to ", e); + return 1; + } + } + private int executeSearchReindexApp( String appName, Set entities, @@ -689,9 +778,8 @@ public Integer reIndexDI( CollectionRegistry.getInstance().loadSeedData(jdbi, config, null, null, null, true); ApplicationHandler.initialize(config); AppScheduler.initialize(config, collectionDAO, searchRepository); - String appName = "DataInsightsApplication"; return executeDataInsightsReindexApp( - appName, batchSize, recreateIndexes, getBackfillConfiguration(startDate, endDate)); + batchSize, recreateIndexes, getBackfillConfiguration(startDate, endDate)); } catch (Exception e) { LOG.error("Failed to reindex due to ", e); return 1; @@ -713,13 +801,10 @@ private BackfillConfiguration getBackfillConfiguration(String startDate, String } private int executeDataInsightsReindexApp( - String appName, - int batchSize, - boolean recreateIndexes, - BackfillConfiguration backfillConfiguration) { + int batchSize, boolean recreateIndexes, BackfillConfiguration backfillConfiguration) { AppRepository appRepository = (AppRepository) Entity.getEntityRepository(Entity.APPLICATION); App originalDataInsightsApp = - appRepository.getByName(null, appName, appRepository.getFields("id")); + appRepository.getByName(null, "DataInsightsApplication", appRepository.getFields("id")); DataInsightsAppConfig storedConfig = JsonUtils.convertValue( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java index b81690dbf342..ecaadd91c7b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ResultList.java @@ -17,6 +17,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import org.openmetadata.schema.system.EntityError; import org.openmetadata.schema.type.Paging; @@ -95,6 +97,11 @@ public ResultList(List data, Integer offset, int total) { paging = new Paging().withBefore(null).withAfter(null).withTotal(total).withOffset(offset); } + /* Conveniently map the data to another type without the need to create a new ResultList */ + public ResultList map(Function mapper) { + return new ResultList<>(data.stream().map(mapper).collect(Collectors.toList()), paging); + } + public ResultList(List data, Integer offset, Integer limit, Integer total) { this.data = data; paging = @@ -106,6 +113,17 @@ public ResultList(List data, Integer offset, Integer limit, Integer total) { .withLimit(limit); } + public ResultList(List data, Paging other) { + this.data = data; + paging = + new Paging() + .withBefore(null) + .withAfter(null) + .withTotal(other.getTotal()) + .withOffset(other.getOffset()) + .withLimit(other.getLimit()); + } + public ResultList( List data, List errors, String beforeCursor, String afterCursor, int total) { this.data = data; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java index 7bef5e5ed20d..bea22fb5cd0f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java @@ -377,12 +377,19 @@ public static Set getTargetsForAlert( // TODO: For Announcement, Immediate Consumer needs to be Notified (find information from // Lineage) case Announcement -> { - receiverUrls.addAll(buildReceivers(action, category, type, event, event.getEntityId())); + receiverUrls.addAll( + buildReceivers( + action, + category, + type, + thread.getEntityRef().getType(), + thread.getEntityRef().getId())); } } } else { EntityInterface entityInterface = getEntity(event); - receiverUrls.addAll(buildReceivers(action, category, type, event, entityInterface.getId())); + receiverUrls.addAll( + buildReceivers(action, category, type, event.getEntityType(), entityInterface.getId())); } return receiverUrls; @@ -392,12 +399,12 @@ private static Set buildReceivers( SubscriptionAction action, SubscriptionDestination.SubscriptionCategory category, SubscriptionDestination.SubscriptionType type, - ChangeEvent event, + String entityType, UUID id) { Set result = new HashSet<>(); result.addAll( buildReceiversListFromActions( - action, category, type, Entity.getCollectionDAO(), id, event.getEntityType())); + action, category, type, Entity.getCollectionDAO(), id, entityType)); return result; } diff --git a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/email-verification.json b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/email-verification.json index adadb7ede94e..af99e0420551 100644 --- a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/email-verification.json +++ b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/email-verification.json @@ -5,7 +5,7 @@ "entityType": "EmailTemplate", "fullyQualifiedName": "email-verification", "data": { - "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tWelcome to ${entity}, the all-in-one platform for data discovery, data quality, observability, governance, data lineage, and team collaboration. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tThanks for joining. Please confirm your email address to complete the registration process. Please click the link below or copy and paste this URL into your web browser to verify your email. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\tConfirm email\n\t\t\t\t\n\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\t\tThe verification link is valid for ${expirationTime} hours only. \n\t\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\n\t\t
 
\n\n\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n Slack \n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n", + "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tWelcome to ${entity}, the all-in-one platform for data discovery, data quality, observability, governance, data lineage, and team collaboration. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tThanks for joining. Please confirm your email address to complete the registration process. Please click the link below or copy and paste this URL into your web browser to verify your email. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\tConfirm email\n\t\t\t\t\n\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\t\tThe verification link is valid for ${expirationTime} hours only. \n\t\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\n\t\t
 
\n\n\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n here \n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n", "placeHolders": [ { "name": "userName", diff --git a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-createPassword.json b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-createPassword.json index dfa5d20b072e..aee694fbf39b 100644 --- a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-createPassword.json +++ b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-createPassword.json @@ -5,7 +5,7 @@ "entityType": "EmailTemplate", "fullyQualifiedName": "invite-createPassword", "data": { - "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tWelcome to ${entity}, the all-in-one platform for data discovery, data quality, observability, governance, data lineage, and team collaboration.  \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tPlease use the following link to create a new password for ${entity}. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\tCreate Password\n\t\t\t\t\n\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n Slack \n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n \n", + "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tWelcome to ${entity}, the all-in-one platform for data discovery, data quality, observability, governance, data lineage, and team collaboration.  \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tPlease use the following link to create a new password for ${entity}. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\tCreate Password\n\t\t\t\t\n\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n here \n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n \n", "placeHolders": [ { "name": "userName", diff --git a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-randompwd.json b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-randompwd.json index d58c3af7250d..b0beea300b98 100644 --- a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-randompwd.json +++ b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/invite-randompwd.json @@ -5,7 +5,7 @@ "entityType": "EmailTemplate", "fullyQualifiedName": "invite-randompwd", "data": { - "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tWelcome to ${entity}, the all-in-one platform for data discovery, data quality, observability, governance, data lineage, and team collaboration.\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tYou have been added as an admin. Please use below password to sign-in. Please change the password on signing in to ${entity}. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t\t
 
\n\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t\t\t\t${password}\n\t\t\t\t\t\t\t\t\t\t\t
\n \t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\tLogin\n\t\t\t\t\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\n\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n Slack \n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n ", + "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tWelcome to ${entity}, the all-in-one platform for data discovery, data quality, observability, governance, data lineage, and team collaboration.\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tYou have been added as an admin. Please use below password to sign-in. Please change the password on signing in to ${entity}. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t\t
 
\n\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t\t\t\t${password}\n\t\t\t\t\t\t\t\t\t\t\t
\n \t\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t\t
\n\t\t\t\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\tLogin\n\t\t\t\t\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\n\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n here \n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n ", "placeHolders": [ { "name": "userName", diff --git a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/reset-link.json b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/reset-link.json index c5b76aa91270..35f2bb359bbe 100644 --- a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/reset-link.json +++ b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/reset-link.json @@ -5,7 +5,7 @@ "entityType": "EmailTemplate", "fullyQualifiedName": "reset-link", "data": { - "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHello ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tYou have requested to reset your ${entity} password. Please click the link below to reset your password. This link is valid for ${expirationTime} minutes only. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\n\t\t\t\t
 
\n\t\t\t\tRest Link\n\t\t\t\t\n\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\tIf you did not request this change, please let us know. You can contact us on Slack for any questions you may have.\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Happy Exploring!
Thanks

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n \n", + "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\t\n\t\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHello ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tYou have requested to reset your ${entity} password. Please click the link below to reset your password. This link is valid for ${expirationTime} minutes only. \n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\n\t\t\t\t
 
\n\t\t\t\tRest Link\n\t\t\t\t\n\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\tIf you did not request this change, please let us know. You can contact us on here for any questions you may have.\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Happy Exploring!
Thanks

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n \n", "placeHolders": [ { "name": "userName", diff --git a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/testMail.json b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/testMail.json index b6fbec20b02b..a29468b2b28e 100644 --- a/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/testMail.json +++ b/openmetadata-service/src/main/resources/json/data/document/emailTemplates/openmetadata/testMail.json @@ -5,7 +5,7 @@ "entityType": "EmailTemplate", "fullyQualifiedName": "testMail", "data": { - "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tThis is a Test Email, receiving this Email Confirms that you have successfully configured OpenMetadata to send Mails.\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t\n\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\n\t\t
 
\n\n\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n Slack\n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n \n", + "template": " \n \n \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t \n\t
\n\t\n\t\n
\n\t
\n\t\n\t
\n\t\t\n\t
\n\t\n
\n
\n\t\n\t\n\t\n\t\t\n\t
\n\t\t\n\t\t
\n\t\t\t\t\t\"\"\n\t\t\t\t
\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tHi ${userName},\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t
 
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t
\n\t\t\t\t\t\t\tThis is a Test Email, receiving this Email Confirms that you have successfully configured OpenMetadata to send Mails.\n\t\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t\t\n\t\t\t
\n\t\t\t\t\n\t\t\t\n\t\t
\n\t\t
 
\n\n\t\t \n\t\t
\n\t\t\t\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n \n Feel free to reach out to us on\n here\n for any questions you may have.\n \n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t
\n\t\t
 
\n\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t
 
\n\t\t\t\n\t\t\t\n\t\t\t
\n\t\t\t\t
\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t
\n\t\t\t\t\t\t

Thanks,
The ${entity} Team

\n\t\t\t\t\t
\n\t\t\t\t
\n\t\t\t
\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t
\n\t
\n\t\n
\n
\n
\n \n \n", "placeHolders": [ { "name": "userName", diff --git a/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json b/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json index f14c8acc5152..ecc066fdd3fb 100644 --- a/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json +++ b/openmetadata-service/src/main/resources/json/data/governance/workflows/GlossaryApprovalWorkflow.json @@ -24,6 +24,12 @@ "name": "ApprovedEnd", "displayName": "Glossary Term Status: Approved" }, + { + "type": "endEvent", + "subType": "endEvent", + "name": "ApprovedEndAfterApproval", + "displayName": "Glossary Term Status: Approved" + }, { "type": "endEvent", "subType": "endEvent", @@ -43,6 +49,9 @@ "displayName": "Check if Glossary Term has Reviewers", "config": { "rules": "{\"and\":[{\"!!\":[{\"var\":\"reviewers\"}]}]}" + }, + "inputNamespaceMap": { + "relatedEntity": "global" } }, { @@ -52,6 +61,9 @@ "displayName": "Check if Glossary Term is Ready to be Reviewed", "config": { "rules": "{\"and\":[{\"!!\":[{\"var\":\"description\"}]}]}" + }, + "inputNamespaceMap": { + "relatedEntity": "global" } }, { @@ -61,6 +73,9 @@ "displayName": "Set Status to 'In Review'", "config": { "glossaryTermStatus": "In Review" + }, + "inputNamespaceMap": { + "relatedEntity": "global" } }, { @@ -70,6 +85,9 @@ "displayName": "Set Status to 'Draft'", "config": { "glossaryTermStatus": "Draft" + }, + "inputNamespaceMap": { + "relatedEntity": "global" } }, { @@ -81,6 +99,22 @@ "assignees": { "addReviewers": true } + }, + "inputNamespaceMap": { + "relatedEntity": "global" + } + }, + { + "type": "automatedTask", + "subType": "setGlossaryTermStatusTask", + "name": "SetGlossaryTermStatusToApprovedAfterApproval", + "displayName": "Set Status to 'Approved'", + "config": { + "glossaryTermStatus": "Approved" + }, + "inputNamespaceMap": { + "relatedEntity": "global", + "updatedBy": "ApproveGlossaryTerm" } }, { @@ -90,6 +124,9 @@ "displayName": "Set Status to 'Approved'", "config": { "glossaryTermStatus": "Approved" + }, + "inputNamespaceMap": { + "relatedEntity": "global" } }, { @@ -99,6 +136,10 @@ "displayName": "Set Status to 'Rejected'", "config": { "glossaryTermStatus": "Rejected" + }, + "inputNamespaceMap": { + "relatedEntity": "global", + "updatedBy": "ApproveGlossaryTerm" } } ], @@ -110,17 +151,17 @@ { "from": "CheckGlossaryTermHasReviewers", "to": "SetGlossaryTermStatusToApproved", - "condition": false + "condition": "false" }, { "from": "CheckGlossaryTermHasReviewers", "to": "CheckGlossaryTermIsReadyToBeReviewed", - "condition": true + "condition": "true" }, { "from": "CheckGlossaryTermIsReadyToBeReviewed", "to": "SetGlossaryTermStatusToDraft", - "condition": false + "condition": "false" }, { "from": "SetGlossaryTermStatusToDraft", @@ -129,7 +170,7 @@ { "from": "CheckGlossaryTermIsReadyToBeReviewed", "to": "SetGlossaryTermStatusToInReview", - "condition": true + "condition": "true" }, { "from": "SetGlossaryTermStatusToInReview", @@ -137,13 +178,17 @@ }, { "from": "ApproveGlossaryTerm", - "to": "SetGlossaryTermStatusToApproved", - "condition": true + "to": "SetGlossaryTermStatusToApprovedAfterApproval", + "condition": "true" }, { "from": "ApproveGlossaryTerm", "to": "SetGlossaryTermStatusToRejected", - "condition": false + "condition": "false" + }, + { + "from": "SetGlossaryTermStatusToApprovedAfterApproval", + "to": "ApprovedEndAfterApproval" }, { "from": "SetGlossaryTermStatusToApproved", diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java index 35fe934705fa..f9fa0d218767 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java @@ -139,6 +139,8 @@ public void createApplication() throws Exception { sqlContainer.withReuse(false); sqlContainer.withStartupTimeoutSeconds(240); sqlContainer.withConnectTimeoutSeconds(240); + sqlContainer.withPassword("password"); + sqlContainer.withUsername("username"); sqlContainer.start(); final String flyWayMigrationScriptsLocation = @@ -218,7 +220,7 @@ public void createApplication() throws Exception { createClient(); } - public static void validateAndRunSystemDataMigrations( + public void validateAndRunSystemDataMigrations( Jdbi jdbi, OpenMetadataApplicationConfig config, ConnectionType connType, @@ -240,7 +242,7 @@ public static void validateAndRunSystemDataMigrations( // Initialize search repository SearchRepository searchRepository = new SearchRepository(getEsConfig()); Entity.setSearchRepository(searchRepository); - Entity.setCollectionDAO(jdbi.onDemand(CollectionDAO.class)); + Entity.setCollectionDAO(getDao(jdbi)); Entity.setJobDAO(jdbi.onDemand(JobDAO.class)); Entity.initializeRepositories(config, jdbi); workflow.loadMigrations(); @@ -248,6 +250,10 @@ public static void validateAndRunSystemDataMigrations( Entity.cleanup(); } + protected CollectionDAO getDao(Jdbi jdbi) { + return jdbi.onDemand(CollectionDAO.class); + } + @NotNull protected DropwizardAppExtension getApp( ConfigOverride[] configOverridesArray) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index c00e20069def..de2e355ba004 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -31,8 +31,6 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.csv.EntityCsvTest.assertSummary; -import static org.openmetadata.schema.type.MetadataOperation.EDIT_ALL; -import static org.openmetadata.schema.type.MetadataOperation.EDIT_TESTS; import static org.openmetadata.schema.type.TaskType.RequestDescription; import static org.openmetadata.service.Entity.*; import static org.openmetadata.service.exception.CatalogExceptionMessage.ENTITY_ALREADY_EXISTS; @@ -1373,7 +1371,9 @@ protected void post_entity_as_non_admin_401(TestInfo test) { assertResponse( () -> createEntity(createRequest(test), TEST_AUTH_HEADERS), FORBIDDEN, - permissionNotAllowed(TEST_USER_NAME, List.of(MetadataOperation.CREATE))); + List.of( + permissionNotAllowed(TEST_USER_NAME, List.of(MetadataOperation.CREATE)), + "User does not have ANY of the required permissions.")); } @Test @@ -1674,12 +1674,13 @@ void put_entityUpdate_as_non_owner_4xx(TestInfo test) throws IOException { // Update description and remove owner as non-owner // Expect to throw an exception since only owner or admin can update resource K updateRequest = createRequest(entity.getName(), "newDescription", "displayName", null); - MetadataOperation operation = entityType.equals(Entity.TEST_CASE) ? EDIT_TESTS : EDIT_ALL; assertResponse( () -> updateEntity(updateRequest, OK, TEST_AUTH_HEADERS), FORBIDDEN, - permissionNotAllowed(TEST_USER_NAME, List.of(operation))); + List.of( + permissionNotAllowed(TEST_USER_NAME, List.of(MetadataOperation.EDIT_ALL)), + "User does not have ANY of the required permissions.")); } @Test @@ -2801,8 +2802,7 @@ protected final WebTarget getFollowerResource(UUID id, UUID userId) { return getFollowersCollection(id).path("/" + userId); } - protected final T getEntity(UUID id, Map authHeaders) - throws HttpResponseException { + public final T getEntity(UUID id, Map authHeaders) throws HttpResponseException { WebTarget target = getResource(id); target = target.queryParam("fields", allFields); return TestUtils.get(target, entityClass, authHeaders); @@ -3217,7 +3217,9 @@ protected T patchEntityAndCheckAuthorization( assertResponse( () -> patchEntity(entity.getId(), originalJson, entity, authHeaders), FORBIDDEN, - permissionNotAllowed(userName, List.of(disallowedOperation))); + List.of( + "User does not have ANY of the required permissions.", + permissionNotAllowed(userName, List.of(disallowedOperation)))); return entity; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 7f5b37276611..b233e0d841b5 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -28,8 +28,10 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.schema.api.teams.CreateTeam.TeamType.GROUP; import static org.openmetadata.schema.type.ColumnDataType.BIGINT; +import static org.openmetadata.schema.type.MetadataOperation.DELETE; import static org.openmetadata.schema.type.MetadataOperation.EDIT_TESTS; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; +import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.Entity.TEST_CASE; import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.exception.CatalogExceptionMessage.permissionNotAllowed; @@ -65,6 +67,7 @@ import org.apache.http.client.HttpResponseException; import org.apache.http.util.EntityUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -75,6 +78,8 @@ import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.feed.CloseTask; import org.openmetadata.schema.api.feed.ResolveTask; +import org.openmetadata.schema.api.policies.CreatePolicy; +import org.openmetadata.schema.api.teams.CreateRole; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.api.tests.CreateTestCase; @@ -83,6 +88,9 @@ import org.openmetadata.schema.api.tests.CreateTestSuite; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.entity.policies.Policy; +import org.openmetadata.schema.entity.policies.accessControl.Rule; +import org.openmetadata.schema.entity.teams.Role; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.tests.DataQualityReport; @@ -108,6 +116,7 @@ import org.openmetadata.schema.type.DataQualityDimensions; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.TableData; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TaskStatus; @@ -116,6 +125,8 @@ import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.resources.feeds.FeedResourceTest; import org.openmetadata.service.resources.feeds.MessageParser; +import org.openmetadata.service.resources.policies.PolicyResourceTest; +import org.openmetadata.service.resources.teams.RoleResourceTest; import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; import org.openmetadata.service.search.SearchAggregation; @@ -141,6 +152,25 @@ public class TestCaseResourceTest extends EntityResourceTest", "non-existent"); } + @BeforeAll + public void setupPoliciesRolesUsers() throws Exception { + // ------------------------------------------------------------------------------------------- + // 1) Create actual Rules to be placed in the Policies + // Each rule grants certain operations on specific Entity types. + // ------------------------------------------------------------------------------------------- + Rule tableEditTestsRule = + new Rule() + .withName("AllowTableEditTests") + .withDescription("Allow EDIT_TESTS on TABLE entities") + .withEffect(Rule.Effect.ALLOW) + .withOperations(List.of(MetadataOperation.EDIT_TESTS)) + .withResources(List.of(TABLE)); + + Rule testCaseCreateRule = + new Rule() + .withName("AllowTestCaseCreate") + .withDescription("Allow CREATE on TEST_CASE entities") + .withEffect(Rule.Effect.ALLOW) + .withOperations(List.of(MetadataOperation.CREATE)) + .withResources(List.of(TEST_CASE)); + + Rule testCaseUpdateRule = + new Rule() + .withName("AllowTestCaseUpdate") + .withDescription("Allow UPDATE on TEST_CASE entities") + .withEffect(Rule.Effect.ALLOW) + .withOperations(List.of(MetadataOperation.EDIT_ALL)) + .withResources(List.of(TEST_CASE)); + + // An empty or do-nothing rule for NoPermissions + Rule noRelevantRule = + new Rule() + .withName("NoRelevantRule") + .withEffect(Rule.Effect.DENY) + .withOperations(List.of()) + .withResources(List.of(TEST_CASE)); + + Rule tableOwnerEditTestsRule = + new Rule() + .withName("tableOwnerEditTestsRule") + .withDescription("Allow EDIT_TESTS on TABLE if user isOwner()") + .withEffect(Rule.Effect.ALLOW) + .withOperations(List.of(MetadataOperation.EDIT_TESTS)) + .withResources(List.of(Entity.TABLE)) + .withCondition("isOwner()"); + + Rule testCaseAllOpsRule = + new Rule() + .withName("testCaseAllOpsRule") + .withDescription("Allow CREATE, UPDATE, DELETE on TEST_CASE entities") + .withEffect(Rule.Effect.ALLOW) + .withOperations( + List.of( + MetadataOperation.CREATE, MetadataOperation.EDIT_ALL, MetadataOperation.DELETE)) + .withResources(List.of(Entity.TEST_CASE)); + + PolicyResourceTest policyResourceTest = new PolicyResourceTest(); + POLICY_TABLE_EDIT_TESTS = + policyResourceTest.createEntity( + new CreatePolicy() + .withName("Policy_TableEditTests") + .withDescription("Policy that allows TABLE:EDIT_TESTS") + .withRules(List.of(tableEditTestsRule)), + ADMIN_AUTH_HEADERS); + + POLICY_TEST_CASE_CREATE = + policyResourceTest.createEntity( + new CreatePolicy() + .withName("Policy_TestCaseCreate") + .withDescription("Policy that allows TEST_CASE:CREATE") + .withRules(List.of(testCaseCreateRule)), + ADMIN_AUTH_HEADERS); + + POLICY_TEST_CASE_UPDATE = + policyResourceTest.createEntity( + new CreatePolicy() + .withName("Policy_TestCaseUpdate") + .withDescription("Policy that allows TEST_CASE:UPDATE") + .withRules(List.of(testCaseUpdateRule)), + ADMIN_AUTH_HEADERS); + + POLICY_NO_PERMS = + policyResourceTest.createEntity( + new CreatePolicy() + .withName("Policy_NoPerms") + .withDescription("Policy that grants no relevant perms") + .withRules(List.of(noRelevantRule)), + ADMIN_AUTH_HEADERS); + + POLICY_TABLE_OWNER_EDIT_TESTS = + policyResourceTest.createEntity( + new CreatePolicy() + .withName("Policy_TableOwnerEditTests") + .withRules(List.of(tableOwnerEditTestsRule)), + ADMIN_AUTH_HEADERS); + + Policy POLICY_TEST_CASE_ALL_OPS = + policyResourceTest.createEntity( + new CreatePolicy() + .withName("Policy_TestCaseAllOps") + .withRules(List.of(testCaseAllOpsRule)), + ADMIN_AUTH_HEADERS); + + RoleResourceTest roleResourceTest = new RoleResourceTest(); + ROLE_TABLE_EDIT_TESTS = + roleResourceTest.createEntity( + new CreateRole() + .withName("Role_TableEditTests") + .withDescription("Role that references POLICY_TABLE_EDIT_TESTS") + .withPolicies(List.of(POLICY_TABLE_EDIT_TESTS.getFullyQualifiedName())), + ADMIN_AUTH_HEADERS); + + ROLE_TEST_CASE_CREATE = + roleResourceTest.createEntity( + new CreateRole() + .withName("Role_TestCaseCreate") + .withPolicies(List.of(POLICY_TEST_CASE_CREATE.getFullyQualifiedName())), + ADMIN_AUTH_HEADERS); + + ROLE_TEST_CASE_UPDATE = + roleResourceTest.createEntity( + new CreateRole() + .withName("Role_TestCaseUpdate") + .withPolicies(List.of(POLICY_TEST_CASE_UPDATE.getFullyQualifiedName())), + ADMIN_AUTH_HEADERS); + + ROLE_NO_PERMS = + roleResourceTest.createEntity( + new CreateRole() + .withName("Role_NoPermissions") + .withPolicies(List.of(POLICY_NO_PERMS.getFullyQualifiedName())), + ADMIN_AUTH_HEADERS); + + Role ROLE_TABLE_OWNER_EDIT_TESTS = + roleResourceTest.createEntity( + new CreateRole() + .withName("Role_TableOwnerEditTests") + .withPolicies(List.of(POLICY_TABLE_OWNER_EDIT_TESTS.getFullyQualifiedName())), + ADMIN_AUTH_HEADERS); + + Role ROLE_TEST_CASE_ALL_OPS = + roleResourceTest.createEntity( + new CreateRole() + .withName("Role_TestCaseAllOps") + .withPolicies(List.of(POLICY_TEST_CASE_ALL_OPS.getFullyQualifiedName())), + ADMIN_AUTH_HEADERS); + + UserResourceTest userResourceTest = new UserResourceTest(); + USER_TABLE_EDIT_TESTS = + userResourceTest.createEntity( + new CreateUser() + .withName("user-table-edit-tests") + .withEmail("user-table-edit-tests@open-metadata.org") + .withRoles(List.of(ROLE_TABLE_EDIT_TESTS.getId())), + ADMIN_AUTH_HEADERS); + + USER_TEST_CASE_CREATE = + userResourceTest.createEntity( + new CreateUser() + .withName("user-test-case-create") + .withEmail("user-test-case-create@open-metadata.org") + .withRoles(List.of(ROLE_TEST_CASE_CREATE.getId())), + ADMIN_AUTH_HEADERS); + + USER_TEST_CASE_UPDATE = + userResourceTest.createEntity( + new CreateUser() + .withName("user-test-case-update") + .withEmail("user-test-case-update@open-metadata.org") + .withRoles(List.of(ROLE_TEST_CASE_UPDATE.getId())), + ADMIN_AUTH_HEADERS); + + USER_NO_PERMISSIONS = + userResourceTest.createEntity( + new CreateUser() + .withName("user-no-perms") + .withEmail("user-no-perms@open-metadata.org") + .withRoles(List.of(ROLE_NO_PERMS.getId())), + ADMIN_AUTH_HEADERS); + + USER_TABLE_OWNER = + userResourceTest.createEntity( + new CreateUser() + .withName("user-table-owner") + .withEmail("user-table-owner@open-metadata.org") + .withRoles(List.of(ROLE_TABLE_OWNER_EDIT_TESTS.getId())), + ADMIN_AUTH_HEADERS); + + CREATE_ALL_OPS_USER = + userResourceTest.createEntity( + new CreateUser() + .withName("user-test-case-all-ops") + .withEmail("user-test-case-all-ops@open-metadata.org") + .withRoles(List.of(ROLE_TEST_CASE_ALL_OPS.getId())), + ADMIN_AUTH_HEADERS); + } + @Test void test_getEntityName(TestInfo test) { assertTrue(getEntityName(test).contains(supportedNameCharacters)); @@ -984,7 +1212,7 @@ public void post_entity_as_non_admin_401(TestInfo test) { assertResponse( () -> createEntity(createRequest(test), TEST_AUTH_HEADERS), FORBIDDEN, - permissionNotAllowed(TEST_USER_NAME, List.of(EDIT_TESTS))); + "User does not have ANY of the required permissions."); } @Test @@ -1029,7 +1257,7 @@ public void delete_entity_as_non_admin_401(TestInfo test) throws HttpResponseExc assertResponse( () -> deleteAndCheckEntity(entity, TEST_AUTH_HEADERS), FORBIDDEN, - permissionNotAllowed(TEST_USER_NAME, List.of(EDIT_TESTS))); + permissionNotAllowed(TEST_USER_NAME, List.of(DELETE))); } @Test @@ -2500,13 +2728,15 @@ void test_resultSummaryCascadeToAllSuites(TestInfo test) throws IOException, Par } deleteEntity(testCase1.getId(), true, true, ADMIN_AUTH_HEADERS); // hard delete - resultList = - getTestCaseResults( - testCase1.getFullyQualifiedName(), - TestUtils.dateToTimestamp("2021-10-01"), - TestUtils.dateToTimestamp("2021-10-30"), - ADMIN_AUTH_HEADERS); - assertEquals(resultList.getData().size(), 0); // hard deletion should delete existing results + assertResponse( + () -> + getTestCaseResults( + testCase1.getFullyQualifiedName(), + TestUtils.dateToTimestamp("2021-10-01"), + TestUtils.dateToTimestamp("2021-10-30"), + ADMIN_AUTH_HEADERS), + NOT_FOUND, + "testCase instance for " + testCase1.getFullyQualifiedName() + " not found"); if (supportsSearchIndex) { getAndValidateTestSummary(testCase.getTestSuite().getId().toString()); @@ -2559,6 +2789,163 @@ void test_createMany(TestInfo test) throws HttpResponseException { } } + @Test + void test_createTestCaseWithOrPermissions() throws Exception { + CreateTestCase createReq = + new CreateTestCase() + .withName("TestCase_OrPerms") + .withDescription("Simple test case") + .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()) + .withEntityLink(TABLE_LINK) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()); + + // 1) user-table-edit-tests -> Allowed + TestCase testCase1 = createEntity(createReq, authHeaders("user-table-edit-tests")); + assertNotNull(testCase1); + + // 2) user-test-case-create -> Allowed + CreateTestCase createReq2 = createReq.withName("TestCase_OrPerms_2"); + TestCase testCase2 = createEntity(createReq2, authHeaders("user-test-case-create")); + assertNotNull(testCase2); + + // 3) user-no-perms -> Forbidden + CreateTestCase createReq3 = createReq.withName("TestCaseNoPermFail"); + TestUtils.assertResponse( + () -> createEntity(createReq3, authHeaders("user-no-perms")), + FORBIDDEN, + "User does not have ANY of the required permissions."); + } + + @Test + void test_updateTestCaseOrPermissions() throws Exception { + + CreateTestCase createReq = + new CreateTestCase() + .withName("MyTestCaseUpdate") + .withDescription("Initial desc") + .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()) + .withEntityLink(TABLE_LINK) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()); + + TestCase testCase = createEntity(createReq, ADMIN_AUTH_HEADERS); + + CreateTestCase updateReq = createReq.withDescription("Updated desc"); + + TestCase updatedByTable = updateEntity(updateReq, OK, authHeaders("user-table-edit-tests")); + assertEquals("Updated desc", updatedByTable.getDescription()); + + CreateTestCase updateReq2 = updateReq.withDescription("Updated again by testCaseUpdate user"); + TestCase updatedByTestCase = updateEntity(updateReq2, OK, authHeaders("user-test-case-update")); + assertEquals("Updated again by testCaseUpdate user", updatedByTestCase.getDescription()); + + CreateTestCase updateReq3 = updateReq.withDescription("Should fail"); + TestUtils.assertResponse( + () -> updateEntity(updateReq3, OK, authHeaders("user-no-perms")), + FORBIDDEN, + "User does not have ANY of the required permissions."); + } + + @Test + void test_testCaseCrudByTableOwner_withTemporaryOwnership() throws Exception { + String tableOwnerUsername = "user-table-owner"; + TableResourceTest tableResourceTest = new TableResourceTest(); + Table tableEntity = + tableResourceTest.getEntity(TEST_TABLE1.getId(), "owners", ADMIN_AUTH_HEADERS); + List originalOwners = + tableEntity.getOwners() == null ? List.of() : tableEntity.getOwners(); + + String originalTableJson = JsonUtils.pojoToJson(tableEntity); + try { + tableEntity.setOwners(List.of(USER_TABLE_OWNER.getEntityReference())); + tableResourceTest.patchEntity( + tableEntity.getId(), originalTableJson, tableEntity, ADMIN_AUTH_HEADERS); + + CreateTestCase createReq = + new CreateTestCase() + .withName("TempOwnerTestCase") + .withDescription("TestCase by temporarily assigned table owner") + .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()) + .withEntityLink(TABLE_LINK) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()); + + TestCase created = createEntity(createReq, authHeaders(tableOwnerUsername)); + assertNotNull(created); + + CreateTestCase updateReq = + createReq.withDescription("Updated description by temporary owner"); + TestCase updated = updateEntity(updateReq, OK, authHeaders(tableOwnerUsername)); + assertEquals("Updated description by temporary owner", updated.getDescription()); + + deleteAndCheckEntity(updated, authHeaders(tableOwnerUsername)); + + } finally { + String modifiedTableJson = JsonUtils.pojoToJson(tableEntity); + tableEntity.setOwners(originalOwners); + tableResourceTest.patchEntity( + tableEntity.getId(), modifiedTableJson, tableEntity, ADMIN_AUTH_HEADERS); + } + } + + @Test + void test_tableOwnerCannotCrudOtherTables() { + // If the user is an owner of "TABLE_LINK" but tries to create + // a testCase referencing a different table they do NOT own, + // then the condition isOwner() => false => no permissions => fail. + CreateTestCase createReq = + new CreateTestCase() + .withName("OwnerFailOtherTableCase") + .withDescription("Fail if referencing a table not owned by user-table-owner") + .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()) + .withEntityLink(TABLE_LINK_2) // A table the user does not own + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()); + + TestUtils.assertResponse( + () -> createEntity(createReq, authHeaders("user-table-owner")), + FORBIDDEN, + "User does not have ANY of the required permissions."); + } + + @Test + void test_testCaseCrudByUserWithDirectTestCasePermissions() throws Exception { + CreateTestCase createReq = + new CreateTestCase() + .withName("AllOpsTestCase") + .withDescription("TestCase with direct testCase perms") + .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()) + .withEntityLink(TABLE_LINK) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()); + + TestCase testCase = createEntity(createReq, authHeaders("user-test-case-all-ops")); + assertNotNull(testCase); + + CreateTestCase updateReq = createReq.withDescription("Updated by direct testCase perms user"); + TestCase updated = updateEntity(updateReq, OK, authHeaders("user-test-case-all-ops")); + assertEquals("Updated by direct testCase perms user", updated.getDescription()); + + deleteAndCheckEntity(updated, authHeaders("user-test-case-all-ops")); + } + + @Test + void test_testCaseCrudByUserWithDirectTestCasePermissions_negative() throws Exception { + // A user who does NOT have CREATE, for example, or is missing one of them -> fails + + CreateTestCase createReq = + new CreateTestCase() + .withName("NoDeleteUserCase") + .withDescription("Will fail on deletion") + .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()) + .withEntityLink(TABLE_LINK) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()); + + TestCase testCase = createEntity(createReq, authHeaders("user-test-case-create")); + assertNotNull(testCase); + + TestUtils.assertResponse( + () -> deleteAndCheckEntity(testCase, authHeaders("user-test-case-create")), + FORBIDDEN, + permissionNotAllowed("user-test-case-create", List.of(DELETE))); + } + // Test utils methods public ResultList listTestCaseResultsFromSearch( diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index 3289b67bf21f..ec167504406d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.services.ingestionPipelines.CreateIngestionPipeline; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.api.tests.CreateLogicalTestCases; @@ -42,8 +43,11 @@ import org.openmetadata.schema.api.tests.CreateTestCaseResult; import org.openmetadata.schema.api.tests.CreateTestSuite; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.metadataIngestion.SourceConfig; +import org.openmetadata.schema.metadataIngestion.TestSuitePipeline; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseStatus; @@ -55,6 +59,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.services.ingestionpipelines.IngestionPipelineResourceTest; import org.openmetadata.service.resources.teams.TeamResourceTest; import org.openmetadata.service.resources.teams.UserResourceTest; import org.openmetadata.service.search.models.IndexMapping; @@ -182,6 +187,15 @@ void put_testCaseResults_200() throws IOException, ParseException { assertEquals(true, deletedTestSuite.getDeleted()); } + @Test + void create_basicTestSuiteWithoutRef(TestInfo test) { + CreateTestSuite createTestSuite = createRequest(test); + assertResponse( + () -> createBasicEmptySuite(createTestSuite, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Cannot create a basic test suite without the BasicEntityReference field informed."); + } + @Test void list_testSuitesIncludeEmpty_200(TestInfo test) throws IOException { List testSuites = new ArrayList<>(); @@ -720,6 +734,80 @@ void delete_LogicalTestSuite_200(TestInfo test) throws IOException { assertEquals(5, actualExecutableTestSuite.getTests().size()); } + @Test + void delete_logicalSuiteWithPipeline(TestInfo test) throws IOException { + TestCaseResourceTest testCaseResourceTest = new TestCaseResourceTest(); + TableResourceTest tableResourceTest = new TableResourceTest(); + CreateTable tableReq = + tableResourceTest + .createRequest(test) + .withColumns( + List.of( + new Column() + .withName(C1) + .withDisplayName("c1") + .withDataType(ColumnDataType.VARCHAR) + .withDataLength(10))); + Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); + CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); + TestSuite executableTestSuite = + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + List testCases1 = new ArrayList<>(); + + // We'll create tests cases for testSuite1 + for (int i = 0; i < 5; i++) { + CreateTestCase createTestCase = + testCaseResourceTest + .createRequest(String.format("test_testSuite_2_%s_", test.getDisplayName()) + i) + .withTestSuite(executableTestSuite.getFullyQualifiedName()); + TestCase testCase = + testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); + testCases1.add(testCase.getEntityReference()); + } + + // We'll create a logical test suite and associate the test cases to it + CreateTestSuite createTestSuite = createRequest(test); + TestSuite testSuite = createEntity(createTestSuite, ADMIN_AUTH_HEADERS); + addTestCasesToLogicalTestSuite( + testSuite, testCases1.stream().map(EntityReference::getId).collect(Collectors.toList())); + TestSuite logicalTestSuite = getEntity(testSuite.getId(), "*", ADMIN_AUTH_HEADERS); + + // Add ingestion pipeline to the database service + IngestionPipelineResourceTest ingestionPipelineResourceTest = + new IngestionPipelineResourceTest(); + CreateIngestionPipeline createIngestionPipeline = + ingestionPipelineResourceTest + .createRequest(test) + .withService(logicalTestSuite.getEntityReference()); + + TestSuitePipeline testSuitePipeline = new TestSuitePipeline(); + + SourceConfig sourceConfig = new SourceConfig().withConfig(testSuitePipeline); + createIngestionPipeline.withSourceConfig(sourceConfig); + IngestionPipeline ingestionPipeline = + ingestionPipelineResourceTest.createEntity(createIngestionPipeline, ADMIN_AUTH_HEADERS); + + // We can GET the Ingestion Pipeline now + IngestionPipeline actualIngestionPipeline = + ingestionPipelineResourceTest.getEntity(ingestionPipeline.getId(), ADMIN_AUTH_HEADERS); + assertNotNull(actualIngestionPipeline); + + // After deleting the test suite, we can't GET the Ingestion Pipeline + deleteEntity(logicalTestSuite.getId(), true, true, ADMIN_AUTH_HEADERS); + + assertResponse( + () -> + ingestionPipelineResourceTest.getEntity(ingestionPipeline.getId(), ADMIN_AUTH_HEADERS), + NOT_FOUND, + String.format( + "ingestionPipeline instance for %s not found", actualIngestionPipeline.getId())); + + // Test Cases are still there + TestCase testCaseInLogical = + testCaseResourceTest.getEntity(testCases1.get(0).getId(), "*", ADMIN_AUTH_HEADERS); + assertNotNull(testCaseInLogical); + } + @Test void get_listTestSuiteFromSearchWithPagination(TestInfo testInfo) throws IOException { if (supportsSearchIndex) { @@ -834,6 +922,12 @@ public TestSuite createBasicTestSuite( return TestUtils.post(target, createTestSuite, TestSuite.class, authHeaders); } + public TestSuite createBasicEmptySuite( + CreateTestSuite createTestSuite, Map authHeaders) throws IOException { + WebTarget target = getResource("dataQuality/testSuites/basic"); + return TestUtils.post(target, createTestSuite, TestSuite.class, authHeaders); + } + public void addTestCasesToLogicalTestSuite(TestSuite testSuite, List testCaseIds) throws IOException { WebTarget target = getResource("dataQuality/testCases/logicalTestCases"); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index 91ab9865baef..f78c47cae195 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -35,6 +35,9 @@ import static org.openmetadata.csv.EntityCsvTest.getFailedRecord; import static org.openmetadata.schema.type.ProviderType.SYSTEM; import static org.openmetadata.schema.type.TaskType.RequestDescription; +import static org.openmetadata.service.governance.workflows.Workflow.GLOBAL_NAMESPACE; +import static org.openmetadata.service.governance.workflows.Workflow.RELATED_ENTITY_VARIABLE; +import static org.openmetadata.service.governance.workflows.WorkflowVariableHandler.getNamespacedVariableName; import static org.openmetadata.service.security.SecurityUtil.authHeaders; import static org.openmetadata.service.util.EntityUtil.fieldAdded; import static org.openmetadata.service.util.EntityUtil.fieldDeleted; @@ -875,7 +878,12 @@ void testGlossaryImportExport() throws IOException { .withConfig( Map.of( "values", - List.of("single1", "single2", "single3", "single4", "\"single5\""), + List.of( + "\"single val with quotes\"", + "single1", + "single2", + "single3", + "single4"), "multiSelect", false))), new CustomProperty() @@ -916,7 +924,7 @@ void testGlossaryImportExport() throws IOException { ",g1,dsp1,\"dsc1,1\",h1;h2;h3,g1.g1t1;g2.g2t1,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,dsc3,h1;h3;h3,g1.g1t1;g2.g2t1,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", + ",g2,dsp2,dsc3,h1;h3;h3,g1.g1t1;g2.g2t1,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val1|val2|val3|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, "Approved"), String.format( "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,g1.g1t1;g2.g2t1,,,user:%s,team:%s,%s,", @@ -929,7 +937,7 @@ void testGlossaryImportExport() throws IOException { ",g1,dsp1,new-dsc1,h1;h2;h3,g1.g1t1;importExportTest.g2;g2.g2t1,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,new-dsc3,h1;h3;h3,importExportTest.g1;g1.g1t1;g2.g2t1,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", + ",g2,dsp2,new-dsc3,h1;h3;h3,importExportTest.g1;g1.g1t1;g2.g2t1,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val1|val2|val3|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"", @@ -1211,6 +1219,8 @@ public static void waitForTaskToBeCreated(String fullyQualifiedName, long timeou () -> WorkflowHandler.getInstance() .isActivityWithVariableExecuting( - "ApproveGlossaryTerm.approvalTask", "relatedEntity", entityLink)); + "ApproveGlossaryTerm.approvalTask", + getNamespacedVariableName(GLOBAL_NAMESPACE, RELATED_ENTITY_VARIABLE), + entityLink)); } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index 9e4a3cc25d05..f3f1d37efee2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -277,6 +277,13 @@ public static void assertResponse( assertEquals(expectedReason, exception.getReasonPhrase()); } + public static void assertResponse( + Executable executable, Response.Status expectedStatus, List expectedReasons) { + HttpResponseException exception = assertThrows(HttpResponseException.class, executable); + assertEquals(expectedStatus.getStatusCode(), exception.getStatusCode()); + assertTrue(expectedReasons.contains(exception.getReasonPhrase())); + } + public static void assertResponseContains( Executable executable, Response.Status expectedStatus, String expectedReason) { HttpResponseException exception = assertThrows(HttpResponseException.class, executable); diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java index 304f5eec867b..cb732360ff38 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowNodeDefinitionInterface.java @@ -7,11 +7,13 @@ import java.util.Map; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CheckEntityAttributesTaskDefinition; -import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition; -import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.PythonWorkflowAutomationTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CreateIngestionPipelineTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunAppTaskDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunIngestionPipelineTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetEntityCertificationTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.SetGlossaryTermStatusTaskDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.endEvent.EndEventDefinition; +import org.openmetadata.schema.governance.workflows.elements.nodes.gateway.ParallelGatewayDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.startEvent.StartEventDefinition; import org.openmetadata.schema.governance.workflows.elements.nodes.userTask.UserApprovalTaskDefinition; @@ -30,9 +32,13 @@ name = "setGlossaryTermStatusTask"), @JsonSubTypes.Type(value = UserApprovalTaskDefinition.class, name = "userApprovalTask"), @JsonSubTypes.Type( - value = PythonWorkflowAutomationTaskDefinition.class, - name = "pythonWorkflowAutomationTask"), - @JsonSubTypes.Type(value = JsonLogicTaskDefinition.class, name = "jsonLogicTask"), + value = CreateIngestionPipelineTaskDefinition.class, + name = "createIngestionPipelineTask"), + @JsonSubTypes.Type( + value = RunIngestionPipelineTaskDefinition.class, + name = "runIngestionPipelineTask"), + @JsonSubTypes.Type(value = RunAppTaskDefinition.class, name = "runAppTask"), + @JsonSubTypes.Type(value = ParallelGatewayDefinition.class, name = "parallelGateway"), }) public interface WorkflowNodeDefinitionInterface { String getType(); @@ -50,16 +56,20 @@ default Object getConfig() { } ; - default List getInputs() { + default List getInput() { return null; } ; - default List getOutputs() { + default List getOutput() { return null; } ; + default Object getInputNamespaceMap() { + return null; + } + void setType(String type); void setSubType(String subType); @@ -74,11 +84,15 @@ default void setConfig(Map config) { /* no-op implementation to be overridden */ } - default void setInputs(List inputs) { + default void setInput(List inputs) { + /* no-op implementation to be overridden */ + } + + default void setOutput(List outputs) { /* no-op implementation to be overridden */ } - default void setOutputs(List outputs) { + default void setInputNamespaceMap(Object inputNamespaceMap) { /* no-op implementation to be overridden */ } diff --git a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java index ebaa8688ef46..cee785461407 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/schema/governance/workflows/elements/WorkflowTriggerInterface.java @@ -2,14 +2,13 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.util.Set; import org.openmetadata.schema.governance.workflows.elements.nodes.trigger.PeriodicBatchEntityTriggerDefinition; -import org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition; import org.openmetadata.schema.governance.workflows.elements.triggers.EventBasedEntityTriggerDefinition; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") @JsonSubTypes({ @JsonSubTypes.Type(value = EventBasedEntityTriggerDefinition.class, name = "eventBasedEntity"), - @JsonSubTypes.Type(value = CustomSignalTriggerDefinition.class, name = "customSignal"), @JsonSubTypes.Type( value = PeriodicBatchEntityTriggerDefinition.class, name = "periodicBatchEntity"), @@ -20,4 +19,6 @@ public interface WorkflowTriggerInterface { String getType(); Object getConfig(); + + Set getOutput(); } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/ingestionPipelines/createIngestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/api/services/ingestionPipelines/createIngestionPipeline.json index ffea6991bcd7..77b0e8f4c8ce 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/services/ingestionPipelines/createIngestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/services/ingestionPipelines/createIngestionPipeline.json @@ -44,6 +44,10 @@ "domain" : { "description": "Fully qualified name of the domain the Table belongs to.", "type": "string" + }, + "ingestionAgent" : { + "description": "The ingestion agent responsible for executing the ingestion pipeline.", + "$ref": "../../../type/entityReference.json" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json b/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json index 72280c724b1c..765b2b1ebf91 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/workflowSettings.json @@ -28,6 +28,11 @@ "type": "integer", "default": 20, "description": "The amount of Tasks that the Workflow Executor is able to pick up each time it looks for more." + }, + "jobLockTimeInMillis": { + "type": "integer", + "default": 300000, + "description": "The amount of time a Job gets locked before being retried." } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json index 122ee052ef92..6dde1cf433e1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/appRunRecord.json @@ -34,7 +34,8 @@ "active", "activeError", "stopped", - "success" + "success", + "pending" ] }, "runType": { @@ -63,6 +64,10 @@ }, "scheduleInfo": { "$ref": "./app.json#/definitions/appSchedule" + }, + "config": { + "descripton": "The configuration used for this application run. It's type will be based on the application type. Old runs might not be compatible with schema of app configuration.", + "$ref": "../../type/basic.json#/definitions/map" } }, "additionalProperties": false diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json index 06abeffb7805..9dcd2a4ecf99 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeDescriptionAction.json @@ -21,12 +21,18 @@ }, "applyToChildren": { "title": "Apply to Children", - "description": "Remove descriptions from all children of the selected assets. E.g., columns, tasks, topic fields,...", + "description": "Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic fields,...", "type": "array", "items": { "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "default": null + }, + "applyToAll": { + "title": "Apply to All", + "description": "Remove descriptions from all the children and parent of the selected assets.", + "type": "boolean", + "default": null } }, "required": ["type"], diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json index a255e0f9551c..12ebeec4a4aa 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/external/automator/removeTagsAction.json @@ -10,6 +10,15 @@ "type": "string", "enum": ["RemoveTagsAction"], "default": "RemoveTagsAction" + }, + "labelType": { + "description" : "Remove tags by its label type", + "type": "string", + "enum": [ + "Manual", + "Propagated", + "Automated" + ] } }, "properties": { @@ -26,16 +35,30 @@ "$ref": "../../../../../type/tagLabel.json" } }, + "labels": { + "description": "Remove tags by its label type", + "type": "array", + "items": { + "$ref": "#/definitions/labelType" + }, + "default": null + }, "applyToChildren": { "title": "Apply to Children", - "description": "Remove tags from all the children of the selected assets. E.g., columns, tasks, topic fields,...", + "description": "Remove tags from the children of the selected assets. E.g., columns, tasks, topic fields,...", "type": "array", "items": { "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "default": null + }, + "applyToAll": { + "title": "Apply to All", + "description": "Remove tags from all the children and parent of the selected assets.", + "type": "boolean", + "default": null } }, - "required": ["type", "tags"], + "required": ["type"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 4ad8fb950ed4..d7e26c644327 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -193,7 +193,8 @@ "PRIMARY_KEY", "FOREIGN_KEY", "SORT_KEY", - "DIST_KEY" + "DIST_KEY", + "CLUSTER_KEY" ] }, "columns": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json index cb8346c5b840..22756827f76d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json @@ -64,6 +64,12 @@ "type": "integer", "default": 120 }, + "queryHistoryTable":{ + "title": "Query History Table", + "description": "Table name to fetch the query history.", + "type": "string", + "default": "system.query.history" + }, "connectionOptions": { "title": "Connection Options", "$ref": "../connectionBasicType.json#/definitions/connectionOptions" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json index 5e757a529a9b..0a6b32bdbb02 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json @@ -68,6 +68,12 @@ "description": "Session query tag used to monitor usage on snowflake. To use a query tag snowflake user should have enough privileges to alter the session.", "type": "string" }, + "accountUsageSchema":{ + "title": "Account Usage Schema Name", + "description": "Full name of the schema where the account usage data is stored.", + "type": "string", + "default": "SNOWFLAKE.ACCOUNT_USAGE" + }, "privateKey": { "title": "Private Key", "description": "Connection to Snowflake instance via Private Key", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionETL.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionETL.json new file mode 100644 index 000000000000..183cbb45013e --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionETL.json @@ -0,0 +1,43 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/matillion/matillionETL.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Matillion ETL Auth Config", + "description": "Matillion ETL Auth Config.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "MatillionETL" + ], + "default": "MatillionETL" + }, + "hostPort": { + "type": "string", + "title": "Host", + "description": "Matillion Host", + "default": "localhost" + }, + "username": { + "title": "Username", + "description": "Username to connect to the Matillion. This user should have privileges to read all the metadata in Matillion.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "Password to connect to the Matillion.", + "type": "string", + "format": "password" + }, + "sslConfig": { + "$ref": "../../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" + } + }, + "required": [ + "hostPort", + "username", + "password" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json index 07598d15cbd8..3b8517c20461 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json @@ -13,45 +13,6 @@ "Matillion" ], "default": "Matillion" - }, - "matillionETL": { - "description": "Matillion ETL Auth Config", - "type": "object", - "title": "Matillion ETL Auth Config", - "properties": { - "type": { - "type": "string", - "enum": [ - "MatillionETL" - ], - "default": "MatillionETL" - }, - "hostPort": { - "type": "string", - "title": "Host", - "description": "Matillion Host", - "default": "localhost" - }, - "username": { - "title": "Username", - "description": "Username to connect to the Matillion. This user should have privileges to read all the metadata in Matillion.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "Password to connect to the Matillion.", - "type": "string", - "format": "password" - }, - "sslConfig": { - "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" - } - }, - "required": [ - "hostPort", - "username", - "password" - ] } }, "properties": { @@ -66,7 +27,7 @@ "description": "Matillion Auth Configuration", "oneOf": [ { - "$ref": "#/definitions/matillionETL" + "$ref": "matillion/matillionETL.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/basicAuth.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/basicAuth.json new file mode 100644 index 000000000000..63512a45f828 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/basicAuth.json @@ -0,0 +1,28 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/nifi/basicAuth.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Nifi Basic Auth", + "description": "Configuration for connecting to Nifi Basic Auth.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.nifi.BasicAuth", + "type": "object", + "properties": { + "username": { + "title": "Username", + "description": "Nifi user to authenticate to the API.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "Nifi password to authenticate to the API.", + "type": "string", + "format": "password" + }, + "verifySSL": { + "title": "Verify SSL", + "description": "Boolean marking if we need to verify the SSL certs for Nifi. False by default.", + "type": "boolean", + "default": false + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json new file mode 100644 index 000000000000..a540eddcaa7b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json @@ -0,0 +1,26 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Nifi Client Certificate Auth", + "description": "Configuration for connecting to Nifi Client Certificate Auth.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.nifi.ClientCertificateAuth", + "type": "object", + "properties": { + "certificateAuthorityPath": { + "title": "Certificat Authority Path", + "description": "Path to the root CA certificate", + "type": "string" + }, + "clientCertificatePath": { + "title": "Client Certificat", + "description": "Path to the client certificate", + "type": "string" + }, + "clientkeyPath": { + "title": "Client Key", + "description": "Path to the client key", + "type": "string" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json index 843e36b8122b..ce90643ba3ff 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/nifiConnection.json @@ -15,6 +15,7 @@ "basicAuthentication": { "title": "Username/Password Authentication", "description": "username/password auth", + "javaType": "org.openmetadata.schema.services.connections.pipeline.NifiBasicAuth", "type":"object", "properties": { "username": { @@ -41,6 +42,7 @@ "title": "Client Certificate Authentication", "description": "client certificate auth", "type":"object", + "javaType": "org.openmetadata.schema.services.connections.pipeline.NifiClientAuth", "properties": { "certificateAuthorityPath":{ "title":"Certificat Authority Path", @@ -80,10 +82,10 @@ "description": "We support username/password or client certificate authentication", "oneOf": [ { - "$ref": "#/definitions/basicAuthentication" + "$ref": "nifi/basicAuth.json" }, { - "$ref": "#/definitions/clientCertificateAuthentication" + "$ref": "nifi/clientCertificateAuth.json" } ] }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json index 314c78b2e555..2bad77cd45e6 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/ingestionPipeline.json @@ -43,6 +43,10 @@ "status": { "description": "Ingestion Pipeline summary status. Informed at the end of the execution.", "$ref": "status.json#/definitions/ingestionStatus" + }, + "config": { + "description": "Pipeline configuration for this particular execution.", + "$ref": "../../../type/basic.json#/definitions/map" } }, "additionalProperties": false @@ -217,6 +221,10 @@ "applicationType": { "description": "Type of the application when pipelineType is 'application'.", "type": "string" + }, + "ingestionAgent" : { + "description": "The ingestion agent responsible for executing the ingestion pipeline.", + "$ref": "../../../type/entityReference.json" } }, "required": [ diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/edge.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/edge.json index e14b6e010704..7a572628788a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/edge.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/edge.json @@ -16,7 +16,7 @@ }, "condition": { "description": "Defines if the edge will follow a path depending on the source node result.", - "type": "boolean" + "type": "string" } }, "required": ["to", "from"], diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json index 8ce4b811a63c..71ef2f61527f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeSubType.json @@ -12,7 +12,9 @@ "setGlossaryTermStatusTask", "endEvent", "startEvent", - "pythonWorkflowAutomationTask", - "jsonLogicTask" + "createIngestionPipelineTask", + "runIngestionPipelineTask", + "runAppTask", + "parallelGateway" ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeType.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeType.json index 10fd2849aa2d..079bd5863644 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeType.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodeType.json @@ -9,6 +9,7 @@ "automatedTask", "userTask", "endEvent", - "startEvent" + "startEvent", + "gateway" ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.json index 7c41984e33d2..a881cc80fa87 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.json @@ -18,21 +18,27 @@ "default": "checkEntityAttributesTask" }, "name": { + "title": "Name", "description": "Name that identifies this Node.", "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "displayName": { + "title": "Display Name", "description": "Display Name that identifies this Node.", "type": "string" }, "description": { + "title": "Description", "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" }, "config": { + "title": "Node Configuration", "type": "object", "properties": { "rules": { + "title": "Rules to Check", + "description": "Define certain set of rules that you would like to check. If all the rules apply, this will be set as 'True' and will continue through the positive flow. Otherwise it will be set to 'False' and continue through the negative flow.", "type": "string", "outputType": "jsonlogic", "format": "queryBuilder" @@ -48,13 +54,24 @@ "minItems": 1, "maxItems": 1 }, - "output": { + "inputNamespaceMap": { + "type": "object", + "properties": { + "relatedEntity": { + "type": "string", + "default": "global" + } + }, + "additionalProperties": false, + "required": ["relatedEntity"] + }, + "branches": { "type": "array", "items": { "type": "string" }, - "default": ["result"], + "default": ["true", "false"], "additionalItems": false, - "minItems": 1, - "maxItems": 1 + "minItems": 2, + "maxItems": 2 } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/createIngestionPipelineTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/createIngestionPipelineTask.json new file mode 100644 index 000000000000..470b49743563 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/createIngestionPipelineTask.json @@ -0,0 +1,89 @@ +{ + "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/createIngestionPipelineTask.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CreateIngestionPipelineTask", + "description": "Creates an Ingestion Pipeline", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" + ], + "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.CreateIngestionPipelineTaskDefinition", + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "automatedTask" + }, + "subType": { + "type": "string", + "default": "createIngestionPipelineTask" + }, + "name": { + "title": "Name", + "description": "Name that identifies this Node.", + "$ref": "../../../../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "title": "Display Name", + "description": "Display Name that identifies this Node.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "Description of the Node.", + "$ref": "../../../../../type/basic.json#/definitions/markdown" + }, + "config": { + "type": "object", + "properties": { + "pipelineType": { + "title": "Pipeline Type", + "description": "Define which ingestion pipeline type should be created", + "$ref": "../../../../../entity/services/ingestionPipelines/ingestionPipeline.json#/definitions/pipelineType" + }, + "deploy": { + "title": "Deploy", + "description": "Set if the created pipeline should also be deployed", + "type": "boolean", + "default": true + } + }, + "additionalProperties": false, + "required": ["pipelineType", "deploy"] + }, + "input": { + "type": "array", + "items": { "type": "string" }, + "default": ["relatedEntity"], + "additionalItems": false, + "minItems": 1, + "maxItems": 1 + }, + "output": { + "type": "array", + "items": { "type": "string" }, + "default": ["ingestionPipelineId"], + "additionalItems": false, + "minItems": 1, + "maxItems": 1 + }, + "inputNamespaceMap": { + "type": "object", + "properties": { + "relatedEntity": { + "type": "string", + "default": "global" + } + }, + "additionalProperties": false, + "required": ["relatedEntity"] + }, + "branches": { + "type": "array", + "items": { "type": "string" }, + "default": ["success", "failure"], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json deleted file mode 100644 index 512e1f21e05e..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/jsonLogicTask.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "JsonLogicTaskDefinition", - "description": "Checks if an Entity attributes fit given rules.", - "javaInterfaces": [ - "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" - ], - "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.JsonLogicTaskDefinition", - "type": "object", - "properties": { - "type": { - "type": "string", - "default": "automatedTask" - }, - "subType": { - "type": "string", - "default": "jsonLogicTask" - }, - "name": { - "description": "Name that identifies this Node.", - "$ref": "../../../../../type/basic.json#/definitions/entityName" - }, - "displayName": { - "description": "Display Name that identifies this Node.", - "type": "string" - }, - "description": { - "description": "Description of the Node.", - "$ref": "../../../../../type/basic.json#/definitions/markdown" - }, - "config": { - "type": "object", - "properties": { - "rules": { - "type": "string", - "outputType": "jsonlogic", - "format": "queryBuilder" - } - }, - "additionalProperties": false - } - } -} diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.json new file mode 100644 index 000000000000..08744a2f6fb1 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.json @@ -0,0 +1,87 @@ +{ + "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RunAppTaskDefinition", + "description": "Runs an App based on its name.", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" + ], + "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunAppTaskDefinition", + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "automatedTask" + }, + "subType": { + "type": "string", + "default": "runAppTask" + }, + "name": { + "title": "Name", + "description": "Name that identifies this Node.", + "$ref": "../../../../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "title": "Display Name", + "description": "Display Name that identifies this Node.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "Description of the Node.", + "$ref": "../../../../../type/basic.json#/definitions/markdown" + }, + "config": { + "type": "object", + "properties": { + "appName": { + "title": "App Name", + "description": "Set which App should Run", + "type": "string" + }, + "waitForCompletion": { + "title": "Wait for Completion", + "description": "Set if this step should wait until the App finishes running", + "type": "boolean", + "default": true + }, + "timeoutSeconds": { + "title": "Timeout Seconds", + "description": "Set the amount of seconds to wait before defining the App has timed out.", + "type": "integer", + "default": 3600 + } + }, + "additionalProperties": false, + "required": ["appName", "waitForCompletion", "timeoutSeconds"] + }, + "input": { + "type": "array", + "items": { "type": "string" }, + "default": ["relatedEntity"], + "additionalItems": false, + "minItems": 1, + "maxItems": 1 + }, + "inputNamespaceMap": { + "type": "object", + "properties": { + "relatedEntity": { + "type": "string", + "default": "global" + } + }, + "additionalProperties": false, + "required": ["relatedEntity"] + }, + "branches": { + "type": "array", + "items": { "type": "string" }, + "default": ["true", "false"], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/runIngestionPipelineTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/runIngestionPipelineTask.json new file mode 100644 index 000000000000..167058537d0b --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/runIngestionPipelineTask.json @@ -0,0 +1,82 @@ +{ + "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/runIngestionPipelineTask.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RunIngestionPipelineTaskDefinition", + "description": "Runs an Ingestion Pipeline based on its ID.", + "javaInterfaces": [ + "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" + ], + "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.RunIngestionPipelineTaskDefinition", + "type": "object", + "properties": { + "type": { + "type": "string", + "default": "automatedTask" + }, + "subType": { + "type": "string", + "default": "runIngestionPipelineTask" + }, + "name": { + "title": "Name", + "description": "Name that identifies this Node.", + "$ref": "../../../../../type/basic.json#/definitions/entityName" + }, + "displayName": { + "title": "Display Name", + "description": "Display Name that identifies this Node.", + "type": "string" + }, + "description": { + "title": "Description", + "description": "Description of the Node.", + "$ref": "../../../../../type/basic.json#/definitions/markdown" + }, + "config": { + "type": "object", + "properties": { + "waitForCompletion": { + "title": "Wait for Completion", + "description": "Set if this step should wait until the Ingestion Pipeline finishes running", + "type": "boolean", + "default": true + }, + "timeoutSeconds": { + "title": "Timeout Seconds", + "description": "Set the amount of seconds to wait before defining the Ingestion Pipeline has timed out.", + "type": "integer", + "default": 3600 + } + }, + "additionalProperties": false, + "required": ["waitForCompletion", "timeoutSeconds"] + }, + "input": { + "type": "array", + "items": { "type": "string" }, + "default": ["ingestionPipelineId"], + "additionalItems": false, + "minItems": 1, + "maxItems": 1 + }, + "inputNamespaceMap": { + "type": "object", + "properties": { + "ingestionPipelineId": { + "type": "string", + "default": null + } + }, + "additionalProperties": false, + "required": ["ingestionPipelineId"] + }, + "branches": { + "type": "array", + "items": { "type": "string" }, + "default": ["true", "false"], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.json index 63f40165b2c8..c01d8763a316 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.json @@ -25,9 +25,12 @@ ] }, "certificationConfiguration": { + "title": "Node Configuration", "type": "object", "properties": { "certification": { + "title": "Certification", + "description": "Choose which Certification to apply to the Data Asset", "$ref": "#/definitions/certificationEnum" } }, @@ -45,14 +48,17 @@ "default": "setEntityCertificationTask" }, "name": { + "title": "Name", "description": "Name that identifies this Node.", "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "displayName": { + "title": "Display Name", "description": "Display Name that identifies this Node.", "type": "string" }, "description": { + "title": "Description", "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" }, @@ -62,10 +68,25 @@ "input": { "type": "array", "items": { "type": "string" }, - "default": ["relatedEntity"], + "default": ["relatedEntity", "updatedBy"], "additionalItems": false, - "minItems": 1, - "maxItems": 1 + "minItems": 2, + "maxItems": 2 + }, + "inputNamespaceMap": { + "type": "object", + "properties": { + "relatedEntity": { + "type": "string", + "default": "global" + }, + "updatedBy": { + "type": "string", + "default": null + } + }, + "additionalProperties": false, + "required": ["relatedEntity"] } } } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json index 16d4d4dff26c..3b43c13d5879 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.json @@ -18,34 +18,55 @@ "default": "setGlossaryTermStatusTask" }, "name": { + "title": "Name", "description": "Name that identifies this Node.", "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "displayName": { + "title": "Display Name", "description": "Display Name that identifies this Node.", "type": "string" }, "description": { + "title": "Description", "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" }, "config": { + "title": "Node Configuration", "type": "object", "properties": { "glossaryTermStatus": { + "title": "Glossary Term Status", + "description": "Choose which Status to apply to the Glossary Term", "$ref": "../../../../../entity/data/glossaryTerm.json#/definitions/status" } }, - "required": [], + "required": ["glossaryTermStatus"], "additionalProperties": false }, "input": { "type": "array", "items": { "type": "string" }, - "default": ["relatedEntity"], + "default": ["relatedEntity", "updatedBy"], "additionalItems": false, - "minItems": 1, - "maxItems": 1 + "minItems": 2, + "maxItems": 2 + }, + "inputNamespaceMap": { + "type": "object", + "properties": { + "relatedEntity": { + "type": "string", + "default": "global" + }, + "updatedBy": { + "type": "string", + "default": null + } + }, + "additionalProperties": false, + "required": ["relatedEntity"] } } } \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/endEvent/endEvent.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/endEvent/endEvent.json index 88a63306a536..14fdbea15d70 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/endEvent/endEvent.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/endEvent/endEvent.json @@ -18,14 +18,17 @@ "default": "endEvent" }, "name": { + "title": "Name", "description": "Name that identifies this Node.", "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "displayName": { + "title": "Display Name", "description": "Display Name that identifies this Node.", "type": "string" }, "description": { + "title": "Description", "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/gateway/parallelGateway.json similarity index 65% rename from openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json rename to openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/gateway/parallelGateway.json index 5812e6cbb59e..4461fb9e5352 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/gateway/parallelGateway.json @@ -1,21 +1,21 @@ { - "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/automatedTask/pythonWorkflowAutomationTask.json", + "$id": "https://open-metadata.org/schema/governance/workflows/elements/nodes/gateway/parallelGateway.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PythonWorkflowAutomationTaskDefinition", - "description": "Sets the GlossaryTerm Status to the configured value.", + "title": "ParallelGatewayDefinition", + "description": "Parallel Gateway. It should be used when we want to Parallelize or sync back Parallelized tasks.", "javaInterfaces": [ "org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface" ], - "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.automatedTask.PythonWorkflowAutomationTaskDefinition", + "javaType": "org.openmetadata.schema.governance.workflows.elements.nodes.gateway.ParallelGatewayDefinition", "type": "object", "properties": { "type": { "type": "string", - "default": "automatedTask" + "default": "gateway" }, "subType": { "type": "string", - "default": "pythonWorkflowAutomationTask" + "default": "parallelGateway" }, "name": { "description": "Name that identifies this Node.", @@ -28,10 +28,6 @@ "description": { "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" - }, - "config": { - "type": "object", - "existingJavaType": "java.util.Map" } } -} \ No newline at end of file +} diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/startEvent/startEvent.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/startEvent/startEvent.json index 09627f0d3a53..8c366186bd18 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/startEvent/startEvent.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/startEvent/startEvent.json @@ -18,14 +18,17 @@ "default": "startEvent" }, "name": { + "title": "Name", "description": "Name that identifies this Node.", "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "displayName": { + "title": "Display Name", "description": "Display Name that identifies this Node.", "type": "string" }, "description": { + "title": "Description", "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" } diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json index b175fba1d6e8..3d9d3f5b7df0 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.json @@ -18,21 +18,26 @@ "default": "userApprovalTask" }, "name": { + "title": "Name", "description": "Name that identifies this Node.", "$ref": "../../../../../type/basic.json#/definitions/entityName" }, "displayName": { + "title": "Display Name", "description": "Display Name that identifies this Node.", "type": "string" }, "description": { + "title": "Description", "description": "Description of the Node.", "$ref": "../../../../../type/basic.json#/definitions/markdown" }, "config": { + "title": "Node Configuration", "type": "object", "properties": { "assignees": { + "title": "Assignees", "description": "People/Teams assigned to the Task.", "type": "object", "properties": { @@ -55,10 +60,29 @@ "minItems": 1, "maxItems": 1 }, + "inputNamespaceMap": { + "type": "object", + "properties": { + "relatedEntity": { + "type": "string", + "default": "global" + } + }, + "additionalProperties": false, + "required": ["relatedEntity"] + }, "output": { "type": "array", "items": { "type": "string" }, - "default": ["result", "resolvedBy"], + "default": ["updatedBy"], + "additionalItems": false, + "minItems": 1, + "maxItems": 1 + }, + "branches": { + "type": "array", + "items": { "type": "string" }, + "default": ["true", "false"], "additionalItems": false, "minItems": 2, "maxItems": 2 diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json deleted file mode 100644 index a2effcf96e9f..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/customSignalTrigger.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/governance/workflows/elements/triggers/customSignalTrigger.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "CustomSignalTriggerDefinition", - "description": "Event Based Entity Trigger.", - "javaType": "org.openmetadata.schema.governance.workflows.elements.triggers.CustomSignalTriggerDefinition", - "javaInterfaces" : ["org.openmetadata.schema.governance.workflows.elements.WorkflowTriggerInterface"], - "type": "object", - "definitions": { - "config": { - "description": "Entity Event Trigger Configuration.", - "type": "object", - "properties": { - "signal": { - "description": "The signal to be listened to.", - "type": "string" - } - }, - "required": ["signal"], - "additionalProperties": false - } - }, - "properties": { - "type": { - "type": "string", - "default": "customSignal" - }, - "config": { - "$ref": "#/definitions/config" - } - }, - "additionalProperties": false -} diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json index 65bf7f099b18..7f0d61bd6376 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.json @@ -18,6 +18,7 @@ ] }, "config": { + "title": "Trigger Configuration", "description": "Entity Event Trigger Configuration.", "type": "object", "properties": { @@ -26,6 +27,8 @@ "type": "string" }, "events": { + "title": "Events", + "descriptions": "Select the events that should trigger this workflow", "type": "array", "items": { "$ref": "#/definitions/event" @@ -33,7 +36,8 @@ "uniqueItems": true }, "exclude": { - "description": "Exclude events that only modify given attributes.", + "title": "Exclude Fields", + "description": "Select fields that should not trigger the workflow if only them are modified.", "type": "array", "items": { "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json index e0817dfc8beb..58843c9a431d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json @@ -10,6 +10,7 @@ "type": "object", "definitions": { "config": { + "title": "Trigger Configuration", "description": "Entity Event Trigger Configuration.", "type": "object", "properties": { @@ -18,14 +19,17 @@ "description": "Defines the schedule of the Periodic Trigger." }, "entityType": { + "title": "Entity Type", "description": "Entity Type for which it should be triggered.", "type": "string" }, "filters": { - "description": "Search Filters to filter down the entities fetched.", + "title": "Filters", + "description": "Select the Search Filters to filter down the entities fetched.", "type": "string" }, "batchSize": { + "title": "Batch Size", "description": "Number of Entities to process at once.", "type": "integer", "default": 500 diff --git a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json index 4b3ae4cef8ac..11ac55dbc4b1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json +++ b/openmetadata-spec/src/main/resources/json/schema/governance/workflows/workflowDefinition.json @@ -12,7 +12,6 @@ "javaType": "org.openmetadata.schema.governance.workflows.TriggerType", "enum": [ "eventBasedEntity", - "customSignal", "periodicBatchEntity" ] } diff --git a/openmetadata-ui/src/main/resources/ui/.husky/pre-commit b/openmetadata-ui/src/main/resources/ui/.husky/pre-commit index 3c3873cd90fc..12ec7633c6cf 100755 --- a/openmetadata-ui/src/main/resources/ui/.husky/pre-commit +++ b/openmetadata-ui/src/main/resources/ui/.husky/pre-commit @@ -13,6 +13,6 @@ else echo "No changes in JSON schema files. Skipping TypeScript generation." fi - cd openmetadata-ui/src/main/resources/ui +yarn generate:app-docs yarn pre-commit diff --git a/openmetadata-ui/src/main/resources/ui/generateApplicationDocs.js b/openmetadata-ui/src/main/resources/ui/generateApplicationDocs.js new file mode 100644 index 000000000000..6b5d2552be8a --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/generateApplicationDocs.js @@ -0,0 +1,89 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* eslint-disable */ +const fs = require('fs'); +const path = require('path'); + +const SCHEMA_DIR = path.join(__dirname, './src/utils/ApplicationSchemas'); +const DOCS_DIR = path.join(__dirname, './public/locales/en-US/Applications'); + +const processProperty = (key, prop) => { + let markdown = `$$section\n`; + markdown += `### ${prop.title || key} $(id="${key}")\n\n`; + + if (prop.description) { + markdown += `${prop.description}\n\n`; + } + + // Handle nested properties if they exist + if (prop.properties) { + for (const [nestedKey, nestedProp] of Object.entries(prop.properties)) { + markdown += processProperty(`${key}.${nestedKey}`, nestedProp); + } + } + + markdown += `$$\n\n`; + return markdown; +}; + +const generateMarkdown = (schema) => { + let markdown = `# ${schema.title || 'Application Configuration'}\n\n`; + + if (schema.description) { + markdown += `${schema.description}\n\n`; + } + + if (schema.properties) { + for (const [key, prop] of Object.entries(schema.properties)) { + markdown += processProperty(key, prop); + } + } + + return markdown.trim(); +}; + +const parseAndGenerateDocs = () => { + // Ensure docs directory exists + if (!fs.existsSync(DOCS_DIR)) { + fs.mkdirSync(DOCS_DIR, { recursive: true }); + } + + // Read all JSON files from schema directory + const schemaFiles = fs + .readdirSync(SCHEMA_DIR) + .filter((file) => file.endsWith('.json')); + + schemaFiles.forEach((schemaFile) => { + try { + const schemaPath = path.join(SCHEMA_DIR, schemaFile); + const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf-8')); + const baseName = path + .basename(schemaFile, '.json') + .replace('-collate', ''); + + const markdown = generateMarkdown(schema); + + fs.writeFileSync( + path.join(DOCS_DIR, `${baseName}.md`), + markdown, + 'utf-8' + ); + + console.log(`✓ Generated documentation for ${baseName}`); + } catch (error) { + console.error(`✗ Error processing ${schemaFile}:`, error); + } + }); +}; + +parseAndGenerateDocs(); diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index 72341062440e..0c92ce5b3d47 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -35,7 +35,8 @@ "check-i18n": "npm run i18n -- --check", "playwright:run": "playwright test", "playwright:open": "playwright test --ui", - "playwright:codegen": "playwright codegen" + "playwright:codegen": "playwright codegen", + "generate:app-docs": "node generateApplicationDocs.js" }, "dependencies": { "@analytics/session-utils": "^0.1.17", @@ -81,7 +82,7 @@ "cronstrue": "^2.53.0", "crypto-random-string-with-promisify-polyfill": "^5.0.0", "diff": "^5.0.0", - "dompurify": "^3.1.5", + "dompurify": "^3.2.4", "elkjs": "^0.9.3", "eventemitter3": "^5.0.1", "fast-json-patch": "^3.1.1", @@ -161,7 +162,6 @@ "@types/codemirror": "^0.0.104", "@types/dagre": "^0.7.47", "@types/diff": "^5.0.2", - "@types/dompurify": "^3.0.5", "@types/jest": "^26.0.23", "@types/katex": "^0.16.7", "@types/lodash": "^4.14.167", @@ -248,7 +248,9 @@ "path-to-regexp": "1.9.0", "terser-webpack-plugin": "5.1.1", "cookie": "0.7.0", - "jsonpath-plus": "10.2.0", - "cross-spawn": "7.0.5" + "jsonpath-plus": "10.3.0", + "cross-spawn": "7.0.5", + "serialize-javascript": "6.0.2", + "dompurify": "3.2.4" } -} +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts index bd4b0f047766..4ea82f005fa7 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts @@ -381,7 +381,9 @@ test.describe('Activity feed', () => { // Task 1 - Resolved the task + const resolveTask2 = page.waitForResponse('/api/v1/feed/tasks/*/resolve'); await page.getByText('Accept Suggestion').click(); + await resolveTask2; await toastNotification(page, /Task resolved successfully/); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts index 9f94b424d7c5..8f04b37c7e93 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/AdvancedSearch.spec.ts @@ -12,9 +12,10 @@ */ import test from '@playwright/test'; import { SidebarItem } from '../../constant/sidebar'; +import { EntityDataClass } from '../../support/entity/EntityDataClass'; import { TableClass } from '../../support/entity/TableClass'; -import { TopicClass } from '../../support/entity/TopicClass'; -import { TagClass } from '../../support/tag/TagClass'; +import { Glossary } from '../../support/glossary/Glossary'; +import { GlossaryTerm } from '../../support/glossary/GlossaryTerm'; import { UserClass } from '../../support/user/UserClass'; import { FIELDS, @@ -23,7 +24,7 @@ import { verifyAllConditions, } from '../../utils/advancedSearch'; import { createNewPage, redirectToHomePage } from '../../utils/common'; -import { addMultiOwner, assignTag, assignTier } from '../../utils/entity'; +import { assignTier } from '../../utils/entity'; import { sidebarClick } from '../../utils/sidebar'; test.describe.configure({ @@ -31,86 +32,200 @@ test.describe.configure({ timeout: 4 * 60 * 1000, }); +const user = new UserClass(); +const table = new TableClass(undefined, 'Regular'); +let glossaryEntity: Glossary; + test.describe('Advanced Search', { tag: '@advanced-search' }, () => { // use the admin user to login test.use({ storageState: 'playwright/.auth/admin.json' }); - const user1 = new UserClass(); - const user2 = new UserClass(); - const table1 = new TableClass(); - const table2 = new TableClass(); - const topic1 = new TopicClass(); - const topic2 = new TopicClass(); - const tierTag1 = new TagClass({ classification: 'Tier' }); - const tierTag2 = new TagClass({ classification: 'Tier' }); - let searchCriteria: Record = {}; test.beforeAll('Setup pre-requests', async ({ browser }) => { const { page, apiContext, afterAction } = await createNewPage(browser); - await Promise.all([ - user1.create(apiContext), - user2.create(apiContext), - table1.create(apiContext), - table2.create(apiContext), - topic1.create(apiContext), - topic2.create(apiContext), - tierTag1.create(apiContext), - tierTag2.create(apiContext), + await EntityDataClass.preRequisitesForTests(apiContext); + await user.create(apiContext); + glossaryEntity = new Glossary(undefined, [ + { + id: user.responseData.id, + type: 'user', + name: user.responseData.name, + displayName: user.responseData.displayName, + }, ]); + const glossaryTermEntity = new GlossaryTerm(glossaryEntity); + + await glossaryEntity.create(apiContext); + await glossaryTermEntity.create(apiContext); + await table.create(apiContext); // Add Owner & Tag to the table - await table1.visitEntityPage(page); - await addMultiOwner({ - page, - ownerNames: [user1.getUserName()], - activatorBtnDataTestId: 'edit-owner', - resultTestId: 'data-assets-header', - endpoint: table1.endpoint, - type: 'Users', + await EntityDataClass.table1.visitEntityPage(page); + await EntityDataClass.table1.patch({ + apiContext, + patchData: [ + { + op: 'add', + value: { + type: 'user', + id: EntityDataClass.user1.responseData.id, + }, + path: '/owners/0', + }, + { + op: 'add', + value: { + tagFQN: 'PersonalData.Personal', + }, + path: '/tags/0', + }, + { + op: 'add', + path: '/domain', + value: { + id: EntityDataClass.domain1.responseData.id, + type: 'domain', + name: EntityDataClass.domain1.responseData.name, + displayName: EntityDataClass.domain1.responseData.displayName, + }, + }, + ], }); - await assignTag(page, 'PersonalData.Personal'); - await table2.visitEntityPage(page); - await addMultiOwner({ - page, - ownerNames: [user2.getUserName()], - activatorBtnDataTestId: 'edit-owner', - resultTestId: 'data-assets-header', - endpoint: table1.endpoint, - type: 'Users', + await EntityDataClass.table2.visitEntityPage(page); + await EntityDataClass.table2.patch({ + apiContext, + patchData: [ + { + op: 'add', + value: { + type: 'user', + id: EntityDataClass.user2.responseData.id, + }, + path: '/owners/0', + }, + { + op: 'add', + value: { + tagFQN: 'PII.None', + }, + path: '/tags/0', + }, + { + op: 'add', + path: '/domain', + value: { + id: EntityDataClass.domain2.responseData.id, + type: 'domain', + name: EntityDataClass.domain2.responseData.name, + displayName: EntityDataClass.domain2.responseData.displayName, + }, + }, + ], }); - await assignTag(page, 'PII.None'); // Add Tier To the topic 1 - await topic1.visitEntityPage(page); - await assignTier(page, tierTag1.data.displayName, topic1.endpoint); + await EntityDataClass.topic1.visitEntityPage(page); + await assignTier( + page, + EntityDataClass.tierTag1.data.displayName, + EntityDataClass.topic1.endpoint + ); // Add Tier To the topic 2 - await topic2.visitEntityPage(page); - await assignTier(page, tierTag2.data.displayName, topic2.endpoint); + await EntityDataClass.topic2.visitEntityPage(page); + await assignTier( + page, + EntityDataClass.tierTag2.data.displayName, + EntityDataClass.topic2.endpoint + ); // Update Search Criteria here searchCriteria = { - 'owners.displayName.keyword': [user1.getUserName(), user2.getUserName()], + 'owners.displayName.keyword': [ + EntityDataClass.user1.getUserName(), + EntityDataClass.user2.getUserName(), + ], 'tags.tagFQN': ['PersonalData.Personal', 'PII.None'], 'tier.tagFQN': [ - tierTag1.responseData.fullyQualifiedName, - tierTag2.responseData.fullyQualifiedName, + EntityDataClass.tierTag1.responseData.fullyQualifiedName, + EntityDataClass.tierTag2.responseData.fullyQualifiedName, + ], + 'service.displayName.keyword': [ + EntityDataClass.table1.service.name, + EntityDataClass.table2.service.name, ], - 'service.displayName.keyword': [table1.service.name, table2.service.name], 'database.displayName.keyword': [ - table1.database.name, - table2.database.name, + EntityDataClass.table1.database.name, + EntityDataClass.table2.database.name, ], 'databaseSchema.displayName.keyword': [ - table1.schema.name, - table2.schema.name, + EntityDataClass.table1.schema.name, + EntityDataClass.table2.schema.name, + ], + 'columns.name.keyword': [ + EntityDataClass.table1.entity.columns[2].name, + EntityDataClass.table2.entity.columns[3].name, ], - 'columns.name.keyword': ['email', 'shop_id'], 'displayName.keyword': [ - table1.entity.displayName, - table2.entity.displayName, + EntityDataClass.table1.entity.displayName, + EntityDataClass.table2.entity.displayName, + ], + serviceType: [ + EntityDataClass.table1.service.serviceType, + EntityDataClass.topic1.service.serviceType, + ], + 'messageSchema.schemaFields.name.keyword': [ + EntityDataClass.topic1.entity.messageSchema.schemaFields[0].name, + EntityDataClass.topic2.entity.messageSchema.schemaFields[1].name, + ], + 'dataModel.columns.name.keyword': [ + EntityDataClass.container1.entity.dataModel.columns[0].name, + EntityDataClass.container2.entity.dataModel.columns[1].name, + ], + dataModelType: [ + EntityDataClass.dashboard1.dataModel.dataModelType, + EntityDataClass.dashboard2.dataModel.dataModelType, + ], + 'fields.name.keyword': [ + EntityDataClass.searchIndex1.entity.fields[1].name, + EntityDataClass.searchIndex2.entity.fields[3].name, + ], + 'tasks.displayName.keyword': [ + EntityDataClass.pipeline1.entity.tasks[0].displayName, + EntityDataClass.pipeline2.entity.tasks[1].displayName, + ], + 'domain.displayName.keyword': [ + EntityDataClass.domain1.data.displayName, + EntityDataClass.domain2.data.displayName, + ], + 'responseSchema.schemaFields.name.keyword': [ + EntityDataClass.apiCollection1.apiEndpoint.responseSchema + .schemaFields[0].name, + EntityDataClass.apiCollection2.apiEndpoint.responseSchema + .schemaFields[1].name, + ], + 'requestSchema.schemaFields.name.keyword': [ + EntityDataClass.apiCollection1.apiEndpoint.requestSchema.schemaFields[0] + .name, + EntityDataClass.apiCollection2.apiEndpoint.requestSchema.schemaFields[1] + .name, + ], + 'name.keyword': [ + EntityDataClass.table1.entity.name, + EntityDataClass.table2.entity.name, + ], + 'project.keyword': [ + EntityDataClass.dashboardDataModel1.entity.project, + EntityDataClass.dashboardDataModel2.entity.project, + ], + status: ['Approved', 'In Review'], + tableType: [table.entity.tableType, 'MaterializedView'], + entityType: ['dashboard', 'mlmodel'], + 'charts.displayName.keyword': [ + EntityDataClass.dashboard1.charts.displayName, + EntityDataClass.dashboard2.charts.displayName, ], }; @@ -119,16 +234,10 @@ test.describe('Advanced Search', { tag: '@advanced-search' }, () => { test.afterAll('Cleanup', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); - await Promise.all([ - user1.delete(apiContext), - user2.delete(apiContext), - table1.delete(apiContext), - table2.delete(apiContext), - topic1.delete(apiContext), - topic2.delete(apiContext), - tierTag1.delete(apiContext), - tierTag2.delete(apiContext), - ]); + await EntityDataClass.postRequisitesForTests(apiContext); + await glossaryEntity.delete(apiContext); + await user.delete(apiContext); + await table.delete(apiContext); await afterAction(); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts index f2f653128712..b35180a99e1b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/QueryEntity.spec.ts @@ -106,7 +106,9 @@ test('Query Entity', async ({ page }) => { }); await test.step('Update owner, description and tag', async () => { - const ownerListResponse = page.waitForResponse('/api/v1/users?*'); + const ownerListResponse = page.waitForResponse( + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' + ); await page .getByTestId( 'entity-summary-resizable-right-panel-container entity-resizable-panel-container' diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts index 9678b28c4301..4491f4b3aa61 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/TableConstraint.spec.ts @@ -31,6 +31,11 @@ test.use({ storageState: 'playwright/.auth/admin.json' }); const table = new TableClass(); test.describe('Table Constraints', {}, () => { + const columnName1 = table.children[0].name; + const columnName2 = table.children[1].name; + const columnName3 = table.children[2].name; + const columnName4 = table.children[3].name; + test.beforeAll('Prerequisite', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); await table.create(apiContext); @@ -76,23 +81,21 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('primary-constraint-type-select') - .locator('div') - .nth(1) - .type('user_id'); + .getByRole('combobox') + .fill(columnName1, { force: true }); // select 1st value from dropdown - const firstPrimaryKeyColumn = page.getByTitle('user_id'); + const firstPrimaryKeyColumn = page.getByTitle(columnName1); await firstPrimaryKeyColumn.hover(); await firstPrimaryKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('primary-constraint-type-select') - .locator('div') - .nth(1) - .type('shop_id'); + .getByRole('combobox') + .fill(columnName2, { force: true }); - const secondPrimaryKeyColumn = page.getByTitle('shop_id'); + const secondPrimaryKeyColumn = page.getByTitle(columnName2); await secondPrimaryKeyColumn.hover(); await secondPrimaryKeyColumn.click(); await clickOutside(page); @@ -100,7 +103,7 @@ test.describe('Table Constraints', {}, () => { await expect( page .getByTestId('primary-constraint-type-select') - .getByText('user_idshop_id') + .getByText(`${columnName1}${columnName2}`) ).toBeVisible(); // Foreign Key Constraint Section @@ -170,29 +173,31 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('unique-constraint-type-select') - .locator('div') - .nth(1) - .type('name'); + .getByRole('combobox') + .fill(columnName3, { force: true }); // select 1st value from dropdown - const firstUniqueKeyColumn = page.getByTitle('name', { exact: true }); + const firstUniqueKeyColumn = page.getByTitle(columnName3, { + exact: true, + }); await firstUniqueKeyColumn.hover(); await firstUniqueKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('unique-constraint-type-select') - .locator('div') - .nth(1) - .type('email'); + .getByRole('combobox') + .fill(columnName4, { force: true }); - const secondUniqueKeyColumn = page.getByTitle('email'); + const secondUniqueKeyColumn = page.getByTitle(columnName4); await secondUniqueKeyColumn.hover(); await secondUniqueKeyColumn.click(); await clickOutside(page); await expect( - page.getByTestId('unique-constraint-type-select').getByText('nameemail') + page + .getByTestId('unique-constraint-type-select') + .getByText(`${columnName3}${columnName4}`) ).toBeVisible(); // Dist Constraint Section @@ -206,23 +211,21 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('dist-constraint-type-select') - .locator('div') - .nth(1) - .type('user_id'); + .getByRole('combobox') + .fill(columnName1, { force: true }); // select 1st value from dropdown - const firstDistKeyColumn = page.getByTitle('user_id'); + const firstDistKeyColumn = page.getByTitle(columnName1); await firstDistKeyColumn.hover(); await firstDistKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('dist-constraint-type-select') - .locator('div') - .nth(1) - .type('shop_id'); + .getByRole('combobox') + .fill(columnName2, { force: true }); - const secondDistKeyColumn = page.getByTitle('shop_id'); + const secondDistKeyColumn = page.getByTitle(columnName2); await secondDistKeyColumn.hover(); await secondDistKeyColumn.click(); await clickOutside(page); @@ -230,7 +233,7 @@ test.describe('Table Constraints', {}, () => { await expect( page .getByTestId('dist-constraint-type-select') - .getByText('user_idshop_id') + .getByText(`${columnName1}${columnName2}`) ).toBeVisible(); // Sort Constraint Section @@ -244,29 +247,29 @@ test.describe('Table Constraints', {}, () => { await page .getByTestId('sort-constraint-type-select') - .locator('div') - .nth(1) - .type('name'); + .getByRole('combobox') + .fill(columnName3, { force: true }); // select 1st value from dropdown - const firstSortKeyColumn = page.getByTitle('name', { exact: true }); + const firstSortKeyColumn = page.getByTitle(columnName3, { exact: true }); await firstSortKeyColumn.hover(); await firstSortKeyColumn.click(); // select 2nd value from dropdown await page .getByTestId('sort-constraint-type-select') - .locator('div') - .nth(1) - .type('email'); + .getByRole('combobox') + .fill(columnName4, { force: true }); - const secondSortKeyColumn = page.getByTitle('email'); + const secondSortKeyColumn = page.getByTitle(columnName4); await secondSortKeyColumn.hover(); await secondSortKeyColumn.click(); await clickOutside(page); await expect( - page.getByTestId('sort-constraint-type-select').getByText('nameemail') + page + .getByTestId('sort-constraint-type-select') + .getByText(`${columnName3}${columnName4}`) ).toBeVisible(); const saveResponse = page.waitForResponse('/api/v1/tables/*'); @@ -288,31 +291,31 @@ test.describe('Table Constraints', {}, () => { // Verify Primary Key await expect(page.getByTestId('PRIMARY_KEY-container')).toContainText( - 'shop_iduser_id' + `${columnName2}${columnName1}` ); await expect(page.getByTestId('PRIMARY_KEY-icon')).toBeVisible(); // Verify Foreign Key await expect(page.getByTestId('FOREIGN_KEY-container')).toContainText( - `user_id${table.additionalEntityTableResponseData[0]?.['columns'][1].fullyQualifiedName}` + `${columnName1}${table.additionalEntityTableResponseData[0]?.['columns'][1].fullyQualifiedName}` ); await expect(page.getByTestId('FOREIGN_KEY-icon')).toBeVisible(); // Verify Unique Key await expect(page.getByTestId('UNIQUE-container')).toContainText( - 'emailname' + `${columnName4}${columnName3}` ); await expect(page.getByTestId('UNIQUE-icon')).toBeVisible(); // Verify Sort Key await expect(page.getByTestId('SORT_KEY-container')).toContainText( - 'emailname' + `${columnName4}${columnName3}` ); await expect(page.getByTestId('SORT_KEY-icon')).toBeVisible(); // Verify Dist Key await expect(page.getByTestId('DIST_KEY-container')).toContainText( - 'shop_iduser_id' + `${columnName2}${columnName1}` ); await expect(page.getByTestId('DIST_KEY-icon')).toBeVisible(); }); @@ -345,11 +348,11 @@ test.describe('Table Constraints', {}, () => { // Verify Sort and Dist Key to be available await expect(page.getByTestId('SORT_KEY-container')).toContainText( - 'emailname' + `${columnName4}${columnName3}` ); await expect(page.getByTestId('SORT_KEY-icon')).toBeVisible(); await expect(page.getByTestId('DIST_KEY-container')).toContainText( - 'shop_iduser_id' + `${columnName2}${columnName1}` ); // Remove the pending constraints diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts index 1ca2de107757..4fab3aa230f2 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Topic.spec.ts @@ -45,6 +45,6 @@ test.describe('Topic entity specific tests ', () => { test('Topic page should show schema tab with count', async ({ page }) => { await topic.visitEntityPage(page); - await expect(page.getByRole('tab', { name: 'Schema' })).toContainText('1'); + await expect(page.getByRole('tab', { name: 'Schema' })).toContainText('2'); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts index 6ceaa212c3e1..cc1a65106d88 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts @@ -45,7 +45,7 @@ test.beforeAll(async ({ browser }) => { ); await table2.createTestCase(apiContext, { name: `email_column_values_to_be_in_set_${uuid()}`, - entityLink: `<#E::table::${table2.entityResponseData?.['fullyQualifiedName']}::columns::email>`, + entityLink: `<#E::table::${table2.entityResponseData?.['fullyQualifiedName']}::columns::${table2.entity?.columns[3].name}>`, parameterValues: [ { name: 'allowedValues', value: '["gmail","yahoo","collate"]' }, ], @@ -170,7 +170,7 @@ test('Column test case', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { const NEW_COLUMN_TEST_CASE = { name: 'email_column_value_lengths_to_be_between', - column: 'email', + column: table1.entity?.columns[3].name, type: 'columnValueLengthsToBeBetween', label: 'Column Value Lengths To Be Between', min: '3', @@ -365,7 +365,9 @@ test( await expect(page.locator('#tableTestForm_table')).toHaveValue( table2.entityResponseData?.['name'] ); - await expect(page.locator('#tableTestForm_column')).toHaveValue('email'); + await expect(page.locator('#tableTestForm_column')).toHaveValue( + table2.entity?.columns[3].name + ); await expect(page.locator('#tableTestForm_name')).toHaveValue( testCaseName ); @@ -486,9 +488,9 @@ test( profileSample: '60', sampleDataCount: '100', profileQuery: 'select * from table', - excludeColumns: 'user_id', - includeColumns: 'shop_id', - partitionColumnName: 'name', + excludeColumns: table1.entity?.columns[0].name, + includeColumns: table1.entity?.columns[1].name, + partitionColumnName: table1.entity?.columns[2].name, partitionIntervalType: 'COLUMN-VALUE', partitionValues: 'test', }; @@ -561,13 +563,13 @@ test( expect(requestBody).toEqual( JSON.stringify({ - excludeColumns: ['user_id'], + excludeColumns: [table1.entity?.columns[0].name], profileQuery: 'select * from table', profileSample: 60, profileSampleType: 'PERCENTAGE', - includeColumns: [{ columnName: 'shop_id' }], + includeColumns: [{ columnName: table1.entity?.columns[1].name }], partitioning: { - partitionColumnName: 'name', + partitionColumnName: table1.entity?.columns[2].name, partitionIntervalType: 'COLUMN-VALUE', partitionValues: ['test'], enablePartitioning: true, @@ -941,10 +943,13 @@ test('TestCase filters', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { await expect(page.locator('[value="tier"]')).not.toBeVisible(); // Apply domain globally - await page.locator('[data-testid="domain-dropdown"]').click(); + await page.getByTestId('domain-dropdown').click(); + await page - .locator(`li[data-menu-id*='${domain.responseData?.['name']}']`) + .getByTestId(`tag-${domain.responseData.fullyQualifiedName}`) .click(); + await page.getByTestId('saveAssociatedTag').click(); + await sidebarClick(page, SidebarItem.DATA_QUALITY); const getTestCaseList = page.waitForResponse( '/api/v1/dataQuality/testCases/search/list?*' diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts index 6d644b2f5ab5..219790217f74 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Domains.spec.ts @@ -240,11 +240,13 @@ test.describe('Domains', () => { await domain.create(apiContext); await page.reload(); await page.getByTestId('domain-dropdown').click(); + await page - .locator( - `[data-menu-id*="${domain.data.name}"] > .ant-dropdown-menu-title-content` - ) + .getByTestId(`tag-${domain.responseData.fullyQualifiedName}`) .click(); + await page.getByTestId('saveAssociatedTag').click(); + + await page.waitForLoadState('networkidle'); await redirectToHomePage(page); @@ -566,14 +568,11 @@ test.describe('Domains Rbac', () => { .click(); await expect( - userPage - .getByRole('menuitem', { name: domain1.data.displayName }) - .locator('span') + userPage.getByTestId(`tag-${domain1.responseData.fullyQualifiedName}`) ).toBeVisible(); + await expect( - userPage - .getByRole('menuitem', { name: domain3.data.displayName }) - .locator('span') + userPage.getByTestId(`tag-${domain3.responseData.fullyQualifiedName}`) ).toBeVisible(); // Visit explore page and verify if domain is passed in the query diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts index 01d983cf9aef..f0aa8825e374 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Entity.spec.ts @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { test } from '@playwright/test'; +import { expect, test } from '@playwright/test'; import { isUndefined } from 'lodash'; import { CustomPropertySupportedEntityList } from '../../constant/customProperty'; import { ApiEndpointClass } from '../../support/entity/ApiEndpointClass'; @@ -65,8 +65,9 @@ test.use({ storageState: 'playwright/.auth/admin.json' }); entities.forEach((EntityClass) => { const entity = new EntityClass(); const deleteEntity = new EntityClass(); + const entityName = entity.getType(); - test.describe(entity.getType(), () => { + test.describe(entityName, () => { test.beforeAll('Setup pre-requests', async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); @@ -192,6 +193,16 @@ entities.forEach((EntityClass) => { ); }); + if (['Dashboard', 'Dashboard Data Model'].includes(entityName)) { + test(`${entityName} page should show the project name`, async ({ + page, + }) => { + await expect( + page.getByText((entity.entity as { project: string }).project) + ).toBeVisible(); + }); + } + test('Update description', async ({ page }) => { await entity.descriptionUpdate(page); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts index a71cf6ad1f52..5d2de7cfc77d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/MyData.spec.ts @@ -89,7 +89,7 @@ test.describe.serial('My Data page', () => { // Verify entities await verifyEntities( page, - '/api/v1/search/query?q=*&index=all&from=0&size=25', + '/api/v1/search/query?q=*&index=all&from=0&size=25*', TableEntities ); }); @@ -105,7 +105,7 @@ test.describe.serial('My Data page', () => { // Verify entities await verifyEntities( page, - '/api/v1/search/query?q=*followers:*&index=all&from=0&size=25', + '/api/v1/search/query?q=*followers:*&index=all&from=0&size=25*', TableEntities ); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ProfilerConfigurationPage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ProfilerConfigurationPage.spec.ts index fae85658d9b7..15ea3bd1e6ed 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ProfilerConfigurationPage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ProfilerConfigurationPage.spec.ts @@ -19,7 +19,7 @@ import { SidebarItem } from '../../constant/sidebar'; import { AdminClass } from '../../support/user/AdminClass'; import { UserClass } from '../../support/user/UserClass'; import { performAdminLogin } from '../../utils/admin'; -import { redirectToHomePage } from '../../utils/common'; +import { redirectToHomePage, toastNotification } from '../../utils/common'; import { sidebarClick } from '../../utils/sidebar'; const user = new UserClass(); @@ -123,7 +123,8 @@ test.describe('Profiler Configuration Page', () => { ); }); - await expect(adminPage.getByRole('alert').first()).toHaveText( + await toastNotification( + adminPage, /Profiler Configuration updated successfully./ ); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts index 62502e889f22..96b0fb05e562 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/SearchIndexApplication.spec.ts @@ -90,6 +90,22 @@ test('Search Index Application', async ({ page }) => { await verifyLastExecutionRun(page); }); + await test.step('View App Run Config', async () => { + await page.getByTestId('app-historical-config').click(); + await page.waitForSelector('[role="dialog"].ant-modal'); + + await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible(); + + await expect(page.locator('.ant-modal-title')).toContainText( + 'Search Indexing Configuration' + ); + + await page.click('[data-testid="app-run-config-close"]'); + await page.waitForSelector('[role="dialog"].ant-modal', { + state: 'detached', + }); + }); + await test.step('Edit application', async () => { await page.click('[data-testid="edit-button"]'); await page.waitForSelector('[data-testid="schedular-card-container"]'); @@ -109,6 +125,12 @@ test('Search Index Application', async ({ page }) => { ); await page.click('[data-testid="configuration"]'); + await page.waitForLoadState('networkidle'); + + await expect(page.locator('#search-indexing-application')).toContainText( + 'Search Indexing Application' + ); + await page.fill('#root\\/batchSize', '0'); await page.getByTestId('tree-select-widget').click(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts index d7ccd9ba9298..b0395c2f6aba 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts @@ -284,7 +284,7 @@ test('Classification Page', async ({ page }) => { tagDisplayName: displayName, tableId: table.entityResponseData?.['id'], columnNumber: 0, - rowName: 'user_id numeric', + rowName: `${table.entity?.columns[0].name} numeric`, }); }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts index c9b69c526bd3..bed113692269 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestCases.spec.ts @@ -16,6 +16,7 @@ import { descriptionBox, getApiContext, redirectToHomePage, + toastNotification, } from '../../utils/common'; import { deleteTestCase, visitDataQualityTab } from '../../utils/testCases'; @@ -79,18 +80,27 @@ test('Table difference test case', async ({ page }) => { .locator('div') .click(); - await page.fill(`#tableTestForm_params_keyColumns_0_value`, 'user_id'); - await page.getByTitle('user_id').click(); + await page.fill( + `#tableTestForm_params_keyColumns_0_value`, + table1.entity?.columns[0].name + ); + await page.getByTitle(table1.entity?.columns[0].name).click(); await page.fill('#tableTestForm_params_threshold', testCase.threshold); - await page.fill('#tableTestForm_params_useColumns_0_value', 'user_id'); - - await expect(page.getByTitle('user_id').nth(2)).toHaveClass( - /ant-select-item-option-disabled/ + await page.fill( + '#tableTestForm_params_useColumns_0_value', + table1.entity?.columns[0].name ); + await expect( + page.getByTitle(table1.entity?.columns[0].name).nth(2) + ).toHaveClass(/ant-select-item-option-disabled/); + await page.locator('#tableTestForm_params_useColumns_0_value').clear(); - await page.fill('#tableTestForm_params_useColumns_0_value', 'shop_id'); - await page.getByTitle('shop_id').click(); + await page.fill( + '#tableTestForm_params_useColumns_0_value', + table1.entity?.columns[1].name + ); + await page.getByTitle(table1.entity?.columns[1].name).click(); await page.fill('#tableTestForm_params_where', 'test'); const createTestCaseResponse = page.waitForResponse( @@ -121,23 +131,29 @@ test('Table difference test case', async ({ page }) => { .filter({ hasText: 'Key Columns' }) .getByRole('button') .click(); - await page.fill('#tableTestForm_params_keyColumns_1_value', 'email'); - await page.getByTitle('email', { exact: true }).click(); + await page.fill( + '#tableTestForm_params_keyColumns_1_value', + table1.entity?.columns[3].name + ); + await page + .getByTitle(table1.entity?.columns[3].name, { exact: true }) + .click(); await page .locator('label') .filter({ hasText: 'Use Columns' }) .getByRole('button') .click(); - await page.fill('#tableTestForm_params_useColumns_1_value', 'name'); - await page.getByTitle('name', { exact: true }).click(); - await page.getByRole('button', { name: 'Submit' }).click(); - - await expect(page.getByRole('alert')).toContainText( - 'Test case updated successfully.' + await page.fill( + '#tableTestForm_params_useColumns_1_value', + table1.entity?.columns[2].name ); + await page + .getByTitle(table1.entity?.columns[2].name, { exact: true }) + .click(); + await page.getByRole('button', { name: 'Submit' }).click(); - await page.getByLabel('close', { exact: true }).click(); + await toastNotification(page, 'Test case updated successfully.'); }); await test.step('Delete', async () => { @@ -233,11 +249,7 @@ test('Custom SQL Query', async ({ page }) => { await page.getByPlaceholder('Enter a Threshold').fill('244'); await page.getByRole('button', { name: 'Submit' }).click(); - await expect(page.getByRole('alert')).toContainText( - 'Test case updated successfully.' - ); - - await page.getByLabel('close', { exact: true }).click(); + await toastNotification(page, 'Test case updated successfully.'); }); await test.step('Delete', async () => { @@ -252,17 +264,17 @@ test('Custom SQL Query', async ({ page }) => { test('Column Values To Be Not Null', async ({ page }) => { test.slow(); + await redirectToHomePage(page); + const { afterAction, apiContext } = await getApiContext(page); + const table = new TableClass(); const NEW_COLUMN_TEST_CASE_WITH_NULL_TYPE = { name: 'id_column_values_to_be_not_null', displayName: 'ID Column Values To Be Not Null', - column: 'user_id', + column: table.entity?.columns[0].name, type: 'columnValuesToBeNotNull', label: 'Column Values To Be Not Null', description: 'New table test case for columnValuesToBeNotNull', }; - await redirectToHomePage(page); - const { afterAction, apiContext } = await getApiContext(page); - const table = new TableClass(); await table.create(apiContext); await visitDataQualityTab(page, table); @@ -333,11 +345,7 @@ test('Column Values To Be Not Null', async ({ page }) => { await page.keyboard.type(' update'); await page.getByRole('button', { name: 'Submit' }).click(); - await expect(page.getByRole('alert')).toContainText( - 'Test case updated successfully.' - ); - - await page.getByLabel('close', { exact: true }).click(); + await toastNotification(page, 'Test case updated successfully.'); }); await test.step('Delete', async () => { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts index 99842af0227a..cd4adcc1cb58 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts @@ -12,14 +12,18 @@ */ import { expect, test } from '@playwright/test'; import { SidebarItem } from '../../constant/sidebar'; +import { Domain } from '../../support/domain/Domain'; import { EntityTypeEndpoint } from '../../support/entity/Entity.interface'; import { TableClass } from '../../support/entity/TableClass'; import { UserClass } from '../../support/user/UserClass'; import { + assignDomain, createNewPage, descriptionBox, redirectToHomePage, + removeDomain, toastNotification, + updateDomain, uuid, } from '../../utils/common'; import { addMultiOwner, removeOwnersFromList } from '../../utils/entity'; @@ -31,6 +35,8 @@ test.use({ storageState: 'playwright/.auth/admin.json' }); const table = new TableClass(); const user1 = new UserClass(); const user2 = new UserClass(); +const domain1 = new Domain(); +const domain2 = new Domain(); test.beforeAll(async ({ browser }) => { const { apiContext, afterAction } = await createNewPage(browser); @@ -39,6 +45,8 @@ test.beforeAll(async ({ browser }) => { await user2.create(apiContext); await table.createTestCase(apiContext); await table.createTestCase(apiContext); + await domain1.create(apiContext); + await domain2.create(apiContext); await afterAction(); }); @@ -47,6 +55,8 @@ test.afterAll(async ({ browser }) => { await table.delete(apiContext); await user1.delete(apiContext); await user2.delete(apiContext); + await domain1.delete(apiContext); + await domain2.delete(apiContext); await afterAction(); }); @@ -100,6 +110,12 @@ test('Logical TestSuite', async ({ page }) => { await testSuiteResponse; }); + await test.step('Domain Add, Update and Remove', async () => { + await assignDomain(page, domain1.responseData); + await updateDomain(page, domain2.responseData); + await removeDomain(page, domain2.responseData); + }); + await test.step( 'User as Owner assign, update & delete for test suite', async () => { @@ -201,7 +217,9 @@ test('Logical TestSuite', async ({ page }) => { await page.waitForSelector("[data-testid='select-owner-tabs']", { state: 'visible', }); - const getOwnerList = page.waitForResponse('/api/v1/users?*isBot=false*'); + const getOwnerList = page.waitForResponse( + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' + ); await page.click('.ant-tabs [id*=tab-users]'); await getOwnerList; await page.waitForSelector(`[data-testid="loader"]`, { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts index b89c2298347a..0fbef8c43c56 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/UserDetails.spec.ts @@ -14,15 +14,23 @@ import { expect, Page, test as base } from '@playwright/test'; import { GlobalSettingOptions } from '../../constant/settings'; import { USER_DESCRIPTION } from '../../constant/user'; +import { TeamClass } from '../../support/team/TeamClass'; import { AdminClass } from '../../support/user/AdminClass'; import { UserClass } from '../../support/user/UserClass'; import { performAdminLogin } from '../../utils/admin'; -import { descriptionBox, redirectToHomePage } from '../../utils/common'; +import { descriptionBox, redirectToHomePage, uuid } from '../../utils/common'; import { settingClick } from '../../utils/sidebar'; +import { redirectToUserPage } from '../../utils/userDetails'; const user1 = new UserClass(); const user2 = new UserClass(); const admin = new AdminClass(); +const team = new TeamClass({ + name: `a-new-team-${uuid()}`, + displayName: `A New Team ${uuid()}`, + description: 'playwright team description', + teamType: 'Group', +}); // Create 2 page and authenticate 1 with admin and another with normal user const test = base.extend<{ @@ -50,6 +58,8 @@ test.describe('User with different Roles', () => { await user1.create(apiContext); await user2.create(apiContext); + await team.create(apiContext); + await afterAction(); }); @@ -59,32 +69,57 @@ test.describe('User with different Roles', () => { await user1.delete(apiContext); await user2.delete(apiContext); + await team.delete(apiContext); + await afterAction(); }); - test('Non admin user should be able to edit display name and description on own profile', async ({ - userPage, + test('Admin user can get all the teams hierarchy which editing teams', async ({ + adminPage, }) => { - await redirectToHomePage(userPage); + await redirectToUserPage(adminPage); - await userPage.getByTestId('dropdown-profile').click(); + // Check if the avatar is visible + await expect( + adminPage + .getByTestId('user-profile-details') + .getByTestId('profile-avatar') + ).toBeVisible(); - // Hover on the profile avatar to close the name tooltip - await userPage.getByTestId('profile-avatar').hover(); + await adminPage + .locator('.user-profile-container [data-icon="right"]') + .click(); + + await expect( + adminPage.getByTestId('user-team-card-container') + ).toBeVisible(); - await userPage.waitForSelector('.profile-dropdown', { state: 'visible' }); + const teamListResponse = adminPage.waitForResponse( + '/api/v1/teams/hierarchy?isJoinable=false' + ); - const getUserDetails = userPage.waitForResponse(`/api/v1/users/name/*`); + await adminPage.getByTestId('edit-teams-button').click(); - await userPage - .locator('.profile-dropdown') - .getByTestId('user-name') - .click(); + await teamListResponse; + + await expect(adminPage.getByTestId('team-select')).toBeVisible(); + + await adminPage.getByTestId('team-select').click(); + + await adminPage.waitForSelector('.ant-tree-select-dropdown', { + state: 'visible', + }); - await getUserDetails; + // Check if newly added team is there or not + await expect(adminPage.locator('.ant-tree-select-dropdown')).toContainText( + team.responseData.displayName + ); + }); - // Close the profile dropdown - await userPage.getByTestId('dropdown-profile').click(); + test('Non admin user should be able to edit display name and description on own profile', async ({ + userPage, + }) => { + await redirectToUserPage(userPage); // Check if the display name is present await expect( @@ -188,6 +223,32 @@ test.describe('User with different Roles', () => { ).toContainText('No description'); }); + test('Non admin user should not be able to edit the persona or roles', async ({ + userPage, + }) => { + await redirectToUserPage(userPage); + + // Check if the display name is present + await expect( + userPage.getByTestId('user-profile-details').getByTestId('user-name') + ).toHaveText(user1.responseData.displayName); + + await userPage + .locator('.user-profile-container [data-icon="right"]') + .click(); + + // Check for Roles field visibility + await expect(userPage.getByTestId('user-profile-roles')).toBeVisible(); + + // Edit Persona icon shouldn't be visible + await expect( + userPage.getByTestId('persona-list').getByTestId('edit-persona') + ).not.toBeVisible(); + + // Edit Roles icon shouldn't be visible + await expect(userPage.getByTestId('edit-roles-button')).not.toBeVisible(); + }); + test('Non logged in user should not be able to edit display name and description on other users', async ({ userPage, adminPage, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts index 9f139664f1fd..fb74e92382d0 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Users.spec.ts @@ -189,6 +189,10 @@ test.describe('User with Admin Roles', () => { }) => { await redirectToHomePage(adminPage); await settingClick(adminPage, GlobalSettingOptions.USERS); + await adminPage.waitForLoadState('networkidle'); + await adminPage.waitForSelector('.user-list-table [data-testid="loader"]', { + state: 'detached', + }); await softDeleteUserProfilePage( adminPage, user.responseData.name, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts index 27edd96029f0..381763afe7d4 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts @@ -44,7 +44,6 @@ export class ApiCollectionClass extends EntityClass { }; private apiEndpointName = `pw-api-endpoint-${uuid()}`; - private fqn = `${this.service.name}.${this.entity.name}.${this.apiEndpointName}`; apiEndpoint = { name: this.apiEndpointName, @@ -54,91 +53,89 @@ export class ApiCollectionClass extends EntityClass { schemaType: 'JSON', schemaFields: [ { - name: 'default', + name: `default${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default.name`, tags: [], children: [ { - name: 'first_name', + name: `first_name${uuid()}`, dataType: 'STRING', description: 'Description for schema field first_name', - fullyQualifiedName: `${this.fqn}.default.name.first_name`, tags: [], }, { - name: 'last_name', + name: `last_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.last_name`, tags: [], }, ], }, { - name: 'age', + name: `age${uuid()}`, dataType: 'INT', - fullyQualifiedName: `${this.fqn}.default.age`, tags: [], }, { - name: 'club_name', + name: `club_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.club_name`, tags: [], }, ], }, + { + name: `secondary${uuid()}`, + dataType: 'RECORD', + tags: [], + }, ], }, responseSchema: { schemaType: 'JSON', schemaFields: [ { - name: 'default', + name: `default${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default.name`, tags: [], children: [ { - name: 'first_name', + name: `first_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.first_name`, tags: [], }, { - name: 'last_name', + name: `last_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.last_name`, tags: [], }, ], }, { - name: 'age', + name: `age${uuid()}`, dataType: 'INT', - fullyQualifiedName: `${this.fqn}.default.age`, tags: [], }, { - name: 'club_name', + name: `club_name${uuid()}`, dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.club_name`, tags: [], }, ], }, + { + name: `secondary${uuid()}`, + dataType: 'RECORD', + tags: [], + }, ], }, }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts index 1ad4c8d4d575..8f54d624f4bf 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ContainerClass.ts @@ -46,6 +46,38 @@ export class ContainerClass extends EntityClass { name: this.containerName, displayName: this.containerName, service: this.service.name, + dataModel: { + isPartitioned: true, + columns: [ + { + name: `merchant${uuid()}`, + dataType: 'VARCHAR', + dataLength: 100, + dataTypeDisplay: 'varchar', + description: 'The merchant for this transaction.', + tags: [], + ordinalPosition: 2, + }, + { + name: `columbia${uuid()}`, + dataType: 'NUMERIC', + dataTypeDisplay: 'numeric', + description: + 'The ID of the executed transaction. This column is the primary key for this table.', + tags: [], + constraint: 'PRIMARY_KEY', + ordinalPosition: 1, + }, + { + name: `delivery${uuid()}`, + dataType: 'TIMESTAMP', + dataTypeDisplay: 'timestamp', + description: 'The time the transaction took place.', + tags: [], + ordinalPosition: 3, + }, + ], + }, }; childContainer = { name: this.childContainerName, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts index 14777bda65ae..b1c8c226be4b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardClass.ts @@ -26,6 +26,7 @@ import { EntityClass } from './EntityClass'; export class DashboardClass extends EntityClass { private dashboardName = `pw-dashboard-${uuid()}`; private dashboardDataModelName = `pw-dashboard-data-model-${uuid()}`; + private projectName = `pw-project-${uuid()}`; service = { name: `pw-dashboard-service-${uuid()}`, serviceType: 'Superset', @@ -51,8 +52,23 @@ export class DashboardClass extends EntityClass { name: this.dashboardName, displayName: this.dashboardName, service: this.service.name, + project: this.projectName, }; children = [ + { + name: 'merchant', + dataType: 'VARCHAR', + dataLength: 256, + dataTypeDisplay: 'varchar', + description: 'merchant', + }, + { + name: 'notes', + dataType: 'VARCHAR', + dataLength: 256, + dataTypeDisplay: 'varchar', + description: 'merchant', + }, { name: 'country_name', dataType: 'VARCHAR', @@ -76,12 +92,13 @@ export class DashboardClass extends EntityClass { {} as ResponseDataWithServiceType; chartsResponseData: ResponseDataType = {} as ResponseDataType; - constructor(name?: string) { + constructor(name?: string, dataModelType = 'SupersetDataModel') { super(EntityTypeEndpoint.Dashboard); this.service.name = name ?? this.service.name; this.type = 'Dashboard'; this.serviceCategory = SERVICE_TYPE.Dashboard; this.serviceType = ServiceTypes.DASHBOARD_SERVICES; + this.dataModel.dataModelType = dataModelType; } async create(apiContext: APIRequestContext) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts index 738460fdb742..d1b32af7d797 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DashboardDataModelClass.ts @@ -25,6 +25,7 @@ import { EntityClass } from './EntityClass'; export class DashboardDataModelClass extends EntityClass { private dashboardDataModelName = `pw-dashboard-data-model-${uuid()}`; + private projectName = `pw-project-${uuid()}`; service = { name: `pw-dashboard-service-${uuid()}`, serviceType: 'Superset', @@ -58,6 +59,7 @@ export class DashboardDataModelClass extends EntityClass { service: this.service.name, columns: this.children, dataModelType: 'SupersetDataModel', + project: this.projectName, }; serviceResponseData: ResponseDataType = {} as ResponseDataType; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts index 691a7ac1201d..03d8e94bcb2e 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts @@ -102,3 +102,15 @@ export interface UserResponseDataType extends ResponseDataType { isBot: boolean; href?: string; } + +export interface EntityReference { + id: string; + type: string; + name: string; + displayName?: string; + deleted?: boolean; + description?: string; + fullyQualifiedName?: string; + href?: string; + inherited?: boolean; +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts index d66920a060b2..44e0eb9036fb 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/EntityDataClass.ts @@ -17,6 +17,15 @@ import { GlossaryTerm } from '../glossary/GlossaryTerm'; import { TagClass } from '../tag/TagClass'; import { TeamClass } from '../team/TeamClass'; import { UserClass } from '../user/UserClass'; +import { ApiCollectionClass } from './ApiCollectionClass'; +import { ContainerClass } from './ContainerClass'; +import { DashboardClass } from './DashboardClass'; +import { DashboardDataModelClass } from './DashboardDataModelClass'; +import { MlModelClass } from './MlModelClass'; +import { PipelineClass } from './PipelineClass'; +import { SearchIndexClass } from './SearchIndexClass'; +import { TableClass } from './TableClass'; +import { TopicClass } from './TopicClass'; export class EntityDataClass { static readonly domain1 = new Domain(); @@ -31,6 +40,25 @@ export class EntityDataClass { static readonly team1 = new TeamClass(); static readonly team2 = new TeamClass(); static readonly tierTag1 = new TagClass({ classification: 'Tier' }); + static readonly tierTag2 = new TagClass({ classification: 'Tier' }); + static readonly table1 = new TableClass(); + static readonly table2 = new TableClass(undefined, 'MaterializedView'); + static readonly topic1 = new TopicClass(); + static readonly topic2 = new TopicClass(); + static readonly dashboard1 = new DashboardClass(); + static readonly dashboard2 = new DashboardClass(undefined, 'LookMlExplore'); + static readonly mlModel1 = new MlModelClass(); + static readonly mlModel2 = new MlModelClass(); + static readonly pipeline1 = new PipelineClass(); + static readonly pipeline2 = new PipelineClass(); + static readonly dashboardDataModel1 = new DashboardDataModelClass(); + static readonly dashboardDataModel2 = new DashboardDataModelClass(); + static readonly apiCollection1 = new ApiCollectionClass(); + static readonly apiCollection2 = new ApiCollectionClass(); + static readonly searchIndex1 = new SearchIndexClass(); + static readonly searchIndex2 = new SearchIndexClass(); + static readonly container1 = new ContainerClass(); + static readonly container2 = new ContainerClass(); static async preRequisitesForTests(apiContext: APIRequestContext) { // Add pre-requisites for tests @@ -46,6 +74,25 @@ export class EntityDataClass { await this.team1.create(apiContext); await this.team2.create(apiContext); await this.tierTag1.create(apiContext); + await this.tierTag2.create(apiContext); + await this.table1.create(apiContext); + await this.table2.create(apiContext); + await this.topic1.create(apiContext); + await this.topic2.create(apiContext); + await this.dashboard1.create(apiContext); + await this.dashboard2.create(apiContext); + await this.mlModel1.create(apiContext); + await this.mlModel2.create(apiContext); + await this.pipeline1.create(apiContext); + await this.pipeline2.create(apiContext); + await this.dashboardDataModel1.create(apiContext); + await this.dashboardDataModel2.create(apiContext); + await this.apiCollection1.create(apiContext); + await this.apiCollection2.create(apiContext); + await this.searchIndex1.create(apiContext); + await this.searchIndex2.create(apiContext); + await this.container1.create(apiContext); + await this.container2.create(apiContext); } static async postRequisitesForTests(apiContext: APIRequestContext) { @@ -61,5 +108,24 @@ export class EntityDataClass { await this.team1.delete(apiContext); await this.team2.delete(apiContext); await this.tierTag1.delete(apiContext); + await this.tierTag2.delete(apiContext); + await this.table1.delete(apiContext); + await this.table2.delete(apiContext); + await this.topic1.delete(apiContext); + await this.topic2.delete(apiContext); + await this.dashboard1.delete(apiContext); + await this.dashboard2.delete(apiContext); + await this.mlModel1.delete(apiContext); + await this.mlModel2.delete(apiContext); + await this.pipeline1.delete(apiContext); + await this.pipeline2.delete(apiContext); + await this.dashboardDataModel1.delete(apiContext); + await this.dashboardDataModel2.delete(apiContext); + await this.apiCollection1.delete(apiContext); + await this.apiCollection2.delete(apiContext); + await this.searchIndex1.delete(apiContext); + await this.searchIndex2.delete(apiContext); + await this.container1.delete(apiContext); + await this.container2.delete(apiContext); } } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts index b959705e3075..119ee9401dae 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/MlModelClass.ts @@ -44,6 +44,11 @@ export class MlModelClass extends EntityClass { dataType: 'numerical', description: 'Sales amount', }, + { + name: 'persona', + dataType: 'categorical', + description: 'type of buyer', + }, ]; entity = { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts index 6f82dd496267..e7c461ae2afc 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/PipelineClass.ts @@ -39,7 +39,10 @@ export class PipelineClass extends EntityClass { }, }; - children = [{ name: 'snowflake_task' }]; + children = [ + { name: 'snowflake_task', displayName: 'Snowflake Task' }, + { name: 'presto_task', displayName: 'Presto Task' }, + ]; entity = { name: this.pipelineName, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts index 46bd8b9c61df..41efd732e915 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/SearchIndexClass.ts @@ -45,43 +45,45 @@ export class SearchIndexClass extends EntityClass { children = [ { - name: 'name', + name: `name${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Table Entity Name.', - fullyQualifiedName: `${this.fqn}.name`, tags: [], }, { - name: 'description', + name: `databaseSchema${uuid()}`, + dataType: 'TEXT', + dataTypeDisplay: 'text', + description: 'Table Entity Database Schema.', + tags: [], + }, + { + name: `description${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Table Entity Description.', - fullyQualifiedName: `${this.fqn}.description`, tags: [], }, { - name: 'columns', + name: `columns${uuid()}`, dataType: 'NESTED', dataTypeDisplay: 'nested', description: 'Table Columns.', - fullyQualifiedName: `${this.fqn}.columns`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Column Name.', - fullyQualifiedName: `${this.fqn}.columns.name`, tags: [], }, { - name: 'description', + name: `description${uuid()}`, dataType: 'TEXT', dataTypeDisplay: 'text', description: 'Column Description.', - fullyQualifiedName: `${this.fqn}.columns.description`, tags: [], }, ], @@ -104,7 +106,7 @@ export class SearchIndexClass extends EntityClass { this.service.name = name ?? this.service.name; this.type = 'SearchIndex'; this.childrenTabId = 'fields'; - this.childrenSelectorId = this.children[0].fullyQualifiedName; + this.childrenSelectorId = `${this.fqn}.${this.children[0].name}`; this.serviceCategory = SERVICE_TYPE.Search; this.serviceType = ServiceTypes.SEARCH_SERVICES; } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts index 130981f888a3..6b8abfd6bd77 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts @@ -56,28 +56,28 @@ export class TableClass extends EntityClass { }; children = [ { - name: 'user_id', + name: `user_id${uuid()}`, dataType: 'NUMERIC', dataTypeDisplay: 'numeric', description: 'Unique identifier for the user of your Shopify POS or your Shopify admin.', }, { - name: 'shop_id', + name: `shop_id${uuid()}`, dataType: 'NUMERIC', dataTypeDisplay: 'numeric', description: 'The ID of the store. This column is a foreign key reference to the shop_id column in the dim.shop table.', }, { - name: 'name', + name: `name${uuid()}`, dataType: 'VARCHAR', dataLength: 100, dataTypeDisplay: 'varchar', description: 'Name of the staff member.', children: [ { - name: 'first_name', + name: `first_name${uuid()}`, dataType: 'STRUCT', dataLength: 100, dataTypeDisplay: @@ -85,7 +85,7 @@ export class TableClass extends EntityClass { description: 'First name of the staff member.', }, { - name: 'last_name', + name: `last_name${uuid()}`, dataType: 'ARRAY', dataLength: 100, dataTypeDisplay: 'array>>', @@ -93,7 +93,7 @@ export class TableClass extends EntityClass { ], }, { - name: 'email', + name: `email${uuid()}`, dataType: 'VARCHAR', dataLength: 100, dataTypeDisplay: 'varchar', @@ -106,6 +106,7 @@ export class TableClass extends EntityClass { displayName: `pw table ${uuid()}`, description: 'description', columns: this.children, + tableType: 'SecureView', databaseSchema: `${this.service.name}.${this.database.name}.${this.schema.name}`, }; @@ -122,13 +123,14 @@ export class TableClass extends EntityClass { queryResponseData: ResponseDataType[] = []; additionalEntityTableResponseData: ResponseDataType[] = []; - constructor(name?: string) { + constructor(name?: string, tableType?: string) { super(EntityTypeEndpoint.Table); this.service.name = name ?? this.service.name; this.serviceCategory = SERVICE_TYPE.Database; this.serviceType = ServiceTypes.DATABASE_SERVICES; this.type = 'Table'; this.childrenTabId = 'schema'; + this.entity.tableType = tableType ?? this.entity.tableType; this.childrenSelectorId = `${this.entity.databaseSchema}.${this.entity.name}.${this.children[0].name}`; } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts index b223f22d4fc2..aa7c6458874b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TopicClass.ts @@ -39,32 +39,27 @@ export class TopicClass extends EntityClass { }, }; private topicName = `pw-topic-${uuid()}`; - private fqn = `${this.service.name}.${this.topicName}`; children = [ { - name: 'default', + name: `default${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default`, tags: [], children: [ { - name: 'name', + name: `name${uuid()}`, dataType: 'RECORD', - fullyQualifiedName: `${this.fqn}.default.name`, tags: [], children: [ { name: 'first_name', dataType: 'STRING', description: 'Description for schema field first_name', - fullyQualifiedName: `${this.fqn}.default.name.first_name`, tags: [], }, { name: 'last_name', dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.name.last_name`, tags: [], }, ], @@ -72,17 +67,21 @@ export class TopicClass extends EntityClass { { name: 'age', dataType: 'INT', - fullyQualifiedName: `${this.fqn}.default.age`, tags: [], }, { name: 'club_name', dataType: 'STRING', - fullyQualifiedName: `${this.fqn}.default.club_name`, tags: [], }, ], }, + { + name: `secondary${uuid()}`, + dataType: 'RECORD', + tags: [], + children: [], + }, ]; entity = { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts index 1c5db09f8e6c..9ac173401452 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/glossary/Glossary.ts @@ -17,6 +17,7 @@ import { uuid, visitGlossaryPage, } from '../../utils/common'; +import { EntityReference } from '../entity/Entity.interface'; import { GlossaryData, GlossaryResponseDataType } from './Glossary.interface'; export class Glossary { @@ -38,8 +39,9 @@ export class Glossary { responseData: GlossaryResponseDataType = {} as GlossaryResponseDataType; - constructor(name?: string) { + constructor(name?: string, reviewers?: EntityReference[]) { this.data.name = name ?? this.data.name; + this.data.reviewers = reviewers ?? this.data.reviewers; } async visitPage(page: Page) { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts index b8d911fa594e..9c7a1f0242f6 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts @@ -63,6 +63,82 @@ export const FIELDS: EntityFields[] = [ localSearch: false, skipConditions: ['isNull', 'isNotNull'], // Null and isNotNull conditions are not present for display name }, + { + id: 'Service Type', + name: 'serviceType', + localSearch: false, + }, + { + id: 'Schema Field', + name: 'messageSchema.schemaFields.name.keyword', + localSearch: false, + }, + { + id: 'Container Column', + name: 'dataModel.columns.name.keyword', + localSearch: false, + }, + { + id: 'Data Model Type', + name: 'dataModelType', + localSearch: false, + }, + { + id: 'Field', + name: 'fields.name.keyword', + localSearch: false, + }, + { + id: 'Task', + name: 'tasks.displayName.keyword', + localSearch: false, + }, + { + id: 'Domain', + name: 'domain.displayName.keyword', + localSearch: false, + }, + { + id: 'Name', + name: 'name.keyword', + localSearch: false, + skipConditions: ['isNull', 'isNotNull'], // Null and isNotNull conditions are not present for name + }, + { + id: 'Project', + name: 'project.keyword', + localSearch: false, + }, + { + id: 'Status', + name: 'status', + localSearch: false, + }, + { + id: 'Table Type', + name: 'tableType', + localSearch: false, + }, + { + id: 'Entity Type', + name: 'entityType', + localSearch: false, + }, + { + id: 'Chart', + name: 'charts.displayName.keyword', + localSearch: false, + }, + { + id: 'Response Schema Field', + name: 'responseSchema.schemaFields.name.keyword', + localSearch: false, + }, + { + id: 'Request Schema Field', + name: 'requestSchema.schemaFields.name.keyword', + localSearch: false, + }, ]; export const OPERATOR = { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts index f9806afb0c2d..7ed13fc4c0ce 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/common.ts @@ -116,8 +116,6 @@ export const toastNotification = async ( page: Page, message: string | RegExp ) => { - await page.waitForSelector('[data-testid="alert-bar"]', { state: 'visible' }); - await expect(page.getByTestId('alert-bar')).toHaveText(message); await expect(page.getByTestId('alert-icon')).toBeVisible(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customMetric.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customMetric.ts index a1febd08a26a..bd73acea4858 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customMetric.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customMetric.ts @@ -169,7 +169,5 @@ export const deleteCustomMetric = async ({ await deleteMetricResponse; // Verifying the deletion - await expect(page.getByRole('alert').first()).toHaveText( - `"${metric.name}" deleted successfully!` - ); + await toastNotification(page, `"${metric.name}" deleted successfully!`); }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts index 8dea2d06f995..6262005adab6 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/domain.ts @@ -344,14 +344,26 @@ export const addAssetsToDomain = async ( } const assetsAddRes = page.waitForResponse(`/api/v1/domains/*/assets/add`); + const searchRes = page.waitForResponse((response) => { + const url = new URL(response.url()); + const queryParams = new URLSearchParams(url.search); + const queryFilter = queryParams.get('query_filter'); + + return ( + response + .url() + .includes('/api/v1/search/query?q=**&index=all&from=0&size=15') && + queryFilter !== null && + queryFilter !== '' + ); + }); await page.getByTestId('save-btn').click(); await assetsAddRes; - const countRes = page.waitForResponse( - '/api/v1/search/query?q=*&index=all&from=0&size=15' - ); + await searchRes; + await page.reload(); - await countRes; + await page.waitForLoadState('networkidle'); await checkAssetsCount(page, assets.length); }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts index 4d5eb618beb0..542dd0533ef8 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/entity.ts @@ -64,7 +64,7 @@ export const addOwner = async ({ await page.getByTestId(initiatorId).click(); if (type === 'Users') { const userListResponse = page.waitForResponse( - '/api/v1/users?limit=*&isBot=false*' + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' ); await page.getByRole('tab', { name: type }).click(); await userListResponse; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts index fe36cca5a152..2f195d73b43b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts @@ -1094,6 +1094,7 @@ export const approveTagsTask = async ( await sidebarClick(page, SidebarItem.GLOSSARY); await glossaryTermsResponse; await selectActiveGlossary(page, entity.data.displayName); + await page.waitForLoadState('networkidle'); const tagVisibility = await page.isVisible( `[data-testid="tag-${value.tag}"]` diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts index 3096c2062347..e5906f554469 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts @@ -63,7 +63,7 @@ export const fillOwnerDetails = async (page: Page, owners: string[]) => { .press('Enter', { delay: 100 }); const userListResponse = page.waitForResponse( - '/api/v1/users?limit=*&isBot=false*' + '/api/v1/search/query?q=*isBot:false*index=user_search_index*' ); await page.getByRole('tab', { name: 'Users' }).click(); await userListResponse; @@ -75,7 +75,7 @@ export const fillOwnerDetails = async (page: Page, owners: string[]) => { await page.locator('[data-testid="owner-select-users-search-bar"]').clear(); await page.keyboard.type(owner); await page.waitForResponse( - `/api/v1/search/query?q=*${owner}*%20AND%20isBot:false&from=0&size=25&index=user_search_index` + `/api/v1/search/query?q=*${owner}*%20AND%20isBot:false*index=user_search_index*` ); await page.getByRole('listitem', { name: owner }).click(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts index 314411d1ff10..56fe71a9f66b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts @@ -15,6 +15,7 @@ import { expect, Page } from '@playwright/test'; import { GlobalSettingOptions } from '../constant/settings'; import { EntityTypeEndpoint } from '../support/entity/Entity.interface'; import { toastNotification } from './common'; +import { escapeESReservedCharacters } from './entity'; export enum Services { Database = GlobalSettingOptions.DATABASES, @@ -78,8 +79,16 @@ export const deleteService = async ( serviceName: string, page: Page ) => { + const serviceResponse = page.waitForResponse( + `/api/v1/search/query?q=*${encodeURIComponent( + escapeESReservedCharacters(serviceName) + )}*` + ); + await page.fill('[data-testid="searchbar"]', serviceName); + await serviceResponse; + // click on created service await page.click(`[data-testid="service-name-${serviceName}"]`); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts index 471f607a0ea6..8cb2c836978b 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/team.ts @@ -303,7 +303,7 @@ export const verifyAssetsInTeamsPage = async ( .locator(`a:has-text("${team.data.displayName}")`) .click(); - const res = page.waitForResponse('/api/v1/search/query?*size=15'); + const res = page.waitForResponse('/api/v1/search/query?*size=15*'); await page.getByTestId('assets').click(); await res; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/testCases.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/testCases.ts index 61b56308b2b9..922c83c10743 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/testCases.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/testCases.ts @@ -12,6 +12,7 @@ */ import { expect, Page } from '@playwright/test'; import { TableClass } from '../support/entity/TableClass'; +import { toastNotification } from './common'; export const deleteTestCase = async (page: Page, testCaseName: string) => { await page.getByTestId(`delete-${testCaseName}`).click(); @@ -25,7 +26,7 @@ export const deleteTestCase = async (page: Page, testCaseName: string) => { await page.getByTestId('confirm-button').click(); await deleteResponse; - await expect(page.getByRole('alert')).toHaveText(/deleted successfully!/); + await toastNotification(page, /deleted successfully!/); }; export const visitDataQualityTab = async (page: Page, table: TableClass) => { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/userDetails.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/userDetails.ts new file mode 100644 index 000000000000..cf8caa7b0fd0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/userDetails.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Page } from '@playwright/test'; +import { clickOutside, redirectToHomePage } from './common'; + +export const redirectToUserPage = async (page: Page) => { + await redirectToHomePage(page); + + await page.getByTestId('dropdown-profile').click(); + + // Hover on the profile avatar to close the name tooltip + await page.getByTestId('profile-avatar').hover(); + + await page.waitForSelector('.profile-dropdown', { state: 'visible' }); + + const getUserDetails = page.waitForResponse(`/api/v1/users/name/*`); + + await page.locator('.profile-dropdown').getByTestId('user-name').click(); + + await getUserDetails; + + // Close the profile dropdown + await clickOutside(page); +}; diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsApplication.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsApplication.md new file mode 100644 index 000000000000..9e37db76e871 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsApplication.md @@ -0,0 +1,29 @@ +# DataInsightsAppConfig + +This schema defines configuration for the Data Insights Application. + +$$section +### Application Type $(id="type") + +Application Type + +$$ + +$$section +### batchSize $(id="batchSize") + +Maximum number of events processed at a time (Default 100). + +$$ + +$$section +### Recreate DataInsights DataAssets Index $(id="recreateDataAssetsIndex") + +Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property Type and are facing errors. Bear in mind that recreating the index will delete your DataAssets and a backfill will be needed. + +$$ + +$$section +### backfillConfiguration $(id="backfillConfiguration") + +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsReportApplication.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsReportApplication.md new file mode 100644 index 000000000000..61783b4a3954 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataInsightsReportApplication.md @@ -0,0 +1,13 @@ +# Data Insights Report Application + +This schema defines configuration for Data Insights Report Application. + +$$section +### Send To Admins $(id="sendToAdmins") + +$$ + +$$section +### Send To Teams $(id="sendToTeams") + +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataRetentionApplication.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataRetentionApplication.md new file mode 100644 index 000000000000..b40cc1335913 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/DataRetentionApplication.md @@ -0,0 +1,10 @@ +# Retention Configuration + +Configure retention policies for each entity. + +$$section +### Change Event Retention Period (days) $(id="changeEventRetentionPeriod") + +Enter the retention period for change event records in days (e.g., 7 for one week, 30 for one month). + +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/SearchIndexingApplication.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/SearchIndexingApplication.md new file mode 100644 index 000000000000..572d07f432cb --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Applications/SearchIndexingApplication.md @@ -0,0 +1,85 @@ +# Search Indexing Application + +This schema defines configuration for Search Reindexing Application. + +$$section +### Batch Size $(id="batchSize") + +Maximum number of events entities in a batch (Default 100). + +$$ + +$$section +### Payload Size $(id="payLoadSize") + +Maximum number of events entities in a batch (Default 100). + +$$ + +$$section +### Number of Producer Threads $(id="producerThreads") + +Number of threads to use for reindexing + +$$ + +$$section +### Number of Consumer Threads $(id="consumerThreads") + +Number of threads to use for reindexing + +$$ + +$$section +### Queue Size to use. $(id="queueSize") + +Queue Size to use internally for reindexing. + +$$ + +$$section +### Max Concurrent Requests $(id="maxConcurrentRequests") + +Maximum number of concurrent requests to the search index + +$$ + +$$section +### Max Retries $(id="maxRetries") + +Maximum number of retries for a failed request + +$$ + +$$section +### Initial Backoff Millis $(id="initialBackoff") + +Initial backoff time in milliseconds + +$$ + +$$section +### Max Backoff Millis $(id="maxBackoff") + +Maximum backoff time in milliseconds + +$$ + +$$section +### Entities $(id="entities") + +List of entities that you need to reindex + +$$ + +$$section +### Recreate Indexes $(id="recreateIndex") + +$$ + +$$section +### Search Index Language $(id="searchIndexMappingLanguage") + +Recreate Indexes with updated Language + +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx index fae6f2ad736e..53506b58890b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppBar/Appbar.tsx @@ -20,6 +20,7 @@ import { useTourProvider } from '../../context/TourProvider/TourProvider'; import { CurrentTourPageType } from '../../enums/tour.enum'; import { useApplicationStore } from '../../hooks/useApplicationStore'; import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation'; +import TokenService from '../../utils/Auth/TokenService/TokenServiceUtil'; import { extractDetailsFromToken, isProtectedRoute, @@ -38,8 +39,7 @@ const Appbar: React.FC = (): JSX.Element => { const { isTourOpen, updateTourPage, updateTourSearch, tourSearchValue } = useTourProvider(); - const { isAuthenticated, searchCriteria, trySilentSignIn } = - useApplicationStore(); + const { isAuthenticated, searchCriteria } = useApplicationStore(); const parsedQueryString = Qs.parse( location.search.startsWith('?') @@ -126,7 +126,7 @@ const Appbar: React.FC = (): JSX.Element => { const { isExpired } = extractDetailsFromToken(getOidcToken()); if (!document.hidden && isExpired) { // force logout - trySilentSignIn(true); + TokenService.getInstance().refreshToken(); } }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx index 27239dfbc370..d062a08981c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AppTour/Tour.tsx @@ -67,7 +67,7 @@ const Tour = ({ steps }: { steps: TourSteps[] }) => { } maskColor="#302E36" playTour={isTourOpen} - stepWaitTimer={300} + stepWaitTimer={900} steps={steps} onRequestClose={handleRequestClose} onRequestSkip={handleModalSubmit} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx index c3687226679e..470af128a46e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AppAuthenticators/MsalAuthenticator.tsx @@ -124,7 +124,7 @@ const MsalAuthenticator = forwardRef( }; const renewIdToken = async () => { - const user = await fetchIdToken(true); + const user = await fetchIdToken(); return user.id_token; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx index 67f2c38587f6..3115099ca1ae 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx @@ -184,6 +184,9 @@ export const AuthProvider = ({ setApplicationLoading(false); + // Clear the refresh flag (used after refresh is complete) + tokenService.current.clearRefreshInProgress(); + // Upon logout, redirect to the login page history.push(ROUTES.SIGNIN); }, [timeoutId]); @@ -517,7 +520,9 @@ export const AuthProvider = ({ if (error.response) { const { status } = error.response; if (status === ClientErrors.UNAUTHORIZED) { - if (error.config.url === '/users/refresh') { + // For login or refresh we don't want to fire another refresh req + // Hence rejecting it + if (['/users/refresh', '/users/login'].includes(error.config.url)) { return Promise.reject(error as Error); } handleStoreProtectedRedirectPath(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx index 003edda57f2f..0f6ec886d9bd 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -131,14 +131,6 @@ export const AssetSelectionModal = ({ >([]); const [quickFilterQuery, setQuickFilterQuery] = useState(); - const [updatedQueryFilter, setUpdatedQueryFilter] = - useState( - getCombinedQueryFilterObject(queryFilter as QueryFilterInterface, { - query: { - bool: {}, - }, - }) - ); const [selectedFilter, setSelectedFilter] = useState([]); const [filters, setFilters] = useState([]); @@ -237,13 +229,17 @@ export const AssetSelectionModal = ({ useEffect(() => { if (open) { + const combinedQueryFilter = getCombinedQueryFilterObject( + queryFilter as unknown as QueryFilterInterface, + quickFilterQuery as QueryFilterInterface + ); fetchEntities({ index: activeFilter, searchText: search, - updatedQueryFilter, + updatedQueryFilter: combinedQueryFilter, }); } - }, [open, activeFilter, search, type, updatedQueryFilter]); + }, [open, activeFilter, search, type, quickFilterQuery, queryFilter]); useEffect(() => { if (open) { @@ -357,18 +353,6 @@ export const AssetSelectionModal = ({ handleSave(); }, [type, handleSave]); - const mergeFilters = useCallback(() => { - const res = getCombinedQueryFilterObject( - queryFilter as QueryFilterInterface, - quickFilterQuery as QueryFilterInterface - ); - setUpdatedQueryFilter(res); - }, [queryFilter, quickFilterQuery]); - - useEffect(() => { - mergeFilters(); - }, [quickFilterQuery, queryFilter]); - useEffect(() => { const updatedQuickFilters = filters .filter((filter) => selectedFilter.includes(filter.key)) @@ -402,22 +386,28 @@ export const AssetSelectionModal = ({ scrollHeight < 501 && items.length < totalCount ) { + const combinedQueryFilter = getCombinedQueryFilterObject( + queryFilter as unknown as QueryFilterInterface, + quickFilterQuery as QueryFilterInterface + ); + !isLoading && fetchEntities({ searchText: search, page: pageNumber + 1, index: activeFilter, - updatedQueryFilter, + updatedQueryFilter: combinedQueryFilter, }); } }, [ pageNumber, - updatedQueryFilter, activeFilter, search, totalCount, items, + quickFilterQuery, + queryFilter, isLoading, fetchEntities, ] diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts index d24ea97dbfe1..5d69fbf24120 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.interface.ts @@ -27,6 +27,7 @@ export interface FetchOptionsResponse { export interface DataAssetAsyncSelectListProps { mode?: 'multiple'; + autoFocus?: boolean; id?: string; className?: string; placeholder?: string; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx index e2872eacdcb7..5b029e749de1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetAsyncSelectList/DataAssetAsyncSelectList.tsx @@ -36,6 +36,7 @@ import { const DataAssetAsyncSelectList: FC = ({ mode, + autoFocus = true, onChange, debounceTimeout = 800, initialOptions, @@ -242,8 +243,8 @@ const DataAssetAsyncSelectList: FC = ({ return (