Merge branch 'main' into improve-certification-feed

.github/actions/prepare-for-docker-build/action.yml

@@ -0,0 +1,59 @@
+name: Prepare for Docker Build
+description: Set up Docker Build dependencies (without pushing) and run Maven build
+
+inputs:
+  image:
+    description: Image name
+    required: true
+  tag:
+    description: Docker tag to use
+    required: true
+  is_ingestion:
+    description: true if we are building an Ingestion image, false otherwise
+    required: true
+    default: "false"
+  release_version:
+    description: OpenMetadata Release Version
+
+outputs:
+  tags:
+    description: Generated Docker Tags
+    value: ${{ steps.meta.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - name: Install Ubuntu dependencies
+      shell: bash
+      run: |
+        # stop relying on apt cache of GitHub runners
+        sudo apt-get update
+        sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
+        librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev jq
+    - name: Set up JDK 17
+      if: inputs.is_ingestion == 'false'
+      uses: actions/setup-java@v4
+      with:
+        java-version: 17
+        distribution: 'temurin'
+
+    - name: Install antlr cli
+      shell: bash
+      run: |
+        sudo make install_antlr_cli
+
+    - name: Build OpenMetadata Server Application
+      if: inputs.is_ingestion == 'false'
+      shell: bash
+      run: |
+        mvn -DskipTests clean package
+
+    - name: Install OpenMetadata Ingestion Dependencies
+      if: inputs.is_ingestion == 'true'
+      shell: bash
+      run: |
+        python3 -m venv env
+        source env/bin/activate
+        pip install --upgrade pip
+        sudo make install_antlr_cli
+        make install_dev generate

.github/trivy/templates/github.tpl

@@ -0,0 +1,35 @@
+{{- range . }}
+  <h2> 🛡️ TRIVY SCAN RESULT 🛡️ </h2>
+  <h4> Target: <code>{{ .Target }}</code></h4>
+  {{- if .Vulnerabilities }}
+    <h4>Vulnerabilities ({{ len .Vulnerabilities }})</h4>
+    <table border="1" cellspacing="0" cellpadding="5">
+      <thead>
+        <tr>
+          <th>Package</th>
+          <th>Vulnerability ID</th>
+          <th>Severity</th>
+          <th>Installed Version</th>
+          <th>Fixed Version</th>
+        </tr>
+      </thead>
+      <tbody>
+      {{- range .Vulnerabilities }}
+        <tr>
+          <td><code>{{ .PkgName }}</code></td>
+          <td><a href="{{ .PrimaryURL }}" target="_blank">{{ .VulnerabilityID }}</a></td>
+          <td>
+            {{- if eq .Severity "CRITICAL" }} 🔥 CRITICAL 
+            {{- else if eq .Severity "HIGH" }} 🚨 HIGH 
+            {{- else }} {{ .Severity }} {{- end }}
+          </td>
+          <td>{{ .InstalledVersion }}</td>
+          <td>{{ if .FixedVersion }}{{ .FixedVersion }}{{ else }}N/A{{ end }}</td>
+        </tr>
+      {{- end }}
+      </tbody>
+    </table>
+  {{- else }}
+    <h4>No Vulnerabilities Found</h4>
+  {{- end }}
+{{- end }}

.github/workflows/py-cli-e2e-tests.yml

@@ -65,7 +65,7 @@ jobs:
 
     - name: configure aws credentials
       if: contains('quicksight', matrix.e2e-test) || contains('datalake_s3', matrix.e2e-test) || contains('athena', matrix.e2e-test)
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v4
       with:
         role-to-assume: ${{ secrets.E2E_AWS_IAM_ROLE_ARN }}
         role-session-name: github-ci-aws-e2e-tests

.github/workflows/py-tests.yml

@@ -55,7 +55,7 @@ jobs:
         docker-images: false
 
     - name: Wait for the labeler
-      uses: lewagon/[email protected].3
+      uses: lewagon/[email protected].4
       if: ${{ github.event_name == 'pull_request_target' }}
       with:
         ref: ${{ github.event.pull_request.head.sha }}

.github/workflows/trivy-scan-ingestion-base-slim-image.yml

@@ -0,0 +1,81 @@
+name: Trivy Scan For OpenMetadata Ingestion Base Slim Docker Image
+
+on:
+    pull_request_target:
+        types: [labeled, opened, synchronize, reopened]
+        paths:
+          - "ingestion/**"
+          - "openmetadata-service/**"
+          - "openmetadata-spec/src/main/resources/json/schema/**"
+          - "pom.xml"
+          - "Makefile"
+
+concurrency: 
+  group: trivy-ingestion-base-slim-scan-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-scan:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Wait for the labeler
+        uses: lewagon/[email protected]
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: Team Label
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 90
+
+      - name: Verify PR labels
+        uses: jesusvasquez333/[email protected]
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'safe to test'
+          pull-request-number: '${{ github.event.pull_request.number }}'
+          disable-reviews: true  # To not auto approve changes
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Prepare for Docker Build
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build
+        with:
+          image: openmetadata-ingestion-base-slim
+          tag: trivy
+          is_ingestion: true
+
+
+      - name: Build Docker Image
+        run: |
+          docker build -t openmetadata-ingestion-base-slim:trivy -f ingestion/operators/docker/Dockerfile.ci .
+
+
+      - name: Run Trivy Image Scan
+        id: trivy_scan
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: "image"
+          image-ref: openmetadata-ingestion-base-slim:trivy
+          hide-progress: false
+          ignore-unfixed: true
+          severity: "HIGH,CRITICAL"
+          skip-dirs: "/opt/airflow/dags,/home/airflow/ingestion/pipelines"
+          scan-ref: .
+          format: 'template'
+          template: "@.github/trivy/templates/github.tpl"
+          output: "trivy-result-ingestion-base-slim.md"
+        env:
+          TRIVY_DISABLE_VEX_NOTICE: "true"
+
+      - name: Comment Trivy Scan Results on PR
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          path: trivy-result-ingestion-base-slim.md
+          header: "trivy-scan-${{ github.workflow }}"

.github/workflows/trivy-scan-ingestion-image.yml

@@ -0,0 +1,81 @@
+name: Trivy Scan For OpenMetadata Ingestion Docker Image
+
+on:
+    pull_request_target:
+        types: [labeled, opened, synchronize, reopened]
+        paths:
+          - "ingestion/**"
+          - "openmetadata-service/**"
+          - "openmetadata-spec/src/main/resources/json/schema/**"
+          - "pom.xml"
+          - "Makefile"
+
+concurrency: 
+  group: trivy-ingestion-scan-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-scan:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Wait for the labeler
+        uses: lewagon/[email protected]
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: Team Label
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 90
+
+      - name: Verify PR labels
+        uses: jesusvasquez333/[email protected]
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'safe to test'
+          pull-request-number: '${{ github.event.pull_request.number }}'
+          disable-reviews: true  # To not auto approve changes
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Prepare for Docker Build
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build
+        with:
+          image: openmetadata-ingestion
+          tag: trivy
+          is_ingestion: true
+
+
+      - name: Build Docker Image
+        run: |
+          docker build -t openmetadata-ingestion:trivy -f ingestion/Dockerfile.ci .
+
+
+      - name: Run Trivy Image Scan
+        id: trivy_scan
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: "image"
+          image-ref: openmetadata-ingestion:trivy
+          hide-progress: false
+          ignore-unfixed: true
+          severity: "HIGH,CRITICAL"
+          skip-dirs: "/opt/airflow/dags,/home/airflow/ingestion/pipelines"
+          scan-ref: .
+          format: 'template'
+          template: "@.github/trivy/templates/github.tpl"
+          output: "trivy-results-ingestion.md"
+        env:
+          TRIVY_DISABLE_VEX_NOTICE: "true"
+
+      - name: Comment Trivy Scan Results on PR
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          path: trivy-results-ingestion.md
+          header: "trivy-scan-${{ github.workflow }}"

.github/workflows/trivy-scan-openmetadta-server.yml

@@ -0,0 +1,80 @@
+name: Trivy Scan For OpenMetadata Server Docker Image
+on:
+    pull_request_target:
+        types: [labeled, opened, synchronize, reopened]
+        paths:
+          - "openmetadata-service/**"
+          - "openmetadata-spec/src/main/resources/json/schema/**"
+          - "openmetadata-dist/**"
+          - "openmetadata-clients/**"
+          - "common/**"
+          - "pom.xml"
+          - "yarn.lock"
+          - "Makefile"
+          - "bootstrap/**"
+concurrency: 
+  group: trivy-server-scan-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+jobs:
+  build-and-scan:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Wait for the labeler
+        uses: lewagon/[email protected]
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: Team Label
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 90
+
+      - name: Verify PR labels
+        uses: jesusvasquez333/[email protected]
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'safe to test'
+          pull-request-number: '${{ github.event.pull_request.number }}'
+          disable-reviews: true  # To not auto approve changes
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Prepare for Docker Build
+        id: prepare
+        uses: ./.github/actions/prepare-for-docker-build
+        with:
+          image: openmetadata-server
+          tag: trivy
+          is_ingestion: false
+
+      - name: Build Docker Image
+        run: |
+          docker build -t openmetadata-server:trivy -f docker/development/Dockerfile .
+
+      - name: Run Trivy Image Scan
+        id: trivy_scan
+        uses: aquasecurity/trivy-action@master
+        with:
+            scan-type: "image"
+            image-ref: openmetadata-server:trivy
+            hide-progress: false
+            ignore-unfixed: true
+            severity: "HIGH,CRITICAL"
+            scan-ref: .
+            format: 'template'
+            template: "@.github/trivy/templates/github.tpl"
+            output: trivy-result-openmetadata-server.md
+        env:
+            TRIVY_DISABLE_VEX_NOTICE: "true"
+
+      - name: Comment Trivy Scan Results on PR
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+            path: trivy-result-openmetadata-server.md
+            header: "trivy-scan-${{ github.workflow }}"
+

README.md

@@ -26,7 +26,7 @@ Contents:
 - [Features](#key-features-of-openmetadata)
 - [Try our Sandbox](#try-our-sandbox)
 - [Install & Run](#install-and-run-openmetadata)
-- [Roadmap](https://docs.open-metadata.org/v1.3.x/roadmap)
+- [Roadmap](https://docs.open-metadata.org/latest/roadmap)
 - [Documentation and Support](#documentation-and-support)
 - [Contributors](#contributors)
 

docker/development/docker-compose.yml

@@ -492,7 +492,7 @@ services:
       DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
       DB_PORT: ${AIRFLOW_DB_PORT:-3306}
       AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
-      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
+      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
       DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
       DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
 

docker/docker-compose-ingestion/docker-compose-ingestion.yml

@@ -26,7 +26,7 @@ services:
       DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
       DB_PORT: ${AIRFLOW_DB_PORT:-3306}
       AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
-      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
+      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
       DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
       DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
       # extra connection-string properties for the database

docker/docker-compose-quickstart/docker-compose.yml

@@ -497,7 +497,7 @@ services:
       DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
       DB_PORT: ${AIRFLOW_DB_PORT:-3306}
       AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
-      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
+      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+mysqldb}
       DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
       DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
       # extra connection-string properties for the database