Skip to content

Commit

Permalink
Update system test for Dataflow
Browse files Browse the repository at this point in the history
  • Loading branch information
MaksYermak committed Dec 7, 2023
1 parent b946c7d commit 4f8be01
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 112 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@

import os
from datetime import datetime
from pathlib import Path

from airflow.models.dag import DAG
from airflow.providers.apache.beam.hooks.beam import BeamRunnerType
Expand All @@ -34,20 +33,18 @@
PubSubCreateTopicOperator,
PubSubDeleteTopicOperator,
)
from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
DAG_ID = "dataflow_native_python_streaming"

RESOURCE_DATA_BUCKET = "airflow-system-tests-resources"
BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"

PYTHON_FILE_NAME = "streaming_wordcount.py"
GCS_TMP = f"gs://{BUCKET_NAME}/temp/"
GCS_STAGING = f"gs://{BUCKET_NAME}/staging/"
GCS_PYTHON_SCRIPT = f"gs://{BUCKET_NAME}/{PYTHON_FILE_NAME}"
PYTHON_FILE_LOCAL_PATH = str(Path(__file__).parent / "resources" / PYTHON_FILE_NAME)
GCS_PYTHON_SCRIPT = f"gs://{RESOURCE_DATA_BUCKET}/dataflow/python/streaming_wordcount.py"
LOCATION = "europe-west3"
TOPIC_ID = f"topic-{DAG_ID}"

Expand All @@ -68,13 +65,6 @@
) as dag:
create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME)

upload_file = LocalFilesystemToGCSOperator(
task_id="upload_file_to_bucket",
src=PYTHON_FILE_LOCAL_PATH,
dst=PYTHON_FILE_NAME,
bucket=BUCKET_NAME,
)

create_pub_sub_topic = PubSubCreateTopicOperator(
task_id="create_topic", topic=TOPIC_ID, project_id=PROJECT_ID, fail_if_exists=False
)
Expand Down Expand Up @@ -114,7 +104,6 @@
(
# TEST SETUP
create_bucket
>> upload_file
>> create_pub_sub_topic
# TEST BODY
>> start_streaming_python_job
Expand Down

This file was deleted.

0 comments on commit 4f8be01

Please sign in to comment.