diff --git a/.drone.yml b/.drone.yml index 04af4068c37b5..98423531a5ae2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -20,4 +20,6 @@ steps: - mkdir ~/.ssh && echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519 && chmod 0600 ~/.ssh/id_ed25519 - ssh-keyscan -H github.com >> ~/.ssh/known_hosts - make -C .terra/superset init - - make -C .terra/superset validate \ No newline at end of file + - make -C .terra/superset validate + - make -C .terra/insights init + - make -C .terra/insights validate \ No newline at end of file diff --git a/.terra/insights/Makefile b/.terra/insights/Makefile new file mode 100644 index 0000000000000..77a642fb3f0c7 --- /dev/null +++ b/.terra/insights/Makefile @@ -0,0 +1,53 @@ +AWS_REGION:=$(if $(AWS_REGION),$(AWS_REGION),us-west-2) +ENVIRONMENT:=qa +GIT_SHA="$(shell git rev-parse --short=7 HEAD)" +TERRAFORM:=terraform +S3_BUCKET_PREFIX:=$(if $(S3_BUCKET_PREFIX),$(S3_BUCKET_PREFIX),zf-terraform) + +tf_files := $(shell find . -name "*.tf") + +all: plan + +check-fmt: + $(TERRAFORM) fmt -check=true + +format: + $(TERRAFORM) fmt + +init: + $(TERRAFORM) init \ + -backend-config="bucket=$(S3_BUCKET_PREFIX)-$(ENVIRONMENT)" \ + -backend-config="region=$(AWS_REGION)" + +validate: ${tf_files} + $(TERRAFORM) validate + +plan: .terraform/terraform.tfstate ${tf_files} + $(TERRAFORM) plan \ + -var "env=$(ENVIRONMENT)" \ + -var "git_sha=$(GIT_SHA)" \ + -out terra.plan + +apply: plan + $(TERRAFORM) apply terra.plan + +clean: + rm -f *.backup *.plan + rm -f .terraform/terraform.tfstate + +clobber: clean + rm -rf .terraform + +plan_destroy: .terraform/terraform.tfstate ${tf_files} + $(TERRAFORM) plan -destroy \ + -var "env=$(ENVIRONMENT)" \ + -var "git_sha=$(GIT_SHA)" \ + -out terra_destroy.plan + +destroy: plan_destroy + $(TERRAFORM) destroy \ + -var "env=$(ENVIRONMENT)" \ + -var "git_sha=$(GIT_SHA)" \ + -auto-approve + +.PHONY: all check-fmt format init validate plan apply clean clobber destroy plan_destroy \ No newline at end of file diff --git a/.terra/insights/celery_beat/celery_beat.nomad.hcl b/.terra/insights/celery_beat/celery_beat.nomad.hcl new file mode 100644 index 0000000000000..4036eb75d1a94 --- /dev/null +++ b/.terra/insights/celery_beat/celery_beat.nomad.hcl @@ -0,0 +1,106 @@ +group "celery-beat-group" { + count = "${count}" + + meta { + group = "$${NOMAD_GROUP_NAME}" + lang = "python" + } + + task "celery-beat" { + driver = "docker" + shutdown_delay = "10s" + + config { + image = "${ecr_url}/zf/insights:${git_sha}" + command = "celery" + args = [ + "--app=superset.tasks.celery_app:app", + "beat", + ] + force_pull = true + } + + resources { + cpu = 1024 + memory = 1024 + + network { + mbits = 1 + port "https" {} + } + } + + service { + name = "$${NOMAD_JOB_NAME}-$${NOMAD_GROUP_NAME}" + port = "https" + + check { + name = "$${NOMAD_JOB_NAME}-$${NOMAD_GROUP_NAME} up check" + type = "script" + command = "celery" + args = [ + "-A", + "superset.tasks.celery_app:app", + "inspect", + "ping" + ] + interval = "30s" + timeout = "10s" + } + } +template { + data = < None: credentials, _ = google.auth.default( @@ -29,7 +30,7 @@ def get_df(self, query: str) -> pd.DataFrame: raise ValueError("query is required") try: - df = gbq.read_gbq(query, project_id="csdataanalysis") + df = gbq.read_gbq(query, project_id=BQ_DATASET) return df except Exception as e: raise ValueError(f"Error getting dataframe from bigquery: {e}") @@ -46,11 +47,11 @@ def get_schema(self, schema_name: str, table_name: str) -> pd.DataFrame: column_name, data_type FROM - `csdataanalysis.{schema}.INFORMATION_SCHEMA.COLUMNS` + `{dataset}.{schema}.INFORMATION_SCHEMA.COLUMNS` WHERE 1 = 1 AND table_name='{table_name}' """.format( - schema=schema_name, table_name=table_name + dataset=BQ_DATASET, schema=schema_name, table_name=table_name ) try: diff --git a/bi_superset/bi_security_manager/services/dashboard_role_access_service.py b/bi_superset/bi_security_manager/services/dashboard_role_access_service.py index 9a73a51522b9c..6cd73484a10dc 100644 --- a/bi_superset/bi_security_manager/services/dashboard_role_access_service.py +++ b/bi_superset/bi_security_manager/services/dashboard_role_access_service.py @@ -5,6 +5,9 @@ from bi_superset.bi_security_manager.models.models import ( DashboardRoleAccessExternal, ) +from flask import current_app + +BQ_DATASET = current_app.config.get("BQ_DATASET") class DashboardRoleAccessService: @@ -16,7 +19,7 @@ def get_dashboard_role_access_external(self): Get dashboard role access for external users """ - query = DASHBOARD_ROLE_ACCESS_EXTERNAL + query = DASHBOARD_ROLE_ACCESS_EXTERNAL.format(dataset=BQ_DATASET) df = self.sql.get_df(query) diff --git a/bi_superset/bi_security_manager/services/data_source_permission_service.py b/bi_superset/bi_security_manager/services/data_source_permission_service.py index 73f8719de824b..080d32b148616 100644 --- a/bi_superset/bi_security_manager/services/data_source_permission_service.py +++ b/bi_superset/bi_security_manager/services/data_source_permission_service.py @@ -4,7 +4,9 @@ from bi_superset.bi_security_manager.models.models import DataSourceAccess from bi_superset.bi_security_manager.sql.queries import DATA_SOURCE_PERMISSIONS_QUERY from bi_superset.bi_security_manager.models.access_method import AccessMethod +from flask import current_app +BQ_DATASET = current_app.config.get("BQ_DATASET") class DataSourcePermissionService: """ @@ -28,7 +30,7 @@ def get_data_sources( raise ValueError("access_method is required") query = DATA_SOURCE_PERMISSIONS_QUERY.format( - role_name=role_name, access_method=access_method + dataset=BQ_DATASET, role_name=role_name, access_method=access_method ) df = self.sql.get_df(query) diff --git a/bi_superset/bi_security_manager/services/role_gatherer_service.py b/bi_superset/bi_security_manager/services/role_gatherer_service.py index 8a15a2eb91418..0773e406de4f1 100644 --- a/bi_superset/bi_security_manager/services/role_gatherer_service.py +++ b/bi_superset/bi_security_manager/services/role_gatherer_service.py @@ -7,10 +7,12 @@ SupersetRolePermission, ) from bi_superset.bi_security_manager.sql.queries import ( - ROLES_PER_JOB_TITLE, ROLE_DEFINITIONS_QUERY, ROLES_QUERY, ) +from flask import current_app + +BQ_DATASET = current_app.config.get("BQ_DATASET") class RoleGathererService: @@ -26,7 +28,7 @@ def get_roles(self, access_method: str) -> List[str]: if not any(method.value == access_method for method in AccessMethod): raise ValueError("access_method is required") - query = ROLES_QUERY.format(access_method=access_method) + query = ROLES_QUERY.format(dataset=BQ_DATASET, access_method=access_method) df = self.sql.get_df(query) @@ -64,7 +66,7 @@ def get_role_permission( raise ValueError("access_method is required") query = ROLE_DEFINITIONS_QUERY.format( - role_name=role_name, access_method=access_method + dataset=BQ_DATASET, role_name=role_name, access_method=access_method ) df = self.sql.get_df(query) diff --git a/bi_superset/bi_security_manager/services/roles_per_job_title_service.py b/bi_superset/bi_security_manager/services/roles_per_job_title_service.py index 2ffcac8d7ccd3..0ca6e34179086 100644 --- a/bi_superset/bi_security_manager/services/roles_per_job_title_service.py +++ b/bi_superset/bi_security_manager/services/roles_per_job_title_service.py @@ -3,8 +3,9 @@ from bi_superset.bi_security_manager.port.a_sql import ASql from bi_superset.bi_security_manager.sql.queries import ROLES_PER_JOB_TITLE from bi_superset.bi_security_manager.models.models import RolesPerJobTitle +from flask import current_app - +BQ_DATASET = current_app.config.get("BQ_DATASET") class RolePerJobTitleService: def __init__(self, sql: ASql): self.sql: ASql = sql @@ -19,7 +20,7 @@ def get_role_per_job_title(self): """ - query = ROLES_PER_JOB_TITLE + query = ROLES_PER_JOB_TITLE.format(dataset=BQ_DATASET) df = self.sql.get_df(query) diff --git a/bi_superset/bi_security_manager/sql/queries.py b/bi_superset/bi_security_manager/sql/queries.py index 166a2f2cf38dd..ed74785b3dd53 100644 --- a/bi_superset/bi_security_manager/sql/queries.py +++ b/bi_superset/bi_security_manager/sql/queries.py @@ -4,7 +4,7 @@ ROLE_DEFINITIONS_QUERY = """ SELECT human_readable as permission_name, -FROM csdataanalysis.bi_superset_access.role_definitions_{access_method} +FROM {dataset}.bi_superset_access.role_definitions_{access_method} WHERE {role_name} = true """ @@ -17,7 +17,7 @@ table_catalog, table_schema, table_name -FROM csdataanalysis.bi_superset_access.datasource_access_{access_method} +FROM {dataset}.bi_superset_access.datasource_access_{access_method} WHERE {role_name} = true """ @@ -28,7 +28,7 @@ SELECT employee, role_name -FROM csdataanalysis.bi_superset_access.roles_per_job_title +FROM {dataset}.bi_superset_access.roles_per_job_title """ @@ -38,7 +38,7 @@ ROLES_QUERY = """ SELECT name -FROM csdataanalysis.bi_superset_access.roles +FROM {dataset}.bi_superset_access.roles WHERE type = '{access_method}' """ @@ -50,5 +50,5 @@ SELECT dashboard_id, role_name -FROM csdataanalysis.bi_superset_access.dashboard_role_access_exterrnal +FROM {dataset}.bi_superset_access.dashboard_role_access_exterrnal """ diff --git a/bi_superset/superset_config.py b/bi_superset/superset_config.py index c4d8b1916939e..3d6524ea37e81 100644 --- a/bi_superset/superset_config.py +++ b/bi_superset/superset_config.py @@ -96,6 +96,8 @@ def get_env_variable(var_name: str, default: Optional[str] = None) -> str: GUEST_TOKEN_JWT_AUDIENCE: None ZF_JWT_PUBLIC_SECRET = get_env_variable("ZF_JWT_PUBLIC_SECRET", None) +BQ_DATASET = os.getenv("BQ_DATASET", None) + FEATURE_FLAGS = { "ENABLE_TEMPLATE_PROCESSING": True, "ESTIMATE_QUERY_COST": True, diff --git a/bi_superset/superset_config_local.py b/bi_superset/superset_config_local.py index 7551486387aa3..e5d4c771b6822 100644 --- a/bi_superset/superset_config_local.py +++ b/bi_superset/superset_config_local.py @@ -65,6 +65,7 @@ def get_env_variable(var_name: str, default: Optional[str] = None) -> str: True if SUPERSET_ACCESS_METHOD == AccessMethod.EXTERNAL.value else False ) +BQ_DATASET = os.getenv("BQ_DATASET", None) FEATURE_FLAGS = { "ENABLE_TEMPLATE_PROCESSING": True,