Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix malware and code_quality workflows. #92

Merged
merged 2 commits into from
May 9, 2024
Merged

Fix malware and code_quality workflows. #92

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e73c391
Fix malware and code_quality workflows.
revit13 May 9, 2024
901dbb1
Minor fix.
revit13 May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions kfp/transform_workflows/code/code_quality/code_quality_wf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@


# the name of the job script
EXEC_SCRIPT_NAME: str = "cq_transform.py"
EXEC_SCRIPT_NAME: str = "code_quality_transform.py"
PREFIX: str = ""

task_image = "quay.io/dataprep1/data-prep-lab/code_quality:0.2.0"
Expand Down Expand Up @@ -69,9 +69,9 @@ def code_quality(
data_max_files: int = -1,
data_num_samples: int = -1,
# orchestrator
actor_options: str = "{'num_cpus': 0.8}",
pipeline_id: str = "pipeline_id",
code_location: str = "{'github': 'github', 'commit_hash': '12345', 'path': 'path'}",
runtime_actor_options: str = "{'num_cpus': 0.8}",
runtime_pipeline_id: str = "runtime_pipeline_id",
runtime_code_location: str = "{'github': 'github', 'commit_hash': '12345', 'path': 'path'}",
# code quality parameters
cq_contents_column_name: str = "contents",
cq_language_column_name: str = "language",
Expand Down Expand Up @@ -108,8 +108,8 @@ def code_quality(
:param data_s3_config - s3 configuration
:param data_max_files - max files to process
:param data_num_samples - num samples to process
:param actor_options - actor options
:param pipeline_id - pipeline id
:param runtime_actor_options - actor options
:param runtime_pipeline_id - pipeline id
:param cq_contents_column_name - Name of the column holds the data to process
:param cq_language_column_name - Name of the column holds the programming language details
:param cq_tokenizer - Name or path to the tokenizer
Expand All @@ -124,7 +124,7 @@ def code_quality(
# compute execution params
compute_exec_params = compute_exec_params_op(
worker_options=ray_worker_options,
actor_options=actor_options,
actor_options=runtime_actor_options,
)
ComponentUtils.add_settings_to_component(compute_exec_params, ONE_HOUR_SEC * 2)
# start Ray cluster
Expand All @@ -148,10 +148,10 @@ def code_quality(
"data_s3_config": data_s3_config,
"data_max_files": data_max_files,
"data_num_samples": data_num_samples,
"num_workers": compute_exec_params.output,
"worker_options": actor_options,
"pipeline_id": pipeline_id,
"job_id": dsl.RUN_ID_PLACEHOLDER,
"runtime_num_workers": compute_exec_params.output,
"runtime_worker_options": runtime_actor_options,
"runtime_pipeline_id": runtime_pipeline_id,
"runtime_job_id": dsl.RUN_ID_PLACEHOLDER,
"cq_contents_column_name": cq_contents_column_name,
"cq_language_column_name": cq_language_column_name,
"cq_tokenizer": cq_tokenizer,
Expand Down
3 changes: 0 additions & 3 deletions kfp/transform_workflows/code/malware/malware_wf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ def malware(
# mallware
malware_input_column: str = "contents",
malware_output_column: str = "virus_detection",
malware_clamd_socket: str = "/var/run/clamav/clamd.ctl",
# additional parameters
additional_params: str = '{"wait_interval": 2, "wait_cluster_ready_tmout": 400, "wait_cluster_up_tmout": 300, "wait_job_ready_tmout": 400, "wait_print_tmout": 30, "http_retries": 5}',
):
Expand Down Expand Up @@ -102,7 +101,6 @@ def malware(
:param runtime_code_location - code location
:param malware_input_column - input column name
:param malware_output_column - output column name
:param malware_clamd_socket - local socket path for clamd
:return: None
"""
# create clean_up task
Expand Down Expand Up @@ -144,7 +142,6 @@ def malware(
"runtime_code_location": runtime_code_location,
"malware_input_column": malware_input_column,
"malware_output_column": malware_output_column,
"malware_clamd_socket": malware_clamd_socket,
},
exec_script_name=EXEC_SCRIPT_NAME,
server_url=server_url,
Expand Down