renamed misc/utils.py to util/workspace.py

cmu-db · Oct 23, 2024 · 610fc17 · 610fc17
1 parent 9e652d1
commit 610fc17
Show file tree

Hide file tree

Showing 35 changed files with 38 additions and 34 deletions.
diff --git a/benchmark/cli.py b/benchmark/cli.py
@@ -1,7 +1,7 @@
 import click
 
 from benchmark.tpch.cli import tpch_group
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 
 
 @click.group(name="benchmark")

diff --git a/benchmark/tpch/cli.py b/benchmark/tpch/cli.py
@@ -3,7 +3,7 @@
 
 import click
 
-from misc.utils import (
+from util.workspace import (
     DBGymConfig,
     get_scale_factor_string,
     link_result,

diff --git a/benchmark/tpch/load_info.py b/benchmark/tpch/load_info.py
@@ -2,7 +2,7 @@
 from typing import Optional
 
 from dbms.load_info_base_class import LoadInfoBaseClass
-from misc.utils import DBGymConfig, get_scale_factor_string
+from util.workspace import DBGymConfig, get_scale_factor_string
 
 TPCH_SCHEMA_FNAME = "tpch_schema.sql"
 TPCH_CONSTRAINTS_FNAME = "tpch_constraints.sql"

diff --git a/dbms/cli.py b/dbms/cli.py
@@ -1,7 +1,7 @@
 import click
 
 from dbms.postgres.cli import postgres_group
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 
 
 @click.group(name="dbms")

diff --git a/dbms/postgres/cli.py b/dbms/postgres/cli.py
@@ -17,7 +17,7 @@
 
 from benchmark.tpch.load_info import TpchLoadInfo
 from dbms.load_info_base_class import LoadInfoBaseClass
-from misc.utils import (
+from util.workspace import (
     WORKSPACE_PATH_PLACEHOLDER,
     DBGymConfig,
     conv_inputpath_to_realabspath,

diff --git a/manage/cli.py b/manage/cli.py
@@ -6,7 +6,7 @@
 
 import click
 
-from misc.utils import (
+from util.workspace import (
     DBGymConfig,
     get_runs_path_from_workspace_path,
     get_symlinks_path_from_workspace_path,

diff --git a/manage/tests/test_clean.py b/manage/tests/test_clean.py
@@ -7,7 +7,7 @@
 from typing import Any, NewType, cast
 
 from manage.cli import MockDBGymConfig, clean_workspace
-from misc.utils import path_exists_dont_follow_symlinks
+from util.workspace import path_exists_dont_follow_symlinks
 
 # This is here instead of on `if __name__ == "__main__"` because we often run individual tests, which
 #   does not go through the `if __name__ == "__main__"` codepath.

diff --git a/misc/__init__.py b/misc/__init__.py
diff --git a/task.py b/task.py
@@ -15,7 +15,7 @@
 from benchmark.cli import benchmark_group
 from dbms.cli import dbms_group
 from manage.cli import manage_group
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.cli import tune_group
 
 # TODO(phw2): Save commit, git diff, and run command.

diff --git a/tune/cli.py b/tune/cli.py
@@ -1,6 +1,6 @@
 import click
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.protox.cli import protox_group
 
 

diff --git a/tune/protox/agent/base_class.py b/tune/protox/agent/base_class.py
@@ -6,7 +6,7 @@
 import numpy as np
 from numpy.typing import NDArray
 
-from misc.utils import TuningMode
+from util.workspace import TuningMode
 from tune.protox.agent.agent_env import AgentEnv
 from tune.protox.agent.noise import ActionNoise
 from tune.protox.env.artifact_manager import ArtifactManager

diff --git a/tune/protox/agent/build_trial.py b/tune/protox/agent/build_trial.py
@@ -16,7 +16,7 @@
 from torch import nn
 from torch.optim import Adam  # type: ignore[attr-defined]
 
-from misc.utils import (
+from util.workspace import (
     DBGymConfig,
     TuningMode,
     make_redis_started,

diff --git a/tune/protox/agent/cli.py b/tune/protox/agent/cli.py
@@ -1,6 +1,6 @@
 import click
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.protox.agent.hpo import hpo
 from tune.protox.agent.replay import replay
 from tune.protox.agent.tune import tune

diff --git a/tune/protox/agent/coerce_config.py b/tune/protox/agent/coerce_config.py
@@ -3,7 +3,7 @@
 
 import yaml
 
-from misc.utils import DBGymConfig, TuningMode, open_and_save
+from util.workspace import DBGymConfig, TuningMode, open_and_save
 
 
 def coerce_config(

diff --git a/tune/protox/agent/hpo.py b/tune/protox/agent/hpo.py
@@ -22,7 +22,7 @@
 from ray.tune.schedulers import FIFOScheduler
 from ray.tune.search.basic_variant import BasicVariantGenerator
 
-from misc.utils import (
+from util.workspace import (
     BENCHMARK_NAME_PLACEHOLDER,
     DEFAULT_BOOT_CONFIG_FPATH,
     DEFAULT_SYSKNOBS_PATH,

diff --git a/tune/protox/agent/off_policy_algorithm.py b/tune/protox/agent/off_policy_algorithm.py
@@ -4,7 +4,7 @@
 import numpy as np
 from numpy.typing import NDArray
 
-from misc.utils import TuningMode
+from util.workspace import TuningMode
 from tune.protox.agent.agent_env import AgentEnv
 from tune.protox.agent.base_class import BaseAlgorithm
 from tune.protox.agent.buffers import ReplayBuffer

diff --git a/tune/protox/agent/replay.py b/tune/protox/agent/replay.py
@@ -18,7 +18,7 @@
 import tqdm
 from dateutil.parser import parse
 
-from misc.utils import (
+from util.workspace import (
     DBGymConfig,
     TuningMode,
     conv_inputpath_to_realabspath,

diff --git a/tune/protox/agent/tune.py b/tune/protox/agent/tune.py
@@ -7,7 +7,7 @@
 import click
 import pandas as pd
 
-from misc.utils import (
+from util.workspace import (
     BENCHMARK_NAME_PLACEHOLDER,
     DEFAULT_BOOT_CONFIG_FPATH,
     WORKLOAD_NAME_PLACEHOLDER,

diff --git a/tune/protox/cli.py b/tune/protox/cli.py
@@ -1,6 +1,6 @@
 import click
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.protox.agent.cli import agent_group
 from tune.protox.embedding.cli import embedding_group
 

diff --git a/tune/protox/embedding/analyze.py b/tune/protox/embedding/analyze.py
@@ -15,7 +15,7 @@
 import tqdm
 import yaml
 
-from misc.utils import DBGymConfig, open_and_save
+from util.workspace import DBGymConfig, open_and_save
 from tune.protox.embedding.loss import CostLoss, get_bias_fn
 from tune.protox.embedding.train_all import (
     create_vae_model,

diff --git a/tune/protox/embedding/cli.py b/tune/protox/embedding/cli.py
@@ -1,6 +1,6 @@
 import click
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.protox.embedding.datagen import datagen
 from tune.protox.embedding.train import train
 

diff --git a/tune/protox/embedding/datagen.py b/tune/protox/embedding/datagen.py
@@ -19,7 +19,7 @@
 from sklearn.preprocessing import quantile_transform
 
 from dbms.postgres.cli import start_postgres, stop_postgres
-from misc.utils import (
+from util.workspace import (
     BENCHMARK_NAME_PLACEHOLDER,
     SCALE_FACTOR_PLACEHOLDER,
     WORKLOAD_NAME_PLACEHOLDER,

diff --git a/tune/protox/embedding/select.py b/tune/protox/embedding/select.py
@@ -9,7 +9,7 @@
 import tqdm
 from pandas import DataFrame
 
-from misc.utils import DBGymConfig, default_embedder_dname, link_result
+from util.workspace import DBGymConfig, default_embedder_dname, link_result
 from tune.protox.embedding.analyze import RANGES_FNAME, STATS_FNAME
 from tune.protox.embedding.train_args import (
     EmbeddingSelectArgs,

diff --git a/tune/protox/embedding/train.py b/tune/protox/embedding/train.py
@@ -7,7 +7,7 @@
 import numpy as np
 import torch
 
-from misc.utils import (
+from util.workspace import (
     BENCHMARK_NAME_PLACEHOLDER,
     DEFAULT_HPO_SPACE_PATH,
     WORKLOAD_NAME_PLACEHOLDER,

diff --git a/tune/protox/embedding/train_all.py b/tune/protox/embedding/train_all.py
@@ -29,7 +29,7 @@
 from torch.utils.data import TensorDataset
 from typing_extensions import ParamSpec
 
-from misc.utils import DBGymConfig, open_and_save, restart_ray, save_file
+from util.workspace import DBGymConfig, open_and_save, restart_ray, save_file
 from tune.protox.embedding.loss import COST_COLUMNS, CostLoss, get_bias_fn
 from tune.protox.embedding.train_args import (
     EmbeddingTrainAllArgs,

diff --git a/tune/protox/env/artifact_manager.py b/tune/protox/env/artifact_manager.py
@@ -12,7 +12,7 @@
 from torch.utils.tensorboard.writer import SummaryWriter
 from typing_extensions import ParamSpec
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from util.log import DBGYM_LOGGER_NAME
 
 P = ParamSpec("P")

diff --git a/tune/protox/env/pg_env.py b/tune/protox/env/pg_env.py
@@ -9,7 +9,7 @@
 import psycopg
 from plumbum import local
 
-from misc.utils import DBGymConfig, TuningMode
+from util.workspace import DBGymConfig, TuningMode
 from tune.protox.env.artifact_manager import ArtifactManager, time_record
 from tune.protox.env.space.holon_space import HolonSpace
 from tune.protox.env.space.state.space import StateSpace

diff --git a/tune/protox/env/space/state/lsc_space.py b/tune/protox/env/space/state/lsc_space.py
@@ -3,7 +3,7 @@
 import psycopg
 from gymnasium.spaces import Box
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.protox.env.lsc.lsc import LSC
 from tune.protox.env.space.holon_space import HolonSpace
 from tune.protox.env.space.state.metric import MetricStateSpace

diff --git a/tune/protox/env/space/state/metric.py b/tune/protox/env/space/state/metric.py
@@ -8,7 +8,7 @@
 from gymnasium.spaces import Box, Space
 from psycopg.rows import dict_row
 
-from misc.utils import DBGymConfig, open_and_save
+from util.workspace import DBGymConfig, open_and_save
 from tune.protox.env.space.state.space import StateSpace
 from util.pg import DBGYM_POSTGRES_DBNAME
 

diff --git a/tune/protox/env/space/state/space.py b/tune/protox/env/space/state/space.py
@@ -5,7 +5,7 @@
 from gymnasium import spaces
 from psycopg import Connection
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 
 
 class StateSpace(ABC, spaces.Space[Any]):

diff --git a/tune/protox/env/space/state/structure.py b/tune/protox/env/space/state/structure.py
@@ -7,7 +7,7 @@
 import torch as th
 from gymnasium import spaces
 
-from misc.utils import DBGymConfig
+from util.workspace import DBGymConfig
 from tune.protox.env.space.holon_space import HolonSpace
 from tune.protox.env.space.latent_space import (
     LatentIndexSpace,

diff --git a/tune/protox/env/util/pg_conn.py b/tune/protox/env/util/pg_conn.py
@@ -12,15 +12,15 @@
 import threading
 import time
 from pathlib import Path
-from typing import Any, Optional, Tuple, Union
+from typing import Any, Optional, Union
 
 import psutil
 import psycopg
 import yaml
 from plumbum import local
 from psycopg.errors import ProgramLimitExceeded, QueryCanceled
 
-from misc.utils import DBGymConfig, link_result, open_and_save, parent_dpath_of_path
+from util.workspace import DBGymConfig, link_result, open_and_save, parent_dpath_of_path
 from tune.protox.env.artifact_manager import ArtifactManager, time_record
 from util.log import DBGYM_LOGGER_NAME
 from util.pg import (

diff --git a/tune/protox/env/workload.py b/tune/protox/env/workload.py
@@ -12,7 +12,7 @@
 import pglast
 from plumbum import local
 
-from misc.utils import DBGymConfig, open_and_save
+from util.workspace import DBGymConfig, open_and_save
 from tune.protox.env.artifact_manager import ArtifactManager, time_record
 from tune.protox.env.space.holon_space import HolonSpace
 from tune.protox.env.space.latent_space import LatentKnobSpace, LatentQuerySpace

diff --git a/util/pg.py b/util/pg.py
@@ -6,7 +6,7 @@
 import sqlalchemy
 from sqlalchemy import create_engine, text
 
-from misc.utils import DBGymConfig, open_and_save
+from util.workspace import DBGymConfig, open_and_save
 
 DBGYM_POSTGRES_USER = "dbgym_user"
 DBGYM_POSTGRES_PASS = "dbgym_pass"

diff --git a/misc/utils.py → util/workspace.py b/misc/utils.py → util/workspace.py
@@ -1,3 +1,7 @@
+"""
+This file contains everything needed to manage the workspace (the dbgym_workspace/ folder).
+TODO: it has some things specific to Proto-X which should be moved somewhere else in the future.
+"""
 import logging
 import os
 import shutil