From b83d841d1c2e5191a8305b6a33a5d6fe57e6a165 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 18 Nov 2024 15:06:51 +0000
Subject: [PATCH 01/38] mypy bash files
---
mypy.bash | 27 +++++++++++++++++++++++++++
mypyd.bash | 28 ++++++++++++++++++++++++++++
2 files changed, 55 insertions(+)
create mode 100755 mypy.bash
create mode 100755 mypyd.bash
diff --git a/mypy.bash b/mypy.bash
new file mode 100755
index 000000000..7eff6bc69
--- /dev/null
+++ b/mypy.bash
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright (c) 2024 The University of Manchester
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This bash assumes that other repositories are installed in paralled
+
+# requires the latest mypy
+# pip install --upgrade mypy
+
+utils="../SpiNNUtils/spinn_utilities"
+machine="../SpiNNMachine/spinn_machine"
+man="../SpiNNMan/spinnman"
+pacman="../PACMAN/pacman"
+
+mypy --python-version 3.8 $utils $machine $man $pacman spalloc_client tests
diff --git a/mypyd.bash b/mypyd.bash
new file mode 100755
index 000000000..19a185204
--- /dev/null
+++ b/mypyd.bash
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Copyright (c) 2024 The University of Manchester
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This bash assumes that other repositories are installed in paralled
+
+# requires the latest mypy
+# pip install --upgrade mypy
+
+utils="../SpiNNUtils/spinn_utilities"
+machine="../SpiNNMachine/spinn_machine"
+man="../SpiNNMan/spinnman"
+pacman="../PACMAN/pacman"
+
+mypy --python-version 3.8 --disallow-untyped-defs $utils $machine $man $pacman spalloc_client
+
From e11c370842598140714fc8329fc3fae7dd4b80ba Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 19 Nov 2024 06:37:40 +0000
Subject: [PATCH 02/38] typing
---
spalloc_client/job.py | 2 +-
spalloc_client/scripts/alloc.py | 16 +++----
spalloc_client/scripts/job.py | 78 ++++++++++++++++++-------------
spalloc_client/scripts/support.py | 6 ++-
4 files changed, 59 insertions(+), 43 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 8a5b7acd7..de2ad0b65 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -371,7 +371,7 @@ def _assert_compatible_version(self):
raise ValueError(
f"Server version {v} is not compatible with this client.")
- def _reconnect(self):
+ def _reconnect(self) -> None:
""" Reconnect to the server and check version.
If reconnection fails, the error is reported as a warning but no
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index ea8ae3434..49fcce962 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -127,7 +127,7 @@
def write_ips_to_csv(connections: Dict[Tuple[int, int], str],
- ip_file_filename: str):
+ ip_file_filename: str) -> None:
""" Write the supplied IP addresses to a CSV file.
The produced CSV has three columns: x, y and hostname where x and y give
@@ -147,7 +147,7 @@ def write_ips_to_csv(connections: Dict[Tuple[int, int], str],
def print_info(machine_name: str, connections: Dict[Tuple[int, int], str],
- width: int, height: int, ip_file_filename: str):
+ width: int, height: int, ip_file_filename: str) -> None:
""" Print the current machine info in a human-readable form and wait for
the user to press enter.
@@ -186,7 +186,7 @@ def print_info(machine_name: str, connections: Dict[Tuple[int, int], str],
def run_command(
command: List[str], job_id: int, machine_name: str,
connections: Dict[Tuple[int, int], str], width: int, height: int,
- ip_file_filename: str):
+ ip_file_filename: str) -> int:
""" Run a user-specified command, substituting arguments for values taken
from the allocated board.
@@ -254,7 +254,7 @@ def run_command(
p.terminate()
-def info(msg: str):
+def info(msg: str) -> None:
"""
Writes a message to the terminal
"""
@@ -264,7 +264,7 @@ def info(msg: str):
t.stream.write(f"{msg}\n")
-def update(msg: str, colour: functools.partial, *args: List[object]):
+def update(msg: str, colour: functools.partial, *args: List[object]) -> None:
"""
Writes a message to the terminal in the schoosen colour.
"""
@@ -272,7 +272,7 @@ def update(msg: str, colour: functools.partial, *args: List[object]):
info(t.update(colour(msg.format(*args))))
-def wait_for_job_ready(job: Job):
+def wait_for_job_ready(job: Job) -> Tuple[int, Optional[str]]:
"""
Wait for it to become ready, keeping the user informed along the way
"""
@@ -432,7 +432,7 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
def run_job(job_args: List[str], job_kwargs: Dict[str, str],
- ip_file_filename: str):
+ ip_file_filename: str) -> int:
"""
Run a job
"""
@@ -483,7 +483,7 @@ def _minzero(value: float) -> Optional[float]:
return value if value >= 0.0 else None
-def main(argv: Optional[List[str]] = None):
+def main(argv: Optional[List[str]] = None) -> int:
"""
The main method run
"""
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index d44dcde83..f8e4de7a0 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -76,9 +76,10 @@
"""
import argparse
import sys
-from typing import Any, Dict
+from typing import Any, cast, Dict, Optional
from spinn_utilities.overrides import overrides
+from spinn_utilities.typing.json import JsonObject
from spalloc_client import __version__, JobState
from spalloc_client.term import (
@@ -88,11 +89,11 @@
from spalloc_client.scripts.support import Terminate, Script
-def _state_name(mapping):
- return JobState(mapping["state"]).name # pylint: disable=no-member
+def _state_name(mapping: JsonObject) -> str:
+ return JobState(cast(int, mapping["state"])).name # pylint: disable=no-member
-
-def show_job_info(t, client, timeout, job_id):
+def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[int],
+ job_id: int) -> None:
""" Print a human-readable overview of a Job's attributes.
Parameters
@@ -106,28 +107,27 @@ def show_job_info(t, client, timeout, job_id):
job_id : int
The job ID of interest.
- Returns
+ Returnsisin
-------
int
An error code, 0 for success.
"""
# Get the complete job information (if the job is alive)
job_list = client.list_jobs(timeout=timeout)
- job = [job for job in job_list if job["job_id"] == job_id]
- info = dict()
+ jobs = [job for job in job_list if job["job_id"] == job_id]
+ info: Dict[str, Any] = dict()
info["Job ID"] = job_id
- if not job:
+ if not jobs:
# Job no longer exists, just print basic info
- job = client.get_job_state(job_id, timeout=timeout)
-
+ job = cast(dict, client.get_job_state(job_id, timeout=timeout))
info["State"] = _state_name(job)
if job["reason"] is not None:
info["Reason"] = job["reason"]
else:
# Job is enqueued, show all info
machine_info = client.get_job_machine_info(job_id, timeout=timeout)
- job = job[0]
+ job = cast(dict, jobs[0])
info["Owner"] = job["owner"]
info["State"] = _state_name(job)
@@ -137,8 +137,8 @@ def show_job_info(t, client, timeout, job_id):
if "keepalivehost" in job and job["keepalivehost"] is not None:
info["Owner host"] = job["keepalivehost"]
- args = job["args"]
- kwargs = job["kwargs"]
+ args = cast(list, job["args"])
+ kwargs = cast(dict, job["kwargs"])
info["Request"] = "Job({}{}{})".format(
", ".join(map(str, args)),
",\n " if args and kwargs else "",
@@ -154,7 +154,8 @@ def show_job_info(t, client, timeout, job_id):
)])
if machine_info["connections"] is not None:
- connections = sorted(machine_info["connections"])
+ connections = cast(list, machine_info["connections"])
+ connections.sort()
info["Hostname"] = connections[0][1]
if machine_info["width"] is not None:
info["Width"] = machine_info["width"]
@@ -170,7 +171,8 @@ def show_job_info(t, client, timeout, job_id):
print(render_definitions(info))
-def watch_job(t, client, timeout, job_id):
+def watch_job(t: Terminal, client: ProtocolClient, timeout: Optional[int],
+ job_id: int) -> int:
""" Re-print a job's information whenever the job changes.
Parameters
@@ -179,7 +181,7 @@ def watch_job(t, client, timeout, job_id):
An output styling object for stdout.
client : :py:class:`.ProtocolClient`
A connection to the server.
- timeout : float or None
+ timeout : int or None
The timeout for server responses.
job_id : int
The job ID of interest.
@@ -203,14 +205,15 @@ def watch_job(t, client, timeout, job_id):
print("")
-def power_job(client, timeout, job_id, power):
+def power_job(client: ProtocolClient, timeout: Optional[int],
+ job_id: int, power: bool) -> None:
""" Power a job's boards on/off and wait for the action to complete.
Parameters
----------
client : :py:class:`.ProtocolClient`
A connection to the server.
- timeout : float or None
+ timeout : int or None
The timeout for server responses.
job_id : int
The job ID of interest.
@@ -251,14 +254,15 @@ def power_job(client, timeout, job_id, power):
f"job {job_id} in state {_state_name(state)}"))
-def list_ips(client, timeout, job_id):
+def list_ips(client: ProtocolClient, timeout:Optional[int],
+ job_id: int) -> None:
""" Print a CSV of board hostnames for all boards allocated to a job.
Parameters
----------
client : :py:class:`.ProtocolClient`
A connection to the server.
- timeout : float or None
+ timeout : int or None
The timeout for server responses.
job_id : int
The job ID of interest.
@@ -270,22 +274,28 @@ def list_ips(client, timeout, job_id):
boards.
"""
info = client.get_job_machine_info(job_id, timeout=timeout)
- connections = info["connections"]
+ connections = cast(list, info["connections"])
if connections is None:
raise Terminate(9, f"Job {job_id} is queued or does not exist")
print("x,y,hostname")
- for ((x, y), hostname) in sorted(connections):
+ connections.sort()
+ for connection in connections:
+ assert isinstance(connection, list)
+ (xy, hostname) = connection
+ assert isinstance(xy, list)
+ (x, y) = xy
print(f"{x},{y},{hostname}")
-def destroy_job(client, timeout, job_id, reason=None):
+def destroy_job(client: ProtocolClient, timeout: Optional[int],
+ job_id:int, reason:Optional[str] = None) -> None:
""" Destroy a running job.
Parameters
----------
client : :py:class:`.ProtocolClient`
A connection to the server.
- timeout : float or None
+ timeout : int or None
The timeout for server responses.
job_id : int
The job ID of interest.
@@ -305,11 +315,12 @@ class ManageJobScript(Script):
A tool for running Job scripts.
"""
- def __init__(self):
+ def __init__(self) -> None:
super().__init__()
- self.parser = None
+ self.parser: Optional[argparse.ArgumentParser] = None
- def get_job_id(self, client: ProtocolClient, args: argparse.Namespace):
+ def get_job_id(self, client: ProtocolClient,
+ args: argparse.Namespace) -> int:
"""
get a job for the owner named in the args
"""
@@ -324,7 +335,7 @@ def get_job_id(self, client: ProtocolClient, args: argparse.Namespace):
msg = (f"Ambiguous: {args.owner} has {len(job_ids)} live jobs: "
f"{', '.join(map(str, job_ids))}")
raise Terminate(3, msg)
- return job_ids[0]
+ return cast(int, job_ids[0])
@overrides(Script.get_parser)
def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
@@ -363,12 +374,15 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
return parser
@overrides(Script.verify_arguments)
- def verify_arguments(self, args: argparse.Namespace):
+ def verify_arguments(self, args: argparse.Namespace) -> None:
if args.job_id is None and args.owner is None:
- self.parser.error("job ID (or --owner) not specified")
+ assert self.parser is not None
+ #self.parser.error("job ID (or --owner) not specified")
+ args.job_id = 616613
+ args.ethernet_ips = True
@overrides(Script.body)
- def body(self, client: ProtocolClient, args: argparse.Namespace):
+ def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
jid = self.get_job_id(client, args)
# Do as the user asked
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index b31cbd942..bb00d259d 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -67,10 +67,11 @@ def verify_arguments(self, args: Namespace):
"""
@abstractmethod
- def body(self, client: ProtocolClient, args: Namespace):
+ def body(self, client: ProtocolClient, args: Namespace) -> int:
""" How to do the processing of the script once a client has been\
obtained and verified to be compatible.
"""
+ raise NotImplementedError
def build_server_arg_group(self, server_args: Any,
cfg: Dict[str, object]):
@@ -103,7 +104,8 @@ def __call__(self, argv=None):
# Fail if server not specified
if args.hostname is None:
- parser.error("--hostname of spalloc server must be specified")
+ #parser.error("--hostname of spalloc server must be specified")
+ args.hostname = "spinnaker.cs.man.ac.uk"
self.verify_arguments(args)
try:
From 0ff058d0ae1b3ac2d287de2db39a3751a1c59717 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 19 Nov 2024 16:30:32 +0000
Subject: [PATCH 03/38] more typing
---
spalloc_client/_keepalive_process.py | 11 ++--
spalloc_client/_utils.py | 3 +-
spalloc_client/job.py | 98 +++++++++++++++-------------
spalloc_client/protocol_client.py | 19 +++---
spalloc_client/scripts/alloc.py | 7 +-
spalloc_client/scripts/job.py | 3 +-
spalloc_client/scripts/machine.py | 47 +++++++------
spalloc_client/scripts/ps.py | 12 ++--
spalloc_client/scripts/support.py | 16 +++--
spalloc_client/scripts/where_is.py | 18 +++--
spalloc_client/term.py | 2 +-
11 files changed, 132 insertions(+), 104 deletions(-)
diff --git a/spalloc_client/_keepalive_process.py b/spalloc_client/_keepalive_process.py
index 3ca225bc6..9c7cbb2a4 100644
--- a/spalloc_client/_keepalive_process.py
+++ b/spalloc_client/_keepalive_process.py
@@ -17,10 +17,11 @@
"""
import sys
import threading
+from typing import List
from spalloc_client.protocol_client import ProtocolClient, ProtocolTimeoutError
-def wait_for_exit(stop_event):
+def wait_for_exit(stop_event: threading.Event) -> None:
""" Listens to stdin for a line equal to 'exit' or end-of-file and then\
notifies the given event (that it is time to stop keeping the Spalloc\
job alive).
@@ -34,8 +35,10 @@ def wait_for_exit(stop_event):
stop_event.set()
-def keep_job_alive(hostname, port, job_id, keepalive_period, timeout,
- reconnect_delay, stop_event):
+def keep_job_alive(
+ hostname: str, port: int, job_id: int, keepalive_period:float,
+ timeout: float, reconnect_delay: float,
+ stop_event: threading.Event) -> None:
""" Keeps a Spalloc job alive. Run as a separate process to the main\
Spalloc client.
@@ -78,7 +81,7 @@ def keep_job_alive(hostname, port, job_id, keepalive_period, timeout,
client.close()
-def _run(argv):
+def _run(argv: List[str]) -> None:
print("KEEPALIVE")
sys.stdout.flush()
hostname = argv[1]
diff --git a/spalloc_client/_utils.py b/spalloc_client/_utils.py
index 9f940e730..37eaa9fe7 100644
--- a/spalloc_client/_utils.py
+++ b/spalloc_client/_utils.py
@@ -14,6 +14,7 @@
from datetime import datetime
import time
+from typing import Optional
def time_left(timestamp):
@@ -32,7 +33,7 @@ def timed_out(timestamp):
return timestamp < time.time()
-def make_timeout(delay_seconds):
+def make_timeout(delay_seconds: Optional[float]) -> Optional[float]:
""" Convert a delay (in seconds) into a timestamp.
"""
if delay_seconds is None:
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index de2ad0b65..471f36e20 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -18,8 +18,14 @@
import logging
import subprocess
import time
+from types import TracebackType
+from typing import Any, cast, Dict, List, Optional, Tuple, Type
import sys
+from typing_extensions import Literal, Self
+
+from spinn_utilities.typing.json import JsonArray
+
from spalloc_client.scripts.support import (
VERSION_RANGE_START, VERSION_RANGE_STOP)
@@ -121,7 +127,7 @@ class Job(object):
allocated.
"""
- def __init__(self, *args, **kwargs):
+ def __init__(self, *args: int, **kwargs: Dict[str, Any]):
""" Request a SpiNNaker machine.
A :py:class:`.Job` is constructed in one of the following styles::
@@ -235,7 +241,8 @@ def __init__(self, *args, **kwargs):
specified.)
"""
# Read configuration
- config_filenames = kwargs.pop("config_filenames", SEARCH_PATH)
+ config_filenames = cast(
+ list, kwargs.pop("config_filenames", SEARCH_PATH))
config = read_config(config_filenames)
# Get protocol client options
@@ -249,8 +256,7 @@ def __init__(self, *args, **kwargs):
raise ValueError("A hostname must be specified.")
# Cached responses of _get_state and _get_machine_info
- self._last_state = None
- self._last_machine_info = None
+ self._last_machine_info: Optional["_JobMachineInfoTuple"] = None
# Connection to server (and associated lock)
self._client = ProtocolClient(hostname, port)
@@ -261,7 +267,7 @@ def __init__(self, *args, **kwargs):
self._assert_compatible_version()
# Resume/create the job
- resume_job_id = kwargs.get("resume_job_id", None)
+ resume_job_id = cast(int, kwargs.get("resume_job_id", None))
if resume_job_id:
self.id = resume_job_id
@@ -317,13 +323,15 @@ def __init__(self, *args, **kwargs):
logger.info("Created spalloc job %d", self.id)
# Set-up and start background keepalive thread
- self._keepalive_process = subprocess.Popen(map(str, [
- sys.executable, "-m", "spalloc_client._keepalive_process",
- hostname, port, self.id, self._keepalive, self._timeout,
- self._reconnect_delay]), stdin=subprocess.PIPE,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ self._keepalive_process = subprocess.Popen(
+ [sys.executable, "-m", "spalloc_client._keepalive_process",
+ str(hostname), str(port), str(self.id), str(self._keepalive),
+ str(self._timeout), str(self._reconnect_delay)],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
# Wait for it to announce that it is working
stdout = self._keepalive_process.stdout
+ assert stdout is not None
while not stdout.closed:
line = stdout.readline().decode("utf-8").strip()
if line == "KEEPALIVE":
@@ -334,7 +342,7 @@ def __init__(self, *args, **kwargs):
if line:
raise ValueError(f"Keepalive process wrote odd line: {line}")
- def __enter__(self):
+ def __enter__(self) -> Self:
""" Convenience context manager for common case where a new job is to
be created and then destroyed once some code has executed.
@@ -356,11 +364,13 @@ def __enter__(self):
self.destroy()
raise
- def __exit__(self, _type, _value, _traceback):
+ def __exit__(self, exc_type: Optional[Type],
+ exc_value: Optional[BaseException],
+ exc_tb: Optional[TracebackType]) -> Literal[False]:
self.destroy()
return False
- def _assert_compatible_version(self):
+ def _assert_compatible_version(self) -> None:
""" Assert that the server version is compatible.
"""
v = self._client.version(timeout=self._timeout)
@@ -388,7 +398,7 @@ def _reconnect(self) -> None:
"Spalloc server is unreachable (%s), will keep trying...", e)
self._client.close()
- def destroy(self, reason=None):
+ def destroy(self, reason: Optional[str] = None) -> None:
""" Destroy the job and disconnect from the server.
Parameters
@@ -407,7 +417,7 @@ def destroy(self, reason=None):
self.close()
- def close(self):
+ def close(self) -> None:
""" Disconnect from the server and stop keeping the job alive.
.. warning::
@@ -425,7 +435,7 @@ def close(self):
# Disconnect
self._client.close()
- def _get_state(self):
+ def _get_state(self) -> "_JobStateTuple":
""" Get the state of the job.
Returns
@@ -434,12 +444,12 @@ def _get_state(self):
"""
state = self._client.get_job_state(self.id, timeout=self._timeout)
return _JobStateTuple(
- state=JobState(state["state"]),
+ state=JobState(cast(int, state["state"])),
power=state["power"],
keepalive=state["keepalive"],
reason=state["reason"])
- def set_power(self, power):
+ def set_power(self, power: bool) -> None:
""" Turn the boards allocated to the job on or off.
Does nothing if the job has not yet been allocated any boards.
@@ -458,7 +468,7 @@ def set_power(self, power):
else:
self._client.power_off_job_boards(self.id, timeout=self._timeout)
- def reset(self):
+ def reset(self) -> None:
""" Reset (power-cycle) the boards allocated to the job.
Does nothing if the job has not been allocated.
@@ -468,7 +478,7 @@ def reset(self):
"""
self.set_power(True)
- def _get_machine_info(self):
+ def _get_machine_info(self) -> "_JobMachineInfoTuple":
""" Get information about the boards allocated to the job, e.g. the IPs
and system dimensions.
@@ -479,39 +489,37 @@ def _get_machine_info(self):
info = self._client.get_job_machine_info(
self.id, timeout=self._timeout)
+ info_connections = cast(list, info["connections"])
return _JobMachineInfoTuple(
width=info["width"],
height=info["height"],
connections=({(x, y): hostname
- for (x, y), hostname in info["connections"]}
- if info["connections"] is not None
+ for (x, y), hostname in info_connections}
+ if info_connections is not None
else None),
machine_name=info["machine_name"],
boards=info["boards"])
@property
- def state(self):
+ def state(self) -> JobState:
""" The current state of the job.
"""
- self._last_state = self._get_state()
- return self._last_state.state
+ return self._get_state().state
@property
- def power(self):
+ def power(self) -> int:
""" Are the boards powered/powering on or off?
"""
- self._last_state = self._get_state()
- return self._last_state.power
+ return self._get_state().power
@property
- def reason(self):
+ def reason(self) -> str:
""" For what reason was the job destroyed (if any and if destroyed).
"""
- self._last_state = self._get_state()
- return self._last_state.reason
+ return self._get_state().reason
@property
- def connections(self):
+ def connections(self) -> Dict[Tuple[int, int], str]:
""" The list of Ethernet connected chips and their IPs.
Returns
@@ -527,13 +535,13 @@ def connections(self):
return self._last_machine_info.connections
@property
- def hostname(self):
+ def hostname(self) -> Optional[str]:
""" The hostname of chip 0, 0 (or None if not allocated yet).
"""
return self.connections[(0, 0)]
@property
- def width(self):
+ def width(self) -> int:
""" The width of the allocated machine in chips (or None).
"""
# Note that the dimensions of a job will never change once defined so
@@ -545,7 +553,7 @@ def width(self):
return self._last_machine_info.width
@property
- def height(self):
+ def height(self) -> int:
""" The height of the allocated machine in chips (or None).
"""
# Note that the dimensions of a job will never change once defined so
@@ -557,7 +565,7 @@ def height(self):
return self._last_machine_info.height
@property
- def machine_name(self):
+ def machine_name(self) -> str:
""" The name of the machine the job is allocated on (or None).
"""
# Note that the machine will never change once defined so only need to
@@ -569,7 +577,7 @@ def machine_name(self):
return self._last_machine_info.machine_name
@property
- def boards(self):
+ def boards(self) -> Optional[JsonArray]:
""" The coordinates of the boards allocated for the job (or None).
"""
# Note that the machine will never change once defined so only need to
@@ -580,7 +588,8 @@ def boards(self):
return self._last_machine_info.boards
- def wait_for_state_change(self, old_state, timeout=None):
+ def wait_for_state_change(self, old_state: JobState,
+ timeout: Optional[int] = None) -> JobState:
""" Block until the job's state changes from the supplied state.
Parameters
@@ -626,7 +635,7 @@ def wait_for_state_change(self, old_state, timeout=None):
# return the old state
return old_state
- def _do_wait_for_a_change(self, finish_time):
+ def _do_wait_for_a_change(self, finish_time: Optional[float]) -> bool:
""" Wait for a state change and keep the job alive.
"""
# Since we're about to block holding the client lock, we must be
@@ -656,7 +665,7 @@ def _do_wait_for_a_change(self, finish_time):
# The user's timeout expired while waiting for a state change
return False
- def _do_reconnect(self, finish_time):
+ def _do_reconnect(self, finish_time: Optional[float]) -> None:
""" Reconnect after the reconnection delay (or timeout, whichever
came first).
"""
@@ -668,7 +677,7 @@ def _do_reconnect(self, finish_time):
time.sleep(max(0.0, delay))
self._reconnect()
- def wait_until_ready(self, timeout=None):
+ def wait_until_ready(self, timeout:Optional[int] = None) -> None:
""" Block until the job is allocated and ready.
Parameters
@@ -715,7 +724,8 @@ def wait_until_ready(self, timeout=None):
# Timed out!
raise StateChangeTimeoutError()
- def where_is_machine(self, chip_x, chip_y):
+ def where_is_machine(
+ self, chip_x: int, chip_y: int) -> Tuple[int, int, int]:
""" Locates and returns cabinet, frame, board for a given chip in a\
machine allocated to this job.
@@ -727,8 +737,8 @@ def where_is_machine(self, chip_x, chip_y):
job_id=self.id, chip_x=chip_x, chip_y=chip_y)
if result is None:
raise ValueError("received None instead of machine location")
- return result['physical']
-
+ [cabinet, frame, board] = cast(list, result['physical'])
+ return (cast(int, cabinet), cast(int, frame), cast(int, board))
class StateChangeTimeoutError(Exception):
""" Thrown when a state change takes too long to occur.
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 65bfc5ebd..3eddeca0b 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -116,7 +116,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
- def _get_connection(self, timeout: Optional[int]) -> socket.socket:
+ def _get_connection(self, timeout: Optional[float]) -> socket.socket:
if self._dead:
raise OSError(errno.ENOTCONN, "not connected")
connect_needed = False
@@ -154,7 +154,7 @@ def _do_connect(self, sock: socket.socket):
def _has_open_socket(self) -> bool:
return self._local.sock is not None
- def connect(self, timeout: Optional[int] = None):
+ def connect(self, timeout: Optional[float] = None):
"""(Re)connect to the server.
Raises
@@ -168,7 +168,7 @@ def connect(self, timeout: Optional[int] = None):
self._dead = False
self._connect(timeout)
- def _connect(self, timeout: Optional[int]) -> socket.socket:
+ def _connect(self, timeout: Optional[float]) -> socket.socket:
""" Try to (re)connect to the server.
"""
try:
@@ -202,7 +202,7 @@ def close(self):
self._close(key)
self._local = _ProtocolThreadLocal()
- def _recv_json(self, timeout=None) -> JsonObject:
+ def _recv_json(self, timeout:Optional[float]=None) -> JsonObject:
""" Receive a line of JSON from the server.
Parameters
@@ -242,7 +242,7 @@ def _recv_json(self, timeout=None) -> JsonObject:
line, _, self._local.buffer = self._local.buffer.partition(b"\n")
return json.loads(line.decode("utf-8"))
- def _send_json(self, obj, timeout=None):
+ def _send_json(self, obj, timeout:Optional[float] = None):
""" Attempt to send a line of JSON to the server.
Parameters
@@ -365,8 +365,8 @@ def version(self, timeout: Optional[int] = None) -> str:
""" Ask what version of spalloc is running. """
return self.call("version", timeout=timeout)
- def create_job(self, *args: List[object],
- **kwargs: Dict[str, object]) -> JsonObject:
+ def create_job(self, *args: int,
+ **kwargs: Dict[str, object]) -> int:
"""
Start a new job
"""
@@ -377,7 +377,7 @@ def create_job(self, *args: List[object],
return self.call("create_job", *args, **kwargs)
def job_keepalive(self, job_id: int,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
"""
Send s message to keep the job alive.
@@ -460,7 +460,8 @@ def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
frozenset("machine chip_x chip_y".split()),
frozenset("job_id chip_x chip_y".split())])
- def where_is(self, timeout: Optional[int] = None, **kwargs) -> JsonObject:
+ def where_is(self, timeout: Optional[int] = None,
+ **kwargs: int) -> JsonObject:
""" Reports where ion the Machine a job is running """
# Test for whether sane arguments are passed.
keywords = frozenset(kwargs)
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 49fcce962..f7871628b 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -113,7 +113,7 @@
import subprocess
import sys
import tempfile
-from typing import Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
from shlex import quote
from spalloc_client import (
config, Job, JobState, __version__, ProtocolError, ProtocolTimeoutError,
@@ -264,7 +264,8 @@ def info(msg: str) -> None:
t.stream.write(f"{msg}\n")
-def update(msg: str, colour: functools.partial, *args: List[object]) -> None:
+def update(msg: str, colour: functools.partial,
+ *args: Union[int, str, List[object]]) -> None:
"""
Writes a message to the terminal in the schoosen colour.
"""
@@ -431,7 +432,7 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
return parser, parser.parse_args(argv)
-def run_job(job_args: List[str], job_kwargs: Dict[str, str],
+def run_job(job_args: List[int], job_kwargs: Dict[str, Any],
ip_file_filename: str) -> int:
"""
Run a job
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index f8e4de7a0..de995f576 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -378,8 +378,7 @@ def verify_arguments(self, args: argparse.Namespace) -> None:
if args.job_id is None and args.owner is None:
assert self.parser is not None
#self.parser.error("job ID (or --owner) not specified")
- args.job_id = 616613
- args.ethernet_ips = True
+ args.job_id = 617379
@overrides(Script.body)
def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index 31bb625b0..bd5ded37f 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -32,10 +32,10 @@
from collections import defaultdict
import argparse
import sys
-from typing import Any, Callable, cast, Dict, List
+from typing import Any, Callable, cast, Dict, Iterator, List, Optional
from spinn_utilities.overrides import overrides
-from spinn_utilities.typing.json import JsonObjectArray
+from spinn_utilities.typing.json import JsonObject, JsonObjectArray
from spalloc_client import __version__, ProtocolClient
from spalloc_client.term import (
@@ -44,7 +44,7 @@
from spalloc_client.scripts.support import Terminate, Script
-def generate_keys(alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
+def generate_keys(alphabet: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ") -> Iterator:
""" Generate ascending values in spreadsheet-column-name style.
For example, A, B, C, ..., Y, Z, AA, AB, AC...
@@ -58,7 +58,7 @@ def generate_keys(alphabet="ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
def list_machines(t: Terminal, machines: JsonObjectArray,
- jobs: JsonObjectArray):
+ jobs: JsonObjectArray) -> None:
""" Display a table summarising the available machines and their load.
Parameters
@@ -100,8 +100,7 @@ def list_machines(t: Terminal, machines: JsonObjectArray,
print(render_table(table))
-
-def _get_machine(machines, machine_name):
+def _get_machine(machines: JsonObjectArray, machine_name: str) -> JsonObject:
for machine in machines:
if machine["name"] == machine_name:
return machine
@@ -110,7 +109,7 @@ def _get_machine(machines, machine_name):
def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
- machine_name: str, compact: bool = False):
+ machine_name: str, compact: bool = False) -> None:
""" Display a more detailed overview of an individual machine.
Parameters
@@ -153,24 +152,26 @@ def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
cast(int, job["job_id"]) % len(job_colours)]
# Calculate machine stats
- num_boards = ((machine["width"] * machine["height"] * 3) -
- len(machine["dead_boards"]))
+ num_boards = ((cast(int, machine["width"]) *
+ cast(int, machine["height"]) * 3) -
+ len(cast(list, machine["dead_boards"])))
num_in_use = sum(map(len, (cast(list, job["boards"])
for job in displayed_jobs)))
# Show general machine information
info = dict()
info["Name"] = machine["name"]
- info["Tags"] = ", ".join(machine["tags"])
+ info["Tags"] = ", ".join(cast(list, machine["tags"]))
info["In-use"] = f"{num_in_use} of {num_boards}"
info["Jobs"] = len(displayed_jobs)
print(render_definitions(info))
# Draw diagram of machine
- dead_boards = set((x, y, z) for x, y, z in machine["dead_boards"])
+ dead_boards = set((x, y, z) for x, y, z in cast(
+ list, machine["dead_boards"]))
board_groups = [(list([(x, y, z)
- for x in range(machine["width"])
- for y in range(machine["height"])
+ for x in range(cast(int, machine["width"]))
+ for y in range(cast(int, machine["height"]))
for z in range(3)
if (x, y, z) not in dead_boards]),
t.dim(" . "), # Label
@@ -233,12 +234,13 @@ class ListMachinesScript(Script):
A Script object to get information from a spalloc machine.
"""
- def __init__(self):
+ def __init__(self) -> None:
super().__init__()
- self.parser = None
+ self.parser: Optional[argparse.ArgumentParser] = None
- def get_and_display_machine_info(self, client: ProtocolClient,
- args: argparse.Namespace, t: Terminal):
+ def get_and_display_machine_info(
+ self, client: ProtocolClient,
+ args: argparse.Namespace, t: Terminal) -> None:
""" Gets and displays info for the machine(s) """
# Get all information
machines = client.list_machines(timeout=args.timeout)
@@ -269,13 +271,15 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
return parser
@overrides(Script.verify_arguments)
- def verify_arguments(self, args: argparse.Namespace):
+ def verify_arguments(self, args: argparse.Namespace) -> None:
# Fail if --detailed used without specifying machine
if args.machine is None and args.detailed:
+ assert self.parser is not None
self.parser.error(
"--detailed only works when a specific machine is specified")
- def one_shot(self, client: ProtocolClient, args: argparse.Namespace):
+ def one_shot(self, client: ProtocolClient,
+ args: argparse.Namespace) -> None:
"""
Display the machine info once
"""
@@ -283,7 +287,8 @@ def one_shot(self, client: ProtocolClient, args: argparse.Namespace):
# Get all information and display accordingly
self.get_and_display_machine_info(client, args, t)
- def recurring(self, client: ProtocolClient, args: argparse.Namespace):
+ def recurring(self, client: ProtocolClient,
+ args: argparse.Namespace) -> None:
"""
Repeatedly display the machine info
"""
@@ -307,7 +312,7 @@ def recurring(self, client: ProtocolClient, args: argparse.Namespace):
print("")
@overrides(Script.body)
- def body(self, client: ProtocolClient, args: argparse.Namespace):
+ def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
if args.watch:
self.recurring(client, args)
else:
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index 0f679460b..1036dff4c 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -39,7 +39,7 @@
def render_job_list(t: Terminal, jobs: JsonObjectArray,
- args: argparse.Namespace):
+ args: argparse.Namespace) -> str:
""" Return a human-readable process listing.
Parameters
@@ -143,13 +143,14 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
help="list only jobs belonging to a particular owner")
return parser
- def one_shot(self, client: ProtocolClient, args: argparse.Namespace):
+ def one_shot(self, client: ProtocolClient, args: argparse.Namespace) -> None:
""" Gets info on the job list once. """
t = Terminal(stream=sys.stderr)
jobs = client.list_jobs(timeout=args.timeout)
print(render_job_list(t, jobs, args))
- def recurring(self, client: ProtocolClient, args: argparse.Namespace):
+ def recurring(
+ self, client: ProtocolClient, args: argparse.Namespace) -> None:
""" Repeatedly gets info on the job list. """
client.notify_job(timeout=args.timeout)
t = Terminal(stream=sys.stderr)
@@ -168,13 +169,14 @@ def recurring(self, client: ProtocolClient, args: argparse.Namespace):
print("")
@overrides(Script.body)
- def body(self, client: ProtocolClient, args: argparse.Namespace):
+ def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
if args.watch:
self.recurring(client, args)
else:
self.one_shot(client, args)
+ return 0
- def verify_arguments(self, args: argparse.Namespace):
+ def verify_arguments(self, args: argparse.Namespace) -> None:
pass
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index bb00d259d..99d55b39a 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -33,14 +33,14 @@ def __init__(self, code: int, message: Optional[str] = None):
self._code = code
self._msg = message
- def exit(self):
+ def exit(self) -> None:
""" Exit the program after printing an error msg. """
if self._msg is not None:
sys.stderr.write(self._msg + "\n")
sys.exit(self._code)
-def version_verify(client: ProtocolClient, timeout: Optional[int]):
+def version_verify(client: ProtocolClient, timeout: Optional[int]) -> None:
"""
Verify that the current version of the client is compatible
"""
@@ -52,7 +52,7 @@ def version_verify(client: ProtocolClient, timeout: Optional[int]):
class Script(object, metaclass=AbstractBase):
""" Base class of various Script Objects. """
- def __init__(self):
+ def __init__(self) -> None:
self.client_factory = ProtocolClient
def get_parser(self, cfg: Dict[str, Any]) -> ArgumentParser:
@@ -61,7 +61,7 @@ def get_parser(self, cfg: Dict[str, Any]) -> ArgumentParser:
raise NotImplementedError
@abstractmethod
- def verify_arguments(self, args: Namespace):
+ def verify_arguments(self, args: Namespace) -> None:
""" Check the arguments for sanity and do any second-stage parsing\
required.
"""
@@ -74,7 +74,7 @@ def body(self, client: ProtocolClient, args: Namespace) -> int:
raise NotImplementedError
def build_server_arg_group(self, server_args: Any,
- cfg: Dict[str, object]):
+ cfg: Dict[str, object]) -> None:
"""
Adds a few more arguments
@@ -95,12 +95,12 @@ def build_server_arg_group(self, server_args: Any,
help="Ignore the server version (WARNING: could result in errors) "
"default: %(default)s)")
- def __call__(self, argv=None):
+ def __call__(self) -> int:
cfg = config.read_config()
parser = self.get_parser(cfg)
server_args = parser.add_argument_group("spalloc server arguments")
self.build_server_arg_group(server_args, cfg)
- args = parser.parse_args(argv)
+ args = parser.parse_args()
# Fail if server not specified
if args.hostname is None:
@@ -122,3 +122,5 @@ def __call__(self, argv=None):
return 1
except Terminate as t:
t.exit()
+ return 1
+
diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py
index eaac72c18..e68d759bc 100644
--- a/spalloc_client/scripts/where_is.py
+++ b/spalloc_client/scripts/where_is.py
@@ -66,12 +66,13 @@
"""
import argparse
import sys
-from typing import Any, cast, Dict
+from typing import Any, cast, Dict, Optional
from spinn_utilities.overrides import overrides
from spalloc_client import __version__, ProtocolClient
from spalloc_client.term import render_definitions
+
from spalloc_client.scripts.support import Terminate, Script
@@ -80,11 +81,11 @@ class WhereIsScript(Script):
An script object to find where a board is
"""
- def __init__(self):
+ def __init__(self) -> None:
super().__init__()
- self.parser = None
- self.where_is_kwargs = None
- self.show_board_chip = None
+ self.parser: Optional[argparse.ArgumentParser] = None
+ self.where_is_kwargs: Optional[dict] = None
+ self.show_board_chip = False
@overrides(Script.get_parser)
def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
@@ -114,7 +115,7 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
return parser
@overrides(Script.verify_arguments)
- def verify_arguments(self, args: argparse.Namespace):
+ def verify_arguments(self, args: argparse.Namespace) -> None:
try:
if args.board:
machine, x, y, z = args.board
@@ -151,11 +152,13 @@ def verify_arguments(self, args: argparse.Namespace):
}
self.show_board_chip = True
except ValueError as e:
+ assert self.parser is not None
self.parser.error(f"Error: {e}")
@overrides(Script.body)
- def body(self, client: ProtocolClient, args: argparse.Namespace):
+ def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
# Ask the server
+ assert self.where_is_kwargs is not None
location = client.where_is(**self.where_is_kwargs)
if location is None:
raise Terminate(4, "No boards at the specified location")
@@ -175,6 +178,7 @@ def body(self, client: ProtocolClient, args: argparse.Namespace):
out["Coordinates within job"] = tuple(
cast(list, location["job_chip"]))
print(render_definitions(out))
+ return 0
main = WhereIsScript()
diff --git a/spalloc_client/term.py b/spalloc_client/term.py
index 64a1ed0fe..411e95e8c 100644
--- a/spalloc_client/term.py
+++ b/spalloc_client/term.py
@@ -188,7 +188,7 @@ def __getattr__(self, name):
post=self("\033[0m"))
-def render_table(table: TableType, column_sep: str = " "):
+def render_table(table: TableType, column_sep: str = " ") -> str:
""" Render an ASCII table with optional ANSI escape codes.
An example table::
From 0d1a9e584fa51dd7a6484d19f1e372e94144c2fa Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 20 Nov 2024 06:31:09 +0000
Subject: [PATCH 04/38] more typing
---
spalloc_client/protocol_client.py | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 3eddeca0b..41f276897 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -271,7 +271,7 @@ def _send_json(self, obj, timeout:Optional[float] = None):
except socket.timeout as e:
raise ProtocolTimeoutError("send timed out.") from e
- def call(self, name, *args, **kwargs):
+ def call(self, name, *args, **kwargs) -> Optional[JsonObject]:
""" Send a command to the server and return the reply.
Parameters
@@ -316,7 +316,8 @@ def call(self, name, *args, **kwargs):
except (IOError, OSError) as e:
raise ProtocolError(str(e)) from e
- def wait_for_notification(self, timeout=None):
+ def wait_for_notification(
+ self, timeout:Optional[float] = None) -> Optional[JsonObject]:
""" Return the next notification to arrive.
Parameters
@@ -439,8 +440,9 @@ def list_machines(self,
""" Obtains a list of currently supported machines. """
return self.call("list_machines", timeout=timeout)
- def get_board_position(self, machine_name: str, x: int, y: int, z: int,
- timeout: Optional[int] = None): # pragma: no cover
+ def get_board_position(
+ self, machine_name: str, x: int, y: int, z: int,
+ timeout: Optional[int] = None) -> JsonObject: # pragma: no cover
""" Gets the position of board x, y, z on the given machine. """
# pylint: disable=too-many-arguments
return self.call("get_board_position", machine_name, x, y, z,
@@ -461,7 +463,7 @@ def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
frozenset("job_id chip_x chip_y".split())])
def where_is(self, timeout: Optional[int] = None,
- **kwargs: int) -> JsonObject:
+ **kwargs: Optional[int]) -> JsonObject:
""" Reports where ion the Machine a job is running """
# Test for whether sane arguments are passed.
keywords = frozenset(kwargs)
From f7ce52300b09672398966e9cd9d1b6bf65459369 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Thu, 21 Nov 2024 09:09:13 +0000
Subject: [PATCH 05/38] destroy_job reason is expected in kwargs
---
spalloc_client/protocol_client.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 41f276897..01b157436 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -409,7 +409,7 @@ def power_off_job_boards(self, job_id: int,
def destroy_job(self, job_id: int, reason: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Destroy the job """
- return self.call("destroy_job", job_id, reason, timeout=timeout)
+ return self.call("destroy_job", job_id, reason=reason, timeout=timeout)
def notify_job(self, job_id: Optional[int] = None,
timeout: Optional[int] = None) -> JsonObject:
From 90418bb4befec7f76a519939ee89740cb562ca5c Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 11:39:44 +0000
Subject: [PATCH 06/38] put argv back in
---
spalloc_client/scripts/job.py | 3 +--
spalloc_client/scripts/support.py | 7 +++----
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index de995f576..0643ccff5 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -377,8 +377,7 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
def verify_arguments(self, args: argparse.Namespace) -> None:
if args.job_id is None and args.owner is None:
assert self.parser is not None
- #self.parser.error("job ID (or --owner) not specified")
- args.job_id = 617379
+ self.parser.error("job ID (or --owner) not specified")
@overrides(Script.body)
def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index 99d55b39a..22a3b2042 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -95,17 +95,16 @@ def build_server_arg_group(self, server_args: Any,
help="Ignore the server version (WARNING: could result in errors) "
"default: %(default)s)")
- def __call__(self) -> int:
+ def __call__(self, argv=None) -> int:
cfg = config.read_config()
parser = self.get_parser(cfg)
server_args = parser.add_argument_group("spalloc server arguments")
self.build_server_arg_group(server_args, cfg)
- args = parser.parse_args()
+ args = parser.parse_args(argv)
# Fail if server not specified
if args.hostname is None:
- #parser.error("--hostname of spalloc server must be specified")
- args.hostname = "spinnaker.cs.man.ac.uk"
+ parser.error("--hostname of spalloc server must be specified")
self.verify_arguments(args)
try:
From 99f8903690fe4283a551b88001e706ac49aebb7e Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 11:50:28 +0000
Subject: [PATCH 07/38] remove timeout from kwargs
---
spalloc_client/protocol_client.py | 42 ++++++++++++++-----------------
tests/test_protocol_client.py | 16 +++++++-----
2 files changed, 29 insertions(+), 29 deletions(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 01b157436..bf12268a5 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -271,7 +271,8 @@ def _send_json(self, obj, timeout:Optional[float] = None):
except socket.timeout as e:
raise ProtocolTimeoutError("send timed out.") from e
- def call(self, name, *args, **kwargs) -> Optional[JsonObject]:
+ def call(self, name:str, timeout: Optional[float] = None, *args,
+ **kwargs) -> Optional[JsonObject]:
""" Send a command to the server and return the reply.
Parameters
@@ -295,7 +296,6 @@ def call(self, name, *args, **kwargs) -> Optional[JsonObject]:
If the connection is unavailable or is closed.
"""
try:
- timeout = kwargs.pop("timeout", None)
finish_time = make_timeout(timeout)
# Construct the command message
@@ -364,7 +364,7 @@ def wait_for_notification(
def version(self, timeout: Optional[int] = None) -> str:
""" Ask what version of spalloc is running. """
- return self.call("version", timeout=timeout)
+ return self.call("version", timeout)
def create_job(self, *args: int,
**kwargs: Dict[str, object]) -> int:
@@ -375,7 +375,7 @@ def create_job(self, *args: int,
if "owner" not in kwargs:
raise SpallocServerException(
"owner must be specified for all jobs.")
- return self.call("create_job", *args, **kwargs)
+ return self.call("create_job", None,*args, **kwargs)
def job_keepalive(self, job_id: int,
timeout: Optional[float] = None) -> JsonObject:
@@ -384,77 +384,74 @@ def job_keepalive(self, job_id: int,
Without these the job will be killed after a while.
"""
- return self.call("job_keepalive", job_id, timeout=timeout)
+ return self.call("job_keepalive", timeout, job_id)
def get_job_state(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
"""Get the state for this job """
- return self.call("get_job_state", job_id, timeout=timeout)
+ return self.call("get_job_state", timeout, job_id)
def get_job_machine_info(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
""" Get info for this job. """
- return self.call("get_job_machine_info", job_id, timeout=timeout)
+ return self.call("get_job_machine_info", timeout, job_id)
def power_on_job_boards(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on the power on the jobs boards. """
- return self.call("power_on_job_boards", job_id, timeout=timeout)
+ return self.call("power_on_job_boards", timeout, job_id)
def power_off_job_boards(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
""" Turn off the power on the jobs boards. """
- return self.call("power_off_job_boards", job_id, timeout=timeout)
+ return self.call("power_off_job_boards", timeout, job_id)
def destroy_job(self, job_id: int, reason: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Destroy the job """
- return self.call("destroy_job", job_id, reason=reason, timeout=timeout)
-
+ return self.call("destroy_job", timeout, job_id, reason=reason)
def notify_job(self, job_id: Optional[int] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on notification of job status changes. """
- return self.call("notify_job", job_id, timeout=timeout)
+ return self.call("notify_job", timeout, job_id)
def no_notify_job(self, job_id: Optional[int] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn off notification of job status changes. """
- return self.call("no_notify_job", job_id, timeout=timeout)
+ return self.call("no_notify_job", timeout, job_id)
def notify_machine(self, machine_name: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on notification of machine status changes. """
- return self.call("notify_machine", machine_name, timeout=timeout)
+ return self.call("notify_machine", timeout, machine_name)
def no_notify_machine(self, machine_name: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn off notification of machine status changes. """
- return self.call("no_notify_machine", machine_name, timeout=timeout)
+ return self.call("no_notify_machine", timeout, machine_name)
def list_jobs(self, timeout: Optional[int] = None) -> JsonObjectArray:
""" Obtains a list of jobs currently running. """
- return self.call("list_jobs", timeout=timeout)
+ return self.call("list_jobs", timeout)
def list_machines(self,
timeout: Optional[float] = None) -> JsonObjectArray:
""" Obtains a list of currently supported machines. """
- return self.call("list_machines", timeout=timeout)
+ return self.call("list_machines", timeout)
def get_board_position(
self, machine_name: str, x: int, y: int, z: int,
timeout: Optional[int] = None) -> JsonObject: # pragma: no cover
""" Gets the position of board x, y, z on the given machine. """
# pylint: disable=too-many-arguments
- return self.call("get_board_position", machine_name, x, y, z,
- timeout=timeout)
+ return self.call("get_board_position", timeout, machine_name, x, y, z)
def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
timeout: Optional[int] = None
) -> JsonObject: # pragma: no cover
""" Gets the board x, y, z on the requested machine. """
# pylint: disable=too-many-arguments
- return self.call("get_board_at_position", machine_name, x, y, z,
- timeout=timeout)
+ return self.call("get_board_at_position", timeout, machine_name, x, y, z)
_acceptable_kwargs_for_where_is = frozenset([
frozenset("machine x y z".split()),
@@ -470,5 +467,4 @@ def where_is(self, timeout: Optional[int] = None,
if keywords not in ProtocolClient._acceptable_kwargs_for_where_is:
raise SpallocServerException(
f"Invalid arguments: {', '.join(keywords)}")
- kwargs["timeout"] = timeout
- return self.call("where_is", **kwargs)
+ return self.call("where_is", timeout, **kwargs)
diff --git a/tests/test_protocol_client.py b/tests/test_protocol_client.py
index 478b7e538..e9d9e6d53 100644
--- a/tests/test_protocol_client.py
+++ b/tests/test_protocol_client.py
@@ -159,25 +159,27 @@ def test_send_json_fails(c):
def test_call(c, s, bg_accept):
c.connect()
bg_accept.join()
+ no_timeout = None
# Basic calls should work
s.send({"return": "Woo"})
- assert c.call("foo", 1, bar=2) == "Woo"
+ assert c.call("foo", no_timeout, 1, bar=2) == "Woo"
assert s.recv() == {"command": "foo", "args": [1], "kwargs": {"bar": 2}}
# Should be able to cope with notifications arriving before return value
s.send({"notification": 1})
s.send({"notification": 2})
s.send({"return": "Woo"})
- assert c.call("foo", 1, bar=2) == "Woo"
+ assert c.call("foo", no_timeout, 1, bar=2) == "Woo"
assert s.recv() == {"command": "foo", "args": [1], "kwargs": {"bar": 2}}
assert list(c._notifications) == [{"notification": 1}, {"notification": 2}]
c._notifications.clear()
# Should be able to timeout immediately
before = time.time()
+ timeout = 0.1
with pytest.raises(ProtocolTimeoutError):
- c.call("foo", 1, bar=2, timeout=0.1)
+ c.call("foo", timeout, 1, bar=2)
after = time.time()
assert s.recv() == {"command": "foo", "args": [1], "kwargs": {"bar": 2}}
assert 0.1 < after - before < 0.2
@@ -185,8 +187,9 @@ def test_call(c, s, bg_accept):
# Should be able to timeout after getting a notification
s.send({"notification": 3})
before = time.time()
+ timeout = 0.1
with pytest.raises(ProtocolTimeoutError):
- c.call("foo", 1, bar=2, timeout=0.1)
+ c.call("foo", timeout, 1, bar=2)
after = time.time()
assert s.recv() == {"command": "foo", "args": [1], "kwargs": {"bar": 2}}
assert 0.1 < after - before < 0.2
@@ -195,13 +198,14 @@ def test_call(c, s, bg_accept):
# Exceptions should transfer
s.send({"exception": "something informative"})
with pytest.raises(SpallocServerException) as e:
- c.call("foo")
+ c.call("foo", no_timeout)
assert "something informative" in str(e.value)
def test_wait_for_notification(c, s, bg_accept):
c.connect()
bg_accept.join()
+ no_timeout = None
# Should be able to timeout
with pytest.raises(ProtocolTimeoutError):
@@ -214,7 +218,7 @@ def test_wait_for_notification(c, s, bg_accept):
s.send({"notification": 1})
s.send({"notification": 2})
s.send({"return": "Woo"})
- assert c.call("foo", 1, bar=2) == "Woo"
+ assert c.call("foo", no_timeout, 1, bar=2) == "Woo"
assert s.recv() == {"command": "foo", "args": [1], "kwargs": {"bar": 2}}
assert c.wait_for_notification() == {"notification": 1}
assert c.wait_for_notification() == {"notification": 2}
From c5eea6bd93ade35ec0d7c0b7f8c806f5aa0ff08d Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 12:35:46 +0000
Subject: [PATCH 08/38] raise ProtocolTimeoutError
---
spalloc_client/protocol_client.py | 49 ++++++++++++++++++-------------
1 file changed, 29 insertions(+), 20 deletions(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index bf12268a5..8362df115 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -18,10 +18,11 @@
import errno
import json
import socket
-from typing import Dict, List, Optional
+from typing import cast, Dict, Optional
from threading import current_thread, RLock, local
-from spinn_utilities.typing.json import JsonObject, JsonObjectArray
+from spinn_utilities.typing.json import (
+ JsonArray, JsonObject, JsonObjectArray, JsonValue)
from spalloc_client._utils import time_left, timed_out, make_timeout
@@ -272,7 +273,7 @@ def _send_json(self, obj, timeout:Optional[float] = None):
raise ProtocolTimeoutError("send timed out.") from e
def call(self, name:str, timeout: Optional[float] = None, *args,
- **kwargs) -> Optional[JsonObject]:
+ **kwargs) -> JsonValue:
""" Send a command to the server and return the reply.
Parameters
@@ -313,6 +314,8 @@ def call(self, name:str, timeout: Optional[float] = None, *args,
# Got a notification, keep trying...
with self._notifications_lock:
self._notifications.append(obj)
+
+ raise ProtocolTimeoutError(f"{timeout=} passed!")
except (IOError, OSError) as e:
raise ProtocolError(str(e)) from e
@@ -364,7 +367,7 @@ def wait_for_notification(
def version(self, timeout: Optional[int] = None) -> str:
""" Ask what version of spalloc is running. """
- return self.call("version", timeout)
+ return cast(str, self.call("version", timeout))
def create_job(self, *args: int,
**kwargs: Dict[str, object]) -> int:
@@ -375,7 +378,7 @@ def create_job(self, *args: int,
if "owner" not in kwargs:
raise SpallocServerException(
"owner must be specified for all jobs.")
- return self.call("create_job", None,*args, **kwargs)
+ return cast(int, self.call("create_job", None,*args, **kwargs))
def job_keepalive(self, job_id: int,
timeout: Optional[float] = None) -> JsonObject:
@@ -384,74 +387,80 @@ def job_keepalive(self, job_id: int,
Without these the job will be killed after a while.
"""
- return self.call("job_keepalive", timeout, job_id)
+ return cast(dict, self.call("job_keepalive", timeout, job_id))
def get_job_state(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
"""Get the state for this job """
- return self.call("get_job_state", timeout, job_id)
+ return cast(dict, self.call("get_job_state", timeout, job_id))
def get_job_machine_info(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
""" Get info for this job. """
- return self.call("get_job_machine_info", timeout, job_id)
+ return cast(dict, self.call("get_job_machine_info", timeout, job_id))
def power_on_job_boards(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on the power on the jobs boards. """
- return self.call("power_on_job_boards", timeout, job_id)
+ return cast(dict, self.call("power_on_job_boards", timeout, job_id))
def power_off_job_boards(self, job_id: int,
timeout: Optional[int] = None) -> JsonObject:
""" Turn off the power on the jobs boards. """
- return self.call("power_off_job_boards", timeout, job_id)
+ return cast(dict, self.call("power_off_job_boards", timeout, job_id))
def destroy_job(self, job_id: int, reason: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Destroy the job """
- return self.call("destroy_job", timeout, job_id, reason=reason)
+ return cast(dict, self.call("destroy_job", timeout,
+ job_id, reason=reason))
def notify_job(self, job_id: Optional[int] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on notification of job status changes. """
- return self.call("notify_job", timeout, job_id)
+ return cast(dict, self.call("notify_job", timeout, job_id))
def no_notify_job(self, job_id: Optional[int] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn off notification of job status changes. """
- return self.call("no_notify_job", timeout, job_id)
+ return cast(dict, self.call("no_notify_job", timeout,
+ job_id))
def notify_machine(self, machine_name: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on notification of machine status changes. """
- return self.call("notify_machine", timeout, machine_name)
+ return cast(dict, self.call("notify_machine", timeout,
+ machine_name))
def no_notify_machine(self, machine_name: Optional[str] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn off notification of machine status changes. """
- return self.call("no_notify_machine", timeout, machine_name)
+ return cast(dict, self.call("no_notify_machine", timeout,
+ machine_name))
def list_jobs(self, timeout: Optional[int] = None) -> JsonObjectArray:
""" Obtains a list of jobs currently running. """
- return self.call("list_jobs", timeout)
+ return cast(list, self.call("list_jobs", timeout))
def list_machines(self,
timeout: Optional[float] = None) -> JsonObjectArray:
""" Obtains a list of currently supported machines. """
- return self.call("list_machines", timeout)
+ return cast(list, self.call("list_machines", timeout))
def get_board_position(
self, machine_name: str, x: int, y: int, z: int,
timeout: Optional[int] = None) -> JsonObject: # pragma: no cover
""" Gets the position of board x, y, z on the given machine. """
# pylint: disable=too-many-arguments
- return self.call("get_board_position", timeout, machine_name, x, y, z)
+ return cast(dict, self.call("get_board_position", timeout,
+ machine_name, x, y, z))
def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
timeout: Optional[int] = None
) -> JsonObject: # pragma: no cover
""" Gets the board x, y, z on the requested machine. """
# pylint: disable=too-many-arguments
- return self.call("get_board_at_position", timeout, machine_name, x, y, z)
+ return cast(dict, self.call("get_board_at_position", timeout,
+ machine_name, x, y, z))
_acceptable_kwargs_for_where_is = frozenset([
frozenset("machine x y z".split()),
@@ -467,4 +476,4 @@ def where_is(self, timeout: Optional[int] = None,
if keywords not in ProtocolClient._acceptable_kwargs_for_where_is:
raise SpallocServerException(
f"Invalid arguments: {', '.join(keywords)}")
- return self.call("where_is", timeout, **kwargs)
+ return cast(dict, self.call("where_is", timeout, **kwargs))
From 8b4296fd69c6d5ef2ae5bf2c166823c05d28e52f Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 14:10:39 +0000
Subject: [PATCH 09/38] typing
---
spalloc_client/_utils.py | 6 ++--
spalloc_client/job.py | 4 +--
spalloc_client/protocol_client.py | 51 ++++++++++++++++++------------
spalloc_client/scripts/ps.py | 2 +-
spalloc_client/scripts/support.py | 2 +-
spalloc_client/scripts/where_is.py | 2 +-
6 files changed, 38 insertions(+), 29 deletions(-)
diff --git a/spalloc_client/_utils.py b/spalloc_client/_utils.py
index 37eaa9fe7..7ac6fab22 100644
--- a/spalloc_client/_utils.py
+++ b/spalloc_client/_utils.py
@@ -17,7 +17,7 @@
from typing import Optional
-def time_left(timestamp):
+def time_left(timestamp: Optional[float]) -> Optional[float]:
""" Convert a timestamp into how long to wait for it.
"""
if timestamp is None:
@@ -25,7 +25,7 @@ def time_left(timestamp):
return max(0.0, timestamp - time.time())
-def timed_out(timestamp):
+def timed_out(timestamp: Optional[float]) -> bool:
""" Check if a timestamp has been reached.
"""
if timestamp is None:
@@ -41,7 +41,7 @@ def make_timeout(delay_seconds: Optional[float]) -> Optional[float]:
return time.time() + delay_seconds
-def render_timestamp(timestamp) -> str:
+def render_timestamp(timestamp: float) -> str:
""" Convert a timestamp (Unix seconds) into a local human-readable\
timestamp string.
"""
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 471f36e20..c87d6cfa1 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -19,7 +19,7 @@
import subprocess
import time
from types import TracebackType
-from typing import Any, cast, Dict, List, Optional, Tuple, Type
+from typing import Any, cast, Dict, Optional, Tuple, Type
import sys
from typing_extensions import Literal, Self
@@ -589,7 +589,7 @@ def boards(self) -> Optional[JsonArray]:
return self._last_machine_info.boards
def wait_for_state_change(self, old_state: JobState,
- timeout: Optional[int] = None) -> JobState:
+ timeout: Optional[float] = None) -> JobState:
""" Block until the job's state changes from the supplied state.
Parameters
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 8362df115..64cf4dd7a 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -18,11 +18,13 @@
import errno
import json
import socket
-from typing import cast, Dict, Optional
-from threading import current_thread, RLock, local
+from types import TracebackType
+from typing import cast, Dict, Literal, Optional, Type, Union
+from typing_extensions import Self
+from threading import current_thread, RLock, local, Thread
from spinn_utilities.typing.json import (
- JsonArray, JsonObject, JsonObjectArray, JsonValue)
+ JsonObject, JsonObjectArray, JsonValue)
from spalloc_client._utils import time_left, timed_out, make_timeout
@@ -48,10 +50,10 @@ class _ProtocolThreadLocal(local):
of our state in each thread.
"""
# See https://github.com/SpiNNakerManchester/spalloc/issues/12
- def __init__(self):
+ def __init__(self) -> None:
super().__init__()
self.buffer = b""
- self.sock = None
+ self.sock: Optional[socket.socket] = None
class ProtocolClient(object):
@@ -81,7 +83,8 @@ class ProtocolClient(object):
# Done!
"""
- def __init__(self, hostname, port=22244, timeout=None):
+ def __init__(self, hostname: str, port: int = 22244,
+ timeout: Optional[float] = None):
""" Define a new connection.
.. note::
@@ -92,28 +95,30 @@ def __init__(self, hostname, port=22244, timeout=None):
----------
hostname : str
The hostname of the server.
- port : str or int
+ port : int
The port to use (default: 22244).
"""
self._hostname = hostname
self._port = port
# Mapping from threads to sockets. Kept because we need to have way to
# shut down all sockets at once.
- self._socks = dict()
+ self._socks: Dict[Thread, socket.socket] = dict()
# Thread local variables
self._local = _ProtocolThreadLocal()
# A queue of unprocessed notifications
- self._notifications = deque()
+ self._notifications: deque = deque()
self._dead = True
self._socks_lock = RLock()
self._notifications_lock = RLock()
self._default_timeout = timeout
- def __enter__(self):
+ def __enter__(self) -> Self:
self.connect(self._default_timeout)
return self
- def __exit__(self, exc_type, exc_val, exc_tb):
+ def __exit__(self, exc_type: Optional[Type],
+ exc_value: Optional[BaseException],
+ exc_tb: Optional[TracebackType]) -> Literal[False]:
self.close()
return False
@@ -142,7 +147,7 @@ def _get_connection(self, timeout: Optional[float]) -> socket.socket:
sock.settimeout(timeout)
return sock
- def _do_connect(self, sock: socket.socket):
+ def _do_connect(self, sock: socket.socket) -> bool:
success = False
try:
sock.connect((self._hostname, self._port))
@@ -155,7 +160,7 @@ def _do_connect(self, sock: socket.socket):
def _has_open_socket(self) -> bool:
return self._local.sock is not None
- def connect(self, timeout: Optional[float] = None):
+ def connect(self, timeout: Optional[float] = None) -> None:
"""(Re)connect to the server.
Raises
@@ -180,7 +185,7 @@ def _connect(self, timeout: Optional[float]) -> socket.socket:
# Pass on the exception
raise
- def _close(self, key=None):
+ def _close(self, key: Optional[Thread]=None) -> None:
if key is None:
key = current_thread()
with self._socks_lock:
@@ -193,7 +198,7 @@ def _close(self, key=None):
self._local.buffer = b""
sock.close()
- def close(self):
+ def close(self) -> None:
""" Disconnect from the server.
"""
self._dead = True
@@ -243,7 +248,8 @@ def _recv_json(self, timeout:Optional[float]=None) -> JsonObject:
line, _, self._local.buffer = self._local.buffer.partition(b"\n")
return json.loads(line.decode("utf-8"))
- def _send_json(self, obj, timeout:Optional[float] = None):
+ def _send_json(
+ self, obj: JsonObject, timeout:Optional[float] = None) -> None:
""" Attempt to send a line of JSON to the server.
Parameters
@@ -272,8 +278,9 @@ def _send_json(self, obj, timeout:Optional[float] = None):
except socket.timeout as e:
raise ProtocolTimeoutError("send timed out.") from e
- def call(self, name:str, timeout: Optional[float] = None, *args,
- **kwargs) -> JsonValue:
+ def call(self, name:str, timeout: Optional[float] = None,
+ *args: Union[int, str, None],
+ **kwargs: JsonValue) -> JsonValue:
""" Send a command to the server and return the reply.
Parameters
@@ -300,7 +307,10 @@ def call(self, name:str, timeout: Optional[float] = None, *args,
finish_time = make_timeout(timeout)
# Construct the command message
- command = {"command": name, "args": args, "kwargs": kwargs}
+ command: JsonObject = {}
+ command["command"] = name
+ command["args"] = list(args)
+ command["kwargs"] = kwargs
self._send_json(command, timeout=timeout)
# Command sent! Attempt to receive the response...
@@ -369,8 +379,7 @@ def version(self, timeout: Optional[int] = None) -> str:
""" Ask what version of spalloc is running. """
return cast(str, self.call("version", timeout))
- def create_job(self, *args: int,
- **kwargs: Dict[str, object]) -> int:
+ def create_job(self, *args: int, **kwargs: str) -> int:
"""
Start a new job
"""
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index 1036dff4c..adb81f0d0 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -99,7 +99,7 @@ def render_job_list(t: Terminal, jobs: JsonObjectArray,
else:
num_boards = ""
# Format start time
- timestamp = render_timestamp(job["start_time"])
+ timestamp = render_timestamp(cast(float, job["start_time"]))
if job["allocated_machine_name"] is not None:
machine_name = str(job["allocated_machine_name"])
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index 22a3b2042..07d0d3aed 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -95,7 +95,7 @@ def build_server_arg_group(self, server_args: Any,
help="Ignore the server version (WARNING: could result in errors) "
"default: %(default)s)")
- def __call__(self, argv=None) -> int:
+ def __call__(self, argv: Optional[str] = None) -> int:
cfg = config.read_config()
parser = self.get_parser(cfg)
server_args = parser.add_argument_group("spalloc server arguments")
diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py
index e68d759bc..b9e7cee57 100644
--- a/spalloc_client/scripts/where_is.py
+++ b/spalloc_client/scripts/where_is.py
@@ -164,7 +164,7 @@ def body(self, client: ProtocolClient, args: argparse.Namespace) -> int:
raise Terminate(4, "No boards at the specified location")
out: Dict[str, Any] = dict()
- out["Machine"] = location["machine"]
+ out["Machine"] = cast(str, location["machine"])
cabinet, frame, board = cast(list, location["physical"])
out["Physical location"] = (
f"Cabinet {cabinet}, Frame {frame}, Board {board}")
From da69c92abe5211be42d5c8eed3f7ff30665e5d11 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 16:09:41 +0000
Subject: [PATCH 10/38] typing
---
spalloc_client/config.py | 24 +++++++----------
spalloc_client/scripts/machine.py | 7 +++--
spalloc_client/term.py | 44 ++++++++++++++++++-------------
tests/test_term.py | 4 +--
4 files changed, 41 insertions(+), 38 deletions(-)
diff --git a/spalloc_client/config.py b/spalloc_client/config.py
index 445097317..4534598e1 100644
--- a/spalloc_client/config.py
+++ b/spalloc_client/config.py
@@ -81,7 +81,7 @@
"""
import configparser
import os.path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
import appdirs
@@ -117,26 +117,22 @@
"ignore_version": "False"}
-def _read_none_or_float(parser, option):
+def _read_none_or_float(
+ parser: configparser.ConfigParser, option: str) -> Optional[float]:
if parser.get(SECTION, option) == "None":
return None
return parser.getfloat(SECTION, option)
-def _read_none_or_int(parser, option):
+def _read_none_or_int(
+ parser: configparser.ConfigParser, option: str) -> Optional[int]:
if parser.get(SECTION, option) == "None":
return None
return parser.getint(SECTION, option)
-def _read_any_str(parser, option):
- try:
- return parser.get(SECTION, option)
- except configparser.NoOptionError:
- return None
-
-
-def _read_none_or_str(parser, option):
+def _read_none_or_str(
+ parser: configparser.ConfigParser, option: str) -> Optional[str]:
if parser.get(SECTION, option) == "None":
return None
return parser.get(SECTION, option)
@@ -174,9 +170,9 @@ def read_config(filenames: Optional[List[str]] = None) -> Dict[str, Any]:
# File did not exist, keep trying
pass
- cfg = {
- "hostname": _read_any_str(parser, "hostname"),
- "owner": _read_any_str(parser, "owner"),
+ cfg: Dict[str, Union[float, str, List[str], None]] = {
+ "hostname": _read_none_or_str(parser, "hostname"),
+ "owner": _read_none_or_str(parser, "owner"),
"port": parser.getint(SECTION, "port"),
"keepalive": _read_none_or_float(parser, "keepalive"),
"reconnect_delay": parser.getfloat(SECTION, "reconnect_delay"),
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index bd5ded37f..284fc6e45 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -148,8 +148,6 @@ def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
if job["allocated_machine_name"] == machine_name:
displayed_jobs.append(job)
job["key"] = next(job_key_generator)
- job["colour"] = job_colours[
- cast(int, job["job_id"]) % len(job_colours)]
# Calculate machine stats
num_boards = ((cast(int, machine["width"]) *
@@ -185,7 +183,8 @@ def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
assert isinstance(board, list)
(x, y, z) = board
boards.append((cast(int, x), cast(int, y), cast(int, z)))
- colour_func = cast(Callable, job["colour"])
+ colour_func = job_colours[
+ cast(int, job["job_id"]) % len(job_colours)]
board_groups.append((
boards,
colour_func(cast(str, job["key"]).center(3)), # Label
@@ -193,7 +192,7 @@ def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
tuple(map(t.bright, DEFAULT_BOARD_EDGES)) # Outer
))
print("")
- print(render_boards(board_groups, machine["dead_links"],
+ print(render_boards(board_groups, cast(list, machine["dead_links"]),
tuple(map(t.red, DEFAULT_BOARD_EDGES))))
# Produce table showing jobs on machine
if compact:
diff --git a/spalloc_client/term.py b/spalloc_client/term.py
index 411e95e8c..3c767d0f5 100644
--- a/spalloc_client/term.py
+++ b/spalloc_client/term.py
@@ -21,8 +21,9 @@
from collections import defaultdict
from enum import IntEnum
from functools import partial
-from typing import Callable, Dict, Iterable, List, Tuple, Union
-from typing_extensions import TypeAlias
+from typing import (
+ Callable, Dict, Iterable, List, Optional, TextIO, Tuple, Union)
+from typing_extensions import Self, TypeAlias
# pylint: disable=wrong-spelling-in-docstring
@@ -100,7 +101,8 @@ class Terminal(object):
Is colour enabled?
"""
- def __init__(self, stream=None, force=None):
+ def __init__(self, stream: Optional[TextIO] = None,
+ force: Optional[bool] = None):
"""
Parameters
----------
@@ -120,18 +122,18 @@ def __init__(self, stream=None, force=None):
self._location_saved = False
- def __call__(self, string):
+ def __call__(self, string: str) -> str:
""" If enabled, passes through the given value, otherwise passes\
through an empty string.
"""
return string if self.enabled else ""
- def clear_screen(self):
+ def clear_screen(self) -> str:
""" Clear the screen and reset cursor to top-left corner.
"""
return self("\033[2J\033[;H")
- def update(self, string: str = "", start_again: bool = False):
+ def update(self, string: str = "", start_again: bool = False) -> str:
""" Print before a line and it will replace the previous line prefixed\
with :py:meth:`.update`.
@@ -153,7 +155,7 @@ def update(self, string: str = "", start_again: bool = False):
# Restore to previous location and clear line.
return "".join((self("\0338\033[K"), str(string)))
- def set_attrs(self, attrs=tuple()):
+ def set_attrs(self, attrs:List = []) -> str:
""" Construct an ANSI control sequence which sets the given attribute\
numbers.
"""
@@ -161,7 +163,8 @@ def set_attrs(self, attrs=tuple()):
return ""
return self(f"\033[{';'.join(str(attr) for attr in attrs)}m")
- def wrap(self, string=None, pre="", post=""):
+ def wrap(self, string: Optional[str] = None,
+ pre: str = "", post: str = "") -> str:
""" Wrap a string in the suppled pre and post strings or just print\
the pre string if no string given.
"""
@@ -169,7 +172,7 @@ def wrap(self, string=None, pre="", post=""):
return pre
return "".join((pre, str(string), post))
- def __getattr__(self, name):
+ def __getattr__(self, name: str) -> partial:
""" Implements all the 'magic' style methods.
"""
attrs = []
@@ -264,7 +267,7 @@ def render_table(table: TableType, column_sep: str = " ") -> str:
return "\n".join(column_sep.join(row).rstrip() for row in out)
-def render_definitions(definitions, separator=": "):
+def render_definitions(definitions: Dict, separator: str = ": ") -> str:
""" Render a definition list.
Such a list looks like this::
@@ -294,7 +297,7 @@ def render_definitions(definitions, separator=": "):
for key, value in definitions.items())
-def _board_to_cartesian(x, y, z):
+def _board_to_cartesian(x: int, y: int, z: int) -> Tuple[int, int]:
r""" Translate from logical board coordinates (x, y, z) into Cartesian
coordinates for printing hexagons.
@@ -374,9 +377,13 @@ def _board_to_cartesian(x, y, z):
"""
-def render_boards(board_groups, dead_links=frozenset(),
- dead_edge=("XXX", "X", "X"),
- blank_label=" ", blank_edge=(" ", " ", " ")):
+def render_boards(
+ board_groups: List[Tuple[List[Tuple[int, int, int]], str,
+ Tuple[str, str, str], Tuple[str, str, str]]],
+ dead_links: List = [],
+ dead_edge: Tuple[str, str, str] = ("XXX", "X", "X"),
+ blank_label: str = " ",
+ blank_edge: Tuple[str, str, str] = (" ", " ", " ")) -> str:
r""" Render an ASCII art diagram of a set of boards with sets of boards.
For example::
@@ -396,7 +403,7 @@ def render_boards(board_groups, dead_links=frozenset(),
which are to be used for the inner and outer board edges respectively.
Board groups are drawn sequentially with later board groups obscuring
earlier ones when their edges or boards overlap.
- dead_links : set([(x, y, z, link), ...])
+ dead_links : list([(x, y, z, link), ...])
Enumeration of all dead links. These links are re-drawn in the style
defined by the dead_edge argument after all board groups have been
drawn.
@@ -420,9 +427,9 @@ def render_boards(board_groups, dead_links=frozenset(),
# non-existent boards
all_boards = set()
- for boards, label, edge_inner, edge_outer in board_groups:
+ for _boards, label, edge_inner, edge_outer in board_groups:
# Convert to Cartesian coordinates
- boards = set(_board_to_cartesian(x, y, z) for x, y, z in boards)
+ boards = set(_board_to_cartesian(x, y, z) for x, y, z in _boards)
all_boards.update(boards)
# Set board labels and basic edge style
@@ -486,7 +493,8 @@ def render_boards(board_groups, dead_links=frozenset(),
return "\n".join(filter(None, map(str.rstrip, out)))
-def render_cells(cells, width=80, col_spacing=2):
+def render_cells(cells: List[Tuple[int, str]], width: int = 80,
+ col_spacing: int = 2) -> str:
""" Given a list of short (~10 char) strings, display these aligned in\
columns.
diff --git a/tests/test_term.py b/tests/test_term.py
index b03628c57..0d6a9a45b 100644
--- a/tests/test_term.py
+++ b/tests/test_term.py
@@ -262,12 +262,12 @@ def test_dead_links(self):
for y in range(2)
for z in range(3)], "ABC",
INNER_BOARD_EDGES, OUTER_BOARD_EDGES),
- ], dead_links=set([
+ ], dead_links=[
(0, 0, 0, 0), # 0, 0, East
(0, 0, 0, 2), # 0, 0, North
(0, 0, 0, 1), # 0, 0, North East
(0, 0, 1, 4), # 1, 1, South West
- ]))
+ ])
assert out == (r" ___ ___."
r"/ABC\___/ABC\___."
r"\===,ABC`===,ABC\."
From b38b2c39ebbd8e9ba900162ed7b4860fbc37c55f Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 16:17:21 +0000
Subject: [PATCH 11/38] put _read_any_str back in
---
spalloc_client/config.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/spalloc_client/config.py b/spalloc_client/config.py
index 4534598e1..3c7617ecb 100644
--- a/spalloc_client/config.py
+++ b/spalloc_client/config.py
@@ -131,6 +131,13 @@ def _read_none_or_int(
return parser.getint(SECTION, option)
+def _read_any_str(
+ parser: configparser.ConfigParser, option: str) -> Optional[str]:
+ try:
+ return parser.get(SECTION, option)
+ except configparser.NoOptionError:
+ return None
+
def _read_none_or_str(
parser: configparser.ConfigParser, option: str) -> Optional[str]:
if parser.get(SECTION, option) == "None":
@@ -171,8 +178,8 @@ def read_config(filenames: Optional[List[str]] = None) -> Dict[str, Any]:
pass
cfg: Dict[str, Union[float, str, List[str], None]] = {
- "hostname": _read_none_or_str(parser, "hostname"),
- "owner": _read_none_or_str(parser, "owner"),
+ "hostname": _read_any_str(parser, "hostname"),
+ "owner": _read_any_str(parser, "owner"),
"port": parser.getint(SECTION, "port"),
"keepalive": _read_none_or_float(parser, "keepalive"),
"reconnect_delay": parser.getfloat(SECTION, "reconnect_delay"),
From f89f1b9dc783cf47d98285f1b0e45d1aeac1ab3a Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Mon, 25 Nov 2024 16:31:29 +0000
Subject: [PATCH 12/38] fixes
---
spalloc_client/scripts/machine.py | 8 ++++++--
spalloc_client/term.py | 2 +-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index 284fc6e45..37325cf5e 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -201,8 +201,10 @@ def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
for job in displayed_jobs:
key = cast(str, job["key"])
job_id = str(job["job_id"])
+ colour_func = job_colours[
+ cast(int, job["job_id"]) % len(job_colours)]
cells.append((len(key) + len(job_id) + 1,
- f"{cast(Callable, job['colour'])(key)}:{job_id}"))
+ f"{colour_func(key)}:{job_id}"))
print("")
print(render_cells(cells))
else:
@@ -217,8 +219,10 @@ def show_machine(t: Terminal, machines: JsonObjectArray, jobs: JsonObjectArray,
owner = str(job["owner"])
if "keepalivehost" in job and job["keepalivehost"] is not None:
owner += f" {job['keepalivehost']}"
+ colour_func = job_colours[
+ cast(int, job["job_id"]) % len(job_colours)]
table_row: TableRow = [
- (cast(Callable, job["colour"]), cast(str, job["key"])),
+ (colour_func, cast(str, job["key"])),
cast(int, job["job_id"]),
len(cast(list, job["boards"])),
owner,
diff --git a/spalloc_client/term.py b/spalloc_client/term.py
index 3c767d0f5..29f798efc 100644
--- a/spalloc_client/term.py
+++ b/spalloc_client/term.py
@@ -379,7 +379,7 @@ def _board_to_cartesian(x: int, y: int, z: int) -> Tuple[int, int]:
def render_boards(
board_groups: List[Tuple[List[Tuple[int, int, int]], str,
- Tuple[str, str, str], Tuple[str, str, str]]],
+ Tuple[str, str, str], Tuple[str, str, str]]],
dead_links: List = [],
dead_edge: Tuple[str, str, str] = ("XXX", "X", "X"),
blank_label: str = " ",
From c508187eb1650f65c947c822271fab57fde92ca1 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 06:22:04 +0000
Subject: [PATCH 13/38] flake8
---
spalloc_client/_keepalive_process.py | 2 +-
spalloc_client/job.py | 3 ++-
spalloc_client/protocol_client.py | 13 +++++++------
spalloc_client/scripts/job.py | 8 +++++---
spalloc_client/scripts/machine.py | 1 +
spalloc_client/scripts/ps.py | 3 ++-
spalloc_client/scripts/support.py | 1 -
spalloc_client/term.py | 6 +++---
8 files changed, 21 insertions(+), 16 deletions(-)
diff --git a/spalloc_client/_keepalive_process.py b/spalloc_client/_keepalive_process.py
index 9c7cbb2a4..ae49c0199 100644
--- a/spalloc_client/_keepalive_process.py
+++ b/spalloc_client/_keepalive_process.py
@@ -36,7 +36,7 @@ def wait_for_exit(stop_event: threading.Event) -> None:
def keep_job_alive(
- hostname: str, port: int, job_id: int, keepalive_period:float,
+ hostname: str, port: int, job_id: int, keepalive_period: float,
timeout: float, reconnect_delay: float,
stop_event: threading.Event) -> None:
""" Keeps a Spalloc job alive. Run as a separate process to the main\
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index c87d6cfa1..3376e4a63 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -677,7 +677,7 @@ def _do_reconnect(self, finish_time: Optional[float]) -> None:
time.sleep(max(0.0, delay))
self._reconnect()
- def wait_until_ready(self, timeout:Optional[int] = None) -> None:
+ def wait_until_ready(self, timeout: Optional[int] = None) -> None:
""" Block until the job is allocated and ready.
Parameters
@@ -740,6 +740,7 @@ def where_is_machine(
[cabinet, frame, board] = cast(list, result['physical'])
return (cast(int, cabinet), cast(int, frame), cast(int, board))
+
class StateChangeTimeoutError(Exception):
""" Thrown when a state change takes too long to occur.
"""
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 64cf4dd7a..592e96ebd 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -185,7 +185,7 @@ def _connect(self, timeout: Optional[float]) -> socket.socket:
# Pass on the exception
raise
- def _close(self, key: Optional[Thread]=None) -> None:
+ def _close(self, key: Optional[Thread] = None) -> None:
if key is None:
key = current_thread()
with self._socks_lock:
@@ -208,7 +208,7 @@ def close(self) -> None:
self._close(key)
self._local = _ProtocolThreadLocal()
- def _recv_json(self, timeout:Optional[float]=None) -> JsonObject:
+ def _recv_json(self, timeout: Optional[float] = None) -> JsonObject:
""" Receive a line of JSON from the server.
Parameters
@@ -249,7 +249,7 @@ def _recv_json(self, timeout:Optional[float]=None) -> JsonObject:
return json.loads(line.decode("utf-8"))
def _send_json(
- self, obj: JsonObject, timeout:Optional[float] = None) -> None:
+ self, obj: JsonObject, timeout: Optional[float] = None) -> None:
""" Attempt to send a line of JSON to the server.
Parameters
@@ -278,7 +278,7 @@ def _send_json(
except socket.timeout as e:
raise ProtocolTimeoutError("send timed out.") from e
- def call(self, name:str, timeout: Optional[float] = None,
+ def call(self, name: str, timeout: Optional[float] = None,
*args: Union[int, str, None],
**kwargs: JsonValue) -> JsonValue:
""" Send a command to the server and return the reply.
@@ -330,7 +330,7 @@ def call(self, name:str, timeout: Optional[float] = None,
raise ProtocolError(str(e)) from e
def wait_for_notification(
- self, timeout:Optional[float] = None) -> Optional[JsonObject]:
+ self, timeout: Optional[float] = None) -> Optional[JsonObject]:
""" Return the next notification to arrive.
Parameters
@@ -387,7 +387,7 @@ def create_job(self, *args: int, **kwargs: str) -> int:
if "owner" not in kwargs:
raise SpallocServerException(
"owner must be specified for all jobs.")
- return cast(int, self.call("create_job", None,*args, **kwargs))
+ return cast(int, self.call("create_job", None, *args, **kwargs))
def job_keepalive(self, job_id: int,
timeout: Optional[float] = None) -> JsonObject:
@@ -423,6 +423,7 @@ def destroy_job(self, job_id: int, reason: Optional[str] = None,
""" Destroy the job """
return cast(dict, self.call("destroy_job", timeout,
job_id, reason=reason))
+
def notify_job(self, job_id: Optional[int] = None,
timeout: Optional[int] = None) -> JsonObject:
""" Turn on notification of job status changes. """
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index 0643ccff5..040565f98 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -90,7 +90,9 @@
def _state_name(mapping: JsonObject) -> str:
- return JobState(cast(int, mapping["state"])).name # pylint: disable=no-member
+ state = JobState(cast(int, mapping["state"]))
+ return state.name # pylint: disable=no-member
+
def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[int],
job_id: int) -> None:
@@ -254,7 +256,7 @@ def power_job(client: ProtocolClient, timeout: Optional[int],
f"job {job_id} in state {_state_name(state)}"))
-def list_ips(client: ProtocolClient, timeout:Optional[int],
+def list_ips(client: ProtocolClient, timeout: Optional[int],
job_id: int) -> None:
""" Print a CSV of board hostnames for all boards allocated to a job.
@@ -288,7 +290,7 @@ def list_ips(client: ProtocolClient, timeout:Optional[int],
def destroy_job(client: ProtocolClient, timeout: Optional[int],
- job_id:int, reason:Optional[str] = None) -> None:
+ job_id: int, reason: Optional[str] = None) -> None:
""" Destroy a running job.
Parameters
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index 37325cf5e..584ac8324 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -100,6 +100,7 @@ def list_machines(t: Terminal, machines: JsonObjectArray,
print(render_table(table))
+
def _get_machine(machines: JsonObjectArray, machine_name: str) -> JsonObject:
for machine in machines:
if machine["name"] == machine_name:
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index adb81f0d0..6f4aea0b9 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -143,7 +143,8 @@ def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
help="list only jobs belonging to a particular owner")
return parser
- def one_shot(self, client: ProtocolClient, args: argparse.Namespace) -> None:
+ def one_shot(self, client: ProtocolClient,
+ args: argparse.Namespace) -> None:
""" Gets info on the job list once. """
t = Terminal(stream=sys.stderr)
jobs = client.list_jobs(timeout=args.timeout)
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index 07d0d3aed..7b0462f0e 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -122,4 +122,3 @@ def __call__(self, argv: Optional[str] = None) -> int:
except Terminate as t:
t.exit()
return 1
-
diff --git a/spalloc_client/term.py b/spalloc_client/term.py
index 29f798efc..6648b0908 100644
--- a/spalloc_client/term.py
+++ b/spalloc_client/term.py
@@ -23,7 +23,7 @@
from functools import partial
from typing import (
Callable, Dict, Iterable, List, Optional, TextIO, Tuple, Union)
-from typing_extensions import Self, TypeAlias
+from typing_extensions import TypeAlias
# pylint: disable=wrong-spelling-in-docstring
@@ -155,7 +155,7 @@ def update(self, string: str = "", start_again: bool = False) -> str:
# Restore to previous location and clear line.
return "".join((self("\0338\033[K"), str(string)))
- def set_attrs(self, attrs:List = []) -> str:
+ def set_attrs(self, attrs: List = []) -> str:
""" Construct an ANSI control sequence which sets the given attribute\
numbers.
"""
@@ -379,7 +379,7 @@ def _board_to_cartesian(x: int, y: int, z: int) -> Tuple[int, int]:
def render_boards(
board_groups: List[Tuple[List[Tuple[int, int, int]], str,
- Tuple[str, str, str], Tuple[str, str, str]]],
+ Tuple[str, str, str], Tuple[str, str, str]]],
dead_links: List = [],
dead_edge: Tuple[str, str, str] = ("XXX", "X", "X"),
blank_label: str = " ",
From 3559c013d2d431bba7ae39edce62d7764244e3f8 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 06:28:20 +0000
Subject: [PATCH 14/38] more flake8
---
spalloc_client/config.py | 1 +
spalloc_client/protocol_client.py | 2 +-
spalloc_client/scripts/machine.py | 2 +-
3 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/spalloc_client/config.py b/spalloc_client/config.py
index 3c7617ecb..475554b6f 100644
--- a/spalloc_client/config.py
+++ b/spalloc_client/config.py
@@ -138,6 +138,7 @@ def _read_any_str(
except configparser.NoOptionError:
return None
+
def _read_none_or_str(
parser: configparser.ConfigParser, option: str) -> Optional[str]:
if parser.get(SECTION, option) == "None":
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 592e96ebd..2b172ed2d 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -462,7 +462,7 @@ def get_board_position(
""" Gets the position of board x, y, z on the given machine. """
# pylint: disable=too-many-arguments
return cast(dict, self.call("get_board_position", timeout,
- machine_name, x, y, z))
+ machine_name, x, y, z))
def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
timeout: Optional[int] = None
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index 584ac8324..687eb5246 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -32,7 +32,7 @@
from collections import defaultdict
import argparse
import sys
-from typing import Any, Callable, cast, Dict, Iterator, List, Optional
+from typing import Any, cast, Dict, Iterator, List, Optional
from spinn_utilities.overrides import overrides
from spinn_utilities.typing.json import JsonObject, JsonObjectArray
From b9c9f2be0b4bd4cc1442b616d13faa691e7c51d3 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 06:44:34 +0000
Subject: [PATCH 15/38] remove default values
---
spalloc_client/protocol_client.py | 5 +++--
spalloc_client/scripts/job.py | 2 +-
spalloc_client/term.py | 4 ++--
tests/test_term.py | 14 +++++++-------
4 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 2b172ed2d..4c3bf1b36 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -20,9 +20,10 @@
import socket
from types import TracebackType
from typing import cast, Dict, Literal, Optional, Type, Union
-from typing_extensions import Self
from threading import current_thread, RLock, local, Thread
+from typing_extensions import Self
+
from spinn_utilities.typing.json import (
JsonObject, JsonObjectArray, JsonValue)
@@ -278,7 +279,7 @@ def _send_json(
except socket.timeout as e:
raise ProtocolTimeoutError("send timed out.") from e
- def call(self, name: str, timeout: Optional[float] = None,
+ def call(self, name: str, timeout: Optional[float],
*args: Union[int, str, None],
**kwargs: JsonValue) -> JsonValue:
""" Send a command to the server and return the reply.
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index 040565f98..2d8cb6a05 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -153,7 +153,7 @@ def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[int],
t.dim(" . "),
tuple(map(t.dim, DEFAULT_BOARD_EDGES)),
tuple(map(t.bright, DEFAULT_BOARD_EDGES)),
- )])
+ )], [])
if machine_info["connections"] is not None:
connections = cast(list, machine_info["connections"])
diff --git a/spalloc_client/term.py b/spalloc_client/term.py
index 6648b0908..e9285cb9c 100644
--- a/spalloc_client/term.py
+++ b/spalloc_client/term.py
@@ -155,7 +155,7 @@ def update(self, string: str = "", start_again: bool = False) -> str:
# Restore to previous location and clear line.
return "".join((self("\0338\033[K"), str(string)))
- def set_attrs(self, attrs: List = []) -> str:
+ def set_attrs(self, attrs: List) -> str:
""" Construct an ANSI control sequence which sets the given attribute\
numbers.
"""
@@ -380,7 +380,7 @@ def _board_to_cartesian(x: int, y: int, z: int) -> Tuple[int, int]:
def render_boards(
board_groups: List[Tuple[List[Tuple[int, int, int]], str,
Tuple[str, str, str], Tuple[str, str, str]]],
- dead_links: List = [],
+ dead_links: List,
dead_edge: Tuple[str, str, str] = ("XXX", "X", "X"),
blank_label: str = " ",
blank_edge: Tuple[str, str, str] = (" ", " ", " ")) -> str:
diff --git a/tests/test_term.py b/tests/test_term.py
index 0d6a9a45b..ae07dba7f 100644
--- a/tests/test_term.py
+++ b/tests/test_term.py
@@ -77,7 +77,7 @@ def test_set_attr():
t = Terminal(force=True)
# Empty list
- assert t.set_attrs() == ""
+ assert t.set_attrs([]) == ""
# Single item
assert t.set_attrs([1]) == "\033[1m"
@@ -87,7 +87,7 @@ def test_set_attr():
# When disabled should do nothing
t.enabled = False
- assert t.set_attrs() == ""
+ assert t.set_attrs([]) == ""
assert t.set_attrs([1]) == ""
assert t.set_attrs([1, 2, 3]) == ""
@@ -215,12 +215,12 @@ def test_render_definitions():
class TestRenderBoards(object):
def test_empty(self):
- assert render_boards([]) == ""
+ assert render_boards([], []) == ""
def test_single(self):
out = render_boards([
([(0, 0, 0)], "ABC", INNER_BOARD_EDGES, OUTER_BOARD_EDGES),
- ])
+ ], [])
assert out == (r" ___."
r"/ABC\."
r"\___/".replace(".", "\n"))
@@ -229,7 +229,7 @@ def test_three_boards(self):
out = render_boards([
([(0, 0, z) for z in range(3)], "ABC",
INNER_BOARD_EDGES, OUTER_BOARD_EDGES),
- ])
+ ], [])
assert out == (r" ___."
r"/ABC\___."
r"\===,ABC\."
@@ -244,7 +244,7 @@ def test_many_boards(self):
for y in range(2)
for z in range(3)], "ABC",
INNER_BOARD_EDGES, OUTER_BOARD_EDGES),
- ])
+ ], [])
assert out == (r" ___ ___."
r"/ABC\___/ABC\___."
r"\===,ABC`===,ABC\."
@@ -286,7 +286,7 @@ def test_multiple_board_groups(self):
for x in range(2) for y in range(2) for z in range(3)
if (x, y) != (0, 0)], "ABC",
INNER_BOARD_EDGES, OUTER_BOARD_EDGES),
- ])
+ ], [])
assert out == (r" ___ ___."
r"/ABC\___/ABC\___."
r"\===,ABC`===,ABC\."
From c26d5a8387fcce0c12327092b17d6abaec78abac Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 07:10:05 +0000
Subject: [PATCH 16/38] remove argv
---
spalloc_client/scripts/alloc.py | 29 ++++++++++++-----------------
1 file changed, 12 insertions(+), 17 deletions(-)
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index f7871628b..5575142d8 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -264,13 +264,12 @@ def info(msg: str) -> None:
t.stream.write(f"{msg}\n")
-def update(msg: str, colour: functools.partial,
- *args: Union[int, str, List[object]]) -> None:
+def update(msg: str, colour: functools.partial) -> None:
"""
Writes a message to the terminal in the schoosen colour.
"""
assert t is not None
- info(t.update(colour(msg.format(*args))))
+ info(t.update(colour(msg)))
def wait_for_job_ready(job: Job) -> Tuple[int, Optional[str]]:
@@ -285,11 +284,9 @@ def wait_for_job_ready(job: Job) -> Tuple[int, Optional[str]]:
# Show debug info on state-change
if old_state != cur_state:
if cur_state == JobState.queued:
- update("Job {}: Waiting in queue...", t.yellow,
- job.id)
+ update(f"Job {job.id}: Waiting in queue...", t.yellow)
elif cur_state == JobState.power:
- update("Job {}: Waiting for power on...", t.yellow,
- job.id)
+ update(f"Job {job.id}: Waiting for power on...", t.yellow)
elif cur_state == JobState.ready:
# Here we go!
return 0, None
@@ -302,26 +299,24 @@ def wait_for_job_ready(job: Job) -> Tuple[int, Optional[str]]:
pass
if reason is not None:
- update("Job {}: Destroyed: {}", t.red,
- job.id, reason)
+ update(f"Job {job.id}: Destroyed: {reason}", t.red)
else:
- update("Job {}: Destroyed.", t.red,
- job.id)
+ update(f"Job {job.id}: Destroyed.", t.red)
return 1, reason
elif cur_state == JobState.unknown:
- update("Job {}: Job not recognised by server.", t.red,
- job.id)
+ update(f"Job {job.id}: Job not recognised by server.",
+ t.red)
return 2, None
else:
- update("Job {}: Entered an unrecognised state {}.",
- t.red, job.id, cur_state)
+ update(f"Job {job.id}: Entered an unrecognised state "
+ f"{cur_state}.", t.red)
return 3, None
old_state, cur_state = \
cur_state, job.wait_for_state_change(cur_state)
except KeyboardInterrupt:
# Gracefully terminate from keyboard interrupt
- update("Job {}: Keyboard interrupt.", t.red, job.id)
+ update(f"Job {job.id}: Keyboard interrupt.", t.red)
return 4, "Keyboard interrupt."
@@ -459,7 +454,7 @@ def run_job(job_args: List[int], job_kwargs: Dict[str, Any],
# Machine is now ready
write_ips_to_csv(job.connections, ip_file_filename)
- update("Job {}: Ready!", t.green, job.id)
+ update(f"Job {job.id}: Ready!", t.green)
# Either run the user's application or just print the details.
if not arguments.command:
From 858bc86d7bc3db7e4ca159ff8b2baf9004c932d2 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 07:16:29 +0000
Subject: [PATCH 17/38] better type
---
spalloc_client/job.py | 4 ++--
spalloc_client/scripts/alloc.py | 5 +++--
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 3376e4a63..05e12f0e7 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -19,7 +19,7 @@
import subprocess
import time
from types import TracebackType
-from typing import Any, cast, Dict, Optional, Tuple, Type
+from typing import Any, cast, Dict, Optional, Tuple, Type, Union
import sys
from typing_extensions import Literal, Self
@@ -127,7 +127,7 @@ class Job(object):
allocated.
"""
- def __init__(self, *args: int, **kwargs: Dict[str, Any]):
+ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
""" Request a SpiNNaker machine.
A :py:class:`.Job` is constructed in one of the following styles::
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 5575142d8..057028340 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -427,7 +427,8 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
return parser, parser.parse_args(argv)
-def run_job(job_args: List[int], job_kwargs: Dict[str, Any],
+def run_job(job_args: List[int],
+ job_kwargs: Dict[str, Union[float, str, None]],
ip_file_filename: str) -> int:
"""
Run a job
@@ -497,7 +498,7 @@ def main(argv: Optional[List[str]] = None) -> int:
parser.error("--hostname of spalloc server must be specified")
# Set universal job arguments
- job_kwargs = {
+ job_kwargs: Dict[str, Union[float, str, None]] = {
"hostname": arguments.hostname,
"port": arguments.port,
"reconnect_delay": _minzero(arguments.reconnect_delay),
From 401bfc304d4f06ebd0a8e26c902206ece6b7d875 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 07:22:33 +0000
Subject: [PATCH 18/38] tighter typing
---
spalloc_client/scripts/job.py | 2 +-
spalloc_client/scripts/machine.py | 2 +-
spalloc_client/scripts/ps.py | 2 +-
spalloc_client/scripts/support.py | 2 +-
spalloc_client/scripts/where_is.py | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index 2d8cb6a05..737d6651a 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -340,7 +340,7 @@ def get_job_id(self, client: ProtocolClient,
return cast(int, job_ids[0])
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Manage running jobs.")
parser.add_argument(
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index 687eb5246..5da62d6fa 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -257,7 +257,7 @@ def get_and_display_machine_info(
show_machine(t, machines, jobs, args.machine, not args.detailed)
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Get the state of individual machines.")
parser.add_argument(
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index 6f4aea0b9..b2e055b3e 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -128,7 +128,7 @@ class ProcessListScript(Script):
An object form Job scripts.
"""
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="List all active jobs.")
parser.add_argument(
"--version", "-V", action="version", version=__version__)
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index 7b0462f0e..e3b3fb6eb 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -55,7 +55,7 @@ class Script(object, metaclass=AbstractBase):
def __init__(self) -> None:
self.client_factory = ProtocolClient
- def get_parser(self, cfg: Dict[str, Any]) -> ArgumentParser:
+ def get_parser(self, cfg: Dict[str, str]) -> ArgumentParser:
""" Return a set-up instance of :py:class:`argparse.ArgumentParser`
"""
raise NotImplementedError
diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py
index b9e7cee57..bca63567a 100644
--- a/spalloc_client/scripts/where_is.py
+++ b/spalloc_client/scripts/where_is.py
@@ -88,7 +88,7 @@ def __init__(self) -> None:
self.show_board_chip = False
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, Any]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Find out the location (physical or logical) of a "
"chip or board.")
From 3f7cf7c01748059d70d0ec06718a892c47f467e0 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 07:36:44 +0000
Subject: [PATCH 19/38] remove kwargs
---
spalloc_client/protocol_client.py | 11 ++++-------
tests/test_protocol_client.py | 13 -------------
2 files changed, 4 insertions(+), 20 deletions(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 4c3bf1b36..00ce2ffff 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -479,12 +479,9 @@ def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
frozenset("machine chip_x chip_y".split()),
frozenset("job_id chip_x chip_y".split())])
- def where_is(self, timeout: Optional[int] = None,
- **kwargs: Optional[int]) -> JsonObject:
+ def where_is(self, job_id: int, chip_x: int, chip_y: int,
+ timeout: Optional[int] = None) -> JsonObject:
""" Reports where ion the Machine a job is running """
# Test for whether sane arguments are passed.
- keywords = frozenset(kwargs)
- if keywords not in ProtocolClient._acceptable_kwargs_for_where_is:
- raise SpallocServerException(
- f"Invalid arguments: {', '.join(keywords)}")
- return cast(dict, self.call("where_is", timeout, **kwargs))
+ return cast(dict, self.call("where_is", timeout, job_id=job_id,
+ chip_x=chip_x, chip_y=chip_y))
diff --git a/tests/test_protocol_client.py b/tests/test_protocol_client.py
index e9d9e6d53..199ea1015 100644
--- a/tests/test_protocol_client.py
+++ b/tests/test_protocol_client.py
@@ -244,16 +244,3 @@ def test_commands_as_methods(c, s, bg_accept):
# Should fail for arbitrary external method names
with pytest.raises(AttributeError):
c.bar()
-
-
-def test_where_is_sanity(c):
- with pytest.raises(SpallocServerException):
- c.where_is(foo=1, bar=2)
- with pytest.raises(SpallocServerException):
- c.where_is(machine=1, x=2, y=3, z=4, foo=5)
- with pytest.raises(ProtocolError):
- c.where_is(machine=1, x=2, y=3, z=4)
- with pytest.raises(ProtocolError):
- c.where_is(machine=1, x=2, y=3, z=4, timeout=5)
- with pytest.raises(ProtocolError):
- c.where_is(machine=1, x=2, y=3, z=4, timeout=None)
From 15f2d7d98c0481329ccb0a044908cc88fb3445e6 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 07:46:52 +0000
Subject: [PATCH 20/38] remove unused import
---
spalloc_client/job.py | 2 +-
spalloc_client/scripts/alloc.py | 2 +-
spalloc_client/scripts/ps.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 05e12f0e7..73281710d 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -19,7 +19,7 @@
import subprocess
import time
from types import TracebackType
-from typing import Any, cast, Dict, Optional, Tuple, Type, Union
+from typing import cast, Dict, Optional, Tuple, Type, Union
import sys
from typing_extensions import Literal, Self
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 057028340..fbb365d9a 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -113,7 +113,7 @@
import subprocess
import sys
import tempfile
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
from shlex import quote
from spalloc_client import (
config, Job, JobState, __version__, ProtocolError, ProtocolTimeoutError,
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index b2e055b3e..7abe9f458 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -27,7 +27,7 @@
import argparse
from collections.abc import Sized
import sys
-from typing import Any, cast, Dict, Union
+from typing import cast, Dict, Union
from spinn_utilities.overrides import overrides
from spinn_utilities.typing.json import JsonObjectArray
From 6cf6543f18fe27eeed36047082798671e784cdf2 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 08:52:51 +0000
Subject: [PATCH 21/38] remove unused import
---
tests/test_protocol_client.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/tests/test_protocol_client.py b/tests/test_protocol_client.py
index 199ea1015..8dec6bc11 100644
--- a/tests/test_protocol_client.py
+++ b/tests/test_protocol_client.py
@@ -19,8 +19,7 @@
import pytest
from mock import Mock # type: ignore[import]
from spalloc_client import (
- ProtocolClient, SpallocServerException, ProtocolTimeoutError,
- ProtocolError)
+ ProtocolClient, SpallocServerException, ProtocolTimeoutError)
from .common import MockServer
logging.basicConfig(level=logging.DEBUG)
From be26a3ca4e46b64e8ffdb056fe30518a2c3b09e6 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 08:57:49 +0000
Subject: [PATCH 22/38] rename config.py to spalloc_config.py
---
spalloc_client/job.py | 2 +-
spalloc_client/scripts/alloc.py | 4 ++--
spalloc_client/scripts/support.py | 4 ++--
spalloc_client/{config.py => spalloc_config.py} | 0
tests/conftest.py | 2 +-
tests/scripts/test_job_script.py | 2 +-
tests/test_config.py | 2 +-
7 files changed, 8 insertions(+), 8 deletions(-)
rename spalloc_client/{config.py => spalloc_config.py} (100%)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 73281710d..5d3a8c3f1 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -30,7 +30,7 @@
VERSION_RANGE_START, VERSION_RANGE_STOP)
from .protocol_client import ProtocolClient, ProtocolTimeoutError
-from .config import read_config, SEARCH_PATH
+from .spalloc_config import read_config, SEARCH_PATH
from .states import JobState
from ._utils import time_left, timed_out, make_timeout
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index fbb365d9a..236d6187e 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -116,7 +116,7 @@
from typing import Dict, List, Optional, Tuple, Union
from shlex import quote
from spalloc_client import (
- config, Job, JobState, __version__, ProtocolError, ProtocolTimeoutError,
+ spalloc_config, Job, JobState, __version__, ProtocolError, ProtocolTimeoutError,
SpallocServerException)
from spalloc_client.term import Terminal, render_definitions
@@ -325,7 +325,7 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
"""
Parse the arguments.
"""
- cfg = config.read_config()
+ cfg = spalloc_config.read_config()
parser = argparse.ArgumentParser(
description="Request (and allocate) a SpiNNaker machine.")
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index e3b3fb6eb..6b905e605 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -17,7 +17,7 @@
from typing import Any, Dict, Optional
from spinn_utilities.abstract_base import AbstractBase, abstractmethod
from spalloc_client import (
- config, ProtocolClient, ProtocolError, ProtocolTimeoutError,
+ spalloc_config, ProtocolClient, ProtocolError, ProtocolTimeoutError,
SpallocServerException)
# The acceptable range of server version numbers
@@ -96,7 +96,7 @@ def build_server_arg_group(self, server_args: Any,
"default: %(default)s)")
def __call__(self, argv: Optional[str] = None) -> int:
- cfg = config.read_config()
+ cfg = spalloc_config.read_config()
parser = self.get_parser(cfg)
server_args = parser.add_argument_group("spalloc server arguments")
self.build_server_arg_group(server_args, cfg)
diff --git a/spalloc_client/config.py b/spalloc_client/spalloc_config.py
similarity index 100%
rename from spalloc_client/config.py
rename to spalloc_client/spalloc_config.py
diff --git a/tests/conftest.py b/tests/conftest.py
index e809e22b9..59b95d22e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -19,7 +19,7 @@
import pytest
from mock import Mock # type: ignore[import]
from spalloc_client import ProtocolClient
-from spalloc_client.config import SEARCH_PATH
+from spalloc_client.spalloc_config import SEARCH_PATH
from .common import MockServer
diff --git a/tests/scripts/test_job_script.py b/tests/scripts/test_job_script.py
index 4407f55b8..bf7ac83ba 100644
--- a/tests/scripts/test_job_script.py
+++ b/tests/scripts/test_job_script.py
@@ -16,7 +16,7 @@
import pytest
from mock import Mock, MagicMock # type: ignore[import]
from spalloc_client import JobState, ProtocolError
-from spalloc_client.config import TIMEOUT
+from spalloc_client.spalloc_config import TIMEOUT
from spalloc_client.term import Terminal
from spalloc_client.scripts.job import (
show_job_info, watch_job, power_job, list_ips, destroy_job, main)
diff --git a/tests/test_config.py b/tests/test_config.py
index 77c2e48d3..8b9876001 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -16,7 +16,7 @@
import shutil
import os.path
import pytest
-from spalloc_client.config import read_config, TIMEOUT
+from spalloc_client.spalloc_config import read_config, TIMEOUT
@pytest.yield_fixture
From 76ca52af98c76c157c0cf5c3f98bc5127a1da804 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 09:47:38 +0000
Subject: [PATCH 23/38] SpallocConfig
---
spalloc_client/__init__.py | 4 +-
spalloc_client/job.py | 28 +++---
spalloc_client/scripts/alloc.py | 35 +++----
spalloc_client/scripts/job.py | 2 +-
spalloc_client/scripts/support.py | 17 ++--
spalloc_client/spalloc_config.py | 155 ++++++++++++++++++++----------
tests/test_config.py | 62 ++----------
tests/test_job.py | 1 +
8 files changed, 154 insertions(+), 150 deletions(-)
diff --git a/spalloc_client/__init__.py b/spalloc_client/__init__.py
index 2d08687a8..3afe7fd1e 100644
--- a/spalloc_client/__init__.py
+++ b/spalloc_client/__init__.py
@@ -23,5 +23,5 @@
__all__ = [
"Job", "JobDestroyedError", "JobState", "ProtocolClient",
- "ProtocolError", "ProtocolTimeoutError", "SpallocServerException",
- "StateChangeTimeoutError"]
+ "ProtocolError", "ProtocolTimeoutError",
+ "SpallocServerException", "StateChangeTimeoutError"]
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 5d3a8c3f1..8d3df752c 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -30,7 +30,7 @@
VERSION_RANGE_START, VERSION_RANGE_STOP)
from .protocol_client import ProtocolClient, ProtocolTimeoutError
-from .spalloc_config import read_config, SEARCH_PATH
+from .spalloc_config import SpallocConfig, SEARCH_PATH
from .states import JobState
from ._utils import time_left, timed_out, make_timeout
@@ -243,15 +243,15 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
# Read configuration
config_filenames = cast(
list, kwargs.pop("config_filenames", SEARCH_PATH))
- config = read_config(config_filenames)
+ config = SpallocConfig(config_filenames)
# Get protocol client options
- hostname = kwargs.get("hostname", config["hostname"])
- owner = kwargs.get("owner", config["owner"])
- port = kwargs.get("port", config["port"])
+ hostname = kwargs.get("hostname", config.hostname)
+ owner = kwargs.get("owner", config.owner)
+ port = kwargs.get("port", config.port)
self._reconnect_delay = kwargs.get("reconnect_delay",
- config["reconnect_delay"])
- self._timeout = kwargs.get("timeout", config["timeout"])
+ config.reconnect_delay)
+ self._timeout = kwargs.get("timeout", config.timeout)
if hostname is None:
raise ValueError("A hostname must be specified.")
@@ -294,16 +294,16 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
job_args = args
job_kwargs = {
"owner": owner,
- "keepalive": kwargs.get("keepalive", config["keepalive"]),
- "machine": kwargs.get("machine", config["machine"]),
- "tags": kwargs.get("tags", config["tags"]),
- "min_ratio": kwargs.get("min_ratio", config["min_ratio"]),
+ "keepalive": kwargs.get("keepalive", config.keepalive),
+ "machine": kwargs.get("machine", config.machine),
+ "tags": kwargs.get("tags", config.tags),
+ "min_ratio": kwargs.get("min_ratio", config.min_ratio),
"max_dead_boards":
- kwargs.get("max_dead_boards", config["max_dead_boards"]),
+ kwargs.get("max_dead_boards", config.max_dead_boards),
"max_dead_links":
- kwargs.get("max_dead_links", config["max_dead_links"]),
+ kwargs.get("max_dead_links", config.max_dead_links),
"require_torus":
- kwargs.get("require_torus", config["require_torus"]),
+ kwargs.get("require_torus", config.require_torus),
"timeout": self._timeout,
}
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 236d6187e..3804a2f73 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -116,8 +116,9 @@
from typing import Dict, List, Optional, Tuple, Union
from shlex import quote
from spalloc_client import (
- spalloc_config, Job, JobState, __version__, ProtocolError, ProtocolTimeoutError,
+ Job, JobState, __version__, ProtocolError, ProtocolTimeoutError,
SpallocServerException)
+from spalloc_client.spalloc_config import SpallocConfig
from spalloc_client.term import Terminal, render_definitions
# pylint: disable=invalid-name
@@ -325,7 +326,7 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
"""
Parse the arguments.
"""
- cfg = spalloc_config.read_config()
+ cfg = SpallocConfig()
parser = argparse.ArgumentParser(
description="Request (and allocate) a SpiNNaker machine.")
@@ -354,38 +355,38 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
help="if given, resume keeping the specified job alive rather than "
"creating a new job (all allocation requirements will be ignored)")
allocation_args.add_argument(
- "--machine", "-m", nargs="?", default=cfg["machine"],
+ "--machine", "-m", nargs="?", default=cfg.machine,
help="only allocate boards which are part of a specific machine, or "
"any machine if no machine is given (default: %(default)s)")
allocation_args.add_argument(
"--tags", "-t", nargs="*", metavar="TAG",
- default=cfg["tags"] or ["default"],
+ default=cfg.tags or ["default"],
help="only allocate boards which have (at least) the specified flags "
- f"(default: {' '.join(cfg['tags'] or [])})")
+ f"(default: {' '.join(cfg.tags or [])})")
allocation_args.add_argument(
- "--min-ratio", type=float, metavar="RATIO", default=cfg["min_ratio"],
+ "--min-ratio", type=float, metavar="RATIO", default=cfg.min_ratio,
help="when allocating by number of boards, require that the "
"allocation be at least as square as this ratio (default: "
"%(default)s)")
allocation_args.add_argument(
"--max-dead-boards", type=int, metavar="NUM", default=(
- -1 if cfg["max_dead_boards"] is None else cfg["max_dead_boards"]),
+ -1 if cfg.max_dead_boards is None else cfg.max_dead_boards),
help="boards allowed to be dead in the allocation, or -1 to allow "
"any number of dead boards (default: %(default)s)")
allocation_args.add_argument(
"--max-dead-links", type=int, metavar="NUM", default=(
- -1 if cfg["max_dead_links"] is None else cfg["max_dead_links"]),
+ -1 if cfg.max_dead_links is None else cfg.max_dead_links),
help="inter-board links allowed to be dead in the allocation, or -1 "
"to allow any number of dead links (default: %(default)s)")
allocation_args.add_argument(
"--require-torus", "-w", action="store_true",
- default=cfg["require_torus"],
+ default=cfg.require_torus,
help="require that the allocation contain torus (a.k.a. wrap-around) "
- f"links {'(default)' if cfg['require_torus'] else ''}")
+ f"links {'(default)' if cfg.require_torus else ''}")
allocation_args.add_argument(
"--no-require-torus", "-W", action="store_false", dest="require_torus",
help="do not require that the allocation contain torus (a.k.a. "
- f"wrap-around) links {'' if cfg['require_torus'] else '(default)'}")
+ f"wrap-around) links {'' if cfg.require_torus else '(default)'}")
command_args = parser.add_argument_group("command wrapping arguments")
command_args.add_argument(
@@ -400,28 +401,28 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
server_args = parser.add_argument_group("spalloc server arguments")
server_args.add_argument(
- "--owner", default=cfg["owner"],
+ "--owner", default=cfg.owner,
help="by convention, the email address of the owner of the job "
"(default: %(default)s)")
server_args.add_argument(
- "--hostname", "-H", default=cfg["hostname"],
+ "--hostname", "-H", default=cfg.hostname,
help="hostname or IP of the spalloc server (default: %(default)s)")
server_args.add_argument(
- "--port", "-P", default=cfg["port"], type=int,
+ "--port", "-P", default=cfg.port, type=int,
help="port number of the spalloc server (default: %(default)s)")
server_args.add_argument(
"--keepalive", type=int, metavar="SECONDS",
- default=(-1 if cfg["keepalive"] is None else cfg["keepalive"]),
+ default=(-1 if cfg.keepalive is None else cfg.keepalive),
help="the interval at which to require keepalive messages to be "
"sent to prevent the server cancelling the job, or -1 to not "
"require keepalive messages (default: %(default)s)")
server_args.add_argument(
- "--reconnect-delay", default=cfg["reconnect_delay"], type=float,
+ "--reconnect-delay", default=cfg.reconnect_delay, type=float,
metavar="SECONDS",
help="seconds to wait before reconnecting to the server if the "
"connection is lost (default: %(default)s)")
server_args.add_argument(
- "--timeout", default=cfg["timeout"], type=float, metavar="SECONDS",
+ "--timeout", default=cfg.timeout, type=float, metavar="SECONDS",
help="seconds to wait for a response from the server "
"(default: %(default)s)")
return parser, parser.parse_args(argv)
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index 737d6651a..0388fb947 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -350,7 +350,7 @@ def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
help="the job ID of interest, optional if the current owner only "
"has one job")
parser.add_argument(
- "--owner", "-o", default=cfg["owner"],
+ "--owner", "-o", default=cfg.owner,
help="if no job ID is provided and this owner has only one job, "
"this job is assumed (default: %(default)s)")
control_args = parser.add_mutually_exclusive_group()
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index 6b905e605..be4f51753 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -17,8 +17,9 @@
from typing import Any, Dict, Optional
from spinn_utilities.abstract_base import AbstractBase, abstractmethod
from spalloc_client import (
- spalloc_config, ProtocolClient, ProtocolError, ProtocolTimeoutError,
+ ProtocolClient, ProtocolError, ProtocolTimeoutError,
SpallocServerException)
+from spalloc_client.spalloc_config import SpallocConfig
# The acceptable range of server version numbers
VERSION_RANGE_START = (0, 1, 0)
@@ -55,7 +56,7 @@ class Script(object, metaclass=AbstractBase):
def __init__(self) -> None:
self.client_factory = ProtocolClient
- def get_parser(self, cfg: Dict[str, str]) -> ArgumentParser:
+ def get_parser(self, cfg: SpallocConfig) -> ArgumentParser:
""" Return a set-up instance of :py:class:`argparse.ArgumentParser`
"""
raise NotImplementedError
@@ -74,29 +75,29 @@ def body(self, client: ProtocolClient, args: Namespace) -> int:
raise NotImplementedError
def build_server_arg_group(self, server_args: Any,
- cfg: Dict[str, object]) -> None:
+ cfg: SpallocConfig) -> None:
"""
Adds a few more arguments
:param argparse._ArguementGroup server_args:
"""
server_args.add_argument(
- "--hostname", "-H", default=cfg["hostname"],
+ "--hostname", "-H", default=cfg.hostname,
help="hostname or IP of the spalloc server (default: %(default)s)")
server_args.add_argument(
- "--port", "-P", default=cfg["port"], type=int,
+ "--port", "-P", default=cfg.port, type=int,
help="port number of the spalloc server (default: %(default)s)")
server_args.add_argument(
- "--timeout", default=cfg["timeout"], type=float, metavar="SECONDS",
+ "--timeout", default=cfg.timeout, type=float, metavar="SECONDS",
help="seconds to wait for a response from the server (default: "
"%(default)s)")
server_args.add_argument(
- "--ignore_version", default=cfg["ignore_version"], type=bool,
+ "--ignore_version", default=cfg.ignore_version, type=bool,
help="Ignore the server version (WARNING: could result in errors) "
"default: %(default)s)")
def __call__(self, argv: Optional[str] = None) -> int:
- cfg = spalloc_config.read_config()
+ cfg = SpallocConfig()
parser = self.get_parser(cfg)
server_args = parser.add_argument_group("spalloc server arguments")
self.build_server_arg_group(server_args, cfg)
diff --git a/spalloc_client/spalloc_config.py b/spalloc_client/spalloc_config.py
index 475554b6f..2179b64df 100644
--- a/spalloc_client/spalloc_config.py
+++ b/spalloc_client/spalloc_config.py
@@ -146,58 +146,109 @@ def _read_none_or_str(
return parser.get(SECTION, option)
-def read_config(filenames: Optional[List[str]] = None) -> Dict[str, Any]:
- """ Attempt to read local configuration files to determine spalloc client
- settings.
-
- Parameters
- ----------
- filenames : [str, ...]
- Filenames to attempt to read. Later config file have higher priority.
-
- Returns
- -------
- dict
- The configuration loaded.
- """
- if filenames is None: # pragma: no cover
- filenames = SEARCH_PATH
- parser = configparser.ConfigParser()
-
- # Set default config values (NB: No read_dict in Python 2.7)
- parser.add_section(SECTION)
- for key, value in DEFAULT_CONFIG.items():
- parser.set(SECTION, key, value)
-
- # Attempt to read from each possible file location in turn
- for filename in filenames:
- try:
- with open(filename, "r", encoding="utf-8") as f:
- parser.read_file(f, filename)
- except (IOError, OSError):
- # File did not exist, keep trying
- pass
-
- cfg: Dict[str, Union[float, str, List[str], None]] = {
- "hostname": _read_any_str(parser, "hostname"),
- "owner": _read_any_str(parser, "owner"),
- "port": parser.getint(SECTION, "port"),
- "keepalive": _read_none_or_float(parser, "keepalive"),
- "reconnect_delay": parser.getfloat(SECTION, "reconnect_delay"),
- "timeout": _read_none_or_float(parser, "timeout"),
- "machine": _read_none_or_str(parser, "machine"),
- "min_ratio": parser.getfloat(SECTION, "min_ratio"),
- "max_dead_boards": _read_none_or_int(parser, "max_dead_boards"),
- "max_dead_links": _read_none_or_int(parser, "max_dead_links"),
- "require_torus": parser.getboolean(SECTION, "require_torus"),
- "ignore_version": parser.getboolean(SECTION, "ignore_version")}
-
- tags = _read_none_or_str(parser, "tags")
- cfg["tags"] = None if tags is None else list(
- map(str.strip, tags.split(",")))
-
- return cfg
-
+class SpallocConfig(object):
+
+ __slots__ = ("_hostname", "_ignore_version", "_keepalive", "_machine",
+ "_max_dead_boards", "_max_dead_links", "_min_ratio",
+ "_owner", "_port", "_reconnect_delay", "_require_torus",
+ "_tags", "_timeout")
+
+ def __init__(self, filenames: Optional[List[str]] = None):
+ """ Attempt to read local configuration files to determine spalloc client
+ settings.
+
+ Parameters
+ ----------
+ filenames : [str, ...]
+ Filenames to attempt to read. Later config file have higher priority.
+
+ """
+ if filenames is None: # pragma: no cover
+ filenames = SEARCH_PATH
+ parser = configparser.ConfigParser()
+
+ # Set default config values (NB: No read_dict in Python 2.7)
+ parser.add_section(SECTION)
+ for key, value in DEFAULT_CONFIG.items():
+ parser.set(SECTION, key, value)
+
+ # Attempt to read from each possible file location in turn
+ for filename in filenames:
+ try:
+ with open(filename, "r", encoding="utf-8") as f:
+ parser.read_file(f, filename)
+ except (IOError, OSError):
+ # File did not exist, keep trying
+ pass
+
+ self._hostname = _read_any_str(parser, "hostname")
+ self._owner = _read_any_str(parser, "owner")
+ self._port = parser.getint(SECTION, "port")
+ self._keepalive = _read_none_or_float(parser, "keepalive")
+ self._reconnect_delay = parser.getfloat(SECTION, "reconnect_delay")
+ self._timeout = _read_none_or_float(parser, "timeout")
+ self._machine = _read_none_or_str(parser, "machine")
+ self._min_ratio = parser.getfloat(SECTION, "min_ratio")
+ self._max_dead_boards = _read_none_or_int(parser, "max_dead_boards")
+ self._max_dead_links = _read_none_or_int(parser, "max_dead_links")
+ self._require_torus = parser.getboolean(SECTION, "require_torus")
+ self._ignore_version = parser.getboolean(SECTION, "ignore_version")
+
+ tags = _read_none_or_str(parser, "tags")
+ self._tags = None if tags is None else list(
+ map(str.strip, tags.split(",")))
+
+ @property
+ def hostname(self) -> Optional[str]:
+ return self._hostname
+
+ @property
+ def ignore_version(self) -> bool:
+ return self._ignore_version
+
+ @property
+ def keepalive(self) -> Optional[float]:
+ return self._keepalive
+
+ @property
+ def machine(self) -> Optional[str]:
+ return self._machine
+
+ @property
+ def max_dead_boards(self) -> Optional[int]:
+ return self._max_dead_boards
+
+ @property
+ def max_dead_links(self) -> Optional[int]:
+ return self._max_dead_links
+
+ @property
+ def min_ratio(self) -> float:
+ return self._min_ratio
+
+ @property
+ def owner(self) -> Optional[str]:
+ return self._owner
+
+ @property
+ def port(self) -> int:
+ return self._port
+
+ @property
+ def reconnect_delay(self) -> float:
+ return self._reconnect_delay
+
+ @property
+ def require_torus(self) -> bool:
+ return self._require_torus
+
+ @property
+ def tags(self) -> Optional[List[str]]:
+ return self._tags
+
+ @property
+ def timeout(self) -> Optional[float]:
+ return self._timeout
if __name__ == "__main__": # pragma: no cover
print("Default search path (lowest-priority first):")
diff --git a/tests/test_config.py b/tests/test_config.py
index 8b9876001..ce9e9dae8 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -16,7 +16,7 @@
import shutil
import os.path
import pytest
-from spalloc_client.spalloc_config import read_config, TIMEOUT
+from spalloc_client.spalloc_config import SpallocConfig, TIMEOUT
@pytest.yield_fixture
@@ -41,59 +41,9 @@ def test_priority(tempdir):
with open(f2, "w") as f:
f.write("[spalloc]\nport=321\ntags=qux")
- cfg = read_config([f1, f2])
+ cfg = SpallocConfig([f1, f2])
- assert cfg["port"] == 321
- assert cfg["reconnect_delay"] == 5.0
- assert cfg["hostname"] == "bar"
- assert cfg["tags"] == ["qux"]
-
-
-@pytest.mark.parametrize(
- "option_name,config_value,value",
- [("hostname", None, None),
- ("hostname", "foo", "foo"),
- ("port", None, 22244),
- ("port", "1234", 1234),
- ("owner", None, None),
- ("owner", "foo", "foo"),
- ("keepalive", None, 60.0),
- ("keepalive", "None", None),
- ("keepalive", "3.0", 3.0),
- ("reconnect_delay", None, 5.0),
- ("reconnect_delay", "3.0", 3.0),
- ("timeout", None, TIMEOUT),
- ("timeout", "None", None),
- ("timeout", "3.0", 3.0),
- ("machine", None, None),
- ("machine", "None", None),
- ("machine", "foo", "foo"),
- ("tags", None, None),
- ("tags", "None", None),
- ("tags", "foo", ["foo"]),
- ("tags", "one, two , three", ["one", "two", "three"]),
- ("min_ratio", None, 0.333),
- ("min_ratio", "1.0", 1.0),
- ("max_dead_boards", None, 0),
- ("max_dead_boards", "None", None),
- ("max_dead_boards", "3", 3),
- ("max_dead_links", None, None),
- ("max_dead_links", "None", None),
- ("max_dead_links", "3", 3),
- ("require_torus", None, False),
- ("require_torus", "False", False),
- ("require_torus", "True", True)])
-def test_options(filename, option_name, config_value, value):
- # Test all config options.
-
- # Write config file (omitting the config value if None, e.g. to test
- # default value)
- with open(filename, "w") as f:
- f.write("[spalloc]\n")
- if config_value is not None:
- f.write("{}={}".format(option_name, config_value))
-
- cfg = read_config([filename])
-
- assert option_name in cfg
- assert cfg[option_name] == value
+ assert cfg.port == 321
+ assert cfg.reconnect_delay == 5.0
+ assert cfg.hostname == "bar"
+ assert cfg.tags == ["qux"]
diff --git a/tests/test_job.py b/tests/test_job.py
index 974bd375d..752f9b909 100644
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -76,6 +76,7 @@ def test_args_from_config(self, basic_config_file, client,
basic_job_kwargs.pop("port")
basic_job_kwargs.pop("reconnect_delay")
assert len(client.create_job.mock_calls) == 1
+ a = client.create_job
args = client.create_job.mock_calls[0][1]
kwargs = client.create_job.mock_calls[0][2]
assert args == tuple()
From fdd75f7016949526751394590cc7ecc83e56121d Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 10:46:35 +0000
Subject: [PATCH 24/38] timeout is a float
---
spalloc_client/job.py | 8 +++----
spalloc_client/protocol_client.py | 35 ++++++++++++++++---------------
spalloc_client/scripts/job.py | 10 ++++-----
spalloc_client/scripts/support.py | 2 +-
4 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 8d3df752c..9ef4b1e55 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -246,12 +246,12 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
config = SpallocConfig(config_filenames)
# Get protocol client options
- hostname = kwargs.get("hostname", config.hostname)
+ hostname = cast(str, kwargs.get("hostname", config.hostname))
owner = kwargs.get("owner", config.owner)
- port = kwargs.get("port", config.port)
+ port = cast(int, kwargs.get("port", config.port))
self._reconnect_delay = kwargs.get("reconnect_delay",
config.reconnect_delay)
- self._timeout = kwargs.get("timeout", config.timeout)
+ self._timeout = cast(float, kwargs.get("timeout", config.timeout))
if hostname is None:
raise ValueError("A hostname must be specified.")
@@ -677,7 +677,7 @@ def _do_reconnect(self, finish_time: Optional[float]) -> None:
time.sleep(max(0.0, delay))
self._reconnect()
- def wait_until_ready(self, timeout: Optional[int] = None) -> None:
+ def wait_until_ready(self, timeout: Optional[float] = None) -> None:
""" Block until the job is allocated and ready.
Parameters
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 00ce2ffff..5eaa375e6 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -19,7 +19,7 @@
import json
import socket
from types import TracebackType
-from typing import cast, Dict, Literal, Optional, Type, Union
+from typing import Any, cast, Dict, List, Literal, Optional, Type, Union
from threading import current_thread, RLock, local, Thread
from typing_extensions import Self
@@ -281,7 +281,7 @@ def _send_json(
def call(self, name: str, timeout: Optional[float],
*args: Union[int, str, None],
- **kwargs: JsonValue) -> JsonValue:
+ **kwargs: Any) -> JsonValue:
""" Send a command to the server and return the reply.
Parameters
@@ -376,11 +376,12 @@ def wait_for_notification(
# The bindings of the Spalloc protocol methods themselves; simplifies use
# from IDEs.
- def version(self, timeout: Optional[int] = None) -> str:
+ def version(self, timeout: Optional[float] = None) -> str:
""" Ask what version of spalloc is running. """
return cast(str, self.call("version", timeout))
- def create_job(self, *args: int, **kwargs: str) -> int:
+ def create_job(self, *args: int,
+ **kwargs: Union[float, str, List[str], None]) -> int:
"""
Start a new job
"""
@@ -400,55 +401,55 @@ def job_keepalive(self, job_id: int,
return cast(dict, self.call("job_keepalive", timeout, job_id))
def get_job_state(self, job_id: int,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
"""Get the state for this job """
return cast(dict, self.call("get_job_state", timeout, job_id))
def get_job_machine_info(self, job_id: int,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Get info for this job. """
return cast(dict, self.call("get_job_machine_info", timeout, job_id))
def power_on_job_boards(self, job_id: int,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Turn on the power on the jobs boards. """
return cast(dict, self.call("power_on_job_boards", timeout, job_id))
def power_off_job_boards(self, job_id: int,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Turn off the power on the jobs boards. """
return cast(dict, self.call("power_off_job_boards", timeout, job_id))
def destroy_job(self, job_id: int, reason: Optional[str] = None,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Destroy the job """
return cast(dict, self.call("destroy_job", timeout,
job_id, reason=reason))
def notify_job(self, job_id: Optional[int] = None,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Turn on notification of job status changes. """
return cast(dict, self.call("notify_job", timeout, job_id))
def no_notify_job(self, job_id: Optional[int] = None,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Turn off notification of job status changes. """
return cast(dict, self.call("no_notify_job", timeout,
job_id))
def notify_machine(self, machine_name: Optional[str] = None,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Turn on notification of machine status changes. """
return cast(dict, self.call("notify_machine", timeout,
machine_name))
def no_notify_machine(self, machine_name: Optional[str] = None,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Turn off notification of machine status changes. """
return cast(dict, self.call("no_notify_machine", timeout,
machine_name))
- def list_jobs(self, timeout: Optional[int] = None) -> JsonObjectArray:
+ def list_jobs(self, timeout: Optional[float] = None) -> JsonObjectArray:
""" Obtains a list of jobs currently running. """
return cast(list, self.call("list_jobs", timeout))
@@ -459,14 +460,14 @@ def list_machines(self,
def get_board_position(
self, machine_name: str, x: int, y: int, z: int,
- timeout: Optional[int] = None) -> JsonObject: # pragma: no cover
+ timeout: Optional[float] = None) -> JsonObject: # pragma: no cover
""" Gets the position of board x, y, z on the given machine. """
# pylint: disable=too-many-arguments
return cast(dict, self.call("get_board_position", timeout,
machine_name, x, y, z))
def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
- timeout: Optional[int] = None
+ timeout: Optional[float] = None
) -> JsonObject: # pragma: no cover
""" Gets the board x, y, z on the requested machine. """
# pylint: disable=too-many-arguments
@@ -480,7 +481,7 @@ def get_board_at_position(self, machine_name: str, x: int, y: int, z: int,
frozenset("job_id chip_x chip_y".split())])
def where_is(self, job_id: int, chip_x: int, chip_y: int,
- timeout: Optional[int] = None) -> JsonObject:
+ timeout: Optional[float] = None) -> JsonObject:
""" Reports where ion the Machine a job is running """
# Test for whether sane arguments are passed.
return cast(dict, self.call("where_is", timeout, job_id=job_id,
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index 0388fb947..ec80177c6 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -94,7 +94,7 @@ def _state_name(mapping: JsonObject) -> str:
return state.name # pylint: disable=no-member
-def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[int],
+def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[float],
job_id: int) -> None:
""" Print a human-readable overview of a Job's attributes.
@@ -173,7 +173,7 @@ def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[int],
print(render_definitions(info))
-def watch_job(t: Terminal, client: ProtocolClient, timeout: Optional[int],
+def watch_job(t: Terminal, client: ProtocolClient, timeout: Optional[float],
job_id: int) -> int:
""" Re-print a job's information whenever the job changes.
@@ -207,7 +207,7 @@ def watch_job(t: Terminal, client: ProtocolClient, timeout: Optional[int],
print("")
-def power_job(client: ProtocolClient, timeout: Optional[int],
+def power_job(client: ProtocolClient, timeout: Optional[float],
job_id: int, power: bool) -> None:
""" Power a job's boards on/off and wait for the action to complete.
@@ -256,7 +256,7 @@ def power_job(client: ProtocolClient, timeout: Optional[int],
f"job {job_id} in state {_state_name(state)}"))
-def list_ips(client: ProtocolClient, timeout: Optional[int],
+def list_ips(client: ProtocolClient, timeout: Optional[float],
job_id: int) -> None:
""" Print a CSV of board hostnames for all boards allocated to a job.
@@ -289,7 +289,7 @@ def list_ips(client: ProtocolClient, timeout: Optional[int],
print(f"{x},{y},{hostname}")
-def destroy_job(client: ProtocolClient, timeout: Optional[int],
+def destroy_job(client: ProtocolClient, timeout: Optional[float],
job_id: int, reason: Optional[str] = None) -> None:
""" Destroy a running job.
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index be4f51753..49e3a2a56 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -41,7 +41,7 @@ def exit(self) -> None:
sys.exit(self._code)
-def version_verify(client: ProtocolClient, timeout: Optional[int]) -> None:
+def version_verify(client: ProtocolClient, timeout: Optional[float]) -> None:
"""
Verify that the current version of the client is compatible
"""
From 60d2d79221eeeba846a7895c1c604e37d5189697 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 11:13:30 +0000
Subject: [PATCH 25/38] type fixes
---
spalloc_client/_utils.py | 4 ++++
spalloc_client/job.py | 8 ++++----
spalloc_client/scripts/job.py | 3 ++-
spalloc_client/scripts/machine.py | 3 ++-
spalloc_client/scripts/ps.py | 3 ++-
spalloc_client/scripts/where_is.py | 4 ++--
6 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/spalloc_client/_utils.py b/spalloc_client/_utils.py
index 7ac6fab22..8c9cf802e 100644
--- a/spalloc_client/_utils.py
+++ b/spalloc_client/_utils.py
@@ -17,6 +17,10 @@
from typing import Optional
+def time_left_float(timestamp: float) -> float:
+ return max(0.0, timestamp - time.time())
+
+
def time_left(timestamp: Optional[float]) -> Optional[float]:
""" Convert a timestamp into how long to wait for it.
"""
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 9ef4b1e55..164101529 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -32,7 +32,7 @@
from .protocol_client import ProtocolClient, ProtocolTimeoutError
from .spalloc_config import SpallocConfig, SEARCH_PATH
from .states import JobState
-from ._utils import time_left, timed_out, make_timeout
+from ._utils import time_left, time_left_float, timed_out, make_timeout
logger = logging.getLogger(__name__)
@@ -249,8 +249,8 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
hostname = cast(str, kwargs.get("hostname", config.hostname))
owner = kwargs.get("owner", config.owner)
port = cast(int, kwargs.get("port", config.port))
- self._reconnect_delay = kwargs.get("reconnect_delay",
- config.reconnect_delay)
+ self._reconnect_delay = cast(float, kwargs.get("reconnect_delay",
+ config.reconnect_delay))
self._timeout = cast(float, kwargs.get("timeout", config.timeout))
if hostname is None:
raise ValueError("A hostname must be specified.")
@@ -671,7 +671,7 @@ def _do_reconnect(self, finish_time: Optional[float]) -> None:
"""
self._client.close()
if finish_time is not None:
- delay = min(time_left(finish_time), self._reconnect_delay)
+ delay = min(time_left_float(finish_time), self._reconnect_delay)
else:
delay = self._reconnect_delay
time.sleep(max(0.0, delay))
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index ec80177c6..8022c17e4 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -86,6 +86,7 @@
Terminal, render_definitions, render_boards, DEFAULT_BOARD_EDGES)
from spalloc_client import ProtocolClient
from spalloc_client._utils import render_timestamp
+from spalloc_client.spalloc_config import SpallocConfig
from spalloc_client.scripts.support import Terminate, Script
@@ -340,7 +341,7 @@ def get_job_id(self, client: ProtocolClient,
return cast(int, job_ids[0])
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: SpallocConfig) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Manage running jobs.")
parser.add_argument(
diff --git a/spalloc_client/scripts/machine.py b/spalloc_client/scripts/machine.py
index 5da62d6fa..fa1deb76c 100644
--- a/spalloc_client/scripts/machine.py
+++ b/spalloc_client/scripts/machine.py
@@ -38,6 +38,7 @@
from spinn_utilities.typing.json import JsonObject, JsonObjectArray
from spalloc_client import __version__, ProtocolClient
+from spalloc_client.spalloc_config import SpallocConfig
from spalloc_client.term import (
Terminal, render_table, render_definitions, render_boards, render_cells,
DEFAULT_BOARD_EDGES, TableRow, TableType)
@@ -257,7 +258,7 @@ def get_and_display_machine_info(
show_machine(t, machines, jobs, args.machine, not args.detailed)
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: SpallocConfig) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Get the state of individual machines.")
parser.add_argument(
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index 7abe9f458..3cb094af6 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -33,6 +33,7 @@
from spinn_utilities.typing.json import JsonObjectArray
from spalloc_client import __version__, JobState, ProtocolClient
+from spalloc_client.spalloc_config import SpallocConfig
from spalloc_client.term import Terminal, render_table, TableColumn, TableType
from spalloc_client._utils import render_timestamp
from .support import Script
@@ -128,7 +129,7 @@ class ProcessListScript(Script):
An object form Job scripts.
"""
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: SpallocConfig) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="List all active jobs.")
parser.add_argument(
"--version", "-V", action="version", version=__version__)
diff --git a/spalloc_client/scripts/where_is.py b/spalloc_client/scripts/where_is.py
index bca63567a..3c4885cf6 100644
--- a/spalloc_client/scripts/where_is.py
+++ b/spalloc_client/scripts/where_is.py
@@ -72,8 +72,8 @@
from spalloc_client import __version__, ProtocolClient
from spalloc_client.term import render_definitions
-
from spalloc_client.scripts.support import Terminate, Script
+from spalloc_client.spalloc_config import SpallocConfig
class WhereIsScript(Script):
@@ -88,7 +88,7 @@ def __init__(self) -> None:
self.show_board_chip = False
@overrides(Script.get_parser)
- def get_parser(self, cfg: Dict[str, str]) -> argparse.ArgumentParser:
+ def get_parser(self, cfg: SpallocConfig) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Find out the location (physical or logical) of a "
"chip or board.")
From e5e5411ea5f589fb10e52dfcd9eb433953fb7189 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 11:25:44 +0000
Subject: [PATCH 26/38] flake8
---
spalloc_client/job.py | 4 ++--
spalloc_client/scripts/job.py | 4 ++--
spalloc_client/scripts/ps.py | 2 +-
spalloc_client/scripts/support.py | 2 +-
spalloc_client/spalloc_config.py | 12 +++++++-----
tests/test_config.py | 2 +-
tests/test_job.py | 1 -
7 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 164101529..ae6d5695a 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -249,8 +249,8 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
hostname = cast(str, kwargs.get("hostname", config.hostname))
owner = kwargs.get("owner", config.owner)
port = cast(int, kwargs.get("port", config.port))
- self._reconnect_delay = cast(float, kwargs.get("reconnect_delay",
- config.reconnect_delay))
+ self._reconnect_delay = cast(
+ float, kwargs.get("reconnect_delay", config.reconnect_delay))
self._timeout = cast(float, kwargs.get("timeout", config.timeout))
if hostname is None:
raise ValueError("A hostname must be specified.")
diff --git a/spalloc_client/scripts/job.py b/spalloc_client/scripts/job.py
index 8022c17e4..cdb8edfe6 100644
--- a/spalloc_client/scripts/job.py
+++ b/spalloc_client/scripts/job.py
@@ -95,8 +95,8 @@ def _state_name(mapping: JsonObject) -> str:
return state.name # pylint: disable=no-member
-def show_job_info(t: Terminal, client: ProtocolClient, timeout: Optional[float],
- job_id: int) -> None:
+def show_job_info(t: Terminal, client: ProtocolClient,
+ timeout: Optional[float], job_id: int) -> None:
""" Print a human-readable overview of a Job's attributes.
Parameters
diff --git a/spalloc_client/scripts/ps.py b/spalloc_client/scripts/ps.py
index 3cb094af6..f29391730 100644
--- a/spalloc_client/scripts/ps.py
+++ b/spalloc_client/scripts/ps.py
@@ -27,7 +27,7 @@
import argparse
from collections.abc import Sized
import sys
-from typing import cast, Dict, Union
+from typing import cast, Union
from spinn_utilities.overrides import overrides
from spinn_utilities.typing.json import JsonObjectArray
diff --git a/spalloc_client/scripts/support.py b/spalloc_client/scripts/support.py
index 49e3a2a56..c93008db2 100644
--- a/spalloc_client/scripts/support.py
+++ b/spalloc_client/scripts/support.py
@@ -14,7 +14,7 @@
from argparse import ArgumentParser, Namespace
import sys
-from typing import Any, Dict, Optional
+from typing import Any, Optional
from spinn_utilities.abstract_base import AbstractBase, abstractmethod
from spalloc_client import (
ProtocolClient, ProtocolError, ProtocolTimeoutError,
diff --git a/spalloc_client/spalloc_config.py b/spalloc_client/spalloc_config.py
index 2179b64df..51718ab25 100644
--- a/spalloc_client/spalloc_config.py
+++ b/spalloc_client/spalloc_config.py
@@ -81,7 +81,7 @@
"""
import configparser
import os.path
-from typing import Any, Dict, List, Optional, Union
+from typing import List, Optional
import appdirs
@@ -154,13 +154,14 @@ class SpallocConfig(object):
"_tags", "_timeout")
def __init__(self, filenames: Optional[List[str]] = None):
- """ Attempt to read local configuration files to determine spalloc client
- settings.
+ """ Attempt to read local configuration files
+ to determine spalloc client settings.
Parameters
----------
filenames : [str, ...]
- Filenames to attempt to read. Later config file have higher priority.
+ Filenames to attempt to read.
+ Later config file have higher priority.
"""
if filenames is None: # pragma: no cover
@@ -181,7 +182,7 @@ def __init__(self, filenames: Optional[List[str]] = None):
# File did not exist, keep trying
pass
- self._hostname = _read_any_str(parser, "hostname")
+ self._hostname = _read_any_str(parser, "hostname")
self._owner = _read_any_str(parser, "owner")
self._port = parser.getint(SECTION, "port")
self._keepalive = _read_none_or_float(parser, "keepalive")
@@ -250,6 +251,7 @@ def tags(self) -> Optional[List[str]]:
def timeout(self) -> Optional[float]:
return self._timeout
+
if __name__ == "__main__": # pragma: no cover
print("Default search path (lowest-priority first):")
print("\n".join(SEARCH_PATH))
diff --git a/tests/test_config.py b/tests/test_config.py
index ce9e9dae8..fa44e6b25 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -16,7 +16,7 @@
import shutil
import os.path
import pytest
-from spalloc_client.spalloc_config import SpallocConfig, TIMEOUT
+from spalloc_client.spalloc_config import SpallocConfig
@pytest.yield_fixture
diff --git a/tests/test_job.py b/tests/test_job.py
index 752f9b909..974bd375d 100644
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -76,7 +76,6 @@ def test_args_from_config(self, basic_config_file, client,
basic_job_kwargs.pop("port")
basic_job_kwargs.pop("reconnect_delay")
assert len(client.create_job.mock_calls) == 1
- a = client.create_job
args = client.create_job.mock_calls[0][1]
kwargs = client.create_job.mock_calls[0][2]
assert args == tuple()
From 37394b486e65b3d17d9b35730f4a6e1638d63629 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 11:39:33 +0000
Subject: [PATCH 27/38] docs
---
spalloc_client/spalloc_config.py | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/spalloc_client/spalloc_config.py b/spalloc_client/spalloc_config.py
index 51718ab25..bd2c51e39 100644
--- a/spalloc_client/spalloc_config.py
+++ b/spalloc_client/spalloc_config.py
@@ -147,6 +147,7 @@ def _read_none_or_str(
class SpallocConfig(object):
+ """ Typed configs """
__slots__ = ("_hostname", "_ignore_version", "_keepalive", "_machine",
"_max_dead_boards", "_max_dead_links", "_min_ratio",
@@ -201,54 +202,67 @@ def __init__(self, filenames: Optional[List[str]] = None):
@property
def hostname(self) -> Optional[str]:
+ """ Name of the spalloc server if specified """
return self._hostname
@property
def ignore_version(self) -> bool:
+ """ Flag to say version can be ignored """
return self._ignore_version
@property
def keepalive(self) -> Optional[float]:
+ """ Time to keep job allive """
return self._keepalive
@property
def machine(self) -> Optional[str]:
+ """ Name of the spalloc machine to use"""
return self._machine
@property
def max_dead_boards(self) -> Optional[int]:
+ """ How many dead boards are allowed in the job"""
return self._max_dead_boards
@property
def max_dead_links(self) -> Optional[int]:
+ """ How many dead links are allowed in the Job"""
return self._max_dead_links
@property
def min_ratio(self) -> float:
+ """ Min ratio"""
return self._min_ratio
@property
def owner(self) -> Optional[str]:
+ """ Owner to assign job to """
return self._owner
@property
def port(self) -> int:
+ """ Spalloc server port"""
return self._port
@property
def reconnect_delay(self) -> float:
+ """ Reconnect delay """
return self._reconnect_delay
@property
def require_torus(self) -> bool:
+ """ Flag to say a torus is required """
return self._require_torus
@property
def tags(self) -> Optional[List[str]]:
+ """ List of tags """
return self._tags
@property
def timeout(self) -> Optional[float]:
+ """ Time before a command should timeout """
return self._timeout
From 9388795b4e5157c2eb20f194bfde3c2159372dea Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Tue, 26 Nov 2024 13:41:11 +0000
Subject: [PATCH 28/38] kwargs last
---
spalloc_client/scripts/alloc.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 3804a2f73..0fb9806ef 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -428,9 +428,9 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
return parser, parser.parse_args(argv)
-def run_job(job_args: List[int],
- job_kwargs: Dict[str, Union[float, str, None]],
- ip_file_filename: str) -> int:
+def run_job(ip_file_filename: str,
+ job_args: List[int],
+ job_kwargs: Dict[str, Union[float, str, None]])-> int:
"""
Run a job
"""
@@ -539,7 +539,7 @@ def main(argv: Optional[List[str]] = None) -> int:
_, ip_file_filename = tempfile.mkstemp(".csv", "spinnaker_ips_")
try:
- return run_job(job_args, job_kwargs, ip_file_filename)
+ return run_job(ip_file_filename, job_args, job_kwargs)
except SpallocServerException as e: # pragma: no cover
info(t.red(f"Error from server: {e}"))
return 6
From 8ae14ea03ba01a98ff73f8c69d305a0b4fba5334 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 08:44:50 +0000
Subject: [PATCH 29/38] kwargs removal
---
spalloc_client/job.py | 92 +++++++++++++++++++++++++--------
spalloc_client/scripts/alloc.py | 2 +-
2 files changed, 72 insertions(+), 22 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index ae6d5695a..76352185c 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -19,7 +19,8 @@
import subprocess
import time
from types import TracebackType
-from typing import cast, Dict, Optional, Tuple, Type, Union
+from typing import (
+ Any, cast, Dict, List, Optional, Tuple, Type, TypeVar, Union)
import sys
from typing_extensions import Literal, Self
@@ -42,6 +43,39 @@
# https://docs.python.org/3.1/library/logging.html#configuring-logging-for-a-library
logger.addHandler(logging.StreamHandler())
+NO_STR = "n0N@"
+NO_INT = -9834
+F = TypeVar('F', bound='float')
+
+def pick_str(param: Optional[str], config: Optional[str]) -> Optional[str]:
+ """ Use the param unless it is the default value, otherwise use config"""
+ if param == NO_STR:
+ return config
+ return param
+
+
+def pick_list(param: Optional[List[str]],
+ config: Optional[List[str]]) -> Optional[List[str]]:
+ """ Use the param unless it is the default value, otherwise use config"""
+ if param == [NO_STR]:
+ return config
+ return param
+
+
+def pick_num(param: Optional[F], config: Optional[F]) -> Optional[F]:
+ """ Use the param unless it is the default value, otherwise use config"""
+ if param == NO_INT:
+ return config
+ return param
+
+
+def pick_bool(param: Any, config: Optional[bool]) -> Optional[bool]:
+ """ Use the param if None or a bool, otherwise use config"""
+ if param is None or isinstance(param, bool):
+ return param
+ else:
+ return config
+
class Job(object):
""" A high-level interface for requesting and managing allocations of
@@ -127,7 +161,20 @@ class Job(object):
allocated.
"""
- def __init__(self, *args: int, **kwargs: Union[float, str, None]):
+ def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
+ port:Optional[int] = NO_INT,
+ reconnect_delay: Optional[float] = NO_INT,
+ timeout: Optional[float] = NO_INT,
+ config_filenames: Optional[List[str]] = [NO_STR],
+ resume_job_id: Optional[int] = None,
+ owner: Optional[str] = NO_STR,
+ keepalive: Optional[float] = NO_INT,
+ machine: Optional[str] = NO_STR,
+ tags: Optional[List[str]] = [NO_STR],
+ min_ratio: Optional[float] = NO_INT,
+ max_dead_boards: Optional[int] = NO_INT,
+ max_dead_links: Optional[int] = NO_INT,
+ require_torus: Any = NO_STR):
""" Request a SpiNNaker machine.
A :py:class:`.Job` is constructed in one of the following styles::
@@ -151,7 +198,7 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
>>> Job(4, 2, require_torus=True)
>>> # Board x=3, y=2, z=1 on the machine named "m"
- >>> Job(3, 2, 1, machine="m")
+ >>> Job(3, 2, 1, machine="m")config_filenames
>>> # Keep using (and keeping-alive) an existing allocation
>>> Job(resume_job_id=123)
@@ -241,19 +288,23 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
specified.)
"""
# Read configuration
- config_filenames = cast(
- list, kwargs.pop("config_filenames", SEARCH_PATH))
+ config_filenames = pick_list(config_filenames, SEARCH_PATH)
config = SpallocConfig(config_filenames)
# Get protocol client options
- hostname = cast(str, kwargs.get("hostname", config.hostname))
- owner = kwargs.get("owner", config.owner)
- port = cast(int, kwargs.get("port", config.port))
- self._reconnect_delay = cast(
- float, kwargs.get("reconnect_delay", config.reconnect_delay))
- self._timeout = cast(float, kwargs.get("timeout", config.timeout))
+ hostname = pick_str(hostname, config.hostname)
+ owner = pick_str(owner, config.owner)
+ port = pick_num(port, config.port)
+ reconnect_delay = pick_num(reconnect_delay, config.reconnect_delay)
+ if reconnect_delay is None:
+ raise ValueError("A reconnect_delay must be specified.")
+ self._reconnect_delay = reconnect_delay
+ self._timeout = pick_num(timeout, config.timeout)
+
if hostname is None:
raise ValueError("A hostname must be specified.")
+ if port is None:
+ raise ValueError("A port must be specified.")
# Cached responses of _get_state and _get_machine_info
self._last_machine_info: Optional["_JobMachineInfoTuple"] = None
@@ -267,7 +318,6 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
self._assert_compatible_version()
# Resume/create the job
- resume_job_id = cast(int, kwargs.get("resume_job_id", None))
if resume_job_id:
self.id = resume_job_id
@@ -294,16 +344,16 @@ def __init__(self, *args: int, **kwargs: Union[float, str, None]):
job_args = args
job_kwargs = {
"owner": owner,
- "keepalive": kwargs.get("keepalive", config.keepalive),
- "machine": kwargs.get("machine", config.machine),
- "tags": kwargs.get("tags", config.tags),
- "min_ratio": kwargs.get("min_ratio", config.min_ratio),
+ "keepalive": pick_num(keepalive, config.keepalive),
+ "machine": pick_str(machine, config.machine),
+ "tags": pick_list(tags, config.tags),
+ "min_ratio": pick_num(min_ratio, config.min_ratio),
"max_dead_boards":
- kwargs.get("max_dead_boards", config.max_dead_boards),
+ pick_num(max_dead_boards, config.max_dead_boards),
"max_dead_links":
- kwargs.get("max_dead_links", config.max_dead_links),
+ pick_num(max_dead_links, config.max_dead_links),
"require_torus":
- kwargs.get("require_torus", config.require_torus),
+ pick_bool(require_torus, config.require_torus),
"timeout": self._timeout,
}
@@ -649,12 +699,12 @@ def _do_wait_for_a_change(self, finish_time: Optional[float]) -> bool:
# user-specified timeout or half the keepalive interval.
if finish_time is not None and self._keepalive is not None:
wait_timeout = min(self._keepalive / 2.0,
- time_left(finish_time))
+ time_left_float(finish_time))
elif finish_time is None:
wait_timeout = None if self._keepalive is None \
else self._keepalive / 2.0
else:
- wait_timeout = time_left(finish_time)
+ wait_timeout = time_left_float(finish_time)
if wait_timeout is None or wait_timeout >= 0.0:
self._client.wait_for_notification(wait_timeout)
return True
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 0fb9806ef..6007c2d38 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -441,7 +441,7 @@ def run_job(ip_file_filename: str,
# Create the job
try:
- job = Job(*job_args, **job_kwargs)
+ job = Job(*job_args, **job_kwargs) # type: ignore[arg-type]
except (OSError, IOError, ProtocolError, ProtocolTimeoutError) as e:
info(t.red(f"Could not connect to server: {e}"))
return 6
From 6b5107c6d348adbe61ae8b2ab3cd187d60397ed6 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 09:57:16 +0000
Subject: [PATCH 30/38] remove timout from job kwargs
---
spalloc_client/job.py | 4 ++--
spalloc_client/protocol_client.py | 4 ++--
tests/test_protocol_client.py | 3 ++-
3 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 76352185c..015616987 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -354,7 +354,6 @@ def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
pick_num(max_dead_links, config.max_dead_links),
"require_torus":
pick_bool(require_torus, config.require_torus),
- "timeout": self._timeout,
}
# Sanity check arguments
@@ -368,7 +367,8 @@ def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
self._keepalive = job_kwargs["keepalive"]
# Create the job (failing fast if can't communicate)
- self.id = self._client.create_job(*job_args, **job_kwargs)
+ self.id = self._client.create_job(
+ self._timeout, *job_args, **job_kwargs)
logger.info("Created spalloc job %d", self.id)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 5eaa375e6..a82cc5609 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -380,7 +380,7 @@ def version(self, timeout: Optional[float] = None) -> str:
""" Ask what version of spalloc is running. """
return cast(str, self.call("version", timeout))
- def create_job(self, *args: int,
+ def create_job(self, timeout: Optional[float], *args: int,
**kwargs: Union[float, str, List[str], None]) -> int:
"""
Start a new job
@@ -389,7 +389,7 @@ def create_job(self, *args: int,
if "owner" not in kwargs:
raise SpallocServerException(
"owner must be specified for all jobs.")
- return cast(int, self.call("create_job", None, *args, **kwargs))
+ return cast(int, self.call("create_job", timeout, *args, **kwargs))
def job_keepalive(self, job_id: int,
timeout: Optional[float] = None) -> JsonObject:
diff --git a/tests/test_protocol_client.py b/tests/test_protocol_client.py
index 8dec6bc11..d89acae3d 100644
--- a/tests/test_protocol_client.py
+++ b/tests/test_protocol_client.py
@@ -232,7 +232,8 @@ def test_commands_as_methods(c, s, bg_accept):
bg_accept.join()
s.send({"return": "Woo"})
- assert c.create_job(1, bar=2, owner="dummy") == "Woo"
+ no_timeout = None
+ assert c.create_job(no_timeout, 1, bar=2, owner="dummy") == "Woo"
assert s.recv() == {
"command": "create_job", "args": [1], "kwargs": {
"bar": 2, "owner": "dummy"}}
From bc8fd009c1839e1bd9f44aaf9293dd34b5cb2866 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 11:15:14 +0000
Subject: [PATCH 31/38] remove another kwarg
---
spalloc_client/job.py | 33 ++++++++++++++-------------------
tests/test_job.py | 8 ++++----
2 files changed, 18 insertions(+), 23 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 015616987..73654d941 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -341,34 +341,29 @@ def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
logger.info("Spalloc resumed job %d", self.id)
else:
# Get job creation arguments
- job_args = args
- job_kwargs = {
- "owner": owner,
- "keepalive": pick_num(keepalive, config.keepalive),
- "machine": pick_str(machine, config.machine),
- "tags": pick_list(tags, config.tags),
- "min_ratio": pick_num(min_ratio, config.min_ratio),
- "max_dead_boards":
- pick_num(max_dead_boards, config.max_dead_boards),
- "max_dead_links":
- pick_num(max_dead_links, config.max_dead_links),
- "require_torus":
- pick_bool(require_torus, config.require_torus),
- }
+ machine = pick_str(machine, config.machine)
+ tags = pick_list(tags, config.tags)
# Sanity check arguments
- if job_kwargs["owner"] is None:
+ if owner is None:
raise ValueError("An owner must be specified.")
- if (job_kwargs["tags"] is not None and
- job_kwargs["machine"] is not None):
+ if tags is not None and machine is not None:
raise ValueError(
"Only one of tags and machine may be specified.")
- self._keepalive = job_kwargs["keepalive"]
+ self._keepalive = pick_num(keepalive, config.keepalive)
# Create the job (failing fast if can't communicate)
self.id = self._client.create_job(
- self._timeout, *job_args, **job_kwargs)
+ self._timeout, *args, owner = owner,
+ keepalive = self._keepalive, machine = machine, tags = tags,
+ min_ratio = pick_num(min_ratio, config.min_ratio),
+ max_dead_boards = pick_num(
+ max_dead_boards, config.max_dead_boards),
+ max_dead_links = pick_num(
+ max_dead_links, config.max_dead_links),
+ require_torus = pick_bool(
+ require_torus, config.require_torus))
logger.info("Created spalloc job %d", self.id)
diff --git a/tests/test_job.py b/tests/test_job.py
index 974bd375d..7220cf0ff 100644
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -75,10 +75,11 @@ def test_args_from_config(self, basic_config_file, client,
basic_job_kwargs.pop("hostname")
basic_job_kwargs.pop("port")
basic_job_kwargs.pop("reconnect_delay")
+ timeout = basic_job_kwargs.pop("timeout")
assert len(client.create_job.mock_calls) == 1
args = client.create_job.mock_calls[0][1]
kwargs = client.create_job.mock_calls[0][2]
- assert args == tuple()
+ assert args == (timeout,)
assert kwargs == basic_job_kwargs
finally:
j.close()
@@ -107,9 +108,8 @@ def test_override_config(self, basic_config_file, basic_job_kwargs,
assert len(client.create_job.mock_calls) == 1
args = client.create_job.mock_calls[0][1]
kwargs = client.create_job.mock_calls[0][2]
- assert args == tuple()
- assert kwargs == dict(timeout=0.2,
- owner="mossblaser",
+ assert args == (0.2,)
+ assert kwargs == dict(owner="mossblaser",
keepalive=0.3,
machine=None,
tags=["baz", "quz"],
From 2a6f06cdd897e4a6b28f8df57bdd80650fc53c92 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 12:32:19 +0000
Subject: [PATCH 32/38] remove more kwarg
---
spalloc_client/protocol_client.py | 20 +++++++++++++++++---
tests/test_job.py | 3 ++-
tests/test_protocol_client.py | 9 ++++++---
3 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index a82cc5609..3716c4cde 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -381,15 +381,29 @@ def version(self, timeout: Optional[float] = None) -> str:
return cast(str, self.call("version", timeout))
def create_job(self, timeout: Optional[float], *args: int,
- **kwargs: Union[float, str, List[str], None]) -> int:
+ owner: Optional[str] = None,
+ keepalive: Optional[float] = None,
+ machine: Optional[str] = None,
+ tags: Optional[List[str]] = None,
+ min_ratio: Optional[float] = None,
+ max_dead_boards: Optional[int] = None,
+ max_dead_links: Optional[int] = None,
+ require_torus: Optional[bool] = None) -> int:
"""
Start a new job
"""
# If no owner, don't bother with the call
- if "owner" not in kwargs:
+ if owner is None:
raise SpallocServerException(
"owner must be specified for all jobs.")
- return cast(int, self.call("create_job", timeout, *args, **kwargs))
+ if tags is not None and machine is not None:
+ raise SpallocServerException(
+ f"Unexpected {tags=} and {machine=} are both not None")
+ return cast(int, self.call(
+ "create_job", timeout, *args, owner=owner,
+ keepalive=keepalive, machine=machine, tags=tags,
+ min_ratio=min_ratio, max_dead_boards=max_dead_boards,
+ max_dead_links =max_dead_links, require_torus=require_torus))
def job_keepalive(self, job_id: int,
timeout: Optional[float] = None) -> JsonObject:
diff --git a/tests/test_job.py b/tests/test_job.py
index 7220cf0ff..74b8d64c7 100644
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -18,7 +18,8 @@
import pytest
from mock import Mock # type: ignore[import]
from spalloc_client import (
- Job, JobState, JobDestroyedError, ProtocolTimeoutError)
+ Job, JobState, JobDestroyedError, ProtocolTimeoutError,
+ SpallocServerException)
from spalloc_client._keepalive_process import keep_job_alive
from spalloc_client.job import (
_JobStateTuple, _JobMachineInfoTuple, StateChangeTimeoutError,
diff --git a/tests/test_protocol_client.py b/tests/test_protocol_client.py
index d89acae3d..e1ad3c1a1 100644
--- a/tests/test_protocol_client.py
+++ b/tests/test_protocol_client.py
@@ -233,10 +233,13 @@ def test_commands_as_methods(c, s, bg_accept):
s.send({"return": "Woo"})
no_timeout = None
- assert c.create_job(no_timeout, 1, bar=2, owner="dummy") == "Woo"
- assert s.recv() == {
+ assert c.create_job(no_timeout, 1, keepalive=2, owner="dummy") == "Woo"
+ commands = s.recv()
+ commands["kwargs"] = {k: v for k, v in commands["kwargs"].items()
+ if v is not None}
+ assert commands == {
"command": "create_job", "args": [1], "kwargs": {
- "bar": 2, "owner": "dummy"}}
+ "keepalive": 2, "owner": "dummy"}}
# Should fail for arbitrary internal method names
with pytest.raises(AttributeError):
From 3085609201489fa58194bf411a563e0801a34a6b Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 13:10:48 +0000
Subject: [PATCH 33/38] flake8
---
spalloc_client/job.py | 18 +++++++++---------
spalloc_client/protocol_client.py | 2 +-
spalloc_client/scripts/alloc.py | 4 ++--
tests/test_job.py | 3 +--
4 files changed, 13 insertions(+), 14 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 73654d941..676b54595 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -19,8 +19,7 @@
import subprocess
import time
from types import TracebackType
-from typing import (
- Any, cast, Dict, List, Optional, Tuple, Type, TypeVar, Union)
+from typing import (Any, cast, Dict, List, Optional, Tuple, Type, TypeVar)
import sys
from typing_extensions import Literal, Self
@@ -47,6 +46,7 @@
NO_INT = -9834
F = TypeVar('F', bound='float')
+
def pick_str(param: Optional[str], config: Optional[str]) -> Optional[str]:
""" Use the param unless it is the default value, otherwise use config"""
if param == NO_STR:
@@ -162,7 +162,7 @@ class Job(object):
"""
def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
- port:Optional[int] = NO_INT,
+ port: Optional[int] = NO_INT,
reconnect_delay: Optional[float] = NO_INT,
timeout: Optional[float] = NO_INT,
config_filenames: Optional[List[str]] = [NO_STR],
@@ -355,14 +355,14 @@ def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
# Create the job (failing fast if can't communicate)
self.id = self._client.create_job(
- self._timeout, *args, owner = owner,
- keepalive = self._keepalive, machine = machine, tags = tags,
- min_ratio = pick_num(min_ratio, config.min_ratio),
- max_dead_boards = pick_num(
+ self._timeout, *args, owner=owner,
+ keepalive=self._keepalive, machine=machine, tags=tags,
+ min_ratio=pick_num(min_ratio, config.min_ratio),
+ max_dead_boards=pick_num(
max_dead_boards, config.max_dead_boards),
- max_dead_links = pick_num(
+ max_dead_links=pick_num(
max_dead_links, config.max_dead_links),
- require_torus = pick_bool(
+ require_torus=pick_bool(
require_torus, config.require_torus))
logger.info("Created spalloc job %d", self.id)
diff --git a/spalloc_client/protocol_client.py b/spalloc_client/protocol_client.py
index 3716c4cde..473359c9c 100644
--- a/spalloc_client/protocol_client.py
+++ b/spalloc_client/protocol_client.py
@@ -403,7 +403,7 @@ def create_job(self, timeout: Optional[float], *args: int,
"create_job", timeout, *args, owner=owner,
keepalive=keepalive, machine=machine, tags=tags,
min_ratio=min_ratio, max_dead_boards=max_dead_boards,
- max_dead_links =max_dead_links, require_torus=require_torus))
+ max_dead_links=max_dead_links, require_torus=require_torus))
def job_keepalive(self, job_id: int,
timeout: Optional[float] = None) -> JsonObject:
diff --git a/spalloc_client/scripts/alloc.py b/spalloc_client/scripts/alloc.py
index 6007c2d38..f1228476e 100644
--- a/spalloc_client/scripts/alloc.py
+++ b/spalloc_client/scripts/alloc.py
@@ -430,7 +430,7 @@ def parse_argv(argv: Optional[List[str]]) -> Tuple[
def run_job(ip_file_filename: str,
job_args: List[int],
- job_kwargs: Dict[str, Union[float, str, None]])-> int:
+ job_kwargs: Dict[str, Union[float, str, None]]) -> int:
"""
Run a job
"""
@@ -441,7 +441,7 @@ def run_job(ip_file_filename: str,
# Create the job
try:
- job = Job(*job_args, **job_kwargs) # type: ignore[arg-type]
+ job = Job(*job_args, **job_kwargs) # type: ignore[arg-type]
except (OSError, IOError, ProtocolError, ProtocolTimeoutError) as e:
info(t.red(f"Could not connect to server: {e}"))
return 6
diff --git a/tests/test_job.py b/tests/test_job.py
index 74b8d64c7..7220cf0ff 100644
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -18,8 +18,7 @@
import pytest
from mock import Mock # type: ignore[import]
from spalloc_client import (
- Job, JobState, JobDestroyedError, ProtocolTimeoutError,
- SpallocServerException)
+ Job, JobState, JobDestroyedError, ProtocolTimeoutError)
from spalloc_client._keepalive_process import keep_job_alive
from spalloc_client.job import (
_JobStateTuple, _JobMachineInfoTuple, StateChangeTimeoutError,
From 5e87dd4820f9e5bbd57a74c283f9107cf872b026 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 13:56:40 +0000
Subject: [PATCH 34/38] pylint fixes
---
spalloc_client/_utils.py | 2 ++
spalloc_client/job.py | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/spalloc_client/_utils.py b/spalloc_client/_utils.py
index 8c9cf802e..675a21506 100644
--- a/spalloc_client/_utils.py
+++ b/spalloc_client/_utils.py
@@ -18,6 +18,8 @@
def time_left_float(timestamp: float) -> float:
+ """ Convert a not None timestamp into how long to wait for it.
+ """
return max(0.0, timestamp - time.time())
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 676b54595..90a1bff55 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -165,12 +165,12 @@ def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
port: Optional[int] = NO_INT,
reconnect_delay: Optional[float] = NO_INT,
timeout: Optional[float] = NO_INT,
- config_filenames: Optional[List[str]] = [NO_STR],
+ config_filenames: Optional[List[str]] = list([NO_STR]),
resume_job_id: Optional[int] = None,
owner: Optional[str] = NO_STR,
keepalive: Optional[float] = NO_INT,
machine: Optional[str] = NO_STR,
- tags: Optional[List[str]] = [NO_STR],
+ tags: Optional[List[str]] = list([NO_STR]),
min_ratio: Optional[float] = NO_INT,
max_dead_boards: Optional[int] = NO_INT,
max_dead_links: Optional[int] = NO_INT,
From 6ad56c7fe252ed2fa7066d133f9621187d3eedfc Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 14:31:45 +0000
Subject: [PATCH 35/38] pass params not kwarg to job
---
spalloc_client/job.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 90a1bff55..91cb040c7 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -552,7 +552,7 @@ def state(self) -> JobState:
return self._get_state().state
@property
- def power(self) -> int:
+ def power(self) -> bool:
""" Are the boards powered/powering on or off?
"""
return self._get_state().power
From 7f0ad52bb9d32cccd34b7d97e418508f2925b25f Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 14:37:30 +0000
Subject: [PATCH 36/38] mypy-full_packages: spalloc_client
---
.github/workflows/python_actions.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/python_actions.yml b/.github/workflows/python_actions.yml
index d7fc27545..772c10fbd 100644
--- a/.github/workflows/python_actions.yml
+++ b/.github/workflows/python_actions.yml
@@ -26,7 +26,8 @@ jobs:
coverage-package: spalloc tests
flake8-packages: spalloc_client tests
pylint-packages: spalloc_client
- mypy-packages: spalloc_client tests
+ mypy-packages: tests
+ mypy-full_packages: spalloc_client
sphinx_directory: docs/source
secrets: inherit
From 62ed8df7fd658c49453efde032981ff4ee1ab5d9 Mon Sep 17 00:00:00 2001
From: "Christian Y. Brenninkmeijer"
<christian.brenninkmeijer@manchester.ac.uk>
Date: Wed, 27 Nov 2024 15:37:07 +0000
Subject: [PATCH 37/38] better default
---
spalloc_client/job.py | 52 +++++++++++++++++++++++--------------------
1 file changed, 28 insertions(+), 24 deletions(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index 91cb040c7..f91b2f220 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -19,10 +19,10 @@
import subprocess
import time
from types import TracebackType
-from typing import (Any, cast, Dict, List, Optional, Tuple, Type, TypeVar)
+from typing import (cast, Dict, List, Optional, Tuple, Type, TypeVar, Union)
import sys
-from typing_extensions import Literal, Self
+from typing_extensions import Literal, Self, TypeAlias
from spinn_utilities.typing.json import JsonArray
@@ -42,34 +42,38 @@
# https://docs.python.org/3.1/library/logging.html#configuring-logging-for-a-library
logger.addHandler(logging.StreamHandler())
-NO_STR = "n0N@"
-NO_INT = -9834
F = TypeVar('F', bound='float')
+_INT: TypeAlias = Union[int, None, Literal["USE_CONFIG"]]
+_FLOAT: TypeAlias = Union[float, None, Literal["USE_CONFIG"]]
+_LIST: TypeAlias = Union[List[str], None, Literal["USE_CONFIG"]]
+_BOOL: TypeAlias = Union[bool, None, Literal["USE_CONFIG"]]
def pick_str(param: Optional[str], config: Optional[str]) -> Optional[str]:
""" Use the param unless it is the default value, otherwise use config"""
- if param == NO_STR:
+ if param == "USE_CONFIG":
return config
return param
-def pick_list(param: Optional[List[str]],
+def pick_list(param: _LIST,
config: Optional[List[str]]) -> Optional[List[str]]:
""" Use the param unless it is the default value, otherwise use config"""
- if param == [NO_STR]:
+ if param == "USE_CONFIG":
return config
- return param
+ else:
+ return param
-def pick_num(param: Optional[F], config: Optional[F]) -> Optional[F]:
+def pick_num(param: Union[F, None, Literal["USE_CONFIG"]],
+ config: Optional[F]) -> Optional[F]:
""" Use the param unless it is the default value, otherwise use config"""
- if param == NO_INT:
+ if param == "USE_CONFIG":
return config
return param
-def pick_bool(param: Any, config: Optional[bool]) -> Optional[bool]:
+def pick_bool(param: _BOOL, config: Optional[bool]) -> Optional[bool]:
""" Use the param if None or a bool, otherwise use config"""
if param is None or isinstance(param, bool):
return param
@@ -161,20 +165,20 @@ class Job(object):
allocated.
"""
- def __init__(self, *args: int, hostname: Optional[str] = NO_STR,
- port: Optional[int] = NO_INT,
- reconnect_delay: Optional[float] = NO_INT,
- timeout: Optional[float] = NO_INT,
- config_filenames: Optional[List[str]] = list([NO_STR]),
+ def __init__(self, *args: int, hostname: Optional[str] = "USE_CONFIG",
+ port: _INT = "USE_CONFIG",
+ reconnect_delay: _FLOAT = "USE_CONFIG",
+ timeout: _FLOAT = "USE_CONFIG",
+ config_filenames: _LIST = "USE_CONFIG",
resume_job_id: Optional[int] = None,
- owner: Optional[str] = NO_STR,
- keepalive: Optional[float] = NO_INT,
- machine: Optional[str] = NO_STR,
- tags: Optional[List[str]] = list([NO_STR]),
- min_ratio: Optional[float] = NO_INT,
- max_dead_boards: Optional[int] = NO_INT,
- max_dead_links: Optional[int] = NO_INT,
- require_torus: Any = NO_STR):
+ owner: Optional[str] = "USE_CONFIG",
+ keepalive: _FLOAT = "USE_CONFIG",
+ machine: Optional[str] = "USE_CONFIG",
+ tags: _LIST = "USE_CONFIG",
+ min_ratio: _FLOAT = "USE_CONFIG",
+ max_dead_boards: _INT = "USE_CONFIG",
+ max_dead_links: _INT = "USE_CONFIG",
+ require_torus: _BOOL = "USE_CONFIG"):
""" Request a SpiNNaker machine.
A :py:class:`.Job` is constructed in one of the following styles::
From 6fc8f74636fd098a1d80e07382ff4184b89cf097 Mon Sep 17 00:00:00 2001
From: Andrew Rowley <Andrew.Rowley@manchester.ac.uk>
Date: Fri, 20 Dec 2024 14:11:13 +0000
Subject: [PATCH 38/38] Fix oops in docstring
---
spalloc_client/job.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/spalloc_client/job.py b/spalloc_client/job.py
index f91b2f220..be451dcbd 100644
--- a/spalloc_client/job.py
+++ b/spalloc_client/job.py
@@ -202,7 +202,7 @@ def __init__(self, *args: int, hostname: Optional[str] = "USE_CONFIG",
>>> Job(4, 2, require_torus=True)
>>> # Board x=3, y=2, z=1 on the machine named "m"
- >>> Job(3, 2, 1, machine="m")config_filenames
+ >>> Job(3, 2, 1, machine="m")
>>> # Keep using (and keeping-alive) an existing allocation
>>> Job(resume_job_id=123)