import artiq.master.scheduler
import artiq.master.experiments
import artiq.master.log
from atomiq.heros import default_heros_realm, heros, LocalHERO, RemoteHERO, event
from artiq import __version__ as artiq_version
from atomiq import __version__ as atomiq_version
from atomiq.atomiq import logger
import collections
import fnmatch
import itertools
import logging
import os
import argparse
[docs]
def argument_factory(arg_description: dict):
"""
Instantiate a ScanObject or BooleanValue, StringValue etc from type and default arguments
"""
import artiq.language.scan
import artiq.language.environment
arg_type = arg_description["ty"]
if arg_type == "Scannable":
inner_arg_type = arg_description["default"][0]["ty"]
inner_description = arg_description["default"][0]
inner_kwargs = {k: v for k, v in inner_description.items() if k != "ty"}
arg_cls = getattr(artiq.language.scan, inner_arg_type)
return arg_cls(**inner_kwargs).describe()
else:
# _SimpleArgProcessor
inner_arg_type = arg_type
inner_description = arg_description
inner_kwargs = {k: v for k, v in inner_description.items() if k != "ty"}
arg_cls = getattr(artiq.language.environment, inner_arg_type)
arg = arg_cls(**inner_kwargs)
# we further need to convert these into "real" values,
# i.e. we need "some_var": {ty: "BooleanValue", ...} -> "some_var": True
# for simplicity, we just take the default
return arg.default()
[docs]
def export_deprecated_flag(value: str | None, flag: str, env_var: str) -> None:
"""
Export the value of a deprecated command line flag as an environment variable.
An environment variable that is already set takes precedence. ARTIQ workers are subprocesses of the master and
inherit the variable.
Args:
value: value given to the flag, or None if the flag is not given
flag: name of the deprecated flag, used in the warnings
env_var: environment variable that replaces the flag
"""
if value is None:
return
logger.warning(f"{flag} is deprecated, set the {env_var} environment variable instead")
if os.environ.get(env_var, "").strip():
logger.warning(f"{env_var} is set, ignoring {flag}")
return
os.environ[env_var] = value
[docs]
def export_heros_connect(connect: str | None) -> None:
"""
Export the deprecated ``--heros-connect`` endpoint as ``HEROS_ZENOH_CONNECT``.
Args:
connect: zenoh endpoint given to ``--heros-connect``, or None if the flag is not given
"""
export_deprecated_flag(connect, "--heros-connect", "HEROS_ZENOH_CONNECT")
[docs]
def export_heros_realm(realm: str | None) -> None:
"""
Export the deprecated ``--heros-realm`` realm as ``HEROS_REALM``.
Args:
realm: realm given to ``--heros-realm``, or None if the flag is not given
"""
export_deprecated_flag(realm, "--heros-realm", "HEROS_REALM")
# If HEROS is available, replace the artiq Scheduler with our HEROS-enabled version
if heros is not None:
_instance_name = None
_heros_realm = None
_git_autocommit = None
log_buffer: collections.deque[dict] = collections.deque([], maxlen=1000)
_ArtiqScheduler = artiq.master.scheduler.Scheduler
_ArtiqExperimentDB = artiq.master.experiments.ExperimentDB
_ArtiqLogForwarder = artiq.master.log.LogForwarder
class Scheduler(_ArtiqScheduler, LocalHERO):
"""
Patches the ARTIQ master Scheduler to (`artiq.master.scheduler.Scheduler`) to become a HERO.
"""
def __init__(self, *args, **kwargs):
global _instance_name
global _heros_realm
global _git_autocommit
self.heros_realm = _heros_realm
_ArtiqScheduler.__init__(self, *args, **kwargs)
postfix = f"-{_instance_name}" if _instance_name is not None else ""
LocalHERO.__init__(self, f"atomiq-scheduler{postfix}", realm=_heros_realm)
self._worker_handlers.update(
{"run_started": self.run_started, "run_created": self.run_created, "run_ended": self.run_ended}
)
@event
def run_created(self, rid, metadata={}):
metadata.update({"rid": rid})
return metadata
@event
def run_started(self, rid, metadata={}):
metadata.update({"rid": rid})
return metadata
@event
def run_ended(self, rid, metadata={}):
metadata.update({"rid": rid})
return metadata
def get_current_run(self) -> RemoteHERO:
"""
Return a handle to the current run or None.
"""
for rid, run in self.get_status().items():
if run["status"] == "running":
# construct HERO
return RemoteHERO(f"atomiq-run-{rid}", realm=self.heros_realm)
return None
def submit(self, pipeline_name, expid, priority=0, due_date=None, flush=False):
"""Submits a new run.
When called through an experiment, the default values of
``pipeline_name``, ``expid`` and ``priority`` correspond to those of
the current run."""
if (
_git_autocommit
and "repo_rev" in expid
and expid["repo_rev"] is None
and hasattr(self._experiment_db.repo_backend, "git")
):
# lazy import
import pygit2
from pygit2.enums import FileStatus
git_repo = self._experiment_db.repo_backend.git
# Check if any unstaged changes in our file
if bool(git_repo.status_file(expid["file"]) & FileStatus.WT_MODIFIED):
git_repo.index.add(expid["file"])
git_repo.index.write()
author = pygit2.Signature(self._name, "service@atomiq.one")
message = f"Updated experiment {expid['class_name']}"
tree = git_repo.index.write_tree()
new_hash = git_repo.create_commit("HEAD", author, author, message, tree, [git_repo.head.target])
logger.info(f"Committing to repository, new hash: {new_hash}")
expid["repo_rev"] = str(new_hash)
# update the edb manager that all dashboards also use the latest revision to reload args
self._experiment_db.cur_rev = self._experiment_db.repo_backend.get_head_rev()
else:
expid["repo_rev"] = str(git_repo.head.target)
expid["heros_realm"] = self.heros_realm
return _ArtiqScheduler.submit(self, pipeline_name, expid, priority, due_date, flush)
def get_expid(self, rid: int) -> dict:
"""
Return the expid dict of a running experiment.
Args:
rid: A run_id.
Returns:
expid dictionary if the run id is currently in the schedule,
else returns an empty dict
"""
rid = int(rid)
status = self.get_status()
return status[rid]["expid"] if rid in status else {}
def get_log(self, maxlen: int | None = None) -> list[dict]:
""" Return the latest log entries.
Args:
maxlen: Maximum number of log entries to return
Returns:
list of log entries, oldest first. Each entry is a dict with the keys ``level``
(level name), ``source`` (e.g. ``master`` or ``worker(<rid>,<file>)``), ``time``
(UNIX timestamp) and ``message`` (``<logger name>:<message>``)
"""
if maxlen is None:
maxlen = log_buffer.maxlen
return list(itertools.islice(log_buffer, max(0, len(log_buffer)-maxlen), None))
class ExperimentDB(_ArtiqExperimentDB, LocalHERO):
"""
Patches the ARTIQ ExperimentDB to (`artiq.master.experiments.ExperimentDB`) to become a HERO.
"""
def __init__(self, *args, **kwargs):
global _instance_name
global _heros_realm
_ArtiqExperimentDB.__init__(self, *args, **kwargs)
postfix = f"-{_instance_name}" if _instance_name is not None else ""
LocalHERO.__init__(self, f"atomiq-experimentdb{postfix}", realm=_heros_realm)
def scan_repository_sync(self, new_cur_rev=None) -> None:
"""
Trigger a reposistory scan on the ARTIQ master.
.. note::
This method is blocking and likely to timeout for larger repositories.
"""
import asyncio
asyncio.run(self.scan_repository(new_cur_rev))
def get_experiment_examination(self, filename, use_repository=True, revision=None) -> dict:
"""
Examine an experiment.
"""
import asyncio
experiment = asyncio.run(self.examine(filename, use_repository, revision))
return experiment
def get_experiment(self, experiment_name: str) -> dict:
"""
Construct a expid dict for a given experiment.
This dict can then be used to start an experiment via the Scheduler-HERO.
"""
assert experiment_name in self.list_experiments(), (
f"Experiment {experiment_name} is not in {list(self.list_experiments.keys())}"
)
full_description = {**self.list_experiments()[experiment_name]} # make a copy
expid = {
"devarg_override": {},
"repo_rev": None,
"log_level": 20, # INFO
"file": full_description["file"],
"class_name": full_description["class_name"],
"arguments": {},
}
# handle args
arguments = {}
for arg_name, arg_descriptions in full_description["arginfo"].items():
arg = argument_factory(arg_descriptions[0])
arguments[arg_name] = arg
expid["arguments"] = arguments
return expid
def list_experiments(self) -> dict:
"""
List all known experiments.
"""
return self.explist.raw_view
def get_device_db(self, key_filter: str | None = None) -> dict:
"""
Return the ARTIQ device database, without the atomiq component definitions.
Args:
key_filter: Case-sensitive Unix shell style glob pattern (see :py:mod:`fnmatch`). Only
devices whose key matches the pattern are returned. If None, all devices are returned.
Returns:
dict mapping the device key to its device db entry
"""
ddb: dict = self.worker_handlers["get_device_db"]().copy()
# filter out atomiq components
ddb.pop("components", None)
if key_filter is not None:
return {key: value for key, value in ddb.items() if fnmatch.fnmatchcase(key, key_filter)}
return ddb
def get_components(self, key_filter: str | None = None) -> dict:
"""
Return the atomiq component definitions held in the ARTIQ device database.
Args:
key_filter: Case-sensitive Unix shell style glob pattern (see :py:mod:`fnmatch`). Only
components whose key matches the pattern are returned. If None, all components are returned.
Returns:
dict mapping the component name to its component definition. Empty if the device database
holds no atomiq components.
"""
ddb: dict = self.worker_handlers["get_device_db"]()
# the "type" key is a marker set by the components helpers, not a component
components = {key: value for key, value in ddb.get("components", {}).items() if key != "type"}
if key_filter is not None:
return {key: value for key, value in components.items() if fnmatch.fnmatchcase(key, key_filter)}
return components
class LogForwarder(_ArtiqLogForwarder):
def emit(self, record):
super().emit(record)
# same fields that ARTIQ forwards to the dashboard log pane
log_buffer.append({
"level": record.levelname,
"source": record.source,
"time": record.created,
"message": self.format(record),
})
# apply patches
artiq.master.scheduler.Scheduler = Scheduler
artiq.master.experiments.ExperimentDB = ExperimentDB
artiq.master.log.LogForwarder = LogForwarder
del artiq
del Scheduler
del ExperimentDB
del LogForwarder
from artiq.frontend import artiq_master # noqa: E402 We need the import after the heros configuration to work.
_artiq_get_argparser = artiq_master.get_argparser
[docs]
def get_argparser():
"""
We extend the artiq master argparser
"""
parser = _artiq_get_argparser()
# delete the artiq defined --version argument to overwrite with our own.
for action in parser._actions:
if isinstance(action, argparse._VersionAction) or action.option_strings == ["--version"]:
parser._actions.remove(action)
del parser._option_string_actions["--version"]
break
parser.add_argument("--heros-connect", help="deprecated, set the HEROS_ZENOH_CONNECT environment variable instead")
parser.add_argument("--heros-realm", help="deprecated, set the HEROS_REALM environment variable instead")
parser.add_argument(
"--version",
action="version",
version=f"ATOMIQ v{atomiq_version}, \n ARTIQ v{artiq_version}",
help="Print the ATOMIQ and ARTIQ version numbers.",
)
parser.add_argument(
"--git-autocommit", help="automatically commit experiments to git on schedule", action="store_true"
)
return parser
if __name__ == "__main__":
args = get_argparser().parse_args()
artiq_master.get_argparser = get_argparser
if heros is not None:
export_heros_connect(args.heros_connect)
export_heros_realm(args.heros_realm)
if "name" in args:
_instance_name = args.name
if args.git_autocommit and not args.git:
logger.error("Setting --git-autocommit requires also --git to be set")
exit(1)
_git_autocommit = args.git_autocommit
_heros_realm = default_heros_realm()
logger.setLevel(logging.WARNING + args.quiet * 10 - args.verbose * 10)
logger.info("Starting ATOMIQ master with version %s, (ARTIQ version %s)", atomiq_version, artiq_version)
artiq_master.main()