Source code for atomiq.frontend.atomiq_master

import artiq.master.scheduler
import artiq.master.experiments
import artiq.master.log

from atomiq.heros import default_heros_realm, heros, LocalHERO, RemoteHERO, event
from artiq import __version__ as artiq_version
from atomiq import __version__ as atomiq_version
from atomiq.atomiq import logger

import collections
import fnmatch
import itertools
import logging
import os
import argparse


[docs] def argument_factory(arg_description: dict): """ Instantiate a ScanObject or BooleanValue, StringValue etc from type and default arguments """ import artiq.language.scan import artiq.language.environment arg_type = arg_description["ty"] if arg_type == "Scannable": inner_arg_type = arg_description["default"][0]["ty"] inner_description = arg_description["default"][0] inner_kwargs = {k: v for k, v in inner_description.items() if k != "ty"} arg_cls = getattr(artiq.language.scan, inner_arg_type) return arg_cls(**inner_kwargs).describe() else: # _SimpleArgProcessor inner_arg_type = arg_type inner_description = arg_description inner_kwargs = {k: v for k, v in inner_description.items() if k != "ty"} arg_cls = getattr(artiq.language.environment, inner_arg_type) arg = arg_cls(**inner_kwargs) # we further need to convert these into "real" values, # i.e. we need "some_var": {ty: "BooleanValue", ...} -> "some_var": True # for simplicity, we just take the default return arg.default()
[docs] def export_deprecated_flag(value: str | None, flag: str, env_var: str) -> None: """ Export the value of a deprecated command line flag as an environment variable. An environment variable that is already set takes precedence. ARTIQ workers are subprocesses of the master and inherit the variable. Args: value: value given to the flag, or None if the flag is not given flag: name of the deprecated flag, used in the warnings env_var: environment variable that replaces the flag """ if value is None: return logger.warning(f"{flag} is deprecated, set the {env_var} environment variable instead") if os.environ.get(env_var, "").strip(): logger.warning(f"{env_var} is set, ignoring {flag}") return os.environ[env_var] = value
[docs] def export_heros_connect(connect: str | None) -> None: """ Export the deprecated ``--heros-connect`` endpoint as ``HEROS_ZENOH_CONNECT``. Args: connect: zenoh endpoint given to ``--heros-connect``, or None if the flag is not given """ export_deprecated_flag(connect, "--heros-connect", "HEROS_ZENOH_CONNECT")
[docs] def export_heros_realm(realm: str | None) -> None: """ Export the deprecated ``--heros-realm`` realm as ``HEROS_REALM``. Args: realm: realm given to ``--heros-realm``, or None if the flag is not given """ export_deprecated_flag(realm, "--heros-realm", "HEROS_REALM")
# If HEROS is available, replace the artiq Scheduler with our HEROS-enabled version if heros is not None: _instance_name = None _heros_realm = None _git_autocommit = None log_buffer: collections.deque[dict] = collections.deque([], maxlen=1000) _ArtiqScheduler = artiq.master.scheduler.Scheduler _ArtiqExperimentDB = artiq.master.experiments.ExperimentDB _ArtiqLogForwarder = artiq.master.log.LogForwarder class Scheduler(_ArtiqScheduler, LocalHERO): """ Patches the ARTIQ master Scheduler to (`artiq.master.scheduler.Scheduler`) to become a HERO. """ def __init__(self, *args, **kwargs): global _instance_name global _heros_realm global _git_autocommit self.heros_realm = _heros_realm _ArtiqScheduler.__init__(self, *args, **kwargs) postfix = f"-{_instance_name}" if _instance_name is not None else "" LocalHERO.__init__(self, f"atomiq-scheduler{postfix}", realm=_heros_realm) self._worker_handlers.update( {"run_started": self.run_started, "run_created": self.run_created, "run_ended": self.run_ended} ) @event def run_created(self, rid, metadata={}): metadata.update({"rid": rid}) return metadata @event def run_started(self, rid, metadata={}): metadata.update({"rid": rid}) return metadata @event def run_ended(self, rid, metadata={}): metadata.update({"rid": rid}) return metadata def get_current_run(self) -> RemoteHERO: """ Return a handle to the current run or None. """ for rid, run in self.get_status().items(): if run["status"] == "running": # construct HERO return RemoteHERO(f"atomiq-run-{rid}", realm=self.heros_realm) return None def submit(self, pipeline_name, expid, priority=0, due_date=None, flush=False): """Submits a new run. When called through an experiment, the default values of ``pipeline_name``, ``expid`` and ``priority`` correspond to those of the current run.""" if ( _git_autocommit and "repo_rev" in expid and expid["repo_rev"] is None and hasattr(self._experiment_db.repo_backend, "git") ): # lazy import import pygit2 from pygit2.enums import FileStatus git_repo = self._experiment_db.repo_backend.git # Check if any unstaged changes in our file if bool(git_repo.status_file(expid["file"]) & FileStatus.WT_MODIFIED): git_repo.index.add(expid["file"]) git_repo.index.write() author = pygit2.Signature(self._name, "service@atomiq.one") message = f"Updated experiment {expid['class_name']}" tree = git_repo.index.write_tree() new_hash = git_repo.create_commit("HEAD", author, author, message, tree, [git_repo.head.target]) logger.info(f"Committing to repository, new hash: {new_hash}") expid["repo_rev"] = str(new_hash) # update the edb manager that all dashboards also use the latest revision to reload args self._experiment_db.cur_rev = self._experiment_db.repo_backend.get_head_rev() else: expid["repo_rev"] = str(git_repo.head.target) expid["heros_realm"] = self.heros_realm return _ArtiqScheduler.submit(self, pipeline_name, expid, priority, due_date, flush) def get_expid(self, rid: int) -> dict: """ Return the expid dict of a running experiment. Args: rid: A run_id. Returns: expid dictionary if the run id is currently in the schedule, else returns an empty dict """ rid = int(rid) status = self.get_status() return status[rid]["expid"] if rid in status else {} def get_log(self, maxlen: int | None = None) -> list[dict]: """ Return the latest log entries. Args: maxlen: Maximum number of log entries to return Returns: list of log entries, oldest first. Each entry is a dict with the keys ``level`` (level name), ``source`` (e.g. ``master`` or ``worker(<rid>,<file>)``), ``time`` (UNIX timestamp) and ``message`` (``<logger name>:<message>``) """ if maxlen is None: maxlen = log_buffer.maxlen return list(itertools.islice(log_buffer, max(0, len(log_buffer)-maxlen), None)) class ExperimentDB(_ArtiqExperimentDB, LocalHERO): """ Patches the ARTIQ ExperimentDB to (`artiq.master.experiments.ExperimentDB`) to become a HERO. """ def __init__(self, *args, **kwargs): global _instance_name global _heros_realm _ArtiqExperimentDB.__init__(self, *args, **kwargs) postfix = f"-{_instance_name}" if _instance_name is not None else "" LocalHERO.__init__(self, f"atomiq-experimentdb{postfix}", realm=_heros_realm) def scan_repository_sync(self, new_cur_rev=None) -> None: """ Trigger a reposistory scan on the ARTIQ master. .. note:: This method is blocking and likely to timeout for larger repositories. """ import asyncio asyncio.run(self.scan_repository(new_cur_rev)) def get_experiment_examination(self, filename, use_repository=True, revision=None) -> dict: """ Examine an experiment. """ import asyncio experiment = asyncio.run(self.examine(filename, use_repository, revision)) return experiment def get_experiment(self, experiment_name: str) -> dict: """ Construct a expid dict for a given experiment. This dict can then be used to start an experiment via the Scheduler-HERO. """ assert experiment_name in self.list_experiments(), ( f"Experiment {experiment_name} is not in {list(self.list_experiments.keys())}" ) full_description = {**self.list_experiments()[experiment_name]} # make a copy expid = { "devarg_override": {}, "repo_rev": None, "log_level": 20, # INFO "file": full_description["file"], "class_name": full_description["class_name"], "arguments": {}, } # handle args arguments = {} for arg_name, arg_descriptions in full_description["arginfo"].items(): arg = argument_factory(arg_descriptions[0]) arguments[arg_name] = arg expid["arguments"] = arguments return expid def list_experiments(self) -> dict: """ List all known experiments. """ return self.explist.raw_view def get_device_db(self, key_filter: str | None = None) -> dict: """ Return the ARTIQ device database, without the atomiq component definitions. Args: key_filter: Case-sensitive Unix shell style glob pattern (see :py:mod:`fnmatch`). Only devices whose key matches the pattern are returned. If None, all devices are returned. Returns: dict mapping the device key to its device db entry """ ddb: dict = self.worker_handlers["get_device_db"]().copy() # filter out atomiq components ddb.pop("components", None) if key_filter is not None: return {key: value for key, value in ddb.items() if fnmatch.fnmatchcase(key, key_filter)} return ddb def get_components(self, key_filter: str | None = None) -> dict: """ Return the atomiq component definitions held in the ARTIQ device database. Args: key_filter: Case-sensitive Unix shell style glob pattern (see :py:mod:`fnmatch`). Only components whose key matches the pattern are returned. If None, all components are returned. Returns: dict mapping the component name to its component definition. Empty if the device database holds no atomiq components. """ ddb: dict = self.worker_handlers["get_device_db"]() # the "type" key is a marker set by the components helpers, not a component components = {key: value for key, value in ddb.get("components", {}).items() if key != "type"} if key_filter is not None: return {key: value for key, value in components.items() if fnmatch.fnmatchcase(key, key_filter)} return components class LogForwarder(_ArtiqLogForwarder): def emit(self, record): super().emit(record) # same fields that ARTIQ forwards to the dashboard log pane log_buffer.append({ "level": record.levelname, "source": record.source, "time": record.created, "message": self.format(record), }) # apply patches artiq.master.scheduler.Scheduler = Scheduler artiq.master.experiments.ExperimentDB = ExperimentDB artiq.master.log.LogForwarder = LogForwarder del artiq del Scheduler del ExperimentDB del LogForwarder from artiq.frontend import artiq_master # noqa: E402 We need the import after the heros configuration to work. _artiq_get_argparser = artiq_master.get_argparser
[docs] def get_argparser(): """ We extend the artiq master argparser """ parser = _artiq_get_argparser() # delete the artiq defined --version argument to overwrite with our own. for action in parser._actions: if isinstance(action, argparse._VersionAction) or action.option_strings == ["--version"]: parser._actions.remove(action) del parser._option_string_actions["--version"] break parser.add_argument("--heros-connect", help="deprecated, set the HEROS_ZENOH_CONNECT environment variable instead") parser.add_argument("--heros-realm", help="deprecated, set the HEROS_REALM environment variable instead") parser.add_argument( "--version", action="version", version=f"ATOMIQ v{atomiq_version}, \n ARTIQ v{artiq_version}", help="Print the ATOMIQ and ARTIQ version numbers.", ) parser.add_argument( "--git-autocommit", help="automatically commit experiments to git on schedule", action="store_true" ) return parser
if __name__ == "__main__": args = get_argparser().parse_args() artiq_master.get_argparser = get_argparser if heros is not None: export_heros_connect(args.heros_connect) export_heros_realm(args.heros_realm) if "name" in args: _instance_name = args.name if args.git_autocommit and not args.git: logger.error("Setting --git-autocommit requires also --git to be set") exit(1) _git_autocommit = args.git_autocommit _heros_realm = default_heros_realm() logger.setLevel(logging.WARNING + args.quiet * 10 - args.verbose * 10) logger.info("Starting ATOMIQ master with version %s, (ARTIQ version %s)", atomiq_version, artiq_version) artiq_master.main()