Xmen Experiment Classes

Defining Experiments

from xmen.experiment import Experiment
import os
import time
from typing import List


class BaseExperiment(Experiment):
    """A basic experiments experiment demonstrating the features of the xmen api."""

    # Parameters are defined as attributes in the class body with the
    # @p identifier

    t = 'cat'    # @p
    w = 3        # @p parameter w has a help message whilst t does not
    h: int = 10  # @p h declared with typing is very concise and neat

    # Parameters can also be defined in the __init__ method
    def __init__(self, *args, **kwargs):
        super(BaseExperiment, self).__init__(*args, **kwargs)

        self.a: str = 'h'  # @p A parameter
        self.b: int = 17   # @p Another parameter

        # Normal attributes are still allowed
        self.c: int = 5    # This is not a parameter


class AnotherExperiment(BaseExperiment):
    m: str = 'Another value'  # @p Multiple inheritance example
    p: str = 'A parameter only in Another Experiment '  # @p


class AnExperiment(BaseExperiment):
                    #     |
                    # Experiments can inherit from other experiments
                    # parameters are inherited too
    """An experiment testing the xmen experiment API. The __docstring__ will
    appear in both the docstring of the class __and__ as the prolog in the
    command line interface."""

    # Feel free to define more parameters
    x: List[float] = [3., 2.]  # @p Parameters can be defined cleanly as class attributes
    y: float = 5  # @p This parameter will have this
    # Parameters can be overridden
    a: float = 0.5  # a's default and type will be changed. Its help will be overridden
    b: int = 17  # @p b's help will be changed

    m: str = 'Defined in AnExperiment'  # @p m is defined in AnExperiment

    def run(self):
        # Experiment execution is defined in the run method
        print(f'The experiment state inside run is {self.status}')

        # recording messaging is super easy
        self.message({'time': time.time()})

        # Each experiment has its own unique directory. You are encourage to
        # write out data accumulated through the execution (snapshots, logs etc.)
        # to this directory.
        with open(os.path.join(self.directory, 'logs.txt'), 'w') as f:
            f.write('This was written from a running experiment')

    def debug(self):
        self.a = 'In debug mode'
        return self

    @property
    def h(self):
        return 'h has been overloaded as propery and will no longer' \
               'considered as a parameter'


# Experiments can inheret from multiple classes
class MultiParentsExperiment(AnotherExperiment, AnExperiment):
    pass

Note: the above is a copy of what is defined in xmen.examplex.inheritance

[1]:
from xmen.examples.inheritance import AnExperiment, AnotherExperiment, MultiParentsExperiment
from notebook.services.config import ConfigManager
cm = ConfigManager().update('notebook', {'limit_output': 10})

Automatic Documentation

[2]:
# documentation is automatically added to the class
help(AnExperiment)
Help on class AnExperiment in module xmen.examples.inheritance:

class AnExperiment(BaseExperiment)
 |  AnExperiment(*args, **kwargs)
 |
 |  An experiment testing the xmen experiment API. The __docstring__ will
 |      appear in both the docstring of the class __and__ as the prolog in the
 |      command line interface.
 |
 |  Parameters:
 |      BaseExperiment
 |       t: None (default=cat)
 |       w: parameter w has a help message whilst t does not (default=3)
 |       a (float): A parameter (default=0.5)
 |      AnExperiment
 |       b: int=17 ~ b's help will be changed
 |       x: List[float]=[3., 2.] ~ Parameters can be defined cleanly as class attributes
 |       y: float=5 ~ This parameter will have this
 |       m: str='Defined in AnExperiment' ~ m is defined in AnExperiment
 |
 |  Method resolution order:
 |      AnExperiment
 |      BaseExperiment
 |      xmen.experiment.Experiment
 |      builtins.object
 |
 |  Methods defined here:
 |
 |  debug(self)
 |      Inherited classes may overload debug. Used to define a set of open_socket for minimum example
 |
 |  run(self)
 |
 |  ----------------------------------------------------------------------
 |  Readonly properties defined here:
 |
 |  h
 |      int([x]) -> integer
 |      int(x, base=10) -> integer
 |
 |      Convert a number or string to an integer, or return 0 if no arguments
 |      are given.  If x is a number, return x.__int__().  For floating point
 |      numbers, this truncates towards zero.
 |
 |      If x is not a number or if base is given, then x must be a string,
 |      bytes, or bytearray instance representing an integer literal in the
 |      given base.  The literal can be preceded by '+' or '-' and be surrounded
 |      by whitespace.  The base defaults to 10.  Valid bases are 0 and 2-36.
 |      Base 0 means to interpret the base from the string as an integer literal.
 |      >>> int('0b100', base=0)
 |      4
 |
 |  ----------------------------------------------------------------------
 |  Data and other attributes defined here:
 |
 |  __annotations__ = {'a': <class 'float'>, 'b': <class 'int'>, 'm': <cla...
 |
 |  a = 0.5
 |
 |  b = 17
 |
 |  m = 'Defined in AnExperiment'
 |
 |  x = [3.0, 2.0]
 |
 |  y = 5
 |
 |  ----------------------------------------------------------------------
 |  Methods inherited from BaseExperiment:
 |
 |  __init__(self, *args, **kwargs)
 |      Create a new experiment object.
 |
 |      Args:
 |          root, name (str): If not None then the experiment will be registered to a folder ``{root}\{name}``
 |          purpose (str): An optional string giving the purpose of the experiment.
 |          copy (bool): If True then parameters are deep copied to the object instance from the class definition.
 |              Mutable attributes will no longer be shared.
 |          **kwargs: Override parameter defaults.
 |
 |  ----------------------------------------------------------------------
 |  Data and other attributes inherited from BaseExperiment:
 |
 |  t = 'cat'
 |
 |  w = 3
 |
 |  ----------------------------------------------------------------------
 |  Methods inherited from xmen.experiment.Experiment:
 |
 |  __call__(self, *args, **kwargs)
 |      Used to run experiment. Upon entering the experiment status is updated to ``'running`` before ``args`` and
 |      ``kwargs`` are passed to ``run()``. If ``run()`` is successful the experiment ``status`` is updated to
 |      ``'finished'`` else it will be given ``status='error'``.
 |
 |      Both *args and **kwargs are passed to self.run.
 |
 |  __enter__(self)
 |
 |  __exit__(self, exc_type, exc_val, exc_tb)
 |
 |  __repr__(self)
 |      Provides a useful help message for the experiment
 |
 |  __setattr__(self, key, value)
 |      Attributes can only be changed when the status of the experiment is default
 |
 |  compare(self, k, v, keep='latest')
 |
 |  detach(self)
 |
 |  from_yml(self, path, copy=False)
 |      Load state from either a ``params.yml`` or ``defaults.yml`` file (inferred from the filename).
 |      The status of the experiment will be updated to ``'default'`` if ``'defaults.yml'``
 |      file else ``'registered'`` if ``params.yml`` file.
 |
 |  get_param_helps(self)
 |      Get help for all attributes in class (including inherited and private).
 |
 |  get_run_script(self, type='set', shell='/usr/bin/env python3', comment='#')
 |
 |  main(self, args=None)
 |      Take the command line args and execute the experiment (see ``parse_args`` for more
 |       information). In order to expose the command line interface::
 |
 |          if __name__ == '__main__':
 |              exp = AnExperiment().main()
 |
 |      Note that for backwards compatibility it is also possible to pass ``args`` as an argument to ``main``.
 |      This allows the experiment to be run from the commandline as::
 |
 |          if __name__ == '__main__':
 |              exp = AnExperiment()
 |              args = exp.parse_args()
 |              exp.main(args)
 |
 |  message(self, messages, keep='latest', leader=None)
 |      Add a message to the experiment (and an experiments params.yml file). If the experiment is not registered to
 |      a root then no messages will be logged.
 |
 |      Args:
 |          messages (dict): A dictionary of messages. Keys are interpreted as subjects and values interpreted as
 |              messages. If the ``defaults.yml`` already contains subject then the message for subject will be
 |              updated.
 |          keep (str): which message to keep in the case of collision. One of ['latest', 'min', 'max']
 |          leader (str): If not None then all messages will be saved if the keep condition is met for the leader key.
 |
 |      Note:
 |          Only messages of type float, int and string are supported. Any other message will be converted to type float
 |          (if possible) then string thereafter.
 |
 |  param_keys(self)
 |
 |  parse_args(self)
 |      Configure the experiment instance from the command line arguments.
 |
 |  link(self, root, name, purpose='', force=True, same_names=100, generate_script=False, header=None)
 |      Register an experiment to an experiment directory. Its status will be updated to ``registered``. If an
 |      experiment called ``name`` exists in ``root`` and ``force==True`` then name will be appended with an int
 |      (eg. ``{name}_0``) until a unique name is found in ``root``. If ``force==False`` a ``ValueError`` will be raised.
 |
 |      Raises:
 |          ValueError: if ``{root}/{name}`` already contains a ``params.yml`` file
 |
 |  stdout_to_txt(self)
 |      Configure stdout to also log to a text file in the experiment directory
 |
 |  to_defaults(self, defaults_dir)
 |      Create a ``defaults.yml`` file from experiment object.
 |      Any base class inheriting from Experiment can create a default file as::
 |
 |         MyExperiment().to_yaml('/dir/to/defaults/root')
 |
 |  to_root(self, root_dir, shell='/bin/bash')
 |      Generate a ``defaults.yml`` file and ``script.sh`` file in ``root_dir``.
 |
 |      Args:
 |          root_dir (str): A path to the root directory in which to load_and_generate a script.sh and defaults.yml to
 |              run the experiment.
 |
 |  update(self, kwargs)
 |      Update the parameters with a given dictionary
 |
 |  update_meta(self)
 |
 |  update_version(self)
 |
 |  ----------------------------------------------------------------------
 |  Static methods inherited from xmen.experiment.Experiment:
 |
 |  convert_type(v)
 |
 |  ----------------------------------------------------------------------
 |  Readonly properties inherited from xmen.experiment.Experiment:
 |
 |  directory
 |      The directory assigned to the experiment
 |
 |  ----------------------------------------------------------------------
 |  Data descriptors inherited from xmen.experiment.Experiment:
 |
 |  __dict__
 |      dictionary for instance variables (if defined)
 |
 |  __weakref__
 |      list of weak references to the object (if defined)
 |
 |  created
 |      The date the experiment parameters were last updated.
 |
 |  messages
 |      A dictionary of messages logged by the experimet.
 |
 |  name
 |      The name of the current experiment
 |
 |  purpose
 |      A string giving a purpose message for the experiment
 |
 |  root
 |      The root directory to which the experiment belongs
 |
 |  status
 |      The status of the experiment. One of ``'default'``, ``'registered'``, ``'running'``, ``'finished'`` or
 |      ``'error'``.
 |
 |  version
 |      A dictionary giving the version information for the experiment

Configuring Experiments

[3]:
# to run an experiment first we initialise it
# the experiment is initialised in 'default' status
print('\nInitialising')
print('---------------------------')
exp = AnExperiment()
print(exp)

Initialising
---------------------------
status: default
created: 11-16-20-16:50:06
messages:
parameters:
  t: cat
  w: 3
  a: h
  b: 17
  x: [3.0, 2.0]
  y: 5
  m: Defined in AnExperiment

Inheritance

[4]:
# Experiments can inheret from multiple classes:
print('\nMultiple Inheritance')
print('----------------------')
print(MultiParentsExperiment())
print('\n Parameters defaults, helps and values are '
      'inherited according to experiments method resolution order '
      '(i.e left to right). Note that m has the value '
      'defined in Another Experiment')

Multiple Inheritance
----------------------
status: default
created: 11-16-20-16:50:07
messages:
parameters:
  t: cat
  w: 3
  a: h
  b: 17
  m: Another value
  p: A parameter only in Another Experiment
  x: [3.0, 2.0]
  y: 5

 Parameters defaults, helps and values are inherited according to experiments method resolution order (i.e left to right). Note that m has the value defined in Another Experiment

Configuring

[5]:
# whilst the status is default the parameters of the
# experiment can be changed
print('\nConfiguring')
print('------------ ')
exp = AnExperiment()
exp.a = 'hello'
exp.update({'t': 'dog', 'w': 100})
# Note parameters are copied from the class during
# instantiation. This way you don't need to worry
# about accidentally changing the mutable class
# types across the entire class.
exp.x += [4.]
print(exp)
assert AnExperiment.x == [3., 2.]
# If this is not desired (or neccessary) initialise
# use exp = AnExperiment(copy=False)

Configuring
------------
status: default
created: 11-16-20-16:50:07
messages:
parameters:
  t: dog
  w: 100
  a: hello
  b: 17
  x: [3.0, 2.0, 4.0]
  y: 5
  m: Defined in AnExperiment

Registering

[6]:
print('\nRegistering')
print('-------------')
# Before being run an experiment needs to be registered
# to a directory
exp.link('/tmp/an_experiment', 'first_experiment',
             purpose='A bit of a test of the xmen experiment api')
print(exp, end='\n')
print('\nGIT, and system information is automatically logged\n')

Registering
-------------
root: /tmp/an_experiment
name: first_experiment_4
status: registered
created: 11-16-20-16:50:07
purpose: A bit of a test of the xmen experiment api
messages:
version:
  module: xmen.examples.inheritance
  class: AnExperiment
  path: /home/robw/projects/xmen/xmen/examples
  git:
    local: /home/robw/projects/xmen
    remote: https://github.com/robw4/xmen.git
    commit: 46b61c87b4001157a408217a1a3534f62029a1db
    branch: master
meta:
  mac: 0x708bcd585ff7
  host: yossarian
  user: robw
  home: /home/robw
parameters:
  t: dog
  w: 100
  a: hello
  b: 17
  x: [3.0, 2.0, 4.0]
  y: 5
  m: Defined in AnExperiment

GIT, and system information is automatically logged

[7]:
# The parameters of the experiment can no longer be changed
try:
    exp.a = 'cat'
except AttributeError:
    print('Parameters can no longer be changed!', end='\n')
    pass
Parameters can no longer be changed!

Running

[8]:
# An experiment can be run either by...
# (1) calling it
print('\nRunning (1)')
print('-------------')
exp()

Running (1)
-------------
The experiment state inside run is running
[9]:
# (2) using it as a context. Just define a main
#     loop like you normally would
print('\nRunning (2)')
print('-------------')
with exp as e:
    # Inside the experiment context the experiment status is 'running'
    print(f'Once again the experiment state is {e.status}')
    # Write the main loop just as you normally would"
    # using the parameters already defined
    results = dict(sum=sum(e.x), max=max(e.x), min=min(e.x))
    # Write results to the expeirment just as before
    e.message(results)

Running (2)
-------------
Once again the experiment state is running
[10]:
# All the information about the current experiment is
# automatically saved in the experiments root directory
# for free
print(f'\nEverything ypu might need to know is logged in {exp.directory}/params.yml')
print('-----------------------------------------------------------------------------------------------')
print('Note that GIT, and system information is automatically logged\n'
      'along with the messages')
with open('/tmp/an_experiment/first_experiment/params.yml', 'r') as f:
    print(f.read())

Everything ypu might need to know is logged in /tmp/an_experiment/first_experiment_4/params.yml
-----------------------------------------------------------------------------------------------
Note that GIT, and system information is automatically logged
along with the messages
_created: 11-16-20-16:27:41  # _created: str=now_time ~ The date the experiment was created
_messages: # _messages: Dict[Any, Any]={} ~ Messages left by the experiment
  time: 1605544061.5061884
  sum: 9.0
  max: 4.0
  min: 2.0
_meta: # _meta: Optional[Dict]=None ~ The global configuration for the experiment manager
  mac: '0x708bcd585ff7'
  host: yossarian
  user: robw
  home: /home/robw
_name: first_experiment # _name: Optional[str]=None ~ The name of the experiment (under root)
_purpose: A bit of a test of the xmen experiment api # _purpose: Optional[str]=None ~ A description of the experiment purpose
_root: /tmp/an_experiment # _root: Optional[str]=None ~ The root directory of the experiment
_status: finished # _status: str='default' ~ One of ['default' | 'created' | 'running' | 'error' | 'finished']
_version: # _version: Optional[Dict[Any, Any]]=None ~ Experiment version information. See `get_version`
  module: __main__
  class: AnExperiment
  path: /home/robw/projects/xmen/xmen/examples
  git:
    local: /home/robw/projects/xmen
    remote: https://github.com/robw4/xmen.git
    commit: 46b61c87b4001157a408217a1a3534f62029a1db
    branch: master
a: hello # a (float): A parameter (default=0.5)
b: 17 # b: int=17 ~ b's help will be changed
m: Defined in AnExperiment # m: str='Defined in AnExperiment' ~ m is defined in AnExperiment
t: dog # t: None (default=cat)
w: 100 # w: parameter w has a help message whilst t does not (default=3)
x: # x: List[float]=[3., 2.] ~ Parameters can be defined cleanly as class attributes
- 3.0
- 2.0
- 4.0
y: 5 # y: float=5 ~ This parameter will have this