22

✏️ Editing: stateconf.py

"""
A flexible renderer that takes a templating engine and a data format

:maintainer: Jack Kuan <kjkuan@gmail.com>
:maturity: new
:platform: all
"""

# See http://docs.saltproject.io/en/latest/ref/renderers/all/salt.renderers.stateconf.html
# for a guide to using this module.
#
# FIXME: I really need to review and simplify this renderer, it's getting out of hand!
#
# TODO:
#   - sls meta/info state: E.g.,
#
#       sls_info:
#         stateconf.set:
#           - author: Jack Kuan
#           - description: what the salt file does...
#           - version: 0.1.0
#
#   - version constraint for 'include'. E.g.,
#
#       include:
#         - apache: >= 0.1.0
#

import copy
import getopt
import io
import logging
import os
import re
from itertools import chain

import salt.utils.files
import salt.utils.stringutils
from salt.exceptions import SaltRenderError

__all__ = ["render"]

log = logging.getLogger(__name__)

__opts__ = {
    "stateconf_end_marker": r"#\s*-+\s*end of state config\s*-+",
    # e.g., something like "# --- end of state config --" works by default.
    "stateconf_start_state": ".start",
    # name of the state id for the generated start state.
    "stateconf_goal_state": ".goal",
    # name of the state id for the generated goal state.
    "stateconf_state_func": "stateconf.set",
    # names the state and the state function to be recognized as a special
    # state from which to gather sls file context variables. It should be
    # specified in the 'state.func' notation, and both the state module and
    # the function must actually exist and the function should be a dummy,
    # no-op state function that simply returns a
    # dict(name=name, result=True, changes={}, comment='')
}

STATE_FUNC = STATE_NAME = ""

def __init__(opts):
    global STATE_NAME, STATE_FUNC
    STATE_FUNC = __opts__["stateconf_state_func"]
    STATE_NAME = STATE_FUNC.split(".", maxsplit=1)[0]

MOD_BASENAME = os.path.basename(__file__)
INVALID_USAGE_ERROR = SaltRenderError(
    "Invalid use of {0} renderer!\n"
    """Usage: #!{0} [-GoSp] [<data_renderer> [options] . <template_renderer> [options]]

where an example <data_renderer> would be yaml and a <template_renderer> might
be jinja. Each renderer can be passed its renderer specific options.

Options(for this renderer):

-G   Do not generate the goal state that requires all other states in the sls.

-o   Indirectly order the states by adding requires such that they will be
       executed in the order they are defined in the sls. Implies using yaml -o.

-s   Generate the start state that gets inserted as the first state in
       the sls. This only makes sense if your high state data dict is ordered.

-p   Assume high state input. This option allows you to pipe high state data
       through this renderer. With this option, the use of stateconf.set state
       in the sls will have no effect, but other features of the renderer still
       apply.

""".format(
        MOD_BASENAME
    )
)

def render(input, saltenv="base", sls="", argline="", **kws):
    gen_start_state = False
    no_goal_state = False
    implicit_require = False

def process_sls_data(data, context=None, extract=False):
        sls_dir = os.path.dirname(sls.replace(".", os.path.sep)) if "." in sls else sls
        ctx = dict(sls_dir=sls_dir if sls_dir else ".")

if context:
            ctx.update(context)

tmplout = render_template(
            io.StringIO(data),
            saltenv,
            sls,
            context=ctx,
            argline=rt_argline.strip(),
            **kws,
        )
        high = render_data(tmplout, saltenv, sls, argline=rd_argline.strip())
        return process_high_data(high, extract)

def process_high_data(high, extract):
        # make a copy so that the original, un-preprocessed highstate data
        # structure can be used later for error checking if anything goes
        # wrong during the preprocessing.
        data = copy.deepcopy(high)
        try:
            rewrite_single_shorthand_state_decl(data)
            rewrite_sls_includes_excludes(data, sls, saltenv)

if not extract and implicit_require:
                sid = has_names_decls(data)
                if sid:
                    raise SaltRenderError(
                        "'names' declaration(found in state id: {}) is "
                        "not supported with implicitly ordered states! You "
                        "should generate the states in a template for-loop "
                        "instead.".format(sid)
                    )
                add_implicit_requires(data)

if gen_start_state:
                add_start_state(data, sls)

if not extract and not no_goal_state:
                add_goal_state(data)

rename_state_ids(data, sls)

# We must extract no matter what so extending a stateconf sls file
            # works!
            extract_state_confs(data)
        except SaltRenderError:
            raise
        except Exception as err:  # pylint: disable=broad-except
            log.exception(
                "Error found while pre-processing the salt file %s:\n%s", sls, err
            )
            from salt.state import State

state = State(__opts__)
            errors = state.verify_high(high)
            if errors:
                raise SaltRenderError("\n".join(errors))
            raise SaltRenderError("sls preprocessing/rendering failed!")
        return data

# ----------------------
    renderers = kws["renderers"]
    opts, args = getopt.getopt(argline.split(), "Gosp")
    argline = " ".join(args) if args else "yaml . jinja"

if ("-G", "") in opts:
        no_goal_state = True
    if ("-o", "") in opts:
        implicit_require = True
    if ("-s", "") in opts:
        gen_start_state = True

if ("-p", "") in opts:
        data = process_high_data(input, extract=False)
    else:
        # Split on the first dot surrounded by spaces but not preceded by a
        # backslash. A backslash preceded dot will be replaced with just dot.
        args = [
            arg.strip().replace("\\.", ".")
            for arg in re.split(r"\s+(?<!\\)\.\s+", argline, 1)
        ]
        try:
            name, rd_argline = (args[0] + " ").split(" ", 1)
            render_data = renderers[name]  # e.g., the yaml renderer
            if implicit_require:
                if name == "yaml":
                    rd_argline = "-o " + rd_argline
                else:
                    raise SaltRenderError(
                        "Implicit ordering is only supported if the yaml renderer "
                        "is used!"
                    )
            name, rt_argline = (args[1] + " ").split(" ", 1)
            render_template = renderers[name]  # e.g., the mako renderer
        except KeyError as err:
            raise SaltRenderError(f"Renderer: {err} is not available!")
        except IndexError:
            raise INVALID_USAGE_ERROR

if isinstance(input, str):
            with salt.utils.files.fopen(input, "r") as ifile:
                sls_templ = salt.utils.stringutils.to_unicode(ifile.read())
        else:  # assume file-like
            sls_templ = salt.utils.stringutils.to_unicode(input.read())

# first pass to extract the state configuration
        match = re.search(__opts__["stateconf_end_marker"], sls_templ)
        if match:
            process_sls_data(sls_templ[: match.start()], extract=True)

# if some config has been extracted then remove the sls-name prefix
        # of the keys in the extracted stateconf.set context to make them easier
        # to use in the salt file.
        if STATE_CONF:
            tmplctx = STATE_CONF.copy()
            if tmplctx:
                prefix = sls + "::"
                tmplctx = {
                    k[len(prefix) :] if k.startswith(prefix) else k: v
                    for k, v in tmplctx.items()
                }
        else:
            tmplctx = {}

# do a second pass that provides the extracted conf as template context
        data = process_sls_data(sls_templ, tmplctx)

if log.isEnabledFor(logging.DEBUG):
        import pprint  # FIXME: pprint OrderedDict

log.debug("Rendered sls: %s", pprint.pformat(data))
    return data

def has_names_decls(data):
    for sid, _, _, args in statelist(data):
        if sid == "extend":
            continue
        for _ in nvlist(args, ["names"]):
            return sid

def rewrite_single_shorthand_state_decl(data):  # pylint: disable=C0103
    """
    Rewrite all state declarations that look like this::

state_id_decl:
        state.func

into::

state_id_decl:
        state.func: []
    """
    for sid, states in data.items():
        if isinstance(states, str):
            data[sid] = {states: []}

def rewrite_sls_includes_excludes(data, sls, saltenv):
    # if the path of the included/excluded sls starts with a leading dot(.)
    # then it's taken to be relative to the including/excluding sls.
    for sid in data:
        if sid == "include":
            includes = data[sid]
            for i, each in enumerate(includes):
                if isinstance(each, dict):
                    slsenv, incl = each.popitem()
                else:
                    slsenv = saltenv
                    incl = each
                if incl.startswith("."):
                    includes[i] = {slsenv: _relative_to_abs_sls(incl, sls)}
        elif sid == "exclude":
            for sdata in data[sid]:
                if "sls" in sdata and sdata["sls"].startswith("."):
                    sdata["sls"] = _relative_to_abs_sls(sdata["sls"], sls)

def _local_to_abs_sid(sid, sls):  # id must starts with '.'
    if "::" in sid:
        return _relative_to_abs_sls(sid, sls)
    else:
        abs_sls = _relative_to_abs_sls(sid, sls + ".")
        return "::".join(abs_sls.rsplit(".", 1))

def _relative_to_abs_sls(relative, sls):
    """
    Convert ``relative`` sls reference into absolute, relative to ``sls``.
    """
    levels, suffix = re.match(r"^(\.+)(.*)$", relative).groups()
    level_count = len(levels)
    p_comps = sls.split(".")
    if level_count > len(p_comps):
        raise SaltRenderError(
            "Attempted relative include goes beyond top level package"
        )
    return ".".join(p_comps[:-level_count] + [suffix])

def nvlist(thelist, names=None):
    """
    Given a list of items::

- whatever
        - name1: value1
        - name2:
          - key: value
          - key: value

return a generator that yields each (item, key, value) tuple, skipping
    items that are not name-value's(dictionaries) or those not in the
    list of matching names. The item in the returned tuple is the single-key
    dictionary.
    """
    # iterate over the list under the state dict.
    for nvitem in thelist:
        if isinstance(nvitem, dict):
            # then nvitem is a name-value item(a dict) of the list.
            name, value = next(iter(nvitem.items()))
            if names is None or name in names:
                yield nvitem, name, value

def nvlist2(thelist, names=None):
    """
    Like nvlist but applied one more time to each returned value.
    So, given a list, args,  of arguments to a state like this::

- name: echo test
      - cwd: /
      - require:
        - file: test.sh

nvlist2(args, ['require']) would yield the tuple,
    (dict_item, 'file', 'test.sh') where dict_item is the single-key
    dictionary of {'file': 'test.sh'}.

"""
    for _, _, value in nvlist(thelist, names):
        yield from nvlist(value)

def statelist(states_dict, sid_excludes=frozenset(["include", "exclude"])):
    for sid, states in states_dict.items():
        if sid.startswith("__"):
            continue
        if sid in sid_excludes:
            continue
        for sname, args in states.items():
            if sname.startswith("__"):
                continue
            yield sid, states, sname, args

REQUISITES = (
    "require",
    "require_in",
    "watch",
    "watch_in",
    "use",
    "use_in",
    "listen",
    "listen_in",
    "onchanges",
    "onchanges_in",
    "onfail",
    "onfail_in",
)

def rename_state_ids(data, sls, is_extend=False):
    # if the .sls file is salt://my/salt/file.sls
    # then rename all state ids defined in it that start with a dot(.) with
    # "my.salt.file::" + the_state_id_without_the_first_dot.

# update "local" references to the renamed states.

if "extend" in data and not is_extend:
        rename_state_ids(data["extend"], sls, True)

for sid, _, _, args in statelist(data):
        for req, sname, sid in nvlist2(args, REQUISITES):
            if sid.startswith("."):
                req[sname] = _local_to_abs_sid(sid, sls)

for sid in list(data):
        if sid.startswith("."):
            newsid = _local_to_abs_sid(sid, sls)
            if newsid in data:
                raise SaltRenderError(
                    "Can't rename state id({}) into {} because the later "
                    "already exists!".format(sid, newsid)
                )
            # add a '- name: sid' to those states without '- name'.
            for sname, args in data[sid].items():
                if state_name(sname) == STATE_NAME:
                    continue
                for arg in args:
                    if isinstance(arg, dict) and next(iter(arg)) == "name":
                        break
                else:
                    # then no '- name: ...' is defined in the state args
                    # add the sid without the leading dot as the name.
                    args.insert(0, dict(name=sid[1:]))
            data[newsid] = data[sid]
            del data[sid]

REQUIRE = ("require", "watch", "listen", "onchanges", "onfail")
REQUIRE_IN = ("require_in", "watch_in", "listen_in", "onchanges_in", "onfail_in")
EXTENDED_REQUIRE = {}
EXTENDED_REQUIRE_IN = {}

# To avoid cycles among states when each state requires the one before it:
#   explicit require/watch/listen/onchanges/onfail can only contain states before it
#   explicit require_in/watch_in/listen_in/onchanges_in/onfail_in can only contain states after it
def add_implicit_requires(data):
    def T(sid, state):  # pylint: disable=C0103
        return f"{sid}:{state_name(state)}"

states_before = set()
    states_after = set()

for sid in data:
        for state in data[sid]:
            states_after.add(T(sid, state))

prev_state = (None, None)  # (state_name, sid)
    for sid, states, sname, args in statelist(data):
        if sid == "extend":
            for esid, _, _, eargs in statelist(states):
                for _, rstate, rsid in nvlist2(eargs, REQUIRE):
                    EXTENDED_REQUIRE.setdefault(T(esid, rstate), []).append(
                        (None, rstate, rsid)
                    )
                for _, rstate, rsid in nvlist2(eargs, REQUIRE_IN):
                    EXTENDED_REQUIRE_IN.setdefault(T(esid, rstate), []).append(
                        (None, rstate, rsid)
                    )
            continue

tag = T(sid, sname)
        states_after.remove(tag)

reqs = nvlist2(args, REQUIRE)
        if tag in EXTENDED_REQUIRE:
            reqs = chain(reqs, EXTENDED_REQUIRE[tag])
        for _, rstate, rsid in reqs:
            if T(rsid, rstate) in states_after:
                raise SaltRenderError(
                    "State({}) can't require/watch/listen/onchanges/onfail a state({})"
                    " defined after it!".format(tag, T(rsid, rstate))
                )

reqs = nvlist2(args, REQUIRE_IN)
        if tag in EXTENDED_REQUIRE_IN:
            reqs = chain(reqs, EXTENDED_REQUIRE_IN[tag])
        for _, rstate, rsid in reqs:
            if T(rsid, rstate) in states_before:
                raise SaltRenderError(
                    "State({}) can't"
                    " require_in/watch_in/listen_in/onchanges_in/onfail_in a state({})"
                    " defined before it!".format(tag, T(rsid, rstate))
                )

# add a (- state: sid) item, at the beginning of the require of this
        # state if there's a state before this one.
        if prev_state[0] is not None:
            try:
                next(nvlist(args, ["require"]))[2].insert(0, dict([prev_state]))
            except StopIteration:  # i.e., there's no require
                args.append(dict(require=[dict([prev_state])]))

states_before.add(tag)
        prev_state = (state_name(sname), sid)

def add_start_state(data, sls):
    start_sid = __opts__["stateconf_start_state"]
    if start_sid in data:
        raise SaltRenderError(
            "Can't generate start state({})! The same state id already exists!".format(
                start_sid
            )
        )
    if not data:
        return

# the start state is either the first state whose id declaration has
    # no __sls__, or it's the first state whose id declaration has a
    # __sls__ == sls.
    non_sids = ("include", "exclude", "extend")
    for sid, states in data.items():
        if sid in non_sids or sid.startswith("__"):
            continue
        if "__sls__" not in states or states["__sls__"] == sls:
            break
    else:
        raise SaltRenderError("Can't determine the first state in the sls file!")
    reqin = {state_name(next(iter(data[sid].keys()))): sid}
    data[start_sid] = {STATE_FUNC: [{"require_in": [reqin]}]}

def add_goal_state(data):
    goal_sid = __opts__["stateconf_goal_state"]
    if goal_sid in data:
        raise SaltRenderError(
            "Can't generate goal state({})! The same state id already exists!".format(
                goal_sid
            )
        )
    else:
        reqlist = []
        for sid, states, state, _ in statelist(data, ("include", "exclude", "extend")):
            if "__sls__" in states:
                # Then id declaration must have been included from a
                # rendered sls. Currently, this is only possible with
                # pydsl's high state output.
                continue
            reqlist.append({state_name(state): sid})
        data[goal_sid] = {STATE_FUNC: [dict(require=reqlist)]}

def state_name(sname):
    """
    Return the name of the state regardless if sname is
    just the state name or a state.func name.
    """
    return sname.split(".", 1)[0]

# Quick and dirty way to get attribute access for dictionary keys.
# So, we can do: ${apache.port} instead of ${apache['port']} when possible.
class Bunch(dict):
    def __getattr__(self, name):
        return self[name]

# With sls:
#
#   state_id:
#     stateconf.set:
#       - name1: value1
#
# STATE_CONF is:
#    { state_id => {name1: value1} }
#
STATE_CONF = {}  # stateconf.set
STATE_CONF_EXT = {}  # stateconf.set under extend: ...

def extract_state_confs(data, is_extend=False):
    for state_id, state_dict in data.items():
        if state_id == "extend" and not is_extend:
            extract_state_confs(state_dict, True)
            continue

if STATE_NAME in state_dict:
            key = STATE_NAME
        elif STATE_FUNC in state_dict:
            key = STATE_FUNC
        else:
            continue

to_dict = STATE_CONF_EXT if is_extend else STATE_CONF
        conf = to_dict.setdefault(state_id, Bunch())
        for sdk in state_dict[key]:
            if not isinstance(sdk, dict):
                continue
            key, val = next(iter(sdk.items()))
            conf[key] = val

if not is_extend and state_id in STATE_CONF_EXT:
            extend = STATE_CONF_EXT[state_id]
            for requisite in "require", "watch", "listen", "onchanges", "onfail":
                if requisite in extend:
                    extend[requisite] += to_dict[state_id].get(requisite, [])
            to_dict[state_id].update(STATE_CONF_EXT[state_id])

MR SKB