diff --git a/src/cfengine_cli/commands.py b/src/cfengine_cli/commands.py index 9e1149c..a9c6795 100644 --- a/src/cfengine_cli/commands.py +++ b/src/cfengine_cli/commands.py @@ -15,9 +15,11 @@ format_policy_fin_fout, ) from cfengine_cli.utils import UserError -from cfengine_cli.up import validate_config +from cfengine_cli.up import validate_config, up_do, resolve_templates from cfbs.commands import build_command from cf_remote.commands import deploy as deploy_command +from cf_remote.paths import cf_remote_dir +from pydantic import ValidationError def _require_cfagent(): @@ -178,12 +180,47 @@ def up(args) -> int: with open(args.config, "r") as f: content = yaml.safe_load(f) except yaml.YAMLError: - raise UserError("'%s' is not a valid yaml config" % args.config) + raise UserError("'%s' is not valid yaml" % args.config) except FileNotFoundError: raise UserError("'%s' doesn't exist" % args.config) - validate_config(content) + g_new = resolve_templates(content) + try: + new_config = validate_config(g_new) + except ValidationError as v: + msgs = [] + for err in v.errors(): + msgs.append( + f"{err['msg']}. Input '{err['input']}' at location '{err['loc']}'" + ) + raise UserError("\n".join(msgs)) + if args.validate: return 0 - print("Starting VMs...") + + g_old = {} + try: + if not args.reset: + with open(os.path.join(cf_remote_dir(), "old_groups.yaml"), "r") as f: + g_old = yaml.safe_load(f) + else: + os.remove(os.path.join(cf_remote_dir(), "old_groups.yaml")) + except: + pass + if g_old != {}: + try: + validate_config(g_old) + except ValidationError as v: + msgs = [] + for err in v.errors(): + msgs.append( + f"{err['msg']}. Input '{err['input']}' at location '{err['loc']}'" + ) + raise UserError("\n".join(msgs)) + + is_ok = up_do(g_old, g_new, new_config) + + if is_ok: + with open(os.path.join(cf_remote_dir(), "old_groups.yaml"), "w") as f: + yaml.dump(g_new, f, default_flow_style=False, sort_keys=False) return 0 diff --git a/src/cfengine_cli/main.py b/src/cfengine_cli/main.py index 2132df7..995f04a 100644 --- a/src/cfengine_cli/main.py +++ b/src/cfengine_cli/main.py @@ -126,6 +126,9 @@ def _get_arg_parser(): up_parser.add_argument( "--validate", action="store_true", help="Validate the given config" ) + up_parser.add_argument( + "--reset", action="store_true", help="Create a fresh new environment" + ) return ap diff --git a/src/cfengine_cli/up.py b/src/cfengine_cli/up.py index 2e404a6..42e289f 100644 --- a/src/cfengine_cli/up.py +++ b/src/cfengine_cli/up.py @@ -1,11 +1,19 @@ -from pydantic import BaseModel, model_validator, ValidationError, Field +from pydantic import BaseModel, model_validator, Field from typing import Union, Literal, Optional, List, Annotated -from functools import reduce -from cf_remote import log import cfengine_cli.validate as validate from cfengine_cli.utils import UserError +from cf_remote import log +from cf_remote import commands +from cf_remote import utils +from cf_remote import paths + +import textwrap +import yaml + +from collections import defaultdict + # Forces pydantic to throw validation error if config contains unknown keys class NoExtra(BaseModel, extra="forbid"): @@ -18,7 +26,8 @@ class Config(NoExtra): class AWSConfig(Config): image: str - size: Literal["micro", "xlarge"] = "micro" + size: Optional[Literal["micro", "xlarge"]] = None + region: Optional[str] = None @model_validator(mode="after") def check_aws_config(self): @@ -49,7 +58,8 @@ class GCPConfig(Config): image: str # There is no list of available GCP platforms to validate against yet network: Optional[str] = None public_ip: bool = True - size: str = "n1-standard-1" + size: Optional[str] = None + region: Optional[str] = None class AWSProvider(Config): @@ -107,6 +117,9 @@ class CFEngineConfig(Config): client_package: Optional[str] = None package: Optional[str] = None demo: bool = False + insecure: bool = False + call_collect: bool = False + trust_keys: Optional[List[str]] = None @model_validator(mode="after") def check_cfengine_config(self): @@ -142,36 +155,6 @@ def check_group_config(self): return self -def rgetattr(obj, attr, *args): - def _getattr(obj, attr): - return getattr(obj, attr, *args) - - return reduce(_getattr, [obj] + attr.split(".")) - - -class Group: - """ - All group-specific data: - - Vagrantfile - Config that declares it: - - provider, count, cfengine version, role, ... - """ - - def __init__(self, config: GroupConfig): - self.config = config - self.hosts = [] - - -class Host: - """ - All host-specific data: - - user, ip, ssh config, OS, uuid, ... - """ - - def __init__(self): - pass - - def _resolve_templates(parent, templates): if not parent: return @@ -186,7 +169,7 @@ def _resolve_templates(parent, templates): _resolve_templates(value, templates) -def validate_config(content): +def resolve_templates(content): if not content: raise UserError("Empty spawn config") @@ -194,28 +177,215 @@ def validate_config(content): raise UserError("Missing 'groups' key in spawn config") groups = content["groups"] + if groups is None: + return {} + templates = content.get("templates") if templates: _resolve_templates(groups, templates) - if not isinstance(groups, list): - groups = [groups] + return groups + + +def validate_config(groups): state = {} - try: - for g in groups: - if len(g) != 1: - raise UserError( - f"Too many keys in group definition: {', '.join(list(g.keys()))}" + for k, v in groups.items(): + state[k] = GroupConfig(**v) + + return state + + +def generate_diff(old_state, new_state): + spawn = [] + destroy = [] + install = [] + uninstall = [] + + to_print = defaultdict(list) + + for key in old_state.keys() | new_state.keys(): + if key in old_state.keys() & new_state.keys(): + if old_state[key]["source"] != new_state[key]["source"]: + destroy.append(key) + spawn.append(key) + + old_text = textwrap.indent( + yaml.dump({"source": old_state[key]["source"]}), "- " ) + new_text = textwrap.indent( + yaml.dump({"source": new_state[key]["source"]}), "+ " + ) + to_print[key].append(old_text + new_text) + + if old_state[key]["role"] != new_state[key]["role"]: + if key not in destroy: + uninstall.append(key) + install.append(key) - for k, v in g.items(): - state[k] = Group(GroupConfig(**v)) + old_text = textwrap.indent( + yaml.dump({"role": old_state[key]["role"]}), "- " + ) + new_text = textwrap.indent( + yaml.dump({"role": new_state[key]["role"]}), "+ " + ) + to_print[key].append(old_text + new_text) + + if old_state[key].get("cfengine", None) != new_state[key].get( + "cfengine", None + ): + if key not in destroy: + uninstall.append(key) + install.append(key) + + old_text = ( + textwrap.indent( + yaml.dump({"cfengine": old_state[key]["cfengine"]}), "- " + ) + if "cfengine" in old_state[key] + else "" + ) + new_text = ( + textwrap.indent( + yaml.dump({"cfengine": new_state[key]["cfengine"]}), "+ " + ) + if "cfengine" in old_state + else "" + ) + to_print[key].append(old_text + new_text) + + elif key not in old_state: + spawn.append(key) + if "cfengine" in new_state[key]: + install.append(key) + + new_text = textwrap.indent(yaml.dump(new_state[key]), "+ ") + to_print[key].append(new_text) + elif key not in new_state: + destroy.append(key) + + old_text = textwrap.indent(yaml.dump(old_state[key]), "- ") + to_print[key].append(old_text) + + for k, v in to_print.items(): + print(f"{k}:") + for w in v: + print(w) + + return spawn, destroy, install, uninstall + + +def spawn_from_config(group_name, config): + match config.source.mode: + case "spawn": + args = { + "group_name": group_name, + "count": config.source.count, + "role": config.role, + } + + match config.source.spawn.provider: + case "vagrant": + args |= { + "provider": commands.Providers.VAGRANT, + "size": config.source.spawn.vagrant.memory, + "platform": config.source.spawn.vagrant.box, + "vagrant_cpus": config.source.spawn.vagrant.cpus, + "vagrant_sync_folder": config.source.spawn.vagrant.sync_folder, + "vagrant_provision": config.source.spawn.vagrant.provision, + } + case "aws": + args |= { + "provider": commands.Providers.AWS, + "platform": config.source.spawn.aws.image, + "size": config.source.spawn.aws.size, + "region": config.source.spawn.aws.region, + } + case "gcp": + args |= { + "provider": commands.Providers.GCP, + "platform": config.source.spawn.gcp.image, + "network": config.source.spawn.gcp.network, + "public_ip": config.source.spawn.gcp.public_ip, + "size": config.source.spawn.gcp.size, + "region": config.source.spawn.gcp.region, + } + + commands.spawn(**args) + case "save": + commands.save(name=group_name, hosts=config.source.hosts, role=config.role) + + +def install_from_config(key, ips, configs): + ip = ips[key] + config = configs[key] + bootstrap = ( + ips[config.cfengine.bootstrap] if config.cfengine.bootstrap in ips else None + ) + + if not config.cfengine: + return - except ValidationError as v: - msgs = [] - for err in v.errors(): - msgs.append( - f"{err['msg']}. Input '{err['input']}' at location '{err['loc']}'" - ) - raise UserError("\n".join(msgs)) + args = { + "bootstrap": bootstrap, + "package": config.cfengine.package, + "hub_package": config.cfengine.hub_package, + "client_package": config.cfengine.client_package, + "version": config.cfengine.version, + "demo": config.cfengine.demo, + "call_collect": config.cfengine.call_collect, + "edition": config.cfengine.edition, + "remote_download": config.cfengine.remote_download, + "insecure": config.cfengine.insecure, + "trust_keys": config.cfengine.trust_keys, + "hubs": None, + "clients": None, + } + if config.role == "hub": + args["hubs"] = ip + else: + args["clients"] = ip + + commands.install(**args) + + +def up_do(old_state, new_state, config): + + spawn, destroy, install, uninstall = generate_diff(old_state, new_state) + + data = utils.read_json(paths.CLOUD_STATE_FPATH) + if data is None: + data = {} + + for key in destroy: + print("Destroying '@%s'" % key) + if f"@{key}" not in data: + raise UserError("Cannot destroy %s: group doesn't exist" % key) + commands.destroy(key) + + for key in spawn: + print("Spawning '@%s'" % key) + spawn_from_config(key, config[key]) + + data = utils.read_json(paths.CLOUD_STATE_FPATH) + if data is None: + data = {} + ips = { + k[1:]: [h_info["public_ips"][0]] + for k, v in data.items() + for h, h_info in v.items() + if h != "meta" + } + + for key in uninstall: + print("Uninstalling CFEngine on '@%s'" % key) + if f"@{key}" not in data: + raise UserError("Cannot uninstall %s: group doesn't exist" % key) + commands.uninstall(ips, purge=True) + + install = sorted(install, key=lambda x: config[x].role)[::-1] + for key in install: + print("Installing CFEngine on '@%s'" % key) + install_from_config(key, ips, config) + + return False diff --git a/tests/up-validate/001_complex_config.yaml b/tests/up-validate/001_complex_config.yaml index 3c0d4ee..c7b060c 100644 --- a/tests/up-validate/001_complex_config.yaml +++ b/tests/up-validate/001_complex_config.yaml @@ -12,60 +12,60 @@ templates: bootstrap: hub2 groups: - - hub1: - role: hub - cfengine: cfengine3.27 - source: - mode: spawn - spawn: - provider: vagrant - vagrant: - box: ubuntu/focal64 - memory: 512 - cpus: 1 - sync_folder: null - provision: null - count: 1 + hub1: + role: hub + cfengine: cfengine3.27 + source: + mode: spawn + spawn: + provider: vagrant + vagrant: + box: ubuntu/focal64 + memory: 512 + cpus: 1 + sync_folder: null + provision: null + count: 1 - - hub2: - role: hub - cfengine: cfengine-master - source: - mode: save - hosts: ["8.8.8.8"] + hub2: + role: hub + cfengine: cfengine-master + source: + mode: save + hosts: ["8.8.8.8"] - - client2: - role: client - cfengine: cfengine3.27 - source: - mode: spawn - spawn: - provider: aws - aws: - image: debian-13 - size: xlarge - count: 3 + client2: + role: client + cfengine: cfengine3.27 + source: + mode: spawn + spawn: + provider: aws + aws: + image: debian-13 + size: xlarge + count: 3 - - client3: - role: client - cfengine: cfengine-master - source: - mode: spawn - spawn: - provider: gcp - gcp: - image: debian-13 - network: null - public_ip: true - size: n1-standard-1 - count: 2 + client3: + role: client + cfengine: cfengine-master + source: + mode: spawn + spawn: + provider: gcp + gcp: + image: debian-13 + network: null + public_ip: true + size: n1-standard-1 + count: 2 - - client4: - role: client - source: - mode: spawn - spawn: - provider: vagrant - vagrant: - box: generic/centos7 - count: 1 + client4: + role: client + source: + mode: spawn + spawn: + provider: vagrant + vagrant: + box: generic/centos7 + count: 1 diff --git a/tests/up-validate/002_version.x.yaml b/tests/up-validate/002_version.x.yaml index 3906581..2b1b11a 100644 --- a/tests/up-validate/002_version.x.yaml +++ b/tests/up-validate/002_version.x.yaml @@ -1,16 +1,16 @@ templates: groups: - - hub1: - role: hub - cfengine: - version: 3.27.foo + hub1: + role: hub + cfengine: + version: 3.27.foo - source: - mode: spawn - spawn: - provider: vagrant - vagrant: - box: ubuntu/focal64 + source: + mode: spawn + spawn: + provider: vagrant + vagrant: + box: ubuntu/focal64 - count: 1 + count: 1 diff --git a/tests/up-validate/003_box.x.yaml b/tests/up-validate/003_box.x.yaml index 6097493..7431faf 100644 --- a/tests/up-validate/003_box.x.yaml +++ b/tests/up-validate/003_box.x.yaml @@ -1,16 +1,16 @@ templates: groups: - - hub1: - role: hub - cfengine: - version: 3.27.0 + hub1: + role: hub + cfengine: + version: 3.27.0 - source: - mode: spawn - spawn: - provider: vagrant - vagrant: - box: foobar + source: + mode: spawn + spawn: + provider: vagrant + vagrant: + box: foobar - count: 1 + count: 1 diff --git a/tests/up-validate/004_count.x.yaml b/tests/up-validate/004_count.x.yaml index bff0ea7..2752761 100644 --- a/tests/up-validate/004_count.x.yaml +++ b/tests/up-validate/004_count.x.yaml @@ -1,16 +1,16 @@ templates: groups: - - hub1: - role: hubzz - cfengine: - version: 3.27.0 + hub1: + role: hubzz + cfengine: + version: 3.27.0 - source: - mode: spawn - spawn: - provider: vagrant - vagrant: - box: ubuntu/focal64 + source: + mode: spawn + spawn: + provider: vagrant + vagrant: + box: ubuntu/focal64 - count: 0 + count: 0