Source code for backend.vm_manage.spawn

# coding: utf-8

import json
import os
import re
import time

from IPy import IP

from ..ans_utils import run_ansible_playbook_cli
from backend.helpers import get_redis_connection
from backend.vm_manage import PUBSUB_MB, EventTopics
from backend.vm_manage.executor import Executor
from ..exceptions import CoprSpawnFailError
from ..helpers import get_redis_logger
from ..vm_manage import terminate

[docs]def get_ip_from_log(ansible_output): """ Parse IP address from ansible log """ match = re.search(r'IP=([^\{\}"\n\\]+)', ansible_output, re.MULTILINE) if not match: raise CoprSpawnFailError("No ip in the result, trying again") return match.group(1)
[docs]def get_vm_name_from_log(ansible_output): """ Parse vm_name from ansible log """ match = re.search(r'vm_name=([^\{\}"\n\\]+)', ansible_output, re.MULTILINE) if not match: raise CoprSpawnFailError("No vm_name in the playbook output") return match.group(1)
[docs]def spawn_instance(spawn_playbook, log): """ Spawn new VM, executing the following steps: - call the spawn playbook to startup/provision a building instance - get an IP and test if the builder responds - repeat this until you get an IP of working builder :type log: logging.Logger :return: dict with ip and name of created VM :raises CoprSpawnFailError: """ log.info("Spawning a builder with pb: {}".format(spawn_playbook)) start = time.time() # Ansible playbook python API does not work here, dunno why. See: # https://groups.google.com/forum/#!topic/ansible-project/DNBD2oHv5k8 spawn_args = "-c ssh {}".format(spawn_playbook) try: result = run_ansible_playbook_cli(spawn_args, comment="spawning instance", log=log) except Exception as err: raise CoprSpawnFailError(str(err.__dict__)) if not result: raise CoprSpawnFailError("No result, trying again") ipaddr = get_ip_from_log(result) vm_name = get_vm_name_from_log(result) try: IP(ipaddr) except ValueError: # if we get here we"re in trouble msg = "Invalid IP: `{}` back from spawn_instance - dumping cache output\n".format(ipaddr) msg += str(result) raise CoprSpawnFailError(msg) log.info("Got VM {} ip: {}. Instance spawn/provision took {} sec" .format(vm_name, ipaddr, time.time() - start)) return {"vm_ip": ipaddr, "vm_name": vm_name}
[docs]def do_spawn_and_publish(opts, spawn_playbook, group): log = get_redis_logger(opts, "spawner.detached", "spawner") try: log.debug("Going to spawn") spawn_result = spawn_instance(spawn_playbook, log) log.debug("Spawn finished") except CoprSpawnFailError as err: log.info("Spawning a builder with pb: {}".format(err.msg)) vm_ip = get_ip_from_log(err.msg) vm_name = get_vm_name_from_log(err.msg) if vm_ip and vm_name: # VM started but failed later during ansible run. try: log.exception("Trying to terminate: {}({}).".format(vm_name, vm_ip)) terminate.terminate_vm(opts, opts.build_groups[int(group)]["terminate_playbook"], group, vm_name, vm_ip) except Exception: # ignore all errors raise log.exception("Error during ansible invocation: {}".format(err.msg)) return except Exception as err: log.exception("[Unexpected] Failed to spawn builder: {}".format(err)) return spawn_result["group"] = group spawn_result["topic"] = EventTopics.VM_SPAWNED try: rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(spawn_result)) except Exception as err: log.exception("Failed to publish msg about new VM: {} with error: {}" .format(spawn_result, err))
[docs]class Spawner(Executor): __name_for_log__ = "spawner" __who_for_log__ = "spawner" def __init__(self, *args, **kwargs): super(Spawner, self).__init__(*args, **kwargs) self.proc_to_group = {} # {proc: Thread -> group: int}
[docs] def after_proc_finished(self, proc): self.proc_to_group.pop(proc)
[docs] def get_proc_num_per_group(self, group): return sum(1 for _, gr in self.proc_to_group.items() if gr == group)
[docs] def start_spawn(self, group): try: spawn_playbook = self.opts.build_groups[group]["spawn_playbook"] except KeyError: msg = "Config missing spawn playbook for group: {}".format(group) raise CoprSpawnFailError(msg) if spawn_playbook is None: msg = "Missing spawn playbook for group: {} for unknown reason".format(group) raise CoprSpawnFailError(msg) if not os.path.exists(spawn_playbook): msg = "Spawn playbook {} is missing".format(spawn_playbook) raise CoprSpawnFailError(msg) proc = self.run_detached(do_spawn_and_publish, args=(self.opts, spawn_playbook, group)) self.proc_to_group[proc] = group