# (c) cavaliba.com - home - migrator.py # start / bootstrap / update import logging import os import sys import time import uuid import yaml from django.conf import settings from django.utils import timezone from app_data.loader import load_broker from app_data.permissions import bootstrap_permissions, permission_all_keynames from app_data.registry import registry_get_key, registry_set_key from app_home.configuration import sync_configuration logger = logging.getLogger(__name__) def cavaliba_start(): """ init DB or apply migration when cavaliba starts called from management command cavaliba start (from start_dev.sh or docker-entrypoint.sh) Creates Registry Entries - DB_VERSION - LAST_START - INSTANCE_UUID """ logger.info("cavaliba_start() ...") instance_uuid = registry_get_key(key="INSTANCE_UUID") logger.info(f"instance UUID: {instance_uuid}") db_version = registry_get_key(key="DB_VERSION") logger.info(f"db version: {db_version}") logger.info(f"code version: {settings.CAVALIBA_VERSION}") # bootstrap once if db_version is None or db_version == 0: db_version = cavaliba_bootstrap() logger.info(f"bootstrap done to {db_version}") # configuration always sync_configuration() logger.info("configuration synced") # migrate / update always db_version = cavaliba_db_update(db_version) # final check if settings.CAVALIBA_VERSION == db_version: timestamp = timezone.now().isoformat() registry_set_key(key="LAST_START", value=timestamp) logger.info(f"start done at {timestamp}") return else: timestamp = timezone.now().isoformat() registry_set_key(key="LAST_START_FAILED", value=timestamp) logger.critical( f"start FAILED - code version {settings.CAVALIBA_VERSION} != db version {db_version}" ) def cavaliba_bootstrap(): """init an empty database with builtin/init yaml files""" initdir = os.path.join(os.path.dirname(__file__), "..", "builtin", "init") logger.info(f"first start, init/bootstraping new DB : {initdir}") # load permissions first perm_file = os.path.join(initdir, "00_permission.yml") logger.info(f"bootstrap - {perm_file}") with open(perm_file) as f: content = yaml.load(f, Loader=yaml.SafeLoader) bootstrap_permissions(content) # load all other yml files in lexicographic order aaa = {"perms": permission_all_keynames()} other_files = sorted( item.path for item in os.scandir(initdir) if item.is_file(follow_symlinks=False) and item.path.endswith(".yml") and os.path.basename(item.path) != "00_permission.yml" ) for filename in other_files: logger.info(f"bootstrap - {filename}") with open(filename) as f: try: content = yaml.load(f, Loader=yaml.SafeLoader) except Exception as e: logger.error(f"bootstrap - SKIP - YAML error in {filename}: {e}") continue if content: load_broker(datalist=content, aaa=aaa) db_version = "4.2.0" registry_set_key(key="DB_VERSION", value=db_version) new_uuid = str(uuid.uuid4()) registry_set_key(key="INSTANCE_UUID", value=new_uuid) return db_version def cavaliba_db_update(db_version): """apply various migration/update until db_version reaches code version""" if db_version == settings.CAVALIBA_VERSION: logger.info("no DB migration needed") # return db_version # initial empty migration # if db_version < "3.0.0": # logger.info(f"updated from {db_version} to 3.0.0") # db_version = "3.0.0" # registry_set_key(key="DB_VERSION", value=db_version) # # migration to 3.33.0 # if db_version < "3.33.0": # logger.info("update to 3.33.0") # # p_pipeline_run, p_task_view/manage # message = "add new builtin permission" # _nouse = reload_builtin(message, "00_permission.yml", ["*"], "init") # # _pipeline:run_permission field # message = "update _pipeline schema with run_permission field" # _nouse = reload_builtin(message, "020_schema.yml", ["_schema:_pipeline"], "create") # # builtin bulk pipelines # message = "add new builtin bulk pipelines (040_pipeline.yml)" # _nouse = reload_builtin(message, "040_pipeline.yml", ["*"], "create") # db_version = "3.33.0" # registry_set_key(key="DB_VERSION", value=db_version) # # migration to 3.33.1 # if db_version < "3.33.1": # logger.info("update to 3.33.1") # message = "add new builtin bulk pipelines (040_pipeline.yml)" # _nouse = reload_builtin(message, "040_pipeline.yml", ["*"], "create") # db_version = "3.33.1" # registry_set_key(key="DB_VERSION", value=db_version) # migration to 4.0.0 - no longer supported, refuse to start if db_version < "4.0.0": logger.critical("=" * 70) logger.critical(f"STARTUP REFUSED - DB version is {db_version}") logger.critical("Direct upgrade from v3 is no longer supported by this build.") logger.critical("Upgrade to a 4.0.X or 4.1.X release first, then to this version.") logger.critical("=" * 70) time.sleep(10) sys.exit(1) # migration to 4.2.0 if db_version < "4.2.0": from app_user.models import SirenePermission logger.info("update to 4.2.0") # p_keyname_rename message = "add p_keyname_rename builtin permission" _nouse = reload_builtin( message, "00_permission.yml", ["_permission:p_keyname_rename"], "init" ) # code_editor sidebar entry message = "add Code Editor builtin sidebar entry" _nouse = reload_builtin(message, "090_dashboard.yml", ["_home:code_editor"], "init") # revision log sidebar entry message = "add Revision Log builtin sidebar entry" _nouse = reload_builtin(message, "090_dashboard.yml", ["_home:revision"], "init") # new ipam_ip schema message = "add ipam_ip builtin schema" _nouse = reload_builtin(message, "220-ipam.yml", ["_schema:ipam_ip"], "init") # new user default dataview message = "add user_default builtin dataview" _nouse = reload_builtin(message, "010_user.yml", ["_dataview:user_default"], "init") # remove permissions marked "_action: delete" in 00_permission.yml # direct ORM delete: permission_delete() refuses builtin permissions SirenePermission.objects.filter(keyname="p_schema_create").delete() SirenePermission.objects.filter(keyname="p_schema_update").delete() SirenePermission.objects.filter(keyname="p_schema_delete").delete() SirenePermission.objects.filter(keyname="p_schemafield_read").delete() SirenePermission.objects.filter(keyname="p_schemafield_create").delete() SirenePermission.objects.filter(keyname="p_schemafield_update").delete() SirenePermission.objects.filter(keyname="p_schemafield_delete").delete() logger.info(" OK - removed obsolete schema/schemafield permissions") db_version = "4.2.0" registry_set_key(key="DB_VERSION", value=db_version) # HERE NEW MIGRATIONS FOR NEXT VERSIONS # always end to code version db_version = settings.CAVALIBA_VERSION registry_set_key(key="DB_VERSION", value=db_version) db_version_date = settings.CAVALIBA_VERSION_DATE registry_set_key(key="DB_VERSION_DATE", value=db_version_date) return db_version # ---- def reload_builtin(message="", filename=None, objects=None, force_action="create"): """ reload builtin/init/040_pipeline.yml for new builtin bulk pipelines objects: [ "schema:keyname", "schema:keyname", ... ] or ['*'] """ initdir = os.path.join(os.path.dirname(__file__), "..", "builtin", "init") pipeline_file = os.path.join(initdir, filename) aaa = {"perms": permission_all_keynames()} try: with open(pipeline_file) as f: content_yml = yaml.load(f, Loader=yaml.SafeLoader) except Exception as e: logger.error(f" KO - {message} : {e}") return False if not content_yml: logger.error(f" KO - {message} : no content") return False if not objects: logger.error(f" KO - {message} : no object provided") return False # filter objects to load if objects == ["*"]: datalist = content_yml else: # objects is a list of "classname:keyname" strings wanted = set() for entry in objects: parts = entry.split(":", 1) if len(parts) == 2: wanted.add((parts[0], parts[1])) datalist = [ item for item in content_yml if (item.get("classname"), item.get("keyname")) in wanted ] if not datalist: logger.error(f" KO - {message} : no matching objects to {objects}") return False try: load_broker(datalist=datalist, aaa=aaa, force_action=force_action) except Exception as e: logger.error(f" KO - {message} : {e}") return False logger.info(f" OK - {message}")