# (c) cavaliba.com - home - cavaliba.py # start / bootstrap / update import logging import os import uuid import yaml from app_data.loader import load_broker from app_data.registry import registry_get_key, registry_set_key from app_home.configuration import sync_configuration from app_user.permission import bootstrap_permissions, permission_all_keynames from django.conf import settings from django.utils import timezone logger = logging.getLogger(__name__) def cavaliba_start(): """ init DB or apply migration when cavaliba starts called from management command cavaliba start (from start_dev.sh or docker-entrypoint.sh) Creates Registry Entries - DB_VERSION - LAST_START - INSTANCE_UUID """ logger.info("starting...") instance_uuid = registry_get_key(key="INSTANCE_UUID") logger.info(f"instance UUID: {instance_uuid}") db_version = registry_get_key(key="DB_VERSION") logger.info(f"db version: {db_version}") logger.info(f"code version: {settings.CAVALIBA_VERSION}") # bootstrap once if db_version is None or db_version == 0: db_version = cavaliba_bootstrap() logger.info(f"bootstrap done to {db_version}") # configuration always sync_configuration() logger.info("configuration synced") # migrate / update always db_version = cavaliba_db_update(db_version) # final check if settings.CAVALIBA_VERSION == db_version: timestamp = timezone.now().isoformat() registry_set_key(key="LAST_START", value=timestamp) logger.info(f"start done at {timestamp}") return else: timestamp = timezone.now().isoformat() registry_set_key(key="LAST_START_FAILED", value=timestamp) logger.critical(f"start FAILED - code version {settings.CAVALIBA_VERSION} != db version {db_version}") def cavaliba_bootstrap(): """init an empty database with builtin/init yaml files""" initdir = os.path.join(os.path.dirname(__file__), "..", "builtin", "init") logger.info(f"first start, init/bootstraping new DB : {initdir}") # load permissions first perm_file = os.path.join(initdir, "00_permission.yml") logger.info(f"bootstrap - {perm_file}") with open(perm_file) as f: content = yaml.load(f, Loader=yaml.SafeLoader) bootstrap_permissions(content) # load all other yml files in lexicographic order aaa = {"perms": permission_all_keynames()} other_files = sorted( item.path for item in os.scandir(initdir) if item.is_file(follow_symlinks=False) and item.path.endswith('.yml') and os.path.basename(item.path) != "00_permission.yml" ) for filename in other_files: logger.info(f"bootstrap - {filename}") with open(filename) as f: try: content = yaml.load(f, Loader=yaml.SafeLoader) except Exception as e: logger.error(f"bootstrap - SKIP - YAML error in {filename}: {e}") continue if content: load_broker(datalist=content, aaa=aaa) registry_set_key(key="DB_VERSION", value="1.0") new_uuid = str(uuid.uuid4()) registry_set_key(key="INSTANCE_UUID", value=new_uuid) db_version = "0" return db_version def cavaliba_db_update(db_version): """ apply various migration/update until db_version reaches code version""" if db_version == settings.CAVALIBA_VERSION: logger.info("no DB migration needed") #return db_version # initial empty migration if db_version < "3.0.0": logger.info(f"updated from {db_version} to 3.0.0") db_version = "3.0.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 3.33.0 if db_version < "3.33.0": logger.info("update to 3.33.0") # p_pipeline_run, p_task_view/manage message = "add new builtin permission" result = reload_builtin(message, '00_permission.yml', ['*'], 'init') # _pipeline:run_permission field message = "update _pipeline schema with run_permission field" result = reload_builtin(message, '020_schema.yml', ['_schema:_pipeline'], 'create') # builtin bulk pipelines message = "add new builtin bulk pipelines (040_pipeline.yml)" result = reload_builtin(message, '040_pipeline.yml', ['*'], 'create') db_version = "3.33.0" registry_set_key(key="DB_VERSION", value=db_version) # migration to 3.33.1 if db_version < "3.33.1": logger.info("update to 3.33.1") message = "add new builtin bulk pipelines (040_pipeline.yml)" result = reload_builtin(message, '040_pipeline.yml', ['*'], 'create') db_version = "3.33.1" registry_set_key(key="DB_VERSION", value=db_version) # always end to code version db_version = settings.CAVALIBA_VERSION registry_set_key(key="DB_VERSION", value=db_version) return db_version # ---- def reload_builtin(message='', filename=None, objects=[], force_action='create'): ''' reload builtin/init/040_pipeline.yml for new builtin bulk pipelines objects: [ "schema:keyname", "schema:keyname", ... ] or ['*'] ''' initdir = os.path.join(os.path.dirname(__file__), "..", "builtin", "init") pipeline_file = os.path.join(initdir, filename) aaa = {"perms": permission_all_keynames()} try: with open(pipeline_file) as f: content_yml = yaml.load(f, Loader=yaml.SafeLoader) except Exception as e: logger.error(f" KO - {message} : {e}") return False if not content_yml: logger.error(f" KO - {message} : no content") return False # filter objects to load if objects == ['*']: datalist = content_yml else: # objects is a list of "classname:keyname" strings wanted = set() for entry in objects: parts = entry.split(':', 1) if len(parts) == 2: wanted.add((parts[0], parts[1])) datalist = [ item for item in content_yml if (item.get('classname'), item.get('keyname')) in wanted ] if not datalist: logger.error(f" KO - {message} : no matching objects to {objects}") return False try: load_broker(datalist=datalist, aaa=aaa, force_action=force_action) except Exception as e: logger.error(f" KO - {message} : {e}") return False logger.info(f" OK - {message}") # # -------------------------------------------- # # Patch / Migration # # -------------------------------------------- # def apply_data_patch(verbose=True): # aaa = {} # aaa["perms"] = permission_all_keynames() # # V3.20.0 - create Schema _options {} # try: # schemas = Schema.listall() # for schema in schemas: # schema.save() # if verbose: # print("Patch 3.20 - UPDATE schema OK: ", schema.classname) # except Exception as e: # print("Patch 3.20 - UPDATE schema FAILED : ", e) # # V3.21.0 - update APIKEy schema, add role # try: # datalist = yaml.safe_load(''' # - classname: _schema # keyname: _apikey # roles: # order: 80 # displayname: Roles # dataformat: role # cardinal_max: 0 # ''') # #aaa = {'perms':['p_data_admin']} # _ = load_broker(datalist=datalist, aaa=aaa) # if verbose: # print("Patch 3.21 - UPDATE schema API Key with role - OK") # except Exception as e: # print("Patch 3.21 - UPDATE schema API Key with role - FAILED:", e) # # v3.26 - add sms_content field in Sirene schema(s) # try: # datalist = yaml.safe_load(''' # - classname: _schema # keyname: sirene_template # sms_content: # displayname: SMS Content # page: SMS Content # order: 402 # dataformat: text # - classname: _schema # keyname: sirene_message # sms_content: # displayname: SMS Content # page: SMS Content # order: 402 # dataformat: text # ''') # _ = load_broker(datalist=datalist, aaa=aaa) # if verbose: # print("Patch 3.26 - UPDATE schema sirene - sms_content - OK") # except Exception as e: # print("Patch 3.21 - UPDATE schema sirene - sms_content - FAILED:", e)