# (c) cavaliba.com - data - data.py import copy import json import app_home.cache as cache import yaml from app_data.models import DataEAV, DataInstance from app_data.permissions import has_schema_write_permission from app_data.related import get_related from app_data.revision import revision_get from app_data.schema import Schema from app_home.configuration import get_configuration from app_home.log import DEBUG, WARNING, log from app_user.models import SireneGroup, SireneUser from django.conf import settings from django.utils import timezone from django.utils.translation import gettext as _ from .fieldtypes.field_boolean import FieldBoolean from .fieldtypes.field_date import FieldDate from .fieldtypes.field_datetime import FieldDatetime from .fieldtypes.field_enumerate import FieldEnumerate from .fieldtypes.field_external import FieldExternal from .fieldtypes.field_file import FieldFile from .fieldtypes.field_float import FieldFloat from .fieldtypes.field_group import FieldGroup from .fieldtypes.field_int import FieldInt from .fieldtypes.field_ipv4 import FieldIPV4 from .fieldtypes.field_password import FieldPassword from .fieldtypes.field_role import FieldRole from .fieldtypes.field_schema import FieldSchema from .fieldtypes.field_string import FieldString from .fieldtypes.field_text import FieldText from .fieldtypes.field_time import FieldTime from .fieldtypes.field_user import FieldUser FIELD_CONSTRUCTOR_TABLE = { "unknown": FieldString, "string": FieldString, "int": FieldInt, "float": FieldFloat, "boolean": FieldBoolean, "ipv4": FieldIPV4, "date": FieldDate, "datetime": FieldDatetime, "time": FieldTime, "text": FieldText, "user": FieldUser, "group": FieldGroup, "role": FieldRole, "schema": FieldSchema, "enumerate": FieldEnumerate, "external": FieldExternal, "file": FieldFile, "password": FieldPassword, } # YAML export class MyYamlDumper(yaml.SafeDumper): def write_line_break(self, data=None): super().write_line_break(data) if len(self.indents) < 2: super().write_line_break() # ------------------------------------------------------------------------- # bigset/count update # ------------------------------------------------------------------------- def count_instance(classname=None): ''' count the number of instances for given classname (schema name)''' return DataInstance.objects.filter(classname=classname).count() # NEXT : cache reply in DataRegistry, add a --force option to recount / update registry def update_bigset(aaa=None): ''' set bigset to true in DB schema if many instances ; trigers ajax in UI ''' bigset_size = int(get_configuration("data","DATA_BIGSET_SIZE")) schemas = Schema.listall() for schema in schemas: count = count_instance(schema.classname) if count > bigset_size: schema.is_bigset = True schema.save() log(DEBUG, aaa=aaa, app="data", view="update_bigset", action="updated", status="OK", data = f"{schema.classname},count={count},bigset={schema.is_bigset} ") #print(f"update_bigset: schema={schema.classname} count={count} bigset={schema.is_bigset}") # ------------------------------------------------------------------------- # Class Helpers # ------------------------------------------------------------------------- # def get_classes(is_enabled=None): # if is_enabled: # return DataClass.objects.order_by("order").filter(is_enabled=is_enabled) # else: # return DataClass.objects.order_by("order").all() # ------------------------------------------------------------------------- # Instance Helpers # ------------------------------------------------------------------------- def get_instances(classname=None, is_enabled=None): ''' => Queryset''' if is_enabled is not None: instances = DataInstance.objects.filter(classname=classname).filter(is_enabled=is_enabled) else: instances = DataInstance.objects.filter(classname=classname).all() return instances # fast query to select one field value # select keyname, fieldname from instance where classname = classname # select keyname, * from instance where classname = classname def get_instances_raw_json(classname=None, is_enabled=True, fieldname=None): reply = {} if is_enabled: instances = DataInstance.objects.filter(classname=classname).filter(is_enabled=is_enabled) else: instances = DataInstance.objects.filter(classname=classname) # extract json for instobj in instances: jsondata = json.loads(instobj.data_json) data = jsondata # filter on fieldname if fieldname: if fieldname in jsondata: data = jsondata[fieldname] reply[instobj.keyname] = data return reply # def get_instance_by_name(keyname=None, classname=None): # if not classname: # return # if not keyname: # return # obj = cache.instance_get(classname, keyname) # if not obj: # obj = DataInstance.objects.filter(classname=classname, keyname=keyname).first() # cache.instance_store(classname, keyname, obj) # return obj # -------------------------------------------------------- # INSTANCE # -------------------------------------------------------- class Instance: def __init__(self, iobj=None, keyname=None, classname=None, expand=False): self.classname = None self.schema = None self.keyname = None self.iobj = None self.is_enabled = True # True / False self.displayname = None self.p_read = None self.p_update = None self.p_delete = None self.json = None self.value = [] self.fields = {} # key = field keyname self.errors = [] self.last_update = None # is_bound => @property # id => @property # NEW EMPTY (or else use from_keyname() classmethod) if not iobj: if classname: self.classname = classname if keyname: self.keyname = keyname # EXISTING else: # check iobj is a DataInstance if type(iobj) is not DataInstance: return # TODO : check iobj is not a new Model self.iobj = iobj self.classname = iobj.classname self.keyname = iobj.keyname self.is_enabled = iobj.is_enabled self.displayname = iobj.displayname self.p_read = iobj.p_read self.p_update = iobj.p_update self.p_delete = iobj.p_delete self.last_update = iobj.last_update try: self.json = json.loads(iobj.data_json) except Exception: self.json = None # create fields, and fill with json if available self.schema = Schema.from_name(self.classname) if self.schema: for fieldname, fieldschema in self.schema.fields.items(): # safety: don't inject a bad external keyname (missing prefix _) if fieldschema["dataformat"] == "external": if not fieldname.startswith('_'): continue constructor = FIELD_CONSTRUCTOR_TABLE.get(fieldschema["dataformat"], FieldString) self.fields[fieldname] = constructor(fieldname, fieldschema, self.json) # expand injected fields if expand: self.expand_injected() # options ... self.set_options() # Done ! @property def id(self): if self.iobj: return self.iobj.id @property def is_bound(self): ''' if object from DB in self.iobj ''' if self.iobj: return True return False # new 3.19 - specific constructors @classmethod def from_keyname(cls, classname=None, keyname=None, expand=False): ''' existing from DB or None''' if not classname: return if not keyname: return # Recompute (no cache) if expand if not expand: cachekey = f"{classname}::{keyname}" instance = cache.cache2_instance.get(cachekey) # cache hit if instance: return instance # cache miss instobj = DataInstance.objects.filter(classname=classname, keyname=keyname).first() if instobj: instance = cls(iobj=instobj, expand=expand) instance.cache_set() return instance @classmethod def from_id(cls, id=None, expand=False): ''' existing from id or None - v3.21.0''' if not id: return # Recompute (no cache) if expand if not expand: cachekey = f"{id}" instance = cache.cache2_instance.get(cachekey) # cache hit if instance: return instance # cache miss try: instobj = DataInstance.objects.get(pk=id) except Exception: return instance = cls(iobj=instobj, expand=expand) if instance: instance.cache_set() return instance @classmethod def from_iobj(cls, iobj=None, expand=False): ''' existing from iobj or None''' if not isinstance(iobj, DataInstance): return # Recompute (no cache) if expand if not expand: try: cachekey = f"{iobj.classname}::{iobj.keyname}" except Exception: return instance = cache.cache2_instance.get(cachekey) # cache hit if instance: return instance # cache miss instance = cls(iobj=iobj, expand=expand) if instance: instance.cache_set() return instance # NEXT: iterator @classmethod def iterate_classname(cls, classname=None, first=None, last=None, enabled=None, expand=False): ''' returns a list[] of Instance()''' if not classname: return [] instances = [] if type(first) is int and type(last) is int: if enabled == "yes": instances = DataInstance.objects.filter(classname=classname, is_enabled=True)[first:last] elif enabled == "no": instances = DataInstance.objects.filter(classname=classname, is_enabled=False)[first:last] else: instances = DataInstance.objects.filter(classname=classname)[first:last] else: if enabled == "yes": instances = DataInstance.objects.filter(classname=classname, is_enabled=True) elif enabled == "no": instances = DataInstance.objects.filter(classname=classname, is_enabled=False) else: instances = DataInstance.objects.filter(classname=classname) reply = [] for instobj in instances: instance = cls.from_iobj(iobj=instobj, expand=expand) if instance: reply.append(instance) return reply def __str__(self): return f"{self.classname}:{self.keyname}" def print(self): print(f"{self.classname}:{self.keyname}") print(f" id: {self.id}") print(f" is_enabled: {self.is_enabled}") print(f" displayname: {self.displayname}") print(f" is_bound: {self.is_bound}") print(f" is_valid: {self.is_valid()}") print(f" p_read: {self.p_read}") print(f" p_update: {self.p_update}") print(f" p_delete: {self.p_delete}") print( " fields:") #pprint(self.schema) for k, v in self.fields.items(): v.print() #pprint(v.fieldschema) print() def to_yaml(self): # NEXT : Rewrite ; aggregate YAML from each field (for various format) data = self.get_dict_for_export() data.pop('id', None) datastr = yaml.dump([data], allow_unicode=True, Dumper=MyYamlDumper, #default_style='|', sort_keys=False) return datastr def cache_get(self): # cache by id if self.id: cachekey = f"{self.id}" return cache.cache2_instance.get(cachekey) # cache by name if self.classname and self.keyname: cachekey = f"{self.classname}::{self.keyname}" return cache.cache2_instance.get(cachekey) def cache_set(self): # cache by name if self.classname and self.keyname: cachekey = f"{self.classname}::{self.keyname}" cache.cache2_instance.set(cachekey, self) # cache by id if self.id: cachekey = f"{self.id}" cache.cache2_instance.set(cachekey, self) def cache_purge(self): # by name if self.classname and self.keyname: cachekey = f"{self.classname}::{self.keyname}" cache.cache2_instance.delete(cachekey) # by id if self.id: cachekey = f"{self.id}" cache.cache2_instance.delete(cachekey) # flush specific cache : enumerate cache when an _enumerate instance is modified if self.classname == "_enumerate" and self.keyname: cache.cache2_enumerate.delete(self.keyname) def related(self): return get_related(classname=self.classname, keyname=self.keyname) def ordered_fields(self): def sort_key(item): return self.fields[item].order return sorted(self.fields, key=sort_key) def is_field_true(self, fieldname=None): if not fieldname: return False if fieldname not in self.fields: return False return self.fields[fieldname].is_field_true() def get_attribute_first(self, fieldname): r1 = self.get_attribute(fieldname) try: return r1[0] except Exception: return "" def get_attribute(self, fieldname): ''' Returns a List [] of attribute value(s) from instance. Convert to obj if FieldSchema or SireneGroup or User ... ''' if fieldname == "displayname": return [self.displayname] if fieldname == "keyname": return [self.keyname] if fieldname == "is_enabled": return [self.is_enabled] if fieldname not in self.fields: return return self.fields[fieldname].get_attribute() def set_field_value_single(self, fieldname=None, value=None): if not fieldname: return False if fieldname not in self.fields: return False self.fields[fieldname].value = [value] return True # instance PERMISSIONS def has_valid_aaa(self, aaa=None): if not self.is_bound: return False try: if type(aaa["perms"]) is list: return True except Exception: pass return False def has_read_permission(self, aaa=None): if not self.has_valid_aaa(aaa=aaa): return False # global admin : all schema, all instances, all operations if "p_data_admin" in aaa["perms"]: return True # schema admin : aaa has p_admin on this schema ? (allow direct & stop inheritance) schema = Schema.from_name(self.classname) if schema.has_admin_permission(aaa=aaa): return True # instance read if self.p_read: if self.p_read in aaa["perms"]: return True else: return False # default schema read > all data read return schema.has_read_permission(aaa=aaa) def has_update_permission(self, aaa=None): if not self.has_valid_aaa(aaa=aaa): return False if "p_data_admin" in aaa["perms"]: return True # schema admin (no override) schema = Schema.from_name(self.classname) if schema.has_admin_permission(aaa=aaa): return True # if defined, instance permission rules if self.p_update: if self.p_update in aaa["perms"]: return True else: return False # default to class permission return schema.has_update_permission(aaa=aaa) def has_delete_permission(self, aaa=None): if not self.has_valid_aaa(aaa=aaa): return False if "p_data_admin" in aaa["perms"]: return True # schema admin (no override) schema = Schema.from_name(self.classname) if schema.has_admin_permission(aaa=aaa): return True # if defined, instance permission rules if self.p_delete: if self.p_delete in aaa["perms"]: return True else: return False # default to class permission return schema.has_delete_permission(aaa=aaa) def get_recursive_content(self, fieldname=None, fieldmember=None, fieldrecurse=None, done=None): ''' get list of Instance() from field "fieldname" , including - self instance - fieldmember *.fieldname - recurse on fieldsubgroup : fieldname, member, sub-subgroups ... ''' # Example # ------- # SiteGroup = { # sirene_notify =[ SireneGroup1, 2, ...] # members = [site1, site2] # subgroups = [sitegroup1, sitegroup2 ...] #} # # for sitegroup_obj in message.notify_sitegroup.all(): # instance = Instance(iobj=sitegroup_obj) # xlist = instance.get_recursive_content( # fieldname="notify_group", # fieldmember="members", # fieldrecurse="subgroups", # done=[]) # # done[] contains list of iobj like self if done is None: done = [] reply = [] # recursive: already done ? if self in done: return [] done.append(self) # get field values from self xlist1 = self.get_attribute(fieldname) for z in xlist1: if z: if z not in done: reply.append(z) #reply += xlist1 # get all members from self xlist2 = [] if fieldmember: # if fieldmember in self.schema: if fieldmember in self.fields: xlist2 = self.fields[fieldmember].get_attribute() # get field values from members for item in xlist2: # try to get an instance struct # NOTA: won't work on SireneGroup objects ; use SireneGroups for that purpose if type(item) is DataInstance: instance = Instance(iobj=item) if instance: xlist3 = instance.get_attribute(fieldname) for z in xlist3: if z: if z not in done: reply.append(z) # get all subgroup (fieldrecurse field content) xlist4 = [] if fieldrecurse: if fieldrecurse in self.fields: xlist4 = self.fields[fieldrecurse].get_attribute() # recurse for item in xlist4: if type(item) is DataInstance: if item in done: continue instance = Instance(iobj=item) if instance: xlist5 = instance.get_recursive_content( fieldname=fieldname, fieldmember=fieldmember, fieldrecurse=fieldrecurse, done=done ) for z in xlist5: if z: if z not in done: reply.append(z) #reply += xlist5 reply = list(set(reply)) return reply # set options # ----------- def set_options(self): # empty instance if not self.schema: return # keyname_mode if self.schema.keyname_mode == 'auto': if not self.keyname: self.keyname = Schema.create_keyname() # expand / compute injected fields # -------------------------------- def expand_injected(self): # Refresh all injected / external / enumerate / schema # if new empty Instance, skip update if not self.keyname: return # 1 - remove injected fields (from enumerate / schema) # self.fields will be modified so separate loop inventory and processing remove = [] for fieldname, field in self.fields.items(): if field.is_injected: remove.append(fieldname) for v in remove: self.fields.pop(v) # 2 - resolve external to source # self.fields will be modified so separate loop inventory and processing external = [] for fieldname, field in self.fields.items(): if field.dataformat == "external": external.append(fieldname) # processing for fieldname in external: #field = self.fields[fieldname] # recurse here ; will inject resolved field as schema fields self.inject_resolved_external_field(fieldname) # 3 - schema : inject subfields self.inject_schema_subfields() # 4 - enumerate : inject subfields self.inject_enumerate_subfields() return # ------------------------------------------------- # compute with recurse for a single field # inject computed field in self.fields # ------------------------------------------------- def inject_resolved_external_field(self, fieldname): try: field = self.fields[fieldname] except Exception: return # fieldname must be an external (to a schema) if field.dataformat != "external": return parent_fieldname = field.get_parent_fieldname() if not parent_fieldname: return parent_field = self.fields[parent_fieldname] if parent_field.dataformat != "schema": return remote_classname = parent_field.get_classname() if not remote_classname: return # NEXT : support multi-value try: remote_instance_name = parent_field.value[0] except Exception: return # NEXT : add depth to avoid recurse loops # don't expand (perf) ; will recurse on non expanded field remote_instance = Instance.from_keyname(classname=remote_classname, keyname=remote_instance_name) if not remote_instance: return if not remote_instance.iobj: return # get remote fieldname remote_fieldname = field.get_remote_fieldname() if not remote_fieldname: return # 2 options here # either remote_fieldname is a schema , or "_"+remote_field is an external if remote_fieldname in remote_instance.fields: # schema remote_fieldname2 = remote_fieldname remote_field = remote_instance.fields[remote_fieldname] elif "_" + remote_fieldname in remote_instance.fields: # external remote_fieldname2 = "_" + remote_fieldname remote_field = remote_instance.fields[remote_fieldname2] else: remote_field = None if not remote_field: return remote_field_dataformat = remote_field.dataformat # remote field is an external => recurse (again) if remote_field_dataformat == "external": remote_instance.inject_resolved_external_field(remote_fieldname2) # access remote resolved field (without prefix _ ) try: # remote_fieldname is without _ newvalue = remote_instance.fields[remote_fieldname].get_value() newclassname = remote_instance.fields[remote_fieldname].get_classname() # bad structure except Exception: return # remote field is a schema => DONE, it's the source elif remote_field_dataformat == "schema": newvalue = remote_field.get_value() newclassname = remote_field.get_classname() # bad structure else: return # inject external schema field w/ name = external fieldname without prefix _ new_fieldname = field.fieldname[1:] new_json = {new_fieldname: newvalue} # get subfields subfields = field.get_subfields() new_dataformat_ext = newclassname + ' ' + ' '.join(subfields) new_schema = { 'displayname': field.displayname, 'description': new_fieldname, 'is_multi': False, 'dataformat': 'schema', 'dataformat_ext': new_dataformat_ext, 'cardinal_min': 0, 'cardinal_max': 1, 'is_injected': True, 'injected_type': 'external', 'page': field.page, 'order': field.order, } self.fields[new_fieldname] = FieldSchema(new_fieldname, new_schema, new_json) return def inject_schema_subfields(self): # collect all schema fields to process (2 stage : fields will be altered by injection) schema_fieldnames = [] for fieldname, field in self.fields.items(): if field.dataformat == "schema": schema_fieldnames.append(fieldname) # loop over identified schema fields for fieldname in schema_fieldnames: field = self.fields[fieldname] subfieldnames = field.get_subfields() if len(subfieldnames) == 0: continue target_classname = field.get_classname() target_instancename = field.get_first_value() # NEXT : multi : field__xx.0, field_xx.1 ... target_instance = Instance.from_keyname(classname=target_classname, keyname=target_instancename) if not target_instance: continue for fn in subfieldnames: new_fieldname = fieldname + "__" + fn new_displayname = field.displayname + f"({fn})" # try a regular target field: if fn in target_instance.fields: target_field = target_instance.fields[fn] # exclude some dataformat we can't inject if target_field.dataformat in ['schema', 'external']: continue new_dataformat = target_field.dataformat new_schema = { 'displayname': new_displayname, 'description': new_displayname, 'dataformat': new_dataformat, 'dataformat_ext': target_field.dataformat_ext, 'cardinal_min': target_field.cardinal_min, 'cardinal_max': target_field.cardinal_max, 'is_injected': True, 'injected_type':'schema', 'page': field.page, 'order': field.order, } try: new_list = target_field.get_value() new_json = {new_fieldname: new_list} except Exception: continue constructor = FIELD_CONSTRUCTOR_TABLE.get(new_dataformat, FieldString) self.fields[new_fieldname] = constructor(new_fieldname, new_schema, new_json) # try a built-in attribute (display_name ...) else: pass # TODO : inject displayname, keyname def inject_enumerate_subfields(self): enumerate = [] for fieldname, field in self.fields.items(): if field.dataformat == "enumerate": enumerate.append(fieldname) for fieldname in enumerate: field = self.fields[fieldname] subfields = field.get_subfields() for k, value in subfields.items(): new_fieldname = fieldname + "__" + k try: (t, v) = value except Exception: continue new_json = {new_fieldname: [v]} new_schema = { 'displayname': new_fieldname, 'description': new_fieldname, 'dataformat': 'string', 'dataformat_ext': 'safe', 'cardinal_min': 0, 'cardinal_max': 1, 'is_injected': True, 'injected_type': 'enumerate', 'page': field.page, 'order': field.order, } if t == "int": new_schema["dataformat"] = "int" self.fields[new_fieldname] = FieldInt(new_fieldname, new_schema, new_json) elif t == "boolean": new_schema["dataformat"] = "boolean" self.fields[new_fieldname] = FieldBoolean(new_fieldname, new_schema, new_json) elif t == "float": new_schema["dataformat"] = "float" self.fields[new_fieldname] = FieldFloat(new_fieldname, new_schema, new_json) elif t == "date": new_schema["dataformat"] = "date" self.fields[new_fieldname] = FieldDate(new_fieldname, new_schema, new_json) else: self.fields[new_fieldname] = FieldString(new_fieldname, new_schema, new_json) # NEXT : ipv4, datetime, time # -------- def get_dict_for_ui_form(self): # used for edit in form : new or edit instance_ui = {} # SPECIAL fields instance_ui["id"] = self.id instance_ui["keyname"] = self.keyname instance_ui["displayname"] = self.displayname instance_ui["p_read"] = self.p_read instance_ui["p_update"] = self.p_update instance_ui["p_delete"] = self.p_delete instance_ui["is_enabled"] = self.is_enabled # loop over instance.fields instance_ui["PAGES"] = {} for fieldname in self.ordered_fields(): field = self.fields[fieldname] # don't edit injected fields if field.is_injected: continue # don't edit external fields if field.dataformat == "external": continue order = field.order page = field.page if not page: page = self.keyname if page not in instance_ui["PAGES"]: instance_ui["PAGES"][page] = {} if order not in instance_ui["PAGES"][page]: instance_ui["PAGES"][page][order]= [] datapoint = self.fields[fieldname].get_datapoint_ui_edit() instance_ui["PAGES"][page][order].append(datapoint) return instance_ui def get_dict_for_ui_detail(self, skip_external=True, skip_injected=True, skip_enumerate=True): ''' dict suited for UI list template ; NESTED PAGE and FLAT ''' # ui = { # keyname:"xxxx" # displayname:"xxx" # is_enabled:True/False" # p_*3 # First Level / Flat mode # ------------------------ # fieldname: { DATAPOINT } # fieldname: { DATAPOINT } # ... # Nested / Page # ------------- # instance_ui["PAGES"][page][order]= [] # # "PAGES": { # "(page)1": { "(order)100": [ {DATAPOINT}, {}, ... ], # "(order)101": [ {}, {}, ... ], # "(page)2": { }, # ... # } # DATAPOINT = { # datapoint["fieldname"] = self.fieldname # datapoint["displayname"] = self.displayname # datapoint["description"] = self.description # datapoint["dataformat"] = self.dataformat # datapoint["dataformat_ext"] = self.dataformat_ext # datapoint["is_multi"] = self.is_multi() # datapoint["bigset"] = False # datapoint["schema"] = CLASSNAME (for schema field) # datapoint["value"] = '' # } # VALUE : depend on field dataformat (ex. ", ".join(values)") instance_ui = {} # SPECIAL fields P7 instance_ui["id"] = self.id instance_ui["keyname"] = self.keyname instance_ui["displayname"] = self.displayname if not self.displayname or len(self.displayname) == 0: instance_ui["displayname"] = self.keyname instance_ui["p_read"] = self.p_read instance_ui["p_update"] = self.p_update instance_ui["p_delete"] = self.p_delete instance_ui["is_enabled"] = self.is_enabled # loop over instance.fields instance_ui["PAGES"] = {} for fieldname in self.ordered_fields(): field = self.fields[fieldname] if skip_external: if field.dataformat == "external": continue if skip_injected: if field.is_injected: continue if skip_enumerate: if field.injected_type == "enumerate": continue order = field.order page = field.page if not page: page = self.keyname if page not in instance_ui["PAGES"]: instance_ui["PAGES"][page] = {} if order not in instance_ui["PAGES"][page]: instance_ui["PAGES"][page][order]= [] datapoint = field.get_datapoint_ui_detail() # nested structure PAGE/ORDER instance_ui["PAGES"][page][order].append(datapoint) # append also at first level instance_ui[fieldname] = copy.deepcopy(datapoint) return instance_ui def get_dict_for_export(self, refs=None, rev=None): # { # classname: XX # keyname: XX # displayname:"xxx" # is_enabled:True/False" # "attribname": DATAPOINT, # "attribname": ... # _refs: # _user: # schema1: # ... # } # DATAPOINT = value / "value" # DATAPOINT = ["v1", "v2", ...] # refs: list of type keys to inline, e.g. ['_user', '_group', 'server'] # '*' expands to all schema names (does not include _user/_group/_role) if refs and '*' in refs: refs = [r for r in refs if r != '*'] + Schema.listall_names() instance_export = {} # SPECIAL fields instance_export["id"] = self.id instance_export["classname"] = self.classname instance_export["keyname"] = self.keyname instance_export["displayname"] = self.displayname if self.p_read: instance_export["p_read"] = self.p_read if self.p_update: instance_export["p_update"] = self.p_update if self.p_delete: instance_export["p_delete"] = self.p_delete instance_export["is_enabled"] = self.is_enabled # REGULAR fields for fieldname in self.ordered_fields(): field = self.fields[fieldname] datapoint = field.get_datapoint_for_export() cardinal, cardinal_min, cardinal_max = field.get_cardinal3() if not (cardinal_min == 0 and cardinal == 0): instance_export[fieldname] = datapoint # Pass 1: collect sets of referenced keys per type buckets = {} if refs: for fieldname in self.ordered_fields(): field = self.fields[fieldname] if not field.value: continue fmt = field.dataformat if fmt == "user" and "_user" in refs: buckets.setdefault("_user", set()).update(field.value) elif fmt == "group" and "_group" in refs: buckets.setdefault("_group", set()).update(field.value) elif fmt == "role" and "_role" in refs: buckets.setdefault("_role", set()).update(field.value) elif fmt == "schema": classname = field.dataformat_ext if classname and classname in refs: buckets.setdefault(classname, set()).update(field.value) # Pass 2: DB queries to fill refs refs = {} if "_user" in buckets: detail = [] for u in SireneUser.objects.filter(login__in=buckets["_user"]): entry = { "login": u.login, "displayname": u.displayname or "", "firstname": u.firstname or "", "lastname": u.lastname or "", "email": u.email or "", "mobile": u.mobile or "", "external_id": u.external_id or "", } detail.append({k: v for k, v in entry.items() if v}) if detail: refs["_user"] = detail if "_group" in buckets: detail = [] for g in SireneGroup.objects.filter(keyname__in=buckets["_group"], is_role=False): entry = {"keyname": g.keyname, "displayname": g.displayname or "", "description": g.description or ""} detail.append({k: v for k, v in entry.items() if v or k == "keyname"}) if detail: refs["_group"] = detail if "_role" in buckets: detail = [] for r in SireneGroup.objects.filter(keyname__in=buckets["_role"], is_role=True): entry = {"keyname": r.keyname, "displayname": r.displayname or "", "description": r.description or ""} detail.append({k: v for k, v in entry.items() if v or k == "keyname"}) if detail: refs["_role"] = detail for classname, keynames in buckets.items(): if classname.startswith("_"): continue detail = [] for i in DataInstance.objects.filter(classname=classname, keyname__in=keynames).values("keyname", "displayname"): entry = {"keyname": i["keyname"], "classname": classname, "displayname": i["displayname"] or ""} detail.append({k: v for k, v in entry.items() if v or k == "keyname"}) if detail: refs[classname] = detail if refs: instance_export["_refs"] = refs if rev: revisions = revision_get(classname=self.classname, keyname=self.keyname, limit=rev) if revisions: instance_export["_revision"] = [ {"login": r.username or "", "date": r.date.strftime("%Y-%m-%dT%H:%M:%S"), "action": r.action or ""} for r in revisions ] return instance_export def get_csv_columns(self): csv_columns = ["classname","keyname","displayname","is_enabled"] # ,"p_read","p_update","p_delete for fieldname in self.ordered_fields(): field = self.fields[fieldname] # if field.is_multi(): # continue if field.is_injected: continue if fieldname.startswith('_'): continue if field.dataformat == "text": continue csv_columns.append(fieldname) return csv_columns def get_csv_line(self, csv_columns): # /!\ must match get_csv_columns line = [self.classname, self.keyname, self.displayname, self.is_enabled] # self.p_read, self.p_update,self.p_delete for fieldname in self.ordered_fields(): if fieldname not in csv_columns: continue line.append(self.fields[fieldname].get_csv_cell()) return line # new 3.20 : merged new & edit method def merge_request(self, request, aaa=None): # keyname : editable only for new objects (PK in DB) if not self.is_bound: if self.schema.keyname_mode == 'edit': self.keyname = request.POST.get("keyname", "") # displayname (check in is_valid + template auto escaping) self.displayname = request.POST.get("displayname", "") # is_enabled if "is_enabled" in request.POST: self.is_enabled = request.POST["is_enabled"] in settings.TRUE_LIST else: self.is_enabled = False # permissions if type(aaa) is dict: if 'perms' in aaa: if "p_data_security_edit" in aaa['perms']: if "p_read" in request.POST: self.p_read = request.POST["p_read"] if "p_update" in request.POST: self.p_update = request.POST["p_update"] if "p_delete" in request.POST: self.p_delete = request.POST["p_delete"] for fieldname, field in self.fields.items(): # don't merge injected fields if field.is_injected: continue # don't merge external fields if field.dataformat == "external": continue self.fields[fieldname].merge_request(request) # injected fields self.expand_injected() # options self.set_options() def merge_import(self, data, aaa=None): # keyname if new instance only, and not auto computed keyname if not self.is_bound: if self.schema.keyname_mode == 'edit': # keyname may be absent # (new Instance with initial keyname, merge additional data w/o keyname) if 'keyname' in data: self.keyname = data.get("keyname", None) # displayname if 'displayname' in data: self.displayname = data["displayname"] # is_enabled if "is_enabled" in data: self.is_enabled = data["is_enabled"] in settings.TRUE_LIST #else: # self.is_enabled = False # permissions if type(aaa) is dict: if 'perms' in aaa: if "p_data_security_edit" in aaa['perms']: if "p_read" in data: self.p_read = data["p_read"] if "p_update" in data: self.p_update = data["p_update"] if "p_delete" in data: self.p_delete = data["p_delete"] for fieldname, field in self.fields.items(): # don't merge injected fields if field.is_injected: continue # don't merge external fields if field.dataformat == "external": continue # special case : Boolean are not in POST request if False if fieldname in data: fielddata = data[fieldname] self.fields[fieldname].merge_import(fielddata) # injected fields self.expand_injected() # options self.set_options() def is_valid(self): reply = True # keyname if not self.keyname: self.errors.append(_("missing keyname")) return False if len(self.keyname) == 0: self.errors.append(_("keyname empty")) return False # classname (schema) if not self.classname: self.errors.append(_("missing classname")) return False if len(self.classname)==0: self.errors.append(_("empty classname")) return False schema = Schema.from_name(classname=self.classname) if not schema: self.errors.append(_("unknown classname")) return False # NEXT: displayname = safe string for display ? # is_enabled if type(self.is_enabled) is not bool: return False # NEXT - check permissions ? for fieldname, field in self.fields.items(): r = field.is_valid() if not r: reply = False #err = fieldname err = field.displayname self.errors.append(str(err)) # NEXT: use field.errors info return reply # ACTIONs : update Instance() and iobj ; call save to DB # ------------------------------------------------------- def enable(self): if not self.is_bound: return False self.iobj.is_enabled = True self.is_enabled = True return self.save() def disable(self): if not self.is_bound: return False self.is_enabled = False self.iobj.is_enabled = False return self.save() def init(self): ''' init only if not existing ''' # already exists in struct? if self.iobj: self.errors.append(_("init - already created - can't recreate")) return False # instance already in DB ? instance = Instance.from_keyname(keyname=self.keyname, classname=self.classname) if instance: # iobj = get_instance_by_name(keyname=self.keyname, classname=self.classname) # if iobj: self.errors.append(_("init - instance already in DB - can't recreate")) return False # create new instance self.iobj = DataInstance() # V3.19 self.iobj.classname = self.classname return self.update() def create(self): ''' Create or update new Instance in DB from 'instance' ''' # compute/update keyname, ... self.set_options() # create if doesn't exist if not self.is_bound: iobj = DataInstance() iobj.classname = self.classname # new V3.19 iobj.keyname = self.keyname iobj.is_enabled = self.is_enabled iobj.p_read = self.p_read iobj.p_update = self.p_update iobj.p_delete = self.p_delete #iobj.save() self.iobj = iobj return self.update() def update(self): ''' update self.iobj with Instance struct , and save to DB''' if not self.is_bound: return False self.set_options() self.iobj.keyname = self.keyname self.iobj.displayname = self.displayname self.iobj.is_enabled = self.is_enabled self.iobj.p_read = self.p_read self.iobj.p_update = self.p_update self.iobj.p_delete = self.p_delete # V3.19 - populate new classname field self.iobj.classname = self.classname data = {} for fieldname, field in self.fields.items(): # don't store injected fields if field.is_injected: continue field_datalist = self.fields[fieldname].get_value() # don't store empty fields in DB if len(field_datalist) == 0: continue # external field : restore fieldname without _ prefix if field.dataformat == "external": original_fieldname = fieldname[1:] data[original_fieldname] = field_datalist else: data[fieldname] = field_datalist # NEXT : strict mode : self.is_valid() must be True try: self.iobj.data_json = json.dumps(data, indent=2, ensure_ascii=False) except Exception: print("Invalid JSON in update()") return False return self.save() # real write to DB # ----------------- # NEXT - bulk write to DataInstanceSearch for searchable fields def delete(self): # place #1 for real write to DB if not self.is_bound: return False self.cache_purge() try: self.iobj.delete() except Exception as e: print(f"DB delete failed for {self.classname}:{self.keyname} - {e}") return False self.iobj = None self.last_update = timezone.now() self.update_eav() return True def save(self): # place #2 for real write to DB if not self.iobj: print("!! no iobj in save()") return False # V3.19 - populate new classname field self.iobj.classname = self.classname self.last_update = timezone.now() self.iobj.last_update = self.last_update self.cache_purge() try: self.iobj.save() except Exception as e: print(f"save() to DB failed: {e}") return False self.update_eav() return True def update_eav(self): # delete older EAV entries DataEAV.objects.filter( classname=self.classname, keyname=self.keyname ).delete() # self.iobj must not be deleted if not self.iobj: return # save at least one entry with keyname eavobj = DataEAV( iid = self.id, classname = self.classname, keyname = self.keyname, displayname = self.displayname, fieldname = 'keyname', format = 'string', value = self.keyname, last_update = self.last_update ) try: eavobj.save() except Exception: pass # Loop over fields (if any) for fieldname,field in self.fields.items(): # don't store injected fields if field.is_injected: continue # external field also if field.dataformat == "external": continue values = field.get_eav_list() format = field.get_eav_format() for v in values: if len(v) > 1000: # v ="<<< removed - too big >>>" continue if len(v) == 0: continue eavobj = DataEAV( iid = self.id, classname = self.classname, keyname = self.keyname, displayname = self.displayname, fieldname = fieldname, format = format, value = v, last_update = self.last_update ) try: eavobj.save() except Exception: pass # -------------------------------------------------------- # LOADER / IMPORT # Global LOADER : class, schema, instance, static # -------------------------------------------------------- def load_schema(datadict=None, verbose=True, aaa=None): ''' IN: datadict['keyname'] : mandatory classname(!) OUT: None or error string ''' if not datadict: return "load_schema: no data" if not has_schema_write_permission(aaa=aaa): log(WARNING, aaa=aaa, app="data", view="schema", action="load", status="DENY", data=_("Not allowed")) return "load_schema: not allowed" # for a _schema, classname is the provided keyname (classname = _schema) classname = datadict.get("keyname", None) if not classname: return f"load_schema: missing classname {datadict}" action = datadict.get("_action", "create") # delete Schema and Fields if action == 'delete': # TODO: check classname is not builtin (_reserved) schema_obj = Schema.delete(classname) if schema_obj: return else: return f"load_schema: failed to delete {classname}" # enable elif action == "enable": # TODO: check classname is not builtin (_reserved) schema_obj = Schema.enable(classname) if schema_obj: return else: return f"load_schema: failed to enable {classname}" # disable elif action == "disable": # TODO: check classname is not builtin (_reserved) schema_obj = Schema.disable(classname) if schema_obj: return else: return f"load_schema: failed to disable {classname}" # init elif action == "init": if Schema.exists(classname): return schema = Schema() schema.update_from_dict(datadict, verbose=verbose) schema.save() # TODO: check save result return # create (or update) elif action == "create": schema = Schema.from_name(classname) if not schema: schema = Schema() schema.update_from_dict(datadict, verbose=verbose) schema.save() return # update (only) elif action == "update": schema = Schema.from_name(classname) if not schema: return f"load_schema: not found: {classname} " schema.update_from_dict(datadict, verbose=verbose) schema.save() return elif action == "noop": return else: log(WARNING, aaa=aaa, app="data", view="schema", action='unknown', status="FAILED", data=f"Unknown load action {action} for schema {classname}") return f"load_schema: unknown action {action} for {classname}" def load_instance(datadict=None, aaa=None): if not datadict: return "load_instance: no data" if not aaa: log(WARNING, aaa=aaa, app="data", view="instance", action="load", status="DENY", data=_("Not allowed")) return "load_instance: not allowed" classname = datadict.get("classname", None) schema = Schema.from_name(classname) if not schema: return f"load_instance: schema not found: {classname}" action = datadict.get("_action", "create") keyname = datadict.get("keyname", None) if action == "delete": instance = Instance.from_keyname(classname=classname, keyname=keyname, expand=False) if instance: if instance.has_delete_permission(aaa=aaa): instance.delete() return else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not allowed")) else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not found")) return f"load_instance: delete failed for {classname}:{keyname}" elif action == "disable": instance = Instance.from_keyname(classname=classname, keyname=keyname, expand=False) if instance: if instance.has_update_permission(aaa=aaa): instance.disable() return else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not allowed")) else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not found")) return f"load_instance: disable failed for {classname}:{keyname}" elif action == "enable": instance = Instance.from_keyname(classname=classname, keyname=keyname, expand=False) if instance: if instance.has_update_permission(aaa=aaa): instance.enable() return else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not allowed")) else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not found")) return f"load_instance: enable failed for {classname}:{keyname}" # init only == new instance if doesn't already exist elif action == "init": instance = Instance.from_keyname(classname=classname, keyname=keyname, expand=False) if instance: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Already exists")) return instance = Instance(classname=classname) if instance: if schema.has_create_permission(aaa): instance.merge_import(datadict) instance.init() return else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not allowed")) return f"load_instance: init not allowed for {classname}:{keyname}" else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Couldn't create")) return f"load_instance: init failed for {classname}:{keyname}" # create and/or update if exists elif action == "create": instance = Instance.from_keyname(classname=classname, keyname=keyname, expand=False) if not instance: instance = Instance(classname=classname) if instance: if schema.has_create_permission(aaa): instance.merge_import(datadict) instance.create() return else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not allowed")) return f"load_instance: create not allowed for {classname}:{keyname}" else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Couldn't create")) return f"load_instance: create failed for {classname}:{keyname}" # don't create, update only if exists elif action == "update": instance = Instance.from_keyname(classname=classname, keyname=keyname, expand=False) if instance: if instance.has_update_permission(aaa=aaa): instance.merge_import(datadict) instance.update() return else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not allowed")) return f"load_instance: update not allowed for {classname}:{keyname}" else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Not found")) return f"load_instance: update failed for {classname}:{keyname}" elif action == "noop": return # unknown action else: log(WARNING, aaa=aaa, app="data", view="instance", action=action, status="DENY", data=_("Unknown action")) return f"load_instance: unknown action {action} for {classname}:{keyname}"