# (c) cavaliba.com - data - data.py """ Instance() class: in-memory representation of a DataInstance (EAV object). Method inventory (grouped by purpose) - read this before grepping the file: Constructors / lookup: __init__, from_keyname, from_id, from_iobj, get_dn, get_all_keyname, count_all, exists, stream_classname, list_classname Properties: id, is_bound Cache: cache_set, cache_purge Display / debug: __str__, print Serialization (read): to_json, to_yaml, to_dict, get_dict_for_export, get_export_refs, get_export_revisions, get_dict_for_ui_form, get_dict_for_ui_detail, get_csv_columns, get_csv_line Field access: ordered_fields, is_field_true, get_attribute, get_attribute_first, set_field_value_single, related Permissions: has_valid_aaa, has_read_permission, has_update_permission, has_delete_permission Injected fields (schema / enumerate / external subfields): set_options, expand_injected, inject_resolved_external_field, inject_schema_subfields, inject_enumerate_subfields Recursive traversal: get_recursive_content Revision helpers: _snapshot, diff_revision, _revision_add Mutation (form/import) - updates in-memory object only, no DB write: merge_request, merge_import, is_valid Actions (mutation + DB write): enable, disable, create, update Low-level DB write: delete, save Module-level helpers (not part of Instance): update_bigset, MyYamlDumper """ import copy import json import yaml from django.conf import settings from django.utils.dateparse import parse_datetime from django.utils.translation import gettext as _ import app_home.cache as cache from app_data.datetimeutil import iso_seconds, now_seconds, trunc_seconds from app_data.eav import EavBatch from app_data.ipam import discover_ipam_ip, get_related_ipam_ip from app_data.models import DataEAV, DataInstance from app_data.related import get_related from app_data.revision import revision_get from app_data.schema import Schema from app_home.configuration import get_configuration from app_home.log import DEBUG, log from .fieldtypes.field_boolean import FieldBoolean from .fieldtypes.field_date import FieldDate from .fieldtypes.field_datetime import FieldDatetime from .fieldtypes.field_enumerate import FieldEnumerate from .fieldtypes.field_external import FieldExternal from .fieldtypes.field_file import FieldFile from .fieldtypes.field_float import FieldFloat from .fieldtypes.field_int import FieldInt from .fieldtypes.field_ipv4 import FieldIPV4 from .fieldtypes.field_password import FieldPassword from .fieldtypes.field_permission import FieldPermission from .fieldtypes.field_schema import FieldSchema from .fieldtypes.field_string import FieldString from .fieldtypes.field_text import FieldText from .fieldtypes.field_time import FieldTime _USER_EXTRA_FIELDS = ("firstname", "lastname", "email", "mobile", "external_id") # Quick fix: fields excluded from revision diffs — they churn on every login/sync # and aren't meaningful edits. Redesign: move last_login (and last_sync/last_update, # already model fields for non-user schemas) off the EAV path entirely. _DIFF_REVISION_EXCLUDED_FIELDS = {"last_login", "last_sync", "last_update"} def field_export_match(fieldname, fields_filter): """True if fieldname passes fields_filter, a grammar for get_dict_for_export()/ get_csv_columns()/DataView column selection: - None, "", or "*" : no filtering, every field passes - "a,b,c*" : whitelist - only exact "a"/"b", or fields starting with "c" - "!a,b,c*" : leading "!" flips the whole list to a blacklist Also used (by name, e.g. "classname"/"is_enabled"/"displayname") to gate builtin instance attributes in get_csv_columns()/get_csv_line() - only `keyname` (row identifier) and `id` (never emitted in CSV) stay outside this filter entirely.""" if not fields_filter: return True spec = fields_filter.strip() if not spec or spec == "*": return True exclude = spec.startswith("!") if exclude: spec = spec[1:] tokens = [t.strip() for t in spec.split(",") if t.strip()] if not tokens: return True matched = any( fieldname.startswith(t[:-1]) if t.endswith("*") else fieldname == t for t in tokens ) return not matched if exclude else matched FIELD_CONSTRUCTOR_TABLE = { "unknown": FieldString, "string": FieldString, "int": FieldInt, "float": FieldFloat, "boolean": FieldBoolean, "ipv4": FieldIPV4, "date": FieldDate, "datetime": FieldDatetime, "time": FieldTime, "text": FieldText, "schema": FieldSchema, "enumerate": FieldEnumerate, "external": FieldExternal, "file": FieldFile, "password": FieldPassword, "permission": FieldPermission, } def keyname_dataformat_is_valid(schema, value): """True if value satisfies schema.keyname_mode's "dataformat [OPT ...]" format (always True if keyname_mode isn't a dataformat mode). Builds a throwaway Field of the requested dataformat/dataformat_ext, forcing a single mandatory value, and delegates to its own is_valid().""" if not schema.keyname_dataformat: return True constructor = FIELD_CONSTRUCTOR_TABLE.get(schema.keyname_dataformat, FieldString) fieldschema = { "dataformat": schema.keyname_dataformat, "dataformat_ext": schema.keyname_dataformat_ext, "cardinal_min": 1, "cardinal_max": 1, } field = constructor("keyname", fieldschema, {"keyname": [value]}) return field.is_valid() # YAML export class MyYamlDumper(yaml.SafeDumper): def write_line_break(self, data=None): super().write_line_break(data) if len(self.indents) < 2: super().write_line_break() # ------------------------------------------------------------------------- # bigset/count update # ------------------------------------------------------------------------- def update_bigset(aaa=None): """ Recompute every schema's is_bigset flag from its instance count: set above DATA_BIGSET_SIZE, cleared again once the count drops back to or below it (a schema that shrank must return to inline dropdowns, not stay on ajax). Only the schemas whose flag actually changes are written - Schema.save() rewrites every field row and flushes the sidebar cache, so a no-op save is far from free. Drives the UI: a bigset schema is edited through ajax lookups instead of a full option list. """ bigset_size = int(get_configuration("data", "DATA_BIGSET_SIZE")) schemas = Schema.listall() for schema in schemas: count = Schema.count_instances(schema.classname) is_bigset = count > bigset_size if is_bigset != schema.is_bigset: schema.is_bigset = is_bigset schema.save() log(DEBUG, aaa=aaa, app="data", view="update_bigset", action="updated", status="OK", data=f"{schema.classname},count={count},bigset={schema.is_bigset} ") # fmt: skip # print(f"update_bigset: schema={schema.classname} count={count} bigset={schema.is_bigset}") # ------------------------------------------------------------------------- # Instance Helpers # ------------------------------------------------------------------------- # -------------------------------------------------------- # INSTANCE base class # -------------------------------------------------------- class Instance: CLASSNAME = None _SUBCLASS_MAP = { "user": ("app_data.user", "User"), "group": ("app_data.group", "Group"), "role": ("app_data.role", "Role"), } @classmethod def _resolve_class(cls, classname): """Return the concrete subclass (User/Group/Role) for classname, or cls if none.""" if classname in cls._SUBCLASS_MAP: import importlib mod_path, cls_name = cls._SUBCLASS_MAP[classname] return getattr(importlib.import_module(mod_path), cls_name) return cls def __init__(self, iobj=None, keyname=None, classname=None, expand=False): """Build from an existing DataInstance (iobj) or as a new empty instance (classname/keyname); loads schema, builds self.fields, and optionally expands injected fields.""" self.classname = None # string self.schema = None # Schema() self.keyname = None self.iobj_id = None # PK of the underlying DataInstance row, or None if unbound self.is_enabled = True # True / False self.displayname = "" self.p_read = "" self.p_update = "" self.p_delete = "" self.json = None # self.value = [] self.fields = {} # key = field keyname self.errors = [] self.last_update = None self.last_sync = None # set externally before save(); never stored directly self.aaa_revision = {} # stored by merge methods; used by _revision_add # is_bound => @property # id => @property # NEW EMPTY (or else use from_keyname() classmethod) if not iobj: if classname: self.classname = classname if keyname: self.keyname = keyname # EXISTING else: # check iobj is a DataInstance if type(iobj) is not DataInstance: return # TODO : check iobj is not a new Model self.iobj_id = iobj.id self.classname = iobj.classname self.keyname = iobj.keyname self.is_enabled = iobj.is_enabled self.displayname = iobj.displayname self.p_read = iobj.p_read self.p_update = iobj.p_update self.p_delete = iobj.p_delete self.last_update = iobj.last_update self.last_sync = iobj.last_sync try: self.json = json.loads(iobj.data_json) except Exception: self.json = None # create fields, and fill with json if available self.schema = Schema.from_name(self.classname) if self.schema: for fieldname, fieldschema in self.schema.fields.items(): # safety: don't inject a bad external keyname (missing prefix _) if fieldschema["dataformat"] == "external": if not fieldname.startswith("_"): continue constructor = FIELD_CONSTRUCTOR_TABLE.get(fieldschema["dataformat"], FieldString) self.fields[fieldname] = constructor(fieldname, fieldschema, self.json) # expand injected fields if expand: self.expand_injected() # options ... self.set_options() # Done ! @property def id(self): """DB primary key of the underlying DataInstance, or None if unbound.""" return self.iobj_id @property def is_bound(self): """True if this instance is backed by a DataInstance row in DB.""" return self.iobj_id is not None # new 3.19 - specific constructors @classmethod def from_keyname(cls, classname=None, keyname=None, expand=False): """existing from DB or None""" classname = classname or getattr(cls, "CLASSNAME", None) if not classname: return if not keyname: return # Recompute (no cache) if expand if not expand: cachekey = f"{classname}::{keyname}" instance = cache.cache2_instance.get(cachekey) # cache hit if instance: return instance # cache miss instobj = DataInstance.objects.filter(classname=classname, keyname=keyname).first() if instobj: derived_class = cls._resolve_class(classname) instance = derived_class(iobj=instobj, expand=expand) instance.cache_set() return instance @classmethod def from_id(cls, id=None, expand=False): """existing from id or None - v3.21.0""" if not id: return # no expand, try cache if not expand: cachekey = f"{id}" instance = cache.cache2_instance.get(cachekey) if instance is not None: derived_class = cls._resolve_class(instance.classname) if isinstance(instance, derived_class): return instance else: print( "!!! instance.from_id with wrong class: ", instance.id, instance.keyname, instance.classname, ) # cache miss / wrong derived_class try: instobj = DataInstance.objects.get(pk=id) except Exception: return derived_class = cls._resolve_class(instobj.classname) instance = derived_class(iobj=instobj, expand=expand) if isinstance(instance, derived_class): instance.cache_set() return instance else: # TODO logger error print( "!!! instance.from_id with wrong class: ", instance.id, instance.keyname, instance.classname, ) # return anyway return instance @classmethod def from_iobj(cls, iobj=None, expand=False): """existing from iobj or None""" if not isinstance(iobj, DataInstance): return # Recompute (no cache) if expand if not expand: try: cachekey = f"{iobj.classname}::{iobj.keyname}" except Exception: return instance = cache.cache2_instance.get(cachekey) if instance: derived_class = cls._resolve_class(iobj.classname) if isinstance(instance, derived_class): return instance # cache miss derived_class = cls._resolve_class(iobj.classname) instance = derived_class(iobj=iobj, expand=expand) if instance: instance.cache_set() return instance @classmethod def get_dn(cls, classname=None, keyname=None): """Return displayname for keyname in this class, or None if not found.""" classname = classname or getattr(cls, "CLASSNAME", None) if not classname or not keyname: return None iobj = ( DataInstance.objects.filter(classname=classname, keyname=keyname) .values("displayname") .first() ) if iobj: return iobj["displayname"] return None @classmethod def get_all_keyname(cls, classname=None): """Return list of all keynames for this class.""" classname = classname or getattr(cls, "CLASSNAME", None) if not classname: return [] return list( DataInstance.objects.filter(classname=classname).values_list("keyname", flat=True) ) @classmethod def count_all(cls, classname=None): """Return the number of instances for this class, or 0 if classname is unknown.""" classname = classname or getattr(cls, "CLASSNAME", None) if classname: return DataInstance.objects.filter(classname=classname).count() else: return 0 @classmethod def exists(cls, classname=None, keyname=None): """Return True if an instance with this classname/keyname exists in DB.""" classname = classname or getattr(cls, "CLASSNAME", None) if not classname or not keyname: return False return DataInstance.objects.filter(classname=classname, keyname=keyname).exists() @classmethod def stream_classname(cls, classname=None, enabled=None, chunk_size=200): """Yields Instance objects one batch at a time; each DB query is bounded by chunk_size.""" classname = classname or getattr(cls, "CLASSNAME", None) if not classname: return offset = 0 while True: filters = {"classname": classname} if enabled == "yes": filters["is_enabled"] = True elif enabled == "no": filters["is_enabled"] = False batch = list(DataInstance.objects.filter(**filters)[offset : offset + chunk_size]) if not batch: break for iobj in batch: instance = cls.from_iobj(iobj=iobj) if instance: yield instance if len(batch) < chunk_size: break offset += chunk_size @classmethod def list_classname(cls, classname=None, first=None, last=None, enabled=None, expand=False): """returns a list[] of Instance(); use stream_classname() for large sets""" classname = classname or getattr(cls, "CLASSNAME", None) if not classname: return [] instances = [] if type(first) is int and type(last) is int: if enabled == "yes": instances = DataInstance.objects.filter(classname=classname, is_enabled=True)[ first:last ] elif enabled == "no": instances = DataInstance.objects.filter(classname=classname, is_enabled=False)[ first:last ] else: instances = DataInstance.objects.filter(classname=classname)[first:last] else: if enabled == "yes": instances = DataInstance.objects.filter(classname=classname, is_enabled=True) elif enabled == "no": instances = DataInstance.objects.filter(classname=classname, is_enabled=False) else: instances = DataInstance.objects.filter(classname=classname) reply = [] for instobj in instances: instance = cls.from_iobj(iobj=instobj, expand=expand) if instance: reply.append(instance) return reply def __str__(self): """Return "classname:keyname".""" return f"{self.classname}:{self.keyname}" def print(self): """Print a human-readable dump of the instance and its fields (debug helper).""" print(f"{self.classname}:{self.keyname}") print(f" id: {self.id}") print(f" is_enabled: {self.is_enabled}") print(f" displayname: {self.displayname}") print(f" is_bound: {self.is_bound}") print(f" is_valid: {self.is_valid()}") print(f" p_read: {self.p_read}") print(f" p_update: {self.p_update}") print(f" p_delete: {self.p_delete}") print(" fields:") # pprint(self.schema) for _nouse, v in self.fields.items(): v.print() # pprint(v.fieldschema) print() def to_json(self, refs=None, rev=None): """Serialize instance to a JSON bytestring (via get_dict_for_export).""" data = self.get_dict_for_export(refs=refs, rev=rev) data.pop("id", None) return json.dumps(data, indent=4, ensure_ascii=False).encode("utf8") def to_yaml(self, refs=None, rev=None): """Serialize instance to a YAML string, as a single-item list (via get_dict_for_export).""" data = self.get_dict_for_export(refs=refs, rev=rev) data.pop("id", None) return yaml.dump([data], allow_unicode=True, Dumper=MyYamlDumper, sort_keys=False) # default_style='|', # cache_get is in classmethod from_id, from_keyname, ... ; instance doesn't exist yet def cache_set(self): """Store self in cache2_instance, keyed by both "classname::keyname" and id.""" # cache by name if self.classname and self.keyname: cachekey = f"{self.classname}::{self.keyname}" cache.cache2_instance.set(cachekey, self) # cache by id if self.id: cachekey = f"{self.id}" cache.cache2_instance.set(cachekey, self) def cache_purge(self): """Remove self from cache2_instance (by name and id); also flushes cache2_enumerate when self is an _enumerate instance, and cache2_ipam_subnet_forest when self is an ipam_subnet instance (the whole forest, not just self - any single create/ update/delete can change other subnets' first-level children, e.g. inserting a new intermediate subnet demotes what used to be a direct child to a grandchild).""" # by name if self.classname and self.keyname: cachekey = f"{self.classname}::{self.keyname}" cache.cache2_instance.delete(cachekey) # by id if self.id: cachekey = f"{self.id}" cache.cache2_instance.delete(cachekey) # flush specific cache : enumerate cache when an _enumerate instance is modified if self.classname == "_enumerate" and self.keyname: cache.cache2_enumerate.delete(self.keyname) # flush specific cache : ipam_subnet parent/child forest if self.classname == "ipam_subnet": cache.cache2_ipam_subnet_forest.delete("forest") def related(self, aaa=None): """Return objects referencing self (via DataEAV related: lookup), for display in the instance_detail "related objects" section. If aaa provided, reply is filtered to schemas aaa can read. ipam_ip is special-cased to app_data.ipam.get_related_ipam_ip() - its keyname is an opaque hexip string, so the generic classname:keyname match finds nothing; what actually references it is an ipv4-format value equal to its displayname (the IP itself). Schemas with `_options.hide_from_related: true` are dropped here, display-only - the underlying get_related()/build_related_reply() stay unfiltered so other callers (e.g. keyname.py's rename(), which needs every real referrer) still see them.""" if self.classname == "ipam_ip": reply = get_related_ipam_ip(self, aaa=aaa) else: reply = get_related(classname=self.classname, keyname=self.keyname, aaa=aaa) return { classname: entry for classname, entry in reply.items() if not getattr(Schema.from_name(classname), "hide_from_related", False) } def ordered_fields(self): """Return field names sorted by their schema-defined display order.""" def sort_key(item): return self.fields[item].order return sorted(self.fields, key=sort_key) def is_field_true(self, fieldname=None): """Return True if fieldname exists and its boolean value is true.""" if not fieldname: return False if fieldname not in self.fields: return False return self.fields[fieldname].is_field_true() def get_attribute_first(self, fieldname): """Return the first value from get_attribute(fieldname), or "" if empty/missing.""" r1 = self.get_attribute(fieldname) try: return r1[0] except Exception: return "" def get_attribute(self, fieldname): """ Returns a List [] of attribute value(s) from instance Convert to obj if FieldSchema or SireneGroup or User ... """ if fieldname == "displayname": return [self.displayname] if fieldname == "keyname": return [self.keyname] if fieldname == "is_enabled": return [self.is_enabled] if fieldname not in self.fields: return return self.fields[fieldname].get_attribute() def set_field_value_single(self, fieldname=None, value=None): """Set a single-value field's value in-memory (no validation, no save). Returns bool ok.""" if not fieldname: return False if fieldname not in self.fields: return False self.fields[fieldname].value = [value] return True # instance PERMISSIONS def has_valid_aaa(self, aaa=None): """Return True if self is bound and aaa carries a usable perms list.""" if not self.is_bound: return False try: if type(aaa["perms"]) is list: return True except Exception: pass return False def has_read_permission(self, aaa=None): """Check read access: global admin > schema admin > instance p_read > schema read.""" if not self.has_valid_aaa(aaa=aaa): return False # global admin : all schema, all instances, all operations if "p_data_admin" in aaa["perms"]: return True # schema admin : aaa has p_admin on this schema ? (allow direct & stop inheritance) schema = Schema.from_name(self.classname) if schema.has_admin_permission(aaa=aaa): return True # instance read if self.p_read: if self.p_read in aaa["perms"]: return True else: return False # default schema read > all data read return schema.has_read_permission(aaa=aaa) def has_update_permission(self, aaa=None): """Check update access: global admin > schema admin > instance p_update > schema update.""" if not self.has_valid_aaa(aaa=aaa): return False if "p_data_admin" in aaa["perms"]: return True # schema admin (no override) schema = Schema.from_name(self.classname) if schema.has_admin_permission(aaa=aaa): return True # if defined, instance permission rules if self.p_update: if self.p_update in aaa["perms"]: return True else: return False # default to class permission return schema.has_update_permission(aaa=aaa) def has_delete_permission(self, aaa=None): """Check delete access: global admin > schema admin > instance p_delete > schema delete.""" if not self.has_valid_aaa(aaa=aaa): return False if "p_data_admin" in aaa["perms"]: return True # schema admin (no override) schema = Schema.from_name(self.classname) if schema.has_admin_permission(aaa=aaa): return True # if defined, instance permission rules if self.p_delete: if self.p_delete in aaa["perms"]: return True else: return False # default to class permission return schema.has_delete_permission(aaa=aaa) def get_recursive_content(self, fieldname=None, fieldmember=None, fieldrecurse=None, done=None): """get list of Instance() from field "fieldname" , including - self instance - fieldmember *.fieldname - recurse on fieldsubgroup : fieldname, member, sub-subgroups ... """ # Example # ------- # SiteGroup = { # sirene_notify =[ SireneGroup1, 2, ...] # members = [site1, site2] # subgroups = [sitegroup1, sitegroup2 ...] # } # # for sitegroup_obj in message.notify_sitegroup.all(): # instance = Instance(iobj=sitegroup_obj) # xlist = instance.get_recursive_content( # fieldname="notify_group", # fieldmember="members", # fieldrecurse="subgroups", # done=[]) # # done[] contains list of iobj like self if done is None: done = [] reply = [] # recursive: already done ? if self in done: return [] done.append(self) # get field values from self xlist1 = self.get_attribute(fieldname) for z in xlist1: if z: if z not in done: reply.append(z) # reply += xlist1 # get all members from self xlist2 = [] if fieldmember: # if fieldmember in self.schema: if fieldmember in self.fields: xlist2 = self.fields[fieldmember].get_attribute() # get field values from members for item in xlist2: # try to get an instance struct # NOTA: won't work on SireneGroup objects ; use SireneGroups for that purpose if type(item) is DataInstance: instance = Instance(iobj=item) if instance: xlist3 = instance.get_attribute(fieldname) for z in xlist3: if z: if z not in done: reply.append(z) # get all subgroup (fieldrecurse field content) xlist4 = [] if fieldrecurse: if fieldrecurse in self.fields: xlist4 = self.fields[fieldrecurse].get_attribute() # recurse for item in xlist4: if type(item) is DataInstance: if item in done: continue instance = Instance(iobj=item) if instance: xlist5 = instance.get_recursive_content( fieldname=fieldname, fieldmember=fieldmember, fieldrecurse=fieldrecurse, done=done, ) for z in xlist5: if z: if z not in done: reply.append(z) # reply += xlist5 reply = list(set(reply)) return reply # set options # ----------- def set_options(self): """Apply schema-level options: auto-generate keyname when keyname_mode is "uuid" (random uuid4, once), or derive it from displayname when keyname_mode is "hexip" (IPv4 address -> hex, recomputed every time displayname is set).""" # empty instance if not self.schema: return # keyname_mode if self.schema.keyname_mode == "uuid": if not self.keyname: self.keyname = Schema.create_keyname_uuid() elif self.schema.keyname_mode == "hexip": newkeyname = Schema.create_keyname_hexip(self.displayname) if newkeyname and newkeyname != self.keyname: oldkeyname = self.keyname self.keyname = newkeyname # purge the stale cache entry under the old keyname - this is a # raw in-memory keyname reassignment (not the cascading # app_data/keyname.py rename()), so nothing else does this if self.is_bound and oldkeyname and self.classname: cache.cache2_instance.delete(f"{self.classname}::{oldkeyname}") # expand / compute injected fields # -------------------------------- def expand_injected(self): """Recompute all injected fields: drop stale ones, then re-inject resolved external, schema subfield, and enumerate subfield fields.""" # Refresh all injected / external / enumerate / schema # if new empty Instance, skip update if not self.keyname: return # 1 - remove injected fields (from enumerate / schema) # self.fields will be modified so separate loop inventory and processing remove = [] for fieldname, field in self.fields.items(): if field.is_injected: remove.append(fieldname) for v in remove: self.fields.pop(v) # 2 - resolve external to source # self.fields will be modified so separate loop inventory and processing external = [] for fieldname, field in self.fields.items(): if field.dataformat == "external": external.append(fieldname) # processing for fieldname in external: # field = self.fields[fieldname] # recurse here ; will inject resolved field as schema fields self.inject_resolved_external_field(fieldname) # 3 - schema : inject subfields self.inject_schema_subfields() # 4 - enumerate : inject subfields self.inject_enumerate_subfields() return # ------------------------------------------------- # compute with recurse for a single field # inject computed field in self.fields # ------------------------------------------------- def inject_resolved_external_field(self, fieldname): """Resolve an "external" field to its source schema field's value (recursing through chained externals) and inject the result as a new schema-typed field in self.fields.""" try: field = self.fields[fieldname] except Exception: return # fieldname must be an external (to a schema) if field.dataformat != "external": return parent_fieldname = field.get_parent_fieldname() if not parent_fieldname: return parent_field = self.fields[parent_fieldname] if parent_field.dataformat != "schema": return remote_classname = parent_field.get_classname() if not remote_classname: return # NEXT : support multi-value try: remote_instance_name = parent_field.value[0] except Exception: return # NEXT : add depth to avoid recurse loops # don't expand (perf) ; will recurse on non expanded field remote_instance = Instance.from_keyname( classname=remote_classname, keyname=remote_instance_name ) if not remote_instance: return if not remote_instance.is_bound: return # get remote fieldname remote_fieldname = field.get_remote_fieldname() if not remote_fieldname: return # 2 options here # either remote_fieldname is a schema , or "_"+remote_field is an external if remote_fieldname in remote_instance.fields: # schema remote_fieldname2 = remote_fieldname remote_field = remote_instance.fields[remote_fieldname] elif "_" + remote_fieldname in remote_instance.fields: # external remote_fieldname2 = "_" + remote_fieldname remote_field = remote_instance.fields[remote_fieldname2] else: remote_field = None if not remote_field: return remote_field_dataformat = remote_field.dataformat # remote field is an external => recurse (again) if remote_field_dataformat == "external": remote_instance.inject_resolved_external_field(remote_fieldname2) # access remote resolved field (without prefix _ ) try: # remote_fieldname is without _ newvalue = remote_instance.fields[remote_fieldname].get_value() newclassname = remote_instance.fields[remote_fieldname].get_classname() # bad structure except Exception: return # remote field is a schema => DONE, it's the source elif remote_field_dataformat == "schema": newvalue = remote_field.get_value() newclassname = remote_field.get_classname() # bad structure else: return # inject external schema field w/ name = external fieldname without prefix _ new_fieldname = field.fieldname[1:] new_json = {new_fieldname: newvalue} # get subfields subfields = field.get_subfields() new_dataformat_ext = newclassname + " " + " ".join(subfields) new_schema = { "displayname": field.displayname, "description": new_fieldname, "is_multi": False, "dataformat": "schema", "dataformat_ext": new_dataformat_ext, "cardinal_min": 0, "cardinal_max": 1, "is_injected": True, "injected_type": "external", "page": field.page, "order": field.order, } self.fields[new_fieldname] = FieldSchema(new_fieldname, new_schema, new_json) return def inject_schema_subfields(self): """For each "schema" field with declared subfields, fetch the referenced instance and inject its subfield values as new fields named "__".""" # collect all schema fields to process (2 stage : fields will be altered by injection) schema_fieldnames = [] for fieldname, field in self.fields.items(): if field.dataformat == "schema": schema_fieldnames.append(fieldname) # loop over identified schema fields for fieldname in schema_fieldnames: field = self.fields[fieldname] subfieldnames = field.get_subfields() if not subfieldnames: continue target_classname = field.get_classname() target_instancename = field.get_first_value() # NEXT : multi : field__xx.0, field_xx.1 ... target_instance = Instance.from_keyname( classname=target_classname, keyname=target_instancename ) if not target_instance: continue for fn in subfieldnames: new_fieldname = fieldname + "__" + fn new_displayname = field.displayname + f"({fn})" # try a regular target field: if fn in target_instance.fields: target_field = target_instance.fields[fn] # exclude some dataformat we can't inject if target_field.dataformat in ["schema", "external"]: continue new_dataformat = target_field.dataformat new_schema = { "displayname": new_displayname, "description": new_displayname, "dataformat": new_dataformat, "dataformat_ext": target_field.dataformat_ext, "cardinal_min": target_field.cardinal_min, "cardinal_max": target_field.cardinal_max, "is_injected": True, "injected_type": "schema", "page": field.page, "order": field.order, } try: new_list = target_field.get_value() new_json = {new_fieldname: new_list} except Exception: continue constructor = FIELD_CONSTRUCTOR_TABLE.get(new_dataformat, FieldString) self.fields[new_fieldname] = constructor(new_fieldname, new_schema, new_json) # try a built-in attribute (display_name ...) else: pass # TODO : inject displayname, keyname def inject_enumerate_subfields(self): """For each "enumerate" field, inject its subfields (typed int/boolean/float/date/string) as new fields named "__". A subfield value of None (invalid/unparsable at the source, see FieldEnumerate.get_subfields()) is injected as an empty field rather than a bogus value.""" enumerate = [] for fieldname, field in self.fields.items(): if field.dataformat == "enumerate": enumerate.append(fieldname) for fieldname in enumerate: field = self.fields[fieldname] subfields = field.get_subfields() for k, value in subfields.items(): new_fieldname = fieldname + "__" + k try: (t, v) = value except Exception: continue new_json = {new_fieldname: [] if v is None else [v]} new_schema = { "displayname": new_fieldname, "description": new_fieldname, "dataformat": "string", "dataformat_ext": "safe", "cardinal_min": 0, "cardinal_max": 1, "is_injected": True, "injected_type": "enumerate", "page": field.page, "order": field.order, } if t == "int": new_schema["dataformat"] = "int" self.fields[new_fieldname] = FieldInt(new_fieldname, new_schema, new_json) elif t == "boolean": new_schema["dataformat"] = "boolean" self.fields[new_fieldname] = FieldBoolean(new_fieldname, new_schema, new_json) elif t == "float": new_schema["dataformat"] = "float" self.fields[new_fieldname] = FieldFloat(new_fieldname, new_schema, new_json) elif t == "date": new_schema["dataformat"] = "date" self.fields[new_fieldname] = FieldDate(new_fieldname, new_schema, new_json) else: self.fields[new_fieldname] = FieldString(new_fieldname, new_schema, new_json) # NEXT : ipv4, datetime, time # -------- def get_dict_for_ui_form(self): """Build the dict consumed by the new/edit form template: editable fields only, grouped in instance_ui["PAGES"][page][order] (injected/external fields excluded).""" instance_ui = {} # SPECIAL fields instance_ui["id"] = self.id instance_ui["keyname"] = self.keyname instance_ui["displayname"] = self.displayname instance_ui["p_read"] = self.p_read instance_ui["p_update"] = self.p_update instance_ui["p_delete"] = self.p_delete instance_ui["is_enabled"] = self.is_enabled # loop over instance.fields instance_ui["PAGES"] = {} for fieldname in self.ordered_fields(): field = self.fields[fieldname] # don't edit injected fields if field.is_injected: continue # don't edit external fields if field.dataformat == "external": continue if fieldname in self.schema.field_hide_from_edit: continue order = field.order page = field.page if not page: page = self.keyname if page not in instance_ui["PAGES"]: instance_ui["PAGES"][page] = {} if order not in instance_ui["PAGES"][page]: instance_ui["PAGES"][page][order] = [] datapoint = self.fields[fieldname].get_datapoint_ui_edit() instance_ui["PAGES"][page][order].append(datapoint) return instance_ui def get_dict_for_ui_detail(self, skip_external=True, skip_injected=True, skip_enumerate=True): """dict suited for UI list template ; NESTED PAGE and FLAT""" # ui = { # keyname:"xxxx" # displayname:"xxx" # is_enabled:True/False" # p_*3 # First Level / Flat mode # ------------------------ # fieldname: { DATAPOINT } # fieldname: { DATAPOINT } # ... # Nested / Page # ------------- # instance_ui["PAGES"][page][order]= [] # # "PAGES": { # "(page)1": { "(order)100": [ {DATAPOINT}, {}, ... ], # "(order)101": [ {}, {}, ... ], # "(page)2": { }, # ... # } # DATAPOINT = { # datapoint["fieldname"] = self.fieldname # datapoint["displayname"] = self.displayname # datapoint["description"] = self.description # datapoint["dataformat"] = self.dataformat # datapoint["dataformat_ext"] = self.dataformat_ext # datapoint["is_multi"] = self.is_multi() # datapoint["bigset"] = False # datapoint["schema"] = CLASSNAME (for schema field) # datapoint["value"] = '' # } # VALUE : depend on field dataformat (ex. ", ".join(values)") instance_ui = {} # SPECIAL fields P7 instance_ui["id"] = self.id instance_ui["keyname"] = self.keyname instance_ui["displayname"] = self.displayname if not self.displayname: instance_ui["displayname"] = self.keyname instance_ui["p_read"] = self.p_read instance_ui["p_update"] = self.p_update instance_ui["p_delete"] = self.p_delete instance_ui["is_enabled"] = self.is_enabled instance_ui["last_update"] = self.last_update instance_ui["last_sync"] = self.last_sync # loop over instance.fields instance_ui["PAGES"] = {} for fieldname in self.ordered_fields(): field = self.fields[fieldname] if skip_external: if field.dataformat == "external": continue if skip_injected: if field.is_injected: continue if skip_enumerate: if field.injected_type == "enumerate": continue if fieldname in self.schema.field_hide_from_detail: continue order = field.order page = field.page if not page: page = self.keyname if page not in instance_ui["PAGES"]: instance_ui["PAGES"][page] = {} if order not in instance_ui["PAGES"][page]: instance_ui["PAGES"][page][order] = [] datapoint = field.get_datapoint_ui_detail() # nested structure PAGE/ORDER instance_ui["PAGES"][page][order].append(datapoint) # append also at first level instance_ui[fieldname] = copy.deepcopy(datapoint) return instance_ui # alias def to_dict(self): """Alias for get_dict_for_export() with no refs/rev expansion.""" return self.get_dict_for_export() def get_dict_for_export(self, refs=None, rev=None, fields=None, action=None): """Build the dict used for JSON/YAML export and API responses: special fields, regular field values, optional inlined `_refs` (see get_export_refs()), and optional `_revision` history (see get_export_revisions()). `fields` (see field_export_match()) filters which schema fields are included (and, via the `selected_fields` passed to get_export_refs(), which fields' values can be inlined into `_refs`); classname/keyname/displayname and other builtin attributes are always included (no numeric `id` - not read back on import). `action`, when given, is set as an "_action" key (e.g. "noop") - a safe-by-default marker so re-importing this export as-is does nothing until the value is replaced with a real action.""" # { # classname: XX # keyname: XX # displayname:"xxx" # is_enabled:True/False" # "attribname": DATAPOINT, # "attribname": ... # _refs: # user: # group: # schema1: # ... # } # DATAPOINT = value / "value" # DATAPOINT = ["v1", "v2", ...] instance_export = {} # SPECIAL fields instance_export["classname"] = self.classname instance_export["keyname"] = self.keyname if field_export_match("displayname", fields): instance_export["displayname"] = self.displayname if self.p_read: instance_export["p_read"] = self.p_read if self.p_update: instance_export["p_update"] = self.p_update if self.p_delete: instance_export["p_delete"] = self.p_delete if not self.is_enabled: instance_export["is_enabled"] = self.is_enabled if self.last_sync is not None and field_export_match("last_sync", fields): instance_export["last_sync"] = iso_seconds(self.last_sync) if self.last_update is not None and field_export_match("last_update", fields): instance_export["last_update"] = iso_seconds(self.last_update) # REGULAR fields selected_fields = [fn for fn in self.ordered_fields() if field_export_match(fn, fields)] for fieldname in selected_fields: field = self.fields[fieldname] datapoint = field.get_datapoint_for_export() cardinal, cardinal_min, cardinal_max = field.get_cardinal3() if not (cardinal_min == 0 and cardinal == 0): instance_export[fieldname] = datapoint refs_dict = self.get_export_refs(refs=refs, selected_fields=selected_fields) if refs_dict: instance_export["_refs"] = refs_dict revisions_list = self.get_export_revisions(rev=rev) if revisions_list: instance_export["_revision"] = revisions_list if action: instance_export["_action"] = action return instance_export def get_export_refs(self, refs=None, selected_fields=None): """Build the `_refs` dict for get_dict_for_export(): for each field in `selected_fields` (real field names to scan - `get_dict_for_export()` passes the same fields its own export already includes; the dataview-driven HTML/JSON export instead passes the active dataview's own column list, so only reference fields actually selected/displayed contribute) that targets one of the `refs` classnames (a list of schema names to inline, e.g. ['user', 'site'] - '*' expands to all schema names, including user/group/role, via Schema.listall_names()), collect the referenced keynames (Pass 1) then resolve their keyname/classname/displayname (+ _USER_EXTRA_FIELDS for 'user') via DataEAV (Pass 2). Returns {classname: [{...}, ...]}, or {} if nothing to inline.""" if not refs: return {} if "*" in refs: refs = [r for r in refs if r != "*"] + Schema.listall_names() # Pass 1: collect sets of referenced keys per type buckets = {} for fieldname in selected_fields or []: if fieldname not in self.fields: continue field = self.fields[fieldname] if not field.value: continue if field.dataformat == "schema": classname = field.dataformat_ext if classname and classname in refs: buckets.setdefault(classname, set()).update(field.value) # Pass 2: DataEAV queries to fill refs result = {} for classname, keynames in buckets.items(): # keyname → displayname from any EAV row per keyname dn_map = {} for row in ( DataEAV.objects.filter(classname=classname, keyname__in=keynames) .values("keyname", "displayname") .distinct() ): dn_map.setdefault(row["keyname"], row["displayname"] or "") # extra EAV fields for 'user' extra_map = {} if classname == "user": for row in DataEAV.objects.filter( classname="user", keyname__in=keynames, fieldname__in=_USER_EXTRA_FIELDS ).values("keyname", "fieldname", "value"): extra_map.setdefault(row["keyname"], {})[row["fieldname"]] = row["value"] or "" detail = [] for keyname, displayname in dn_map.items(): entry = {"keyname": keyname, "classname": classname, "displayname": displayname} if classname == "user": for fn in _USER_EXTRA_FIELDS: v = extra_map.get(keyname, {}).get(fn, "") if v: entry[fn] = v detail.append({k: v for k, v in entry.items() if v or k == "keyname"}) if detail: result[classname] = detail return result def get_export_revisions(self, rev=None): """Return the `_revision` list for get_dict_for_export(): up to `rev` most recent revision entries ({login, date, action}) for this instance, or [] if none/rev is falsy.""" if not rev: return [] revisions = revision_get(classname=self.classname, keyname=self.keyname, limit=rev) if not revisions: return [] return [ { "login": r.username or "", "date": r.date.strftime("%Y-%m-%dT%H:%M:%S"), "action": r.action or "", } for r in revisions ] def get_csv_columns(self, fields=None): """Return column names for CSV export: `_schema` (classname) and `keyname` - both always present, not gated by `fields` - followed by non-injected, non-text, non-underscore-prefixed fields. `fields` (see field_export_match()) filters which schema fields are included, and also gates `displayname`, `is_enabled`, `last_sync` and `last_update` - opt-out via `fields` exactly like a regular field, not fixed columns. Unlike YAML/JSON's get_dict_for_export(), `last_sync`/`last_update` are not gated on having a value: CSV has one fixed header for every row, so the column is always present (empty cell when unset) rather than presence depending on the first instance's value. There is never a numeric `id` column.""" csv_columns = ["_schema", "keyname"] if field_export_match("displayname", fields): csv_columns.append("displayname") if field_export_match("is_enabled", fields): csv_columns.append("is_enabled") if field_export_match("last_sync", fields): csv_columns.append("last_sync") if field_export_match("last_update", fields): csv_columns.append("last_update") # ,"p_read","p_update","p_delete for fieldname in self.ordered_fields(): field = self.fields[fieldname] # if field.is_multi(): # continue if field.is_injected: continue if fieldname.startswith("_"): continue if field.dataformat == "text": continue if not field_export_match(fieldname, fields): continue csv_columns.append(fieldname) return csv_columns def get_csv_line(self, csv_columns): """Return one CSV row (list of cell values) matching the columns from get_csv_columns().""" # /!\ must match get_csv_columns line = [self.classname, self.keyname] if "displayname" in csv_columns: line.append(self.displayname) if "is_enabled" in csv_columns: line.append(self.is_enabled) if "last_sync" in csv_columns: line.append(iso_seconds(self.last_sync) if self.last_sync else "") if "last_update" in csv_columns: line.append(iso_seconds(self.last_update) if self.last_update else "") # self.p_read, self.p_update,self.p_delete for fieldname in self.ordered_fields(): if fieldname not in csv_columns: continue line.append(self.fields[fieldname].get_csv_cell()) return line # ------------------------------------------------------------------ # Revision helpers # ------------------------------------------------------------------ def _snapshot(self): """Return a comparable dict of keyname/displayname/is_enabled/fields, used as a before/after baseline for diff_revision() (excludes injected, external, and churn-prone fields).""" return { "keyname": self.keyname, "displayname": self.displayname, "is_enabled": self.is_enabled, "fields": { fname: f.get_snapshot_value() for fname, f in self.fields.items() if not f.is_injected and f.dataformat != "external" and fname not in _DIFF_REVISION_EXCLUDED_FIELDS }, } def diff_revision(self, aaa=None): """Diff self's current in-memory state against what's currently persisted in DB (reloaded fresh by pk, bypassing the instance cache so the baseline can't be cross-process-stale). Return {} if unbound or unchanged, else a dict of changed keys/fields each mapping to {"before": ..., "after": ...}.""" if aaa is not None: self.aaa_revision = aaa if not self.is_bound: return {} prev_iobj = DataInstance.objects.filter(pk=self.iobj_id).first() if prev_iobj is None: # stale binding: iobj_id points at a row that no longer exists # (e.g. a cache entry surviving past the row's deletion). Nothing # to diff against — and self isn't really bound to anything real, # so let save() treat it like a fresh insert instead of a no-op. self.iobj_id = None return {} before = self.__class__(iobj=prev_iobj)._snapshot() after = self._snapshot() diff = {} for key in ("keyname", "displayname", "is_enabled"): b = before.get(key) a = after.get(key) if b != a: diff[key] = {"before": b, "after": a} b_fields = before.get("fields", {}) a_fields = after.get("fields", {}) for fname in set(b_fields) | set(a_fields): b = b_fields.get(fname, []) a = a_fields.get(fname, []) if b != a: diff.setdefault("fields", {})[fname] = {"before": b, "after": a} return diff def _revision_add(self, action="update", diff=None): """Record a revision entry for this instance if aaa_revision carries a username.""" if not self.aaa_revision.get("username"): return from app_data.revision import revision_add revision_add(aaa=self.aaa_revision, instance=self, action=action, diff=diff) # new 3.20 : merged new & edit method def merge_request(self, request, aaa=None): """Merge a Django form POST into self (keyname/displayname/is_enabled/permissions/fields), then re-expand injected fields. In-memory only; call save() to persist. Note: aaa is used here only for permission-gated fields — pass aaa= to the following save() call for revision attribution, it isn't stashed here.""" # keyname : editable only for new objects (PK in DB) if not self.is_bound: if self.schema.keyname_mode == "edit" or self.schema.keyname_dataformat: self.keyname = request.POST.get("keyname", "") # displayname (check in is_valid + template auto escaping) self.displayname = request.POST.get("displayname", "") # is_enabled if "is_enabled" in request.POST: self.is_enabled = request.POST["is_enabled"] in settings.TRUE_LIST else: self.is_enabled = False # permissions if type(aaa) is dict: if "perms" in aaa: if "p_data_security_edit" in aaa["perms"]: if "p_read" in request.POST: self.p_read = request.POST["p_read"] if "p_update" in request.POST: self.p_update = request.POST["p_update"] if "p_delete" in request.POST: self.p_delete = request.POST["p_delete"] for fieldname, field in self.fields.items(): # don't merge injected fields if field.is_injected: continue # don't merge external fields if field.dataformat == "external": continue self.fields[fieldname].merge_request(request) # injected fields self.expand_injected() # options self.set_options() def merge_import(self, data, aaa=None, mode="replace"): """Merge a dict (import/API payload) into self (keyname/displayname/is_enabled/ last_sync/permissions/fields), then re-expand injected fields. In-memory only; call save() to persist. Note: aaa is used here only for permission-gated fields — pass aaa= to the following save() call for revision attribution, it isn't stashed here. mode "replace" (default) overwrites each field's value; "append"/ "unappend" union/subtract incoming values into multivalued fields instead (single-valued fields always behave like "replace").""" # keyname if new instance only, and not auto computed keyname if not self.is_bound: if self.schema.keyname_mode == "edit" or self.schema.keyname_dataformat: # keyname may be absent # (new Instance with initial keyname, merge additional data w/o keyname) if "keyname" in data: candidate = data.get("keyname") or "" # dataformat mode: reject a malformed keyname here rather than # silently accepting it - loader.py never calls is_valid() on # this path, unlike the UI form (merge_request + views.py) if self.schema.keyname_dataformat: if keyname_dataformat_is_valid(self.schema, candidate): self.keyname = candidate else: self.errors.append(_("invalid keyname format")) else: self.keyname = candidate # displayname if "displayname" in data: self.displayname = data["displayname"] # is_enabled if "is_enabled" in data: self.is_enabled = data["is_enabled"] in settings.TRUE_LIST # else: # self.is_enabled = False # last_sync (model field, not EAV) — only accept timezone-aware values to avoid # collision with EAV string fields that happen to be named last_sync if "last_sync" in data: val = data["last_sync"] if val is None: self.last_sync = None elif isinstance(val, str): parsed = parse_datetime(val) if parsed and parsed.tzinfo is not None: self.last_sync = trunc_seconds(parsed) elif hasattr(val, "tzinfo") and val.tzinfo is not None: self.last_sync = trunc_seconds(val) # permissions if type(aaa) is dict: if "perms" in aaa: if "p_data_security_edit" in aaa["perms"]: if "p_read" in data: self.p_read = data["p_read"] if "p_update" in data: self.p_update = data["p_update"] if "p_delete" in data: self.p_delete = data["p_delete"] for fieldname, field in self.fields.items(): # don't merge injected fields if field.is_injected: continue # don't merge external fields if field.dataformat == "external": continue # special case : Boolean are not in POST request if False if fieldname in data: fielddata = data[fieldname] if mode == "append": field.merge_append(fielddata, remove=False) elif mode == "unappend": field.merge_append(fielddata, remove=True) else: field.merge_import(fielddata) # injected fields self.expand_injected() # options self.set_options() def is_valid(self): """Validate keyname/classname/is_enabled and all non-injected fields; appends messages to self.errors and returns False on the first structural failure or any invalid field. Injected fields (schema/enumerate/external subfields) are skipped: they are not user-editable and are recomputed on every expand_injected(), so a stale or malformed source (e.g. a badly-authored enumerate subfield) must never block saving the instance.""" reply = True # keyname if not self.keyname: self.errors.append(_("missing keyname")) return False if len(self.keyname) == 0: self.errors.append(_("keyname empty")) return False # classname (schema) if not self.classname: self.errors.append(_("missing classname")) return False if len(self.classname) == 0: self.errors.append(_("empty classname")) return False schema = Schema.from_name(classname=self.classname) if not schema: self.errors.append(_("unknown classname")) return False if not keyname_dataformat_is_valid(schema, self.keyname): self.errors.append(_("invalid keyname format")) reply = False # NEXT: displayname = safe string for display ? # is_enabled if type(self.is_enabled) is not bool: return False # NEXT - check permissions ? for _fieldname, field in self.fields.items(): if field.is_injected: continue r = field.is_valid() if not r: reply = False if field.errors: err = f"{field.displayname}: {'; '.join(field.errors)}" else: err = field.displayname self.errors.append(str(err)) return reply # ----------------- # helper methods # ----------------- def enable(self, aaa=None): """Set is_enabled=True, save. No-op (returns True, no write, no revision) if already enabled.""" if not self.is_enabled: self.is_enabled = True return self.save(aaa=aaa, action="enable") return True def disable(self, aaa=None): """Set is_enabled=False, save. No-op (returns True, no write, no revision) if already disabled.""" if self.is_enabled: self.is_enabled = False return self.save(aaa=aaa, action="disable") return True # ----------------- # real write to DB # ----------------- def delete(self, aaa=None): """ First write path to DB: delete the underlying DataInstance row (+ EAV rows). """ if not self.is_bound: return False if aaa is not None: self.aaa_revision = aaa self.cache_purge() iobj = DataInstance.objects.filter(pk=self.iobj_id).first() if iobj is not None: try: iobj.delete() except Exception as e: print(f"DB delete failed for {self.classname}:{self.keyname} - {e}") return False self._revision_add(action="delete") # before iobj_id cleared self.iobj_id = None self.last_update = now_seconds() EavBatch.save(self) # is_bound now False → clears EAV rows, no re-insert return True def save(self, aaa=None, action=None, force=False, skip_revision=False): """ Second write path to DB: load-or-create the DataInstance row, serialize self.fields to data_json, write the row, rebuild the EAV cache, and record a revision. Upserts: iobj_id set → update; unset → new row (bound on return). force=True bypasses the diff/no-op check (e.g. a rebuild-EAV-only resave where the keyname changed outside self, so there's no in-memory diff to detect). skip_revision=True writes normally but never records a revision — for system-computed changes that aren't a real user decision (e.g. autogroup recompute, last_login touch). A write with no detectable diff and no create (e.g. force=True on an unchanged instance, or a sync_mode-only last_sync touch) never records a revision either way, regardless of skip_revision. The diff itself is computed by diff_revision() against the row currently in DB (fresh read, not the instance cache) — no snapshot bookkeeping needed before mutating self.fields/self.is_enabled/etc. directly. """ want_db_write = force need_revision = False self.set_options() if not self.keyname or not self.classname: # TODO: error : self is invalid and can't produce write to DB return False # revision ? last_update # diff_revision() clears self.iobj_id if it discovers a stale binding # (iobj_id pointing at a row that no longer exists) — self.is_bound # reflects that immediately, and stays stable until self.iobj_id is # re-set at the very end of this method (after the DB write below) diff = self.diff_revision(aaa=aaa) # dict if action: diff_action = action else: diff_action = "update" if self.is_bound else "create" # a genuinely new (or stale-binding-resurrected) instance is always # written, even with no detectable diff if diff or not self.is_bound: want_db_write = True need_revision = True # last update self.last_update = now_seconds() # last_sync sync_mode = (aaa or {}).get("sync_mode", False) sync_only_touch = False if sync_mode: self.last_sync = now_seconds() # no last_update here if not want_db_write: # no diff, not force: only last_sync changes — skip the full # row rewrite + EAV rebuild below, do a single-column UPDATE sync_only_touch = True want_db_write = True # fast path: sync_mode-only touch, nothing else changed if sync_only_touch: DataInstance.objects.filter(pk=self.iobj_id).update(last_sync=self.last_sync) self.cache_purge() return True # get/create DB object and write if want_db_write: iobj = DataInstance.objects.filter(pk=self.iobj_id).first() or DataInstance() iobj.classname = self.classname iobj.keyname = self.keyname iobj.displayname = self.displayname iobj.is_enabled = self.is_enabled iobj.p_read = self.p_read iobj.p_update = self.p_update iobj.p_delete = self.p_delete iobj.last_update = self.last_update iobj.last_sync = self.last_sync data = {} for fieldname, field in self.fields.items(): if field.is_injected: continue field_datalist = field.get_value() # don't store empty fields in DB if len(field_datalist) == 0: continue # external field : restore fieldname without _ prefix if field.dataformat == "external": original_fieldname = fieldname[1:] data[original_fieldname] = field_datalist else: data[fieldname] = field_datalist try: iobj.data_json = json.dumps(data, indent=2, ensure_ascii=False) except Exception: # TODO: log error print("Invalid JSON in save()") return False try: iobj.save() self.iobj_id = iobj.id # bind (create case) / re-affirm except Exception as e: print(f"save() to DB failed: {e}") # TODO: log error return False self.cache_purge() EavBatch.save(self) # reads self.is_bound / self.id / self.fields # discover all IPs and update specific ipam_ip schema discover_ipam_ip(self, aaa=aaa) # a write with no real diff and no create (force=True on an unchanged # instance, or a sync_mode-only last_sync touch) never gets a revision if not skip_revision and need_revision: self._revision_add(action=diff_action, diff=diff) return True