# c(c) cavaliba.com - data - dataview.py """ DataView: named column layout for a schema's list UI ("_dataview" Instances), resolving which columns to show and how to extract/format each Instance's cell values. dataview_for_request() picks the active view (request > session > single-choice > CLASSNAME_default > built-in default) for a given classname. Method inventory: Module functions: get_dataview_dict, get_dataview_content_by_name, dataview_for_request DataView: __init__, get_columns, filter, print """ import re import yaml from django.utils import timezone from django.utils.translation import gettext as _ from app_data.data import Instance, field_export_match from app_data.datetimeutil import iso_seconds from app_data.models import DataEAV # columns not backed by a schema field, always kept regardless of the fields filter # (matches id/classname/keyname being unaffected everywhere else - see # field_export_match()). displayname/last_update/last_sync are deliberately NOT # here even though they're builtin attributes too: they route through # field_export_match() like a regular field, so an explicit "!displayname" (etc.) # filter can exclude them. _DATAVIEW_BUILTIN_SOURCES = {"keyname", "classname", "is_enabled"} # --------------------------------------------------------------------- # Helper # --------------------------------------------------------------------- def get_dataview_dict(schemaname): """ get available dataviews for schema use DataEAV cache IN: schema name (string OUT: dict { keyname:displayname , ... } """ reply = {} eav_entries = DataEAV.objects.filter( classname="_dataview", fieldname="target_class", value=schemaname ).values("keyname", "displayname") # session > single > CLASS_default > DEFAULT built-in dataview_name = None dv_found = False dataview_default = classname + "_default" dataview_selector = get_dataview_dict(classname) # request dataview_name = request.GET.get("dv", None) if not dataview_name: dataview_name = request.POST.get("dv", None) if dataview_name: if dataview_name == dataview_default: dv_found = True else: m = re.compile(r"[a-zA-Z0-9()_/.-]*$") if m.match(dataview_name): if dataview_name in dataview_selector: dv_found = True # in session ? if not dv_found: try: dataview_name = request.session["dataviews"][classname] if dataview_name: m = re.compile(r"[a-zA-Z0-9()_/.-]*$") if m.match(dataview_name): if dataview_name in dataview_selector: dv_found = True except Exception: pass # single entry in selector ? if not dv_found: if len(dataview_selector) == 1: # dataview_name = dataview_selector[0] dataview_name = next(iter(dataview_selector)) dv_found = True # CLASSNAME_default if not dv_found: if dataview_default in dataview_selector: dataview_name = dataview_default dv_found = True # get or built-in default if dv_found: dataview = DataView(keyname=dataview_name) else: dataview = DataView() if dataview.target_class != classname: # wrong class, use a default view dataview = DataView() # store dataview in user session if update_session: if dataview_name: if "dataviews" not in request.session: request.session["dataviews"] = {classname: dataview_name} # request.session.modified = True else: request.session["dataviews"][classname] = dataview_name request.session.modified = True # store request specifics in dataview object if dataview_default not in dataview_selector: dataview.default = dataview_default # dataview.selector = {} dataview.selector = dataview_selector # for k,v in dataview_selector.items(): # #dataview.selector[k] = v.displayname # dataview.selector[k] = v return dataview # ------------------------------------------- # DataView CLASS # ------------------------------------------- class DataView: def __init__(self, keyname=None): """Build the built-in default view (keyname/displayname/last_update columns) if keyname is None or not found/bound; otherwise load and parse the "_dataview" Instance's target_class/content/columns. self.columns is a list of {"label": ..., "fieldname": ..., "popover": ..., "filter": ...} dicts (one per declared column): "label" is the header text (a column's "from:" rename, or the plain field name), "fieldname" is the real underlying field name (see _column_source()) - None for a dict entry with no "from:" and no "popover:"/"filter:" override. "popover"/"filter" are optional per-column YAML booleans (instance_list's column-header "?" popup and its "Filter" button) - both default to True regardless of fieldname, so an existing dataview's display is unchanged; a dataview author sets "filter: no" (or "popover: no") on a column whose fieldname isn't actually searchable (e.g. an injected/external subfield) to suppress a button that would otherwise silently produce a query matching nothing.""" self.instance_id = None self.target_class = None self.keyname = keyname self.displayname = _("Default View") self.is_enabled = True self.content = {"columns": ["keyname", "displayname", "last_update"]} self.columns = [ {"label": "keyname", "fieldname": "keyname", "popover": True, "filter": True}, {"label": "displayname", "fieldname": "displayname", "popover": True, "filter": True}, {"label": "last_update", "fieldname": "last_update", "popover": True, "filter": True}, ] # request specifics self.default = None # name of default if not in selector self.selector = None # list of available dataviews for target_class if not keyname: return # query _dataview Schema for keyname instance = Instance.from_keyname(classname="_dataview", keyname=keyname) if not instance: return if not instance.is_bound: return self.instance_id = instance.id try: self.target_class = instance.fields["target_class"].value[0] except Exception: self.target_class = None self.displayname = instance.displayname self.is_enabled = instance.is_enabled try: raw = instance.fields["content"].value[0] except Exception: raw = "" try: self.content = yaml.safe_load(raw) except Exception as e: print(f"ERR - can't parse dataview content ({keyname}): {e}") return if type(self.content) is not dict: print(f"ERR - dataview content is not a mapping ({keyname})") self.content = {"columns": ["keyname", "displayname", "last_update"]} return # Extract columns # columns: # - nice name: # from: keyname # - col1 # - site: # from: mysitename_col2 # yaml_columns = self.content.get("columns", None) computed_columns = [] if isinstance(yaml_columns, list): for head in yaml_columns: fieldname = self._column_source(head) popover, show_filter = self._column_flags(head) if type(head) is str: computed_columns.append( { "label": head, "fieldname": fieldname, "popover": popover, "filter": show_filter, } ) elif type(head) is dict: for col, _nouse in head.items(): computed_columns.append( { "label": col, "fieldname": fieldname, "popover": popover, "filter": show_filter, } ) elif yaml_columns: print(f"ERR - dataview 'columns' is not a list ({keyname})") self.columns = computed_columns # NEXT: if no keyname/displayname , add keyname ? def _column_source(self, head): """Return the underlying instance field name for a raw column head entry (str column, or dict {label: {"from": source, ...}}), or None if unresolvable. A dict entry's own key is used as the fieldname when it carries a "popover:"/"filter:" override but no "from:" (same field, just flags on its plain name); a dict entry with neither "from:" nor those overrides is a pure label with no real field behind it.""" if type(head) is str: return head if type(head) is dict: for label, operator_line in head.items(): if not isinstance(operator_line, dict): continue if "from" in operator_line: return operator_line["from"] if "popover" in operator_line or "filter" in operator_line: return label return None def _column_flags(self, head): """Return (popover, filter) booleans for a raw column head entry, from optional "popover:"/"filter:" YAML keys (dict form only) - both default to True (for a plain string column, or when the dict form omits them), so an existing dataview's popover/Filter-button display is unchanged unless a column explicitly opts out.""" if type(head) is dict: for _label, operator_line in head.items(): if not isinstance(operator_line, dict): continue return bool(operator_line.get("popover", True)), bool( operator_line.get("filter", True) ) return True, True def _column_passes_filter(self, head, fields): """True if a raw column head should be kept: columns whose source is in _DATAVIEW_BUILTIN_SOURCES (keyname/classname/is_enabled) and columns with no resolvable source field always pass; other columns (including displayname/last_update/last_sync) are matched against fields via field_export_match().""" source = self._column_source(head) if source is None or source in _DATAVIEW_BUILTIN_SOURCES: return True return field_export_match(source, fields) @staticmethod def _format_datetime_cell(dt, mode): """Format a last_sync/last_update cell: user-friendly "YYYY-MM-DD HH:MM:SS" in the active timezone for on-screen mode="datapoint" (matching the |date template filter used on instance_detail - a plain strftime() would print dt's own tzinfo, i.e. UTC, instead of the active timezone), strict ISO 8601 with second precision for mode="value"/"csv" (machine-readable export - also keeps JSON serializable, since a raw datetime isn't). "" when dt is None.""" if not dt: return "" if mode == "datapoint": return timezone.localtime(dt).strftime("%Y-%m-%d %H:%M:%S") return iso_seconds(dt) def _cell_for_field(self, instance, name, mode): """Return one export/UI cell for a plain field/attribute name: a builtin instance attribute (keyname/classname/displayname/is_enabled/last_update/ last_sync - the last two via _format_datetime_cell()) or a real schema field (value depends on mode: datapoint/value/csv). Returns "" for anything else (e.g. a stale/renamed field no longer on the schema). Shared by filter() (per the view's own column defs, both the plain and the "from:"-rename column forms) and get_csv_export_row() (per an explicit flat field name list).""" if name == "keyname": return instance.keyname if name == "classname": return instance.classname if name == "displayname": return instance.displayname if name == "is_enabled": return "X" if instance.is_enabled else "" if name == "last_update": return self._format_datetime_cell(instance.last_update, mode) if name == "last_sync": return self._format_datetime_cell(instance.last_sync, mode) if name in instance.fields: if mode == "value": return instance.fields[name].get_value() if mode == "datapoint": return instance.fields[name].get_datapoint_ui_detail() if mode == "csv": return instance.fields[name].get_csv_cell() return "" def get_columns(self, fields=None, override_columns=None): """Return the list of column names from self.content["columns"], for export: the underlying source field name (see _column_source()) rather than the view's display label/rename, so exported files stay importable (a dataview labeling a "keyname"-backed column "Login" still exports a "keyname" header/key, not "Login"). Falls back to the label only when the column has no resolvable source (e.g. a dict entry with no "from"). `fields` (see field_export_match()) drops columns whose underlying instance field doesn't match; builtin columns and columns with no resolvable source field are always kept. `classname`/`keyname` are guaranteed present (prepended if the columns considered don't already cover them, by source - a renamed `from: keyname` column counts) so the export stays re-importable via `cavaliba load` even if they're never displayed. Columns sharing the same source (e.g. two different labels both `from: classname`) are de-duplicated, keeping only the first, to avoid two identical headers. `override_columns`, when given (e.g. from the export modal's dv_fields textbox), replaces this view's own declared columns entirely - each entry is a plain real field name (not resolved against any "from:" rename), still subject to the same `fields` filter and keyname/classname guarantee as the view's own columns would be.""" reply = [] sources_seen = set() raw = ( override_columns if override_columns is not None else (self.content.get("columns") or []) ) for entry in raw: if not self._column_passes_filter(entry, fields): continue source = self._column_source(entry) if source: if source in sources_seen: continue sources_seen.add(source) reply.append(source) elif type(entry) is dict: reply.append(list(entry.keys())[0]) else: reply.append("") if "keyname" not in sources_seen: reply = ["keyname"] + reply if "classname" not in sources_seen: reply = ["classname"] + reply return reply def filter( self, instance=None, mode="datapoint", value_only=False, fields=None, override_columns=None ): """ IN: * dataview() object, instance() object * mode: datapoint|value|csv OUT: * array ["val1","val2", {}, {},{} ...] values: * string if keyname/displayname/... * mode=datapoint : datapoint dict {} * mode=value {'fieldname': 'mystring', 'displayname': 'MyString', 'description': 'A string', 'dataformat': 'string', 'dataformat_ext': None, 'is_multi': False, 'bigset': False, 'value': 'data for test1-01' {} for more complex fields like enumerate, user } * mode=csv : value or val1^val2^val3 ... if multi `fields` (see field_export_match()) drops columns whose underlying instance field doesn't match, mirroring get_columns(fields=...) so the returned array stays aligned with its header. For mode in value/csv (export), `classname`/`keyname` are guaranteed present and duplicate-source columns are skipped, mirroring get_columns() - see its docstring - so exported rows stay aligned with get_columns()'s header. mode=datapoint (UI table) skips that guarantee instead, since its header is self.columns, which carries no such guarantee - adding hidden columns there would desync row cells from headers. `override_columns`, when given, replaces this view's own declared columns the same way as get_columns(override_columns=...) - pass the same value used to build get_columns() to stay aligned. """ if not instance: return data = [] sources_seen = set() # get dataview requested columns columns_array = ( override_columns if override_columns is not None else (self.content.get("columns") or []) ) for head in columns_array: if not self._column_passes_filter(head, fields): continue source = self._column_source(head) if source: if source in sources_seen: continue sources_seen.add(source) # no operator, no rename, direct column name if type(head) is str: data.append(self._cell_for_field(instance, head, mode)) # operator, rename, ... # - newcol: <= head is dict (col==head) # from: old_col <= operator_line = operator: operator_value # popover/filter: consumed only for column-header metadata, # not a cell-value source - see _column_flags() elif type(head) is dict: # `source` (computed above via _column_source()) already # resolves both the explicit "from:" case and the "popover:"/ # "filter:"-only, same-field case - same builtin-attribute / # real-field resolution as the plain (non-renamed) column form cell = self._cell_for_field(instance, source, mode) if source else "" data.append(cell) else: data.append("") if mode != "datapoint": if "keyname" not in sources_seen: data = [instance.keyname] + data if "classname" not in sources_seen: data = [instance.classname] + data return data def get_csv_fieldnames(self): """Deduped real source field names from this view's own columns definition, in view order - no forced keyname/classname/is_enabled, no fields filter. Basis for the CSV-export dataview textbox default and its server-side fallback when that textbox is left empty.""" reply = [] seen = set() for entry in self.content.get("columns") or []: source = self._column_source(entry) name = source or (list(entry.keys())[0] if type(entry) is dict else None) if name and name not in seen: seen.add(name) reply.append(name) return reply @staticmethod def parse_dv_fields(fields_text): """Parse the export modal's "dv_fields" textbox into a deduped list of raw column tokens (real field names), in the order typed, or None if blank. This is the uniform field-list override applied to get_columns()/filter() (YAML/JSON) and get_csv_export_columns() (CSV) alike - each token is used as a real field name directly, not resolved against the view's own "from:" renames.""" if not fields_text or not fields_text.strip(): return None seen = set() reply = [] for tok in fields_text.split(","): tok = tok.strip() if tok and tok not in seen: seen.add(tok) reply.append(tok) return reply def get_csv_export_columns(self, fields_text=None): """Resolve the column list for the pure-passthrough CSV dataview export: always starts with `_schema` (classname), then either an explicit override from parse_dv_fields(fields_text) if given, else this view's own get_csv_fieldnames(). No forced classname/id/keyname beyond that - unlike get_columns(), this never adds columns the view (or the user's override) didn't ask for.""" override = self.parse_dv_fields(fields_text) reply = override if override is not None else self.get_csv_fieldnames() return ["_schema"] + reply def get_csv_export_row(self, instance, columns): """One CSV row for get_csv_export_columns(): `columns[0]` is always `_schema` (see get_csv_export_columns()), resolved directly to `instance.classname`; the rest is one cell per name via _cell_for_field(), independent of this view's own column definitions (no filtering, no keyname/classname guarantee).""" # /!\ must match get_csv_export_columns return [instance.classname] + [ self._cell_for_field(instance, name, "csv") for name in columns[1:] ] def print(self): """Print a human-readable dump of the dataview (debug helper).""" print("_dataview") print(f" keyname: {self.keyname}") print(f" instance_id: {self.instance_id}") print(f" target_class: {self.target_class}") print(f" is_enabled: {self.is_enabled}") print(f" displayname: {self.displayname}") print(f" columns: {self.columns}") print(f" content: {self.content}")