# (c) cavaliba.com - data - views_exporter.py """ The Exporter tool page and the export Download page. Views only: every option is parsed and every byte is produced by app_data.exporter, so the same export can later be driven from the API or the CLI. Every export is an async DataTask (app_data.tasks.submit_export) followed in a progress modal, whatever its size (5.4.0; there used to be a row-count threshold, the EXPORT_INTERACTIVE_MAX_SIZE configuration entry, below which the file was built inside the request and streamed straight back) - so progress, abort and the result download are the same UI for a 5-row export and a 500k-row one. Method inventory: export_tool, export_fields, export_progress, export_abort, export_result, download_folder. Private helpers: _form_context, _schema_choices, _checked_columns, _readable_schema_name, _run_and_submit. Task-modal ownership and progress figures are shared with the Import Tool - see app_data/task_ui.py. """ from django.conf import settings from django.contrib import messages from django.http import FileResponse, Http404, HttpResponseNotAllowed from django.shortcuts import redirect, render from django.utils import timezone from django.utils.translation import gettext as _ from app_data.aaa import start_view from app_data.dataview import DataView from app_data.exporter import ( EXPORT_FORMATS, EXPORT_MANDATORY_FIELDS, ExportSpec, export_column_catalog, get_file_content, list_export_folder, send_file, spec_from_post, ) from app_data.filestore import uuid_to_queryset from app_data.schema import Schema from app_data.task_manager import abort_task from app_data.task_ui import own_task, progress_context from app_data.tasks import submit_export from app_home.configuration import get_configuration from app_home.log import INFO, WARNING, log2 def _schema_choices(aaa): """Schemas the caller may read, for the Exporter's schema selector.""" return [s for s in Schema.listall() if s.has_read_permission(aaa)] def _readable_schema_name(schema_name, aaa): """Return `schema_name` if the caller may read that schema, else None. The schema reaches the form from ?schema= (a list page's Export link, or a hand written URL), not from the gated selector, so it is checked here rather than trusted. Without this the column selector would list a schema's field names and displaynames to a caller who cannot read the schema at all - the export itself is refused later by _run_and_submit(), but the field list would already be out. """ if not schema_name: return None schema = Schema.from_name(schema_name) if schema and schema.has_read_permission(aaa): return schema_name log2.deny("export.form", reason="permission_denied", target=(schema_name, "")) return None def _checked_columns(catalog, schema_name=None, dataview_name=None, columns=None): """Names the column selector starts with checked, as a set. Three cases, in order: an explicit `columns` list (a re-rendered form echoing what was submitted), a dataview whose own columns pre-check the selector (the user arrived from a list page with ?dv=), or the default - every builtin and every declared schema field, injected subfields left unchecked (they are computed, so they never make it back on import). Only names the catalog actually offers are returned, plus the mandatory ones, so the checked set can never disagree with the checkboxes rendered next to it. DataView(keyname=...) is used directly rather than dataview_for_request(), which would rewrite request.session["dataviews"] and so silently change which view the user's list page shows next. Parameters: catalog (list): export_column_catalog() output schema_name (str): the selected classname, if any dataview_name (str): a view to pre-check from, if any columns (str): a submitted comma-separated column list, if any Returns: set: column names to render checked """ offered = {entry["name"] for entry in catalog} checked = set(EXPORT_MANDATORY_FIELDS) if columns: wanted = DataView.parse_dv_fields(columns) or [] return checked | (set(wanted) & offered) if schema_name and dataview_name: dataview = DataView(keyname=dataview_name) if dataview.target_class == schema_name: wanted = dataview.get_csv_fieldnames() if wanted: return checked | (set(wanted) & offered) return checked | {e["name"] for e in catalog if e["group"] in ("builtin", "schema")} def _form_context(context, aaa, spec=None, schema_name=None, dataview_name=None, columns=None): """Fill the Exporter form context: schema selector, the column selector and its checked set, the option choices, and `values` - every field's current value, so a re-rendered form (rejected input, or a submitted async export) keeps what the user set. `spec` is the validated request when there is one; otherwise the ExportSpec defaults are what the blank form shows. A schema the caller may not read is dropped here (see _readable_schema_name()), leaving the blank form.""" defaults = spec or ExportSpec(schema=schema_name or "") schema_name = _readable_schema_name(schema_name or defaults.schema, aaa) context["schemas"] = _schema_choices(aaa) context["schema_name"] = schema_name or "" context["formats"] = EXPORT_FORMATS context["actions"] = settings.ACTION_LIST context["supported_encoding"] = settings.SUPPORTED_ENCODING context["default_csv_delimiter"] = get_configuration(keyname="CSV_DELIMITER") context["values"] = { "query": defaults.query, "format": defaults.format, "delimiter": defaults.delimiter if spec is not None else "", "encoding": defaults.encoding, "action": defaults.action or "", "refs": ",".join(defaults.refs) if defaults.refs else "", "rev": defaults.rev or "", } catalog = export_column_catalog(schema_name) if schema_name else [] # a submitted "*" is the ExportSpec default, not a user choice - it carries no # column names, so the selector falls back to its own default checked set if columns is None and spec is not None and spec.columns != "*": columns = spec.columns context["catalog"] = catalog context["checked"] = _checked_columns( catalog, schema_name=schema_name, dataview_name=dataview_name, columns=columns ) return context # ------------------------------------------------------------------------- # EXPORTER TOOL # ------------------------------------------------------------------------- def export_tool(request): """GET: render the Exporter form, prepopulated from ?schema=/?dv=/?query= when the user arrived from a schema list page. POST: validate, then submit the export as a DataTask and re-render with its handle, which opens the progress modal.""" context = start_view( request, app="data", view="export_tool", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] # always present, so the template (and tests) can read it unconditionally: # only a submitted export sets it, and only then does the modal open context["task_handle"] = None if request.method == "POST": context["task_handle"] = _run_and_submit(request, context, aaa) return render(request, "app_data/exporter.html", context) schema_name = (request.GET.get("schema") or "").strip() or None spec = ( ExportSpec(schema=schema_name, query=(request.GET.get("query") or "").strip()) if schema_name else None ) # ?dv= is read but never rendered as a selector: it only decides which columns # start checked, so arriving from a list page exports what that page showed _form_context( context, aaa, spec=spec, schema_name=schema_name, dataview_name=(request.GET.get("dv") or "").strip() or None, ) log2.debug("render export tool") return render(request, "app_data/exporter.html", context) def _run_and_submit(request, context, aaa): """Handle an Exporter POST: validate the spec, then submit the export as a DataTask. Returns the task handle (str), or None when nothing was submitted (rejected input, unknown schema, no read permission). Either way the caller re-renders the page with the bound values, so the form keeps what the user asked for - and a handle is also what opens the progress modal.""" spec, err = spec_from_post(request.POST) if err: _form_context( context, aaa, schema_name=(request.POST.get("schema") or "").strip() or None, columns=",".join(request.POST.getlist("columns")), ) messages.add_message(request, messages.ERROR, _("Export failed") + f" - {err}") log2.fail("export.run", reason="invalid_spec", message=str(err), level=WARNING) return None # keep the form showing what was asked for, whatever happens next _form_context(context, aaa, spec=spec, schema_name=spec.schema) schema = Schema.from_name(spec.schema) if not schema: messages.add_message(request, messages.ERROR, _("Export failed") + " - " + _("Unknown schema")) # fmt: skip return None if not schema.has_read_permission(aaa): messages.add_message(request, messages.ERROR, _("Not allowed")) log2.deny("export.run", reason="permission_denied", target=(spec.schema, "")) return None # no row count here: the worker counts the queryset itself (and reports it as # the task's `total`), so counting again in the request would only duplicate # the query handle = submit_export( spec_dict=spec.to_dict(), aaa=aaa, owner_type="user", owner_id=aaa.get("username", ""), ) log2.ok("export.submit", target=("task", handle), extra={"schema": spec.schema, "format": spec.format}) # fmt: skip return handle # ------------------------------------------------------------------------- # EXPORTER - htmx partial : schema changed # ------------------------------------------------------------------------- def export_fields(request): """htmx partial: rebuild the column selector for the schema picked in the form. No dataview is involved here - a schema change starts from the selector's own default checked set (every builtin and schema field), not from a view.""" context = start_view( request, app="data", view="export_fields", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] schema_name = (request.GET.get("schema") or "").strip() or None _form_context(context, aaa, schema_name=schema_name) return render(request, "app_data/exporter_columns.html", context) # ------------------------------------------------------------------------- # EXPORTER - async task modal : progress / abort / result # ------------------------------------------------------------------------- def export_progress(request, handle): """htmx partial: one render of the export progress modal body. Re-requests itself every 2s while the task is live and stops polling once it is terminal.""" context = start_view( request, app="data", view="export_progress", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] task = own_task(handle, aaa) if not task: raise Http404 progress_context(context, task) return render(request, "app_data/exporter_progress.html", context) def export_abort(request, handle): """POST from the modal's Abort button: request the task stop, then re-render the modal body. The worker notices on its next batch and discards its partial file.""" context = start_view( request, app="data", view="export_abort", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] task = own_task(handle, aaa) if not task: raise Http404 if request.method != "POST": return HttpResponseNotAllowed(["POST"]) stopped = abort_task(handle, aaa=aaa) if stopped: log2.ok("export.abort", target=("task", handle)) else: log2.fail("export.abort", reason="not_running", target=("task", handle), level=WARNING) task.refresh_from_db() progress_context(context, task) return render(request, "app_data/exporter_progress.html", context) def export_result(request, handle): """Download a finished async export's file. Not served by views.file_display: that resolves the owning instance through the DataEAV "file" entry, and an export result belongs to no instance, so it would always be refused. Gated here on the task instead - owner (or p_task_all), the task actually DONE with an attachment, and the file not past its expiry. """ context = start_view( request, app="data", view="export_result", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] task = own_task(handle, aaa) if not task: raise Http404 attachment = task.attachment or {} fileid = attachment.get("fileid") if task.state != "DONE" or not fileid: messages.add_message(request, messages.ERROR, _("Export result not available")) return redirect("app_data:data_exporter") datafile = uuid_to_queryset(fileid) if not datafile or not datafile.filepath: messages.add_message(request, messages.ERROR, _("Export result not available")) log2.fail("export.download", reason="file_missing", level=WARNING, target=("task", handle), extra={"fileid": str(fileid)}) # fmt: skip return redirect("app_data:data_exporter") if datafile.not_after and datafile.not_after < timezone.now(): messages.add_message(request, messages.ERROR, _("Export result has expired")) return redirect("app_data:data_exporter") log2.access("export.download", target=("task", handle), level=INFO) try: # as_attachment: FileResponse defaults to "inline", which makes a browser # open the csv/yaml/json in a tab instead of saving it return FileResponse( open(datafile.filepath, "rb"), as_attachment=True, filename=datafile.filename ) except OSError: messages.add_message(request, messages.ERROR, _("Export result not available")) return redirect("app_data:data_exporter") # ------------------------------------------------------------------------- # EXPORT DOWNLOAD (export folder viewer) # ------------------------------------------------------------------------- def download_folder(request): """List, preview (?view=) and download (?dl=) the files produced in the export folder by the exporter cron script. Addressed by md5 of the filename, matched against the folder listing - never by a path from the request.""" context = start_view( request, app="data", view="download_folder", perm="p_data_export_viewer", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) dl_md5 = request.GET.get("dl") view_md5 = request.GET.get("view") context["export_files"] = list_export_folder() if dl_md5: for export_file in context["export_files"]: if export_file["filename_md5"] == dl_md5: response = send_file(export_file["filename"]) if response: return response messages.add_message(request, messages.ERROR, _("Error file not found")) return render(request, "app_data/download.html", context) if view_md5: for export_file in context["export_files"]: if export_file["filename_md5"] == view_md5: content = get_file_content(export_file["filename"]) if content: context["file_content"] = content context["file_name"] = export_file["filename"] return render(request, "app_data/download.html", context) messages.add_message(request, messages.ERROR, _("Error file not found")) return render(request, "app_data/download.html", context) return render(request, "app_data/download.html", context)