# (c) cavaliba.com - data - views_exporter.py """ The Exporter tool page and the export Download page. Views only: every option is parsed and every byte is produced by app_data.exporter, so the same export can later be driven from the API or the CLI. Exports whose matched row count exceeds the EXPORT_INTERACTIVE_MAX_SIZE configuration entry are handed off to an async DataTask (app_data.tasks.submit_export) and followed in a progress modal instead of being built inside the request. Method inventory: export_tool, export_fields, export_progress, export_abort, export_result, download_folder. Private helpers: _form_context, _schema_choices, _checked_columns. """ from django.conf import settings from django.contrib import messages from django.http import FileResponse, Http404, HttpResponse, HttpResponseNotAllowed from django.shortcuts import redirect, render from django.utils import timezone from django.utils.translation import gettext as _ from app_data.aaa import start_view from app_data.dataview import DataView from app_data.exporter import ( EXPORT_FORMATS, EXPORT_MANDATORY_FIELDS, ExportSpec, export_column_catalog, export_count, export_render, get_file_content, list_export_folder, send_file, spec_from_post, ) from app_data.filestore import uuid_to_queryset from app_data.models import DataTask from app_data.schema import Schema from app_data.task_manager import abort_task from app_home.configuration import get_configuration from app_home.log import DEBUG, ERROR, INFO, WARNING, log def _schema_choices(aaa): """Schemas the caller may read, for the Exporter's schema selector.""" return [s for s in Schema.listall() if s.has_read_permission(aaa)] def _readable_schema_name(schema_name, aaa): """Return `schema_name` if the caller may read that schema, else None. The schema reaches the form from ?schema= (a list page's Export link, or a hand written URL), not from the gated selector, so it is checked here rather than trusted. Without this the column selector would list a schema's field names and displaynames to a caller who cannot read the schema at all - the export itself is refused later by _run_export(), but the field list would already be out. """ if not schema_name: return None schema = Schema.from_name(schema_name) if schema and schema.has_read_permission(aaa): return schema_name log(WARNING, aaa=aaa, app="data", view="export_form", action="schema", status="KO", data=f"Not allowed on {schema_name}") # fmt: skip return None def _checked_columns(catalog, schema_name=None, dataview_name=None, columns=None): """Names the column selector starts with checked, as a set. Three cases, in order: an explicit `columns` list (a re-rendered form echoing what was submitted), a dataview whose own columns pre-check the selector (the user arrived from a list page with ?dv=), or the default - every builtin and every declared schema field, injected subfields left unchecked (they are computed, so they never make it back on import). Only names the catalog actually offers are returned, plus the mandatory ones, so the checked set can never disagree with the checkboxes rendered next to it. DataView(keyname=...) is used directly rather than dataview_for_request(), which would rewrite request.session["dataviews"] and so silently change which view the user's list page shows next. Parameters: catalog (list): export_column_catalog() output schema_name (str): the selected classname, if any dataview_name (str): a view to pre-check from, if any columns (str): a submitted comma-separated column list, if any Returns: set: column names to render checked """ offered = {entry["name"] for entry in catalog} checked = set(EXPORT_MANDATORY_FIELDS) if columns: wanted = DataView.parse_dv_fields(columns) or [] return checked | (set(wanted) & offered) if schema_name and dataview_name: dataview = DataView(keyname=dataview_name) if dataview.target_class == schema_name: wanted = dataview.get_csv_fieldnames() if wanted: return checked | (set(wanted) & offered) return checked | {e["name"] for e in catalog if e["group"] in ("builtin", "schema")} def _form_context(context, aaa, spec=None, schema_name=None, dataview_name=None, columns=None): """Fill the Exporter form context: schema selector, the column selector and its checked set, the option choices, and `values` - every field's current value, so a re-rendered form (rejected input, or a submitted async export) keeps what the user set. `spec` is the validated request when there is one; otherwise the ExportSpec defaults are what the blank form shows. A schema the caller may not read is dropped here (see _readable_schema_name()), leaving the blank form.""" defaults = spec or ExportSpec(schema=schema_name or "") schema_name = _readable_schema_name(schema_name or defaults.schema, aaa) context["schemas"] = _schema_choices(aaa) context["schema_name"] = schema_name or "" context["formats"] = EXPORT_FORMATS context["actions"] = settings.ACTION_LIST context["supported_encoding"] = settings.SUPPORTED_ENCODING context["default_csv_delimiter"] = get_configuration(keyname="CSV_DELIMITER") context["max_sync"] = int(get_configuration("data", "EXPORT_INTERACTIVE_MAX_SIZE")) context["values"] = { "query": defaults.query, "format": defaults.format, "delimiter": defaults.delimiter if spec is not None else "", "encoding": defaults.encoding, "action": defaults.action or "", "refs": ",".join(defaults.refs) if defaults.refs else "", "rev": defaults.rev or "", } catalog = export_column_catalog(schema_name) if schema_name else [] # a submitted "*" is the ExportSpec default, not a user choice - it carries no # column names, so the selector falls back to its own default checked set if columns is None and spec is not None and spec.columns != "*": columns = spec.columns context["catalog"] = catalog context["checked"] = _checked_columns( catalog, schema_name=schema_name, dataview_name=dataview_name, columns=columns ) return context # ------------------------------------------------------------------------- # EXPORTER TOOL # ------------------------------------------------------------------------- def export_tool(request): """GET: render the Exporter form, prepopulated from ?schema=/?dv=/?query= when the user arrived from a schema list page. POST: validate, then either stream the file back (small export) or submit an async DataTask and re-render with the progress modal open (large export).""" context = start_view( request, app="data", view="export_tool", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] if request.method == "POST": response = _run_export(request, context, aaa) if response: return response return render(request, "app_data/exporter.html", context) schema_name = (request.GET.get("schema") or "").strip() or None spec = ( ExportSpec(schema=schema_name, query=(request.GET.get("query") or "").strip()) if schema_name else None ) # ?dv= is read but never rendered as a selector: it only decides which columns # start checked, so arriving from a list page exports what that page showed _form_context( context, aaa, spec=spec, schema_name=schema_name, dataview_name=(request.GET.get("dv") or "").strip() or None, ) log(DEBUG, aaa=aaa, app="data", view="export_tool", action="GET", status="OK") return render(request, "app_data/exporter.html", context) def _run_export(request, context, aaa): """Handle an Exporter POST. Returns a ready-made response (the downloaded file), or None to stay on the page - the caller re-renders it with the bound values, so a rejected input or a submitted async task keeps the form filled in.""" spec, err = spec_from_post(request.POST) if err: _form_context( context, aaa, schema_name=(request.POST.get("schema") or "").strip() or None, columns=",".join(request.POST.getlist("columns")), ) messages.add_message(request, messages.ERROR, _("Export failed") + f" - {err}") log(WARNING, aaa=aaa, app="data", view="export", action="post", status="KO", data=str(err)) # fmt: skip return None # keep the form showing what was asked for, whatever happens next _form_context(context, aaa, spec=spec, schema_name=spec.schema) schema = Schema.from_name(spec.schema) if not schema: messages.add_message(request, messages.ERROR, _("Export failed") + " - " + _("Unknown schema")) # fmt: skip return None if not schema.has_read_permission(aaa): messages.add_message(request, messages.ERROR, _("Not allowed")) log(ERROR, aaa=aaa, app="data", view="export", action="post", status="KO", data=f"Not allowed on {spec.schema}") # fmt: skip return None count = export_count(spec) max_sync = int(get_configuration("data", "EXPORT_INTERACTIVE_MAX_SIZE")) if count > max_sync: from app_data.tasks import submit_export handle = submit_export( spec_dict=spec.to_dict(), aaa=aaa, owner_type="user", owner_id=aaa.get("username", ""), ) context["task_handle"] = handle log(INFO, aaa=aaa, app="data", view="export", action="post", status="OK", data=f"async export submitted, handle={handle}, rows={count}") # fmt: skip return None raw, filename, content_type, stats = export_render(spec, aaa) if raw is None: # on failure export_render() returns the error message in place of stats messages.add_message(request, messages.ERROR, _("Export failed") + f" - {stats}") log(ERROR, aaa=aaa, app="data", view="export", action="post", status="KO", data=str(stats)) # fmt: skip return None for error in stats["errors"][:5]: messages.add_message(request, messages.WARNING, str(error)) log(INFO, aaa=aaa, app="data", view="export", action="post", status="OK", data=f"{spec.schema} {spec.format} ok={stats['count_ok']} ko={stats['count_ko']}") # fmt: skip response = HttpResponse(raw, content_type=content_type) response["Content-Disposition"] = f'attachment; filename="{filename}"' return response # ------------------------------------------------------------------------- # EXPORTER - htmx partial : schema changed # ------------------------------------------------------------------------- def export_fields(request): """htmx partial: rebuild the column selector for the schema picked in the form. No dataview is involved here - a schema change starts from the selector's own default checked set (every builtin and schema field), not from a view.""" context = start_view( request, app="data", view="export_fields", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] schema_name = (request.GET.get("schema") or "").strip() or None _form_context(context, aaa, schema_name=schema_name) return render(request, "app_data/exporter_columns.html", context) # ------------------------------------------------------------------------- # EXPORTER - async task modal : progress / abort / result # ------------------------------------------------------------------------- def _own_task(handle, aaa): """Resolve an export DataTask the caller is allowed to see, or None. Gated on ownership rather than on the task permissions: the progress modal is the only UI a user without p_task_view has for their own large export, so p_task_view must not be required here. p_task_all still widens it to anyone's task, same rule as views_task. """ task = DataTask.objects.filter(handle=handle).first() if not task: return None if task.owner_id != aaa.get("username", "") and "p_task_all" not in aaa.get("perms", []): return None return task def _progress_context(context, task): """Derive the modal's live figures from the task row. rate and eta are computed here rather than stored: DataTask already carries the elapsed duration and the progress counters, so deriving keeps the task's write path cheap and the stored payload small (same approach as task_detail's `estimated`). """ progress = task.progress or {} output = task.output or {} is_running = task.state in ("QUEUED", "RUNNING") duration = task.duration or 0 # an export shorter than exporter.BATCH_SIZE never calls on_batch, so it has # no `progress` at all - every figure must fall back to the final stats in # `output`, which finish_task() always writes percent = progress.get("percent") or (0 if is_running else 100) count_done = progress.get("count") or output.get("count_done") or 0 context["task"] = task context["percent"] = percent # the bar's width comes from a .cv-bar-NN class (5% steps) rather than an # inline style, so the template keeps all styling in cavaliba.css context["percent_step"] = min(100, max(0, int(round(percent / 5.0)) * 5)) context["total"] = progress.get("total") or output.get("total") or 0 context["count_done"] = count_done context["count_ok"] = progress.get("count_ok", output.get("count_ok", 0)) context["count_ko"] = progress.get("count_ko", output.get("count_ko", 0)) context["rate"] = round(count_done / duration, 1) if duration and count_done else None context["eta"] = ( round(duration * (100 - percent) / percent, 1) if is_running and duration and percent else None ) context["errors_sample"] = (output.get("errors") or progress.get("errors_sample") or [])[:5] context["attachment"] = task.attachment or None context["is_running"] = is_running return context def export_progress(request, handle): """htmx partial: one render of the export progress modal body. Re-requests itself every 2s while the task is live and stops polling once it is terminal.""" context = start_view( request, app="data", view="export_progress", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] task = _own_task(handle, aaa) if not task: raise Http404 _progress_context(context, task) return render(request, "app_data/exporter_progress.html", context) def export_abort(request, handle): """POST from the modal's Abort button: request the task stop, then re-render the modal body. The worker notices on its next batch and discards its partial file.""" context = start_view( request, app="data", view="export_abort", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] task = _own_task(handle, aaa) if not task: raise Http404 if request.method != "POST": return HttpResponseNotAllowed(["POST"]) stopped = abort_task(handle, aaa=aaa) log(INFO, aaa=aaa, app="data", view="export_abort", action="POST", status="OK" if stopped else "KO", data=f"handle={handle}") # fmt: skip task.refresh_from_db() _progress_context(context, task) return render(request, "app_data/exporter_progress.html", context) def export_result(request, handle): """Download a finished async export's file. Not served by views.file_display: that resolves the owning instance through the DataEAV "file" entry, and an export result belongs to no instance, so it would always be refused. Gated here on the task instead - owner (or p_task_all), the task actually DONE with an attachment, and the file not past its expiry. """ context = start_view( request, app="data", view="export_result", perm="p_data_export", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) aaa = context["aaa"] task = _own_task(handle, aaa) if not task: raise Http404 attachment = task.attachment or {} fileid = attachment.get("fileid") if task.state != "DONE" or not fileid: messages.add_message(request, messages.ERROR, _("Export result not available")) return redirect("app_data:data_exporter") datafile = uuid_to_queryset(fileid) if not datafile or not datafile.filepath: messages.add_message(request, messages.ERROR, _("Export result not available")) log(WARNING, aaa=aaa, app="data", view="export_result", action="GET", status="KO", data=f"missing datafile {fileid} for handle={handle}") # fmt: skip return redirect("app_data:data_exporter") if datafile.not_after and datafile.not_after < timezone.now(): messages.add_message(request, messages.ERROR, _("Export result has expired")) return redirect("app_data:data_exporter") log(DEBUG, aaa=aaa, app="data", view="export_result", action="GET", status="OK", data=f"handle={handle}") # fmt: skip try: # as_attachment: FileResponse defaults to "inline", which makes a browser # open the csv/yaml/json in a tab instead of saving it return FileResponse( open(datafile.filepath, "rb"), as_attachment=True, filename=datafile.filename ) except OSError: messages.add_message(request, messages.ERROR, _("Export result not available")) return redirect("app_data:data_exporter") # ------------------------------------------------------------------------- # EXPORT DOWNLOAD (export folder viewer) # ------------------------------------------------------------------------- def download_folder(request): """List, preview (?view=) and download (?dl=) the files produced in the export folder by the exporter cron script. Addressed by md5 of the filename, matched against the folder listing - never by a path from the request.""" context = start_view( request, app="data", view="download_folder", perm="p_data_export_viewer", noauthz="app_home:private", ) if context["redirect"]: return redirect(context["redirect"]) dl_md5 = request.GET.get("dl") view_md5 = request.GET.get("view") context["export_files"] = list_export_folder() if dl_md5: for export_file in context["export_files"]: if export_file["filename_md5"] == dl_md5: response = send_file(export_file["filename"]) if response: return response messages.add_message(request, messages.ERROR, _("Error file not found")) return render(request, "app_data/download.html", context) if view_md5: for export_file in context["export_files"]: if export_file["filename_md5"] == view_md5: content = get_file_content(export_file["filename"]) if content: context["file_content"] = content context["file_name"] = export_file["filename"] return render(request, "app_data/download.html", context) messages.add_message(request, messages.ERROR, _("Error file not found")) return render(request, "app_data/download.html", context) return render(request, "app_data/download.html", context)