# (c) cavaliba.com - tests / data / exporter column modes and options import io import json import yaml from django.test import TestCase, override_settings import app_home.cache as cache from app_data.exporter import ExportSpec, export_write from tests import helper AAA = {"username": "unittest", "perms": ["p_data_admin"]} ENUM_CONTENT = ( '- value: "A"\n' ' widget: "green_circle"\n' " enumstring: string AAA\n" '- value: "B"\n' ' widget: "yellow_circle"\n' ) def _csv(spec): out = io.StringIO() stats = export_write(spec, AAA, out) lines = out.getvalue().splitlines() return lines, stats class ExporterColumnsTest(TestCase): def setUp(self): cache.clear() helper.add_admin_user(login="unittest") helper.add_instance( classname="_enumerate", keyname="exp_enum", fields={"content": ENUM_CONTENT}, ) helper.add_schema( classname="expref", field_definition={ "refstring": {"dataformat": "string", "displayname": "RefString"}, }, ) helper.add_instance( classname="expref", keyname="expref-01", fields={"refstring": "from the referenced object"}, ) helper.add_schema( classname="expcol", field_definition={ "mystring": {"dataformat": "string", "displayname": "MyString", "order": 1}, "mytext": {"dataformat": "text", "displayname": "MyText", "order": 2}, "myref": { "dataformat": "schema", "displayname": "MyRef", "dataformat_ext": "expref refstring", "order": 3, }, "myenum": { "dataformat": "enumerate", "displayname": "MyEnum", "dataformat_ext": "exp_enum", "order": 4, }, }, ) helper.add_instance( classname="expcol", keyname="expcol-01", fields={ "mystring": "hello", "mytext": "a long text", "myref": "expref-01", "myenum": "A", }, ) # --------------------------------------------------------------- # CSV - filter mode ("*") # --------------------------------------------------------------- @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_star_starts_with_schema_and_keyname(self): lines, stats = _csv(ExportSpec(schema="expcol", format="csv")) header = lines[0].split(",") self.assertEqual(header[0], "_schema") self.assertEqual(header[1], "keyname") self.assertEqual(stats["count_ok"], 1) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_excludes_text_fields(self): lines, _stats = _csv(ExportSpec(schema="expcol", format="csv")) self.assertNotIn("mytext", lines[0].split(",")) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_filter_mode_never_includes_injected_subfields(self): # a filter-mode header is resolved from the first exported row, and an # injected subfield only exists on a row whose reference is set - including # it would shift every later row that has none. Explicit mode is the way to # ask for one (see test_csv_explicit_mode_keeps_typed_order). lines, _stats = _csv(ExportSpec(schema="expcol", format="csv")) header = lines[0].split(",") self.assertNotIn("myref__refstring", header) self.assertIn("myref", header) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_never_includes_enumerate_subfields(self): lines, _stats = _csv(ExportSpec(schema="expcol", format="csv")) header = lines[0].split(",") self.assertNotIn("myenum__widget", header) self.assertNotIn("myenum__enumstring", header) self.assertIn("myenum", header) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_action_is_the_first_column_and_filled(self): lines, _stats = _csv(ExportSpec(schema="expcol", format="csv", action="noop")) header = lines[0].split(",") self.assertEqual(header[0], "_action") self.assertEqual(header[1], "_schema") self.assertEqual(lines[1].split(",")[0], "noop") self.assertEqual(lines[1].split(",")[1], "expcol") @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_no_action_column_without_action(self): lines, _stats = _csv(ExportSpec(schema="expcol", format="csv")) self.assertNotIn("_action", lines[0].split(",")) # --------------------------------------------------------------- # CSV - filter mode ("!a,b") and explicit mode # --------------------------------------------------------------- @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_blacklist_mode_excludes_named_columns(self): lines, _stats = _csv(ExportSpec(schema="expcol", format="csv", columns="!mystring")) header = lines[0].split(",") self.assertNotIn("mystring", header) self.assertIn("myref", header) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_explicit_mode_keeps_typed_order(self): spec = ExportSpec(schema="expcol", format="csv", columns="myref__refstring,mystring") lines, _stats = _csv(spec) self.assertEqual(lines[0].split(","), ["_schema", "myref__refstring", "mystring"]) self.assertEqual(lines[1].split(","), ["expcol", "from the referenced object", "hello"]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_explicit_mode_can_name_an_enumerate_subfield(self): # explicit mode emits what the user named, injected or not spec = ExportSpec(schema="expcol", format="csv", columns="keyname,myenum__enumstring") lines, _stats = _csv(spec) self.assertEqual(lines[0].split(","), ["_schema", "keyname", "myenum__enumstring"]) self.assertIn("AAA", lines[1]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_explicit_mode_always_emits_the_schema_column_once(self): # _schema is what makes an export re-importable: it is prepended whether or # not it was named, and naming it does not duplicate it spec = ExportSpec(schema="expcol", format="csv", columns="_schema,keyname,mystring") lines, _stats = _csv(spec) self.assertEqual(lines[0].split(","), ["_schema", "keyname", "mystring"]) self.assertEqual(lines[1].split(","), ["expcol", "expcol-01", "hello"]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_custom_delimiter(self): spec = ExportSpec(schema="expcol", format="csv", columns="keyname,mystring", delimiter=";") lines, _stats = _csv(spec) self.assertEqual(lines[0], "_schema;keyname;mystring") @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_csv_empty_export_still_has_a_header(self): spec = ExportSpec(schema="expcol", format="csv", query="mystring:nothingmatches") lines, stats = _csv(spec) self.assertEqual(stats["count_ok"], 0) self.assertEqual(len(lines), 1) self.assertEqual(lines[0].split(",")[0], "_schema") # --------------------------------------------------------------- # YAML / JSON # --------------------------------------------------------------- @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_yaml_is_a_valid_top_level_list(self): out = io.StringIO() export_write(ExportSpec(schema="expcol", format="yaml"), AAA, out) data = yaml.safe_load(out.getvalue()) self.assertIsInstance(data, list) self.assertEqual(data[0]["keyname"], "expcol-01") self.assertEqual(data[0]["classname"], "expcol") @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_json_is_a_valid_top_level_list(self): out = io.StringIO() export_write(ExportSpec(schema="expcol", format="json"), AAA, out) data = json.loads(out.getvalue()) self.assertIsInstance(data, list) self.assertEqual(data[0]["keyname"], "expcol-01") @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_yaml_empty_export_is_an_empty_list(self): out = io.StringIO() spec = ExportSpec(schema="expcol", format="yaml", query="mystring:nothingmatches") export_write(spec, AAA, out) self.assertEqual(yaml.safe_load(out.getvalue()), []) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_json_empty_export_is_an_empty_list(self): out = io.StringIO() spec = ExportSpec(schema="expcol", format="json", query="mystring:nothingmatches") export_write(spec, AAA, out) self.assertEqual(json.loads(out.getvalue()), []) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_yaml_keeps_injected_schema_subfield_by_default(self): out = io.StringIO() export_write(ExportSpec(schema="expcol", format="yaml"), AAA, out) data = yaml.safe_load(out.getvalue()) self.assertIn("myref__refstring", data[0]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_yaml_carries_enumerate_subfields_of_an_exported_enumerate(self): out = io.StringIO() export_write(ExportSpec(schema="expcol", format="yaml"), AAA, out) data = yaml.safe_load(out.getvalue()) self.assertIn("myenum", data[0]) self.assertIn("myenum__widget", data[0]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_yaml_explicit_mode_forces_classname_and_keyname(self): out = io.StringIO() spec = ExportSpec(schema="expcol", format="yaml", columns="mystring") export_write(spec, AAA, out) data = yaml.safe_load(out.getvalue()) self.assertEqual(list(data[0]), ["classname", "keyname", "mystring"]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_yaml_explicit_mode_maps_the_schema_column_to_classname(self): # the column selector posts the CSV column name "_schema"; in YAML/JSON that # is the "classname" key, not a field to resolve (which would be empty) out = io.StringIO() spec = ExportSpec(schema="expcol", format="yaml", columns="_schema,keyname,mystring") export_write(spec, AAA, out) data = yaml.safe_load(out.getvalue()) self.assertEqual(list(data[0]), ["classname", "keyname", "mystring"]) self.assertEqual(data[0]["classname"], "expcol") self.assertNotIn("_schema", data[0]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_json_explicit_mode_maps_the_schema_column_to_classname(self): out = io.StringIO() spec = ExportSpec(schema="expcol", format="json", columns="_schema,keyname,mystring") export_write(spec, AAA, out) data = json.loads(out.getvalue()) self.assertEqual(list(data[0]), ["classname", "keyname", "mystring"]) self.assertNotIn("_schema", data[0]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_json_filter_mode_has_no_schema_key(self): out = io.StringIO() export_write(ExportSpec(schema="expcol", format="json"), AAA, out) data = json.loads(out.getvalue()) self.assertEqual(data[0]["classname"], "expcol") self.assertNotIn("_schema", data[0]) @override_settings(CAVALIBA_AUTH_MODE="unittest") def test_json_action_key_added(self): out = io.StringIO() export_write(ExportSpec(schema="expcol", format="json", action="noop"), AAA, out) data = json.loads(out.getvalue()) self.assertEqual(data[0]["_action"], "noop")