peter/daybook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add json and csv artifact kinds with interactive viewers

Browse source 
Gap 1 from planning/2026-09-02-artifact_kind_gaps_for_frequent_agent_output.md.
Chose new first-class kinds over riding on html_fragment, resolving that
doc's open design question in favor of filtering support, raw-source
preservation (like markdown's .md sidecar), and server-owned validation.

- json: validated with json.loads at post time (400 on invalid JSON),
  rendered with renderjson (collapsible tree), vendored locally.
- csv: parsed server-side with the stdlib csv module into columns/rows and
  handed to Grid.js as plain data - no client-side CSV-parsing plugin
  needed, keeping normalization server-owned. Vendored locally (MIT).
- Both preserve their original text as a sidecar file, generalizing what
  was previously markdown-only handling (storage.py's SIDECAR_SUFFIXES) so
  disk-size accounting, eviction cleanup, and ?raw=1 all cover json/csv too.
- Raw JSON is embedded in a <script type="application/json"> block with
  `</script` occurrences escaped (safe because JSON's \/ escape round-trips
  correctly) - verified with a live </script>-injection payload against the
  actual running server, not just unit tests.

New migration (0012) for the KIND_CHOICES change - SQLite doesn't enforce
choices at the schema level, but Django's migration state tracks it.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
peter 2026-09-02 14:56:20 +01:00
parent af9ecf28a9
commit e5d2f192df
12 changed files with 476 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

18
artifacts/migrations/0012_alter_artifact_kind.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 5.2.15 on 2026-09-02 13:49
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('artifacts', '0011_add_asset'),
]
operations = [
migrations.AlterField(
model_name='artifact',
name='kind',
field=models.CharField(choices=[('html_fragment', 'HTML fragment'), ('html_full', 'Full HTML'), ('markdown', 'Markdown'), ('deck', 'Deck'), ('image', 'Image'), ('pdf', 'PDF'), ('binary', 'Binary'), ('json', 'JSON'), ('csv', 'CSV')], max_length=20),
),
]

4
artifacts/models.py
View file

@ -10,6 +10,8 @@ class Artifact(models.Model):
IMAGE = "image"
PDF = "pdf"
BINARY = "binary"
JSON = "json"
CSV = "csv"
AGENT = "agent"
PERSON = "person"
@ -23,6 +25,8 @@ class Artifact(models.Model):
(IMAGE, "Image"),
(PDF, "PDF"),
(BINARY, "Binary"),
(JSON, "JSON"),
(CSV, "CSV"),
]
SOURCE_TYPE_CHOICES = [

29
artifacts/storage.py
View file

@ -1,8 +1,9 @@
"""Disk-usage accounting, per-user quota, and automatic eviction.
An artifact's on-disk footprint is the primary stored file plus, for markdown,
the preserved `.md` sidecar. We record it on the row (`size_bytes`) at ingest so
usage is a cheap SUM rather than a stat of every file.
An artifact's on-disk footprint is the primary stored file plus, for kinds
that preserve their original source (see `SIDECAR_SUFFIXES`), the sidecar
file. We record it on the row (`size_bytes`) at ingest so usage is a cheap
SUM rather than a stat of every file.
"""
import shutil
@ -19,22 +20,31 @@ class QuotaExceeded(Exception):
"""Raised when a post cannot fit even after evicting every unkept artifact."""
# Kinds that preserve their original source alongside the rendered .html file.
SIDECAR_SUFFIXES = {
Artifact.MARKDOWN: ".md",
Artifact.JSON: ".json",
Artifact.CSV: ".csv",
}
def artifact_disk_size(relative_path, kind):
"""Bytes an artifact occupies on disk, including the markdown sidecar."""
"""Bytes an artifact occupies on disk, including any sidecar file."""
root = settings.ARTIFACTS_ROOT
total = 0
primary = root / relative_path
if primary.exists():
total += primary.stat().st_size
if kind == Artifact.MARKDOWN:
sidecar = primary.with_suffix(".md")
sidecar_suffix = SIDECAR_SUFFIXES.get(kind)
if sidecar_suffix:
sidecar = primary.with_suffix(sidecar_suffix)
if sidecar.exists():
total += sidecar.stat().st_size
return total
def delete_artifact_files(artifact):
"""Remove an artifact's files from disk (primary, markdown sidecar, assets).
"""Remove an artifact's files from disk (primary, sidecar, assets).
Assets are cascade-deleted from the DB by the FK, but their files must be
removed here explicitly.
@ -42,8 +52,9 @@ def delete_artifact_files(artifact):
root = settings.ARTIFACTS_ROOT
primary = root / artifact.path
primary.unlink(missing_ok=True)
if artifact.kind == Artifact.MARKDOWN:
primary.with_suffix(".md").unlink(missing_ok=True)
sidecar_suffix = SIDECAR_SUFFIXES.get(artifact.kind)
if sidecar_suffix:
primary.with_suffix(sidecar_suffix).unlink(missing_ok=True)
for asset in artifact.assets.all():
delete_asset_file(asset)

124
artifacts/views.py
View file

@ -1,5 +1,7 @@
import csv
import hashlib
import html
import io
import json
import mimetypes
import re
@ -34,6 +36,7 @@ from .forms import DaybookLoginForm
from .models import Artifact, Asset, TagColor
from .storage import (
QuotaExceeded,
SIDECAR_SUFFIXES,
artifact_disk_size,
delete_artifact_files,
disk_free_bytes,
@ -45,7 +48,14 @@ from .storage import (
)
TEXT_KINDS = {Artifact.HTML_FRAGMENT, Artifact.HTML_FULL, Artifact.MARKDOWN, Artifact.DECK}
TEXT_KINDS = {
Artifact.HTML_FRAGMENT,
Artifact.HTML_FULL,
Artifact.MARKDOWN,
Artifact.DECK,
Artifact.JSON,
Artifact.CSV,
}
# Affects storage layout, renderer choice, or feed ordering - never editable after create.
IMMUTABLE_PATCH_FIELDS = {"content", "source", "kind", "day"}
@ -104,7 +114,10 @@ class ArtifactListCreateView(APIView):
data = request.data
kind = data.get("kind")
if kind not in TEXT_KINDS:
return Response({"detail": "kind must be html_fragment, html_full, markdown, or deck"}, status=400)
return Response(
{"detail": "kind must be html_fragment, html_full, markdown, deck, json, or csv"},
status=400,
)
required = ["source", "title", "content"]
missing = [field for field in required if not data.get(field)]
@ -285,7 +298,7 @@ class CapabilitiesView(APIView):
"artifact_kinds": [choice[0] for choice in Artifact.KIND_CHOICES],
"filters": ["source", "label", "day", "kind", "q"],
"views": ["rendered", "raw"],
"raw_supported_for": [Artifact.MARKDOWN, Artifact.HTML_FULL],
"raw_supported_for": [Artifact.MARKDOWN, Artifact.HTML_FULL, Artifact.JSON, Artifact.CSV],
"source_types": [choice[0] for choice in Artifact.SOURCE_TYPE_CHOICES],
"auth": "bearer_token",
"methods": {
@ -512,12 +525,19 @@ def create_text_artifact(data, request, user):
theme,
request,
)
(target_dir / f"{artifact_id}.md").write_text(content, encoding="utf-8")
elif kind == Artifact.DECK:
rendered = wrap_deck(content, title, request)
elif kind == Artifact.JSON:
rendered = wrap_json_viewer(content, title, request)
elif kind == Artifact.CSV:
rendered = wrap_csv_viewer(content, title, request)
else:
rendered = wrap_html(content, title, theme, request)
sidecar_suffix = SIDECAR_SUFFIXES.get(kind)
if sidecar_suffix:
(target_dir / f"{artifact_id}{sidecar_suffix}").write_text(content, encoding="utf-8")
stored_path = directory / f"{artifact_id}.html"
(settings.ARTIFACTS_ROOT / stored_path).write_text(rendered, encoding="utf-8")
@ -638,6 +658,92 @@ def wrap_deck(body, title, request):
"""
SCRIPT_CLOSE_RE = re.compile(r"</script", re.IGNORECASE)
def escape_script_body(text):
"""Prevent literal `</script` in embedded raw text from prematurely
closing the <script> tag it's placed inside.
Safe specifically for JSON text: JSON's grammar treats `\\/` as an
escaped '/', so this round-trips correctly through JSON.parse - it is
not a general-purpose HTML/JS escaping technique.
"""
return SCRIPT_CLOSE_RE.sub("<\\/script", text)
def wrap_json_viewer(content, title, request):
renderjson_src = request.build_absolute_uri("/static/vendor/renderjson/renderjson.js")
safe_json = escape_script_body(content)
return f"""<!doctype html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{html.escape(title)}</title>
<script src="{renderjson_src}"></script>
<style>
body {{ margin: 0; padding: 1.2rem; background: #fffaf0; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.9rem; }}
</style>
</head>
<body class="artifact">
<script type="application/json" id="daybook-data">{safe_json}</script>
<div id="daybook-json-view"></div>
<script>
(function () {{
var raw = document.getElementById("daybook-data").textContent;
var data = JSON.parse(raw);
renderjson.set_show_to_level(2);
document.getElementById("daybook-json-view").appendChild(renderjson(data));
}})();
</script>
</body>
</html>
"""
def wrap_csv_viewer(content, title, request):
"""Parses CSV server-side into columns/rows and hands Grid.js plain data -
no client-side CSV parsing plugin needed, keeping normalization server-owned.
"""
gridjs_js = request.build_absolute_uri("/static/vendor/gridjs/gridjs.umd.js")
gridjs_css = request.build_absolute_uri("/static/vendor/gridjs/mermaid.min.css")
rows = [row for row in csv.reader(io.StringIO(content)) if row]
columns = rows[0] if rows else []
data = rows[1:] if len(rows) > 1 else []
table_json = escape_script_body(json.dumps({"columns": columns, "data": data}))
return f"""<!doctype html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{html.escape(title)}</title>
<link rel="stylesheet" href="{gridjs_css}">
<script src="{gridjs_js}"></script>
<style>
body {{ margin: 0; padding: 1.2rem; background: #fffaf0; }}
</style>
</head>
<body class="artifact">
<script type="application/json" id="daybook-data">{table_json}</script>
<div id="daybook-csv-view"></div>
<script>
(function () {{
var table = JSON.parse(document.getElementById("daybook-data").textContent);
new gridjs.Grid({{
columns: table.columns,
data: table.data,
search: true,
sort: true,
pagination: {{ limit: 20 }},
}}).render(document.getElementById("daybook-csv-view"));
}})();
</script>
</body>
</html>
"""
def generate_id():
while True:
artifact_id = secrets.token_hex(4)
@ -757,6 +863,11 @@ def validate_content_shape(kind, content):
)
if kind == Artifact.MARKDOWN and not (content or "").strip():
return Response({"detail": "markdown content cannot be blank"}, status=400)
if kind == Artifact.JSON:
try:
json.loads(content)
except (TypeError, ValueError) as exc:
return Response({"detail": f"content must be valid JSON: {exc}"}, status=400)
return None
@ -875,10 +986,11 @@ def safe_artifact_path(relative_path):
def raw_artifact_path(artifact):
path = safe_artifact_path(artifact.path)
if artifact.kind == Artifact.MARKDOWN:
return path.with_suffix(".md")
if artifact.kind == Artifact.HTML_FULL:
return path
sidecar_suffix = SIDECAR_SUFFIXES.get(artifact.kind)
if sidecar_suffix:
return path.with_suffix(sidecar_suffix)
return None