Convert uploaded .xlsx files to csv artifacts server-side

Uploading a .xlsx file no longer becomes a generic downloadable binary -
it's read server-side with openpyxl and converted to a csv artifact,
rendering in the same interactive grid as a native CSV post. Detected by
content type or .xlsx extension (some clients send generic/incorrect
content types for multipart uploads). Only the active sheet converts;
formula cells use whatever cached result the spreadsheet software last
saved (openpyxl doesn't evaluate formulas itself). Corrupt/unreadable
files return 400.

Since the existing posting script already forwards any non-text file via
multipart with no special-casing, `daybook_post.py balance_sheet.xlsx`
already works with zero script changes - verified live.

Two real bugs found and fixed while building this, both now covered by
regression tests:
- {**a_querydict, ...} silently wrapped every value (including title) in
  a list - Python's ** spread on a dict subclass bypasses QueryDict's
  __getitem__ override and exposes its internal one-list-per-key storage
  directly. Fixed by building the delegated dict explicitly via .get().
- Pre-setting None for absent optional fields (e.g. source_prompt) defeated
  create_text_artifact's own data.get(field, default) calls, since .get()
  only falls through to its default when the key is *missing*, not when
  the value is None - caused a NOT NULL constraint failure. Fixed by only
  setting keys that were actually provided.
- csv.writer's default "\r\n" terminator was getting doubled by
  Path.write_text()'s own newline translation on Windows, producing a
  blank line between every row. Fixed with lineterminator="\n".

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
peter 2026-09-02 16:29:39 +01:00
parent 4145c37dfe
commit 4b5ec09ae2
6 changed files with 232 additions and 6 deletions

View file

@ -6,12 +6,12 @@ import json
import mimetypes
import re
import secrets
import time
from datetime import timedelta
from pathlib import Path
from urllib.parse import urlencode
import markdown
import openpyxl
from django.conf import settings
from django.contrib import messages
from django.contrib.auth import views as auth_views
@ -107,7 +107,7 @@ class ArtifactListCreateView(APIView):
return HttpResponseForbidden("Invalid or missing API token")
if request.content_type and request.content_type.startswith("multipart/"):
artifact = create_binary_artifact(request.data, user)
artifact = create_binary_artifact(request.data, request, user)
if isinstance(artifact, Response):
return artifact
apply_tag_colors(request.data)
@ -616,7 +616,36 @@ def create_text_artifact(data, request, user):
return artifact
def create_binary_artifact(data, user):
XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
def is_xlsx_upload(uploaded, content_type):
if content_type == XLSX_CONTENT_TYPE:
return True
return Path(uploaded.name).suffix.lower() == ".xlsx"
def xlsx_to_csv_text(uploaded):
"""Converts the active sheet of an uploaded .xlsx workbook to CSV text.
Only the active sheet is used - other sheets in a multi-sheet workbook
are silently dropped. data_only=True reads cached formula *results*
rather than formula source, since a results grid is what's useful here.
"""
workbook = openpyxl.load_workbook(io.BytesIO(uploaded.read()), data_only=True, read_only=True)
sheet = workbook.active
buffer = io.StringIO()
# lineterminator="\n": csv.writer defaults to "\r\n", which then gets a
# second newline translation from Path.write_text()'s own text-mode
# handling when this string is saved to disk, producing a blank line
# between every row on Windows.
writer = csv.writer(buffer, lineterminator="\n")
for row in sheet.iter_rows(values_only=True):
writer.writerow(["" if cell is None else cell for cell in row])
return buffer.getvalue()
def create_binary_artifact(data, request, user):
uploaded = data.get("file")
required = ["source", "title"]
missing = [field for field in required if not data.get(field)]
@ -628,11 +657,36 @@ def create_binary_artifact(data, user):
if source_error:
return source_error
title = data["title"]
content_type = getattr(uploaded, "content_type", "") or mimetypes.guess_type(uploaded.name)[0] or ""
if is_xlsx_upload(uploaded, content_type):
try:
content = xlsx_to_csv_text(uploaded)
except Exception as exc:
return Response({"detail": f"Could not read .xlsx file: {exc}"}, status=400)
# Built as a plain dict via .get() rather than {**data, ...}: `data` is
# a QueryDict for multipart uploads, and spreading a QueryDict with **
# bypasses its __getitem__ override and exposes its internal
# one-value-per-list storage directly, silently wrapping every value
# (title included) in a list.
text_data = {"source": data.get("source"), "title": data.get("title")}
for optional_field in (
"tags", "source_type", "source_name", "source_id", "source_prompt", "metadata",
):
# Only set if actually provided - create_text_artifact's own
# data.get(field, default) calls need the key absent, not None,
# to fall through to their intended defaults.
value = data.get(optional_field)
if value is not None:
text_data[optional_field] = value
text_data["kind"] = Artifact.CSV
text_data["content"] = content
return create_text_artifact(text_data, request, user)
now = timezone.localtime()
artifact_id = generate_id()
source = slugify(data["source"]) or "unknown"
title = data["title"]
content_type = getattr(uploaded, "content_type", "") or mimetypes.guess_type(uploaded.name)[0] or ""
kind = binary_kind(content_type)
directory = Path(source) / now.date().isoformat()