Convert uploaded .xlsx files to csv artifacts server-side
Uploading a .xlsx file no longer becomes a generic downloadable binary -
it's read server-side with openpyxl and converted to a csv artifact,
rendering in the same interactive grid as a native CSV post. Detected by
content type or .xlsx extension (some clients send generic/incorrect
content types for multipart uploads). Only the active sheet converts;
formula cells use whatever cached result the spreadsheet software last
saved (openpyxl doesn't evaluate formulas itself). Corrupt/unreadable
files return 400.
Since the existing posting script already forwards any non-text file via
multipart with no special-casing, `daybook_post.py balance_sheet.xlsx`
already works with zero script changes - verified live.
Two real bugs found and fixed while building this, both now covered by
regression tests:
- {**a_querydict, ...} silently wrapped every value (including title) in
a list - Python's ** spread on a dict subclass bypasses QueryDict's
__getitem__ override and exposes its internal one-list-per-key storage
directly. Fixed by building the delegated dict explicitly via .get().
- Pre-setting None for absent optional fields (e.g. source_prompt) defeated
create_text_artifact's own data.get(field, default) calls, since .get()
only falls through to its default when the key is *missing*, not when
the value is None - caused a NOT NULL constraint failure. Fixed by only
setting keys that were actually provided.
- csv.writer's default "\r\n" terminator was getting doubled by
Path.write_text()'s own newline translation on Windows, producing a
blank line between every row. Fixed with lineterminator="\n".
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
4145c37dfe
commit
4b5ec09ae2
6 changed files with 232 additions and 6 deletions
|
|
@ -6,12 +6,12 @@ import json
|
|||
import mimetypes
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import markdown
|
||||
import openpyxl
|
||||
from django.conf import settings
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth import views as auth_views
|
||||
|
|
@ -107,7 +107,7 @@ class ArtifactListCreateView(APIView):
|
|||
return HttpResponseForbidden("Invalid or missing API token")
|
||||
|
||||
if request.content_type and request.content_type.startswith("multipart/"):
|
||||
artifact = create_binary_artifact(request.data, user)
|
||||
artifact = create_binary_artifact(request.data, request, user)
|
||||
if isinstance(artifact, Response):
|
||||
return artifact
|
||||
apply_tag_colors(request.data)
|
||||
|
|
@ -616,7 +616,36 @@ def create_text_artifact(data, request, user):
|
|||
return artifact
|
||||
|
||||
|
||||
def create_binary_artifact(data, user):
|
||||
XLSX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
|
||||
|
||||
def is_xlsx_upload(uploaded, content_type):
|
||||
if content_type == XLSX_CONTENT_TYPE:
|
||||
return True
|
||||
return Path(uploaded.name).suffix.lower() == ".xlsx"
|
||||
|
||||
|
||||
def xlsx_to_csv_text(uploaded):
|
||||
"""Converts the active sheet of an uploaded .xlsx workbook to CSV text.
|
||||
|
||||
Only the active sheet is used - other sheets in a multi-sheet workbook
|
||||
are silently dropped. data_only=True reads cached formula *results*
|
||||
rather than formula source, since a results grid is what's useful here.
|
||||
"""
|
||||
workbook = openpyxl.load_workbook(io.BytesIO(uploaded.read()), data_only=True, read_only=True)
|
||||
sheet = workbook.active
|
||||
buffer = io.StringIO()
|
||||
# lineterminator="\n": csv.writer defaults to "\r\n", which then gets a
|
||||
# second newline translation from Path.write_text()'s own text-mode
|
||||
# handling when this string is saved to disk, producing a blank line
|
||||
# between every row on Windows.
|
||||
writer = csv.writer(buffer, lineterminator="\n")
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
writer.writerow(["" if cell is None else cell for cell in row])
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def create_binary_artifact(data, request, user):
|
||||
uploaded = data.get("file")
|
||||
required = ["source", "title"]
|
||||
missing = [field for field in required if not data.get(field)]
|
||||
|
|
@ -628,11 +657,36 @@ def create_binary_artifact(data, user):
|
|||
if source_error:
|
||||
return source_error
|
||||
|
||||
title = data["title"]
|
||||
content_type = getattr(uploaded, "content_type", "") or mimetypes.guess_type(uploaded.name)[0] or ""
|
||||
|
||||
if is_xlsx_upload(uploaded, content_type):
|
||||
try:
|
||||
content = xlsx_to_csv_text(uploaded)
|
||||
except Exception as exc:
|
||||
return Response({"detail": f"Could not read .xlsx file: {exc}"}, status=400)
|
||||
# Built as a plain dict via .get() rather than {**data, ...}: `data` is
|
||||
# a QueryDict for multipart uploads, and spreading a QueryDict with **
|
||||
# bypasses its __getitem__ override and exposes its internal
|
||||
# one-value-per-list storage directly, silently wrapping every value
|
||||
# (title included) in a list.
|
||||
text_data = {"source": data.get("source"), "title": data.get("title")}
|
||||
for optional_field in (
|
||||
"tags", "source_type", "source_name", "source_id", "source_prompt", "metadata",
|
||||
):
|
||||
# Only set if actually provided - create_text_artifact's own
|
||||
# data.get(field, default) calls need the key absent, not None,
|
||||
# to fall through to their intended defaults.
|
||||
value = data.get(optional_field)
|
||||
if value is not None:
|
||||
text_data[optional_field] = value
|
||||
text_data["kind"] = Artifact.CSV
|
||||
text_data["content"] = content
|
||||
return create_text_artifact(text_data, request, user)
|
||||
|
||||
now = timezone.localtime()
|
||||
artifact_id = generate_id()
|
||||
source = slugify(data["source"]) or "unknown"
|
||||
title = data["title"]
|
||||
content_type = getattr(uploaded, "content_type", "") or mimetypes.guess_type(uploaded.name)[0] or ""
|
||||
kind = binary_kind(content_type)
|
||||
|
||||
directory = Path(source) / now.date().isoformat()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue