From 85679b5e781113a62cb5411323f163af2e66da61 Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 7 Oct 2026 17:02:40 +0100 Subject: [PATCH] Schema 1.5.0: holiday calendar registry, citext names; drop num_records For the coverage-aware holiday-calendar API in TSAPI core (db_base holiday_calendar__get / __set_coverage / __set_metadata / __upsert). - meta.holiday_calendars (name, description, covered_from, covered_to, source, updated_at): a calendar exists when it has a row here, not when it has holidays (retires the 9999-12-31 sentinel row). covered_from/to: the range over which the closure list is complete; both set or both NULL, from <= to (holiday_calendars_coverage_check). - Calendar names are citext in every table (registry, holidays.calendar_name, exchanges.default_holiday_calendar, instruments.holiday_calendar): stored as given, unique and matched case-insensitively -- also in the foreign keys -- as the native TSAPI backend does with COLLATE NOCASE. New extension citext (00_extensions.sql). Name check: non-empty, trimmed, <= 50 characters. - FKs: holidays -> registry ON DELETE CASCADE; exchanges/instruments -> registry ON DELETE RESTRICT (a calendar in use cannot be dropped); ON UPDATE CASCADE everywhere. - meta.series.num_records dropped. Only some write paths maintained it (never C++), it sometimes held an estimate, and it was NULL for a series never written to. Clients ask the table: count(*), or SELECT EXISTS (SELECT 1 FROM ) for "is it empty?". - find_schema_problems() covers the new table, columns, constraints and extension, and runs with search_path = pg_catalog so format_type() reports 'public.citext' whatever the caller's search_path. Verified on a fresh database (PG 17): run_setup.py installs it and find_schema_problems() returns zero rows, also for a caller without public on the search_path; constraint behaviour checked by hand (case-insensitive duplicate refused, FK matches any spelling, RESTRICT and CASCADE, coverage and name checks). Re-provision databases (no migration). Co-Authored-By: Claude Opus 5.5 --- README.md | 13 +++++++++++-- VERSION | 2 +- sql/00_extensions.sql | 4 ++++ sql/03_tables.sql | 25 +++++++++++++++++++++---- sql/04_indexes.sql | 22 ++++++++++++++++++++++ sql/09_schema_check.sql | 26 +++++++++++++++++++++----- sql/10_schema_version.sql | 2 +- 7 files changed, 81 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6d2c750..d8b9c23 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ timeseries database. Both client libraries target the schema defined here: Neither project should keep its own copy of the DDL. When the schema changes, it changes **here**, and both projects re-provision / re-test against it. -Current schema version: **1.3.0** (see [`VERSION`](VERSION); recorded at runtime +Current schema version: **1.5.0** (see [`VERSION`](VERSION); recorded at runtime in the `meta.schema_version` table). --- @@ -23,7 +23,7 @@ TSAPI_SERIESDB_SCHEMA/ ├── db_config.json.example # copy to db_config.json and edit (gitignored) ├── README.md └── sql/ - ├── 00_extensions.sql # pg_trgm + ├── 00_extensions.sql # pg_trgm, citext ├── 01_schemas.sql # meta / series / log schemas ├── 02_sequences.sql # id sequences ├── 03_tables.sql # meta.* and log.* tables @@ -78,6 +78,15 @@ database. - **Reference tables:** `meta.currencies`, `meta.timezones`, `meta.exchanges`, `meta.instruments`, `meta.sessions`, `meta.holidays`, `meta.security_types`, `meta.schema_version`. +- **Holiday calendars (1.5.0):** `meta.holiday_calendars` is the registry -- a + calendar exists when it has a row there. It records the coverage range + (`covered_from`/`covered_to`: the closure list is complete over it), + `description` and `source`. Calendar names are `citext` in every table: + stored as given, unique and matched case-insensitively. Foreign keys from + `meta.holidays` (cascade on delete), `meta.exchanges` and `meta.instruments` + (a calendar still in use cannot be dropped). Clients pass names as untyped + parameters (never `::text`) and keep `public` on the `search_path`, so + comparisons use the case-insensitive `citext` operators. - **Procedures:** `meta.drop_series` / `_if_exists` / `drop_all_series`, `meta.update_series_last_timestamp`, `meta.refresh_series_timestamps`, `meta.read_series_batch[_cursor|_ohlc]`, `log.notify_event_log`, diff --git a/VERSION b/VERSION index 88c5fb8..bc80560 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.0 +1.5.0 diff --git a/sql/00_extensions.sql b/sql/00_extensions.sql index 30d94b2..3139386 100644 --- a/sql/00_extensions.sql +++ b/sql/00_extensions.sql @@ -2,3 +2,7 @@ -- Generated: 2025-10-15T12:24:56.026074 CREATE EXTENSION IF NOT EXISTS pg_trgm SCHEMA public; + +-- Case-insensitive text for holiday calendar names (1.5.0): stored as given, +-- compared and keyed case-insensitively -- including in foreign keys. +CREATE EXTENSION IF NOT EXISTS citext SCHEMA public; diff --git a/sql/03_tables.sql b/sql/03_tables.sql index 2346b43..fb2ca17 100644 --- a/sql/03_tables.sql +++ b/sql/03_tables.sql @@ -38,15 +38,29 @@ CREATE TABLE meta.exchanges ( currency character varying(3), default_sec_type character varying(18), default_session character varying(40), - default_holiday_calendar character varying(40), + default_holiday_calendar public.citext, default_tick_size double precision, default_tick_value double precision, default_extended_hours boolean DEFAULT false, country character varying(3) ); +-- Holiday calendar registry (1.5.0). A calendar exists when it has a row +-- here -- not when it has holidays. covered_from/covered_to: the range over +-- which its list of closures is complete (both set, or both NULL = unknown). +-- Names are citext: stored as given, matched case-insensitively ('NYSE' and +-- 'nyse' are one calendar), the same rule as the native TSAPI backend. +CREATE TABLE meta.holiday_calendars ( + name public.citext NOT NULL, + description text, + covered_from date, + covered_to date, + source text, + updated_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP +); + CREATE TABLE meta.holidays ( - calendar_name character varying(50) NOT NULL, + calendar_name public.citext NOT NULL, date date NOT NULL, description character varying(255), early_close boolean NOT NULL DEFAULT false, @@ -64,7 +78,7 @@ CREATE TABLE meta.instruments ( strike double precision, currency character varying(3), session character varying(40), - holiday_calendar character varying(50), + holiday_calendar public.citext, sec_id character varying(50), sec_id_type character varying(20), description character varying(255), @@ -92,6 +106,10 @@ CREATE TABLE meta.schema_version ( description text ); +-- No record count is stored (removed in 1.5.0): "num_records" was maintained by +-- some write paths only, sometimes held an estimate, and was NULL for a series +-- never written to. Ask the table itself: SELECT count(*), or for "is it empty?" +-- SELECT EXISTS (SELECT 1 FROM
) -- fast, stops at the first row. CREATE TABLE meta.series ( table_id integer DEFAULT nextval('meta._table_meta__table_id_seq'::regclass) NOT NULL, db_schema character varying(70) NOT NULL, @@ -108,7 +126,6 @@ CREATE TABLE meta.series ( sec_id character varying(30), sec_id_type character varying(30), create_timestamp timestamp without time zone DEFAULT CURRENT_TIMESTAMP, - num_records bigint, first_timestamp timestamp without time zone, last_timestamp timestamp without time zone, bar_type character varying(30), diff --git a/sql/04_indexes.sql b/sql/04_indexes.sql index 6f192df..9d14885 100644 --- a/sql/04_indexes.sql +++ b/sql/04_indexes.sql @@ -38,6 +38,28 @@ ALTER TABLE meta.series ADD CONSTRAINT series_db_schema_fkey ALTER TABLE meta.series ADD CONSTRAINT series_table_name_check CHECK (table_name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(table_name) <= 63); +-- Holiday calendar registry (1.5.0). citext keys: unique and referenced +-- case-insensitively, stored as given. +ALTER TABLE meta.holiday_calendars ADD CONSTRAINT holiday_calendars_pkey PRIMARY KEY (name); +ALTER TABLE meta.holiday_calendars ADD CONSTRAINT holiday_calendars_name_check + CHECK (name <> '' AND name::text = btrim(name::text) AND char_length(name::text) <= 50); +ALTER TABLE meta.holiday_calendars ADD CONSTRAINT holiday_calendars_coverage_check + CHECK ((covered_from IS NULL) = (covered_to IS NULL) + AND (covered_from IS NULL OR covered_from <= covered_to)); + +-- Holidays belong to their calendar: dropping the calendar drops them. +ALTER TABLE meta.holidays ADD CONSTRAINT holidays_calendar_fkey + FOREIGN KEY (calendar_name) REFERENCES meta.holiday_calendars (name) + ON UPDATE CASCADE ON DELETE CASCADE; + +-- A calendar still named by an exchange or instrument cannot be dropped. +ALTER TABLE meta.exchanges ADD CONSTRAINT exchanges_holiday_calendar_fkey + FOREIGN KEY (default_holiday_calendar) REFERENCES meta.holiday_calendars (name) + ON UPDATE CASCADE ON DELETE RESTRICT; +ALTER TABLE meta.instruments ADD CONSTRAINT instruments_holiday_calendar_fkey + FOREIGN KEY (holiday_calendar) REFERENCES meta.holiday_calendars (name) + ON UPDATE CASCADE ON DELETE RESTRICT; + CREATE INDEX idx_event_log_series_timestamp ON log.event_log USING btree (series_name, "timestamp" DESC); CREATE INDEX idx_event_log_timestamp ON log.event_log USING btree ("timestamp" DESC); diff --git a/sql/09_schema_check.sql b/sql/09_schema_check.sql index c55b6f6..07d0c09 100644 --- a/sql/09_schema_check.sql +++ b/sql/09_schema_check.sql @@ -31,12 +31,17 @@ -- The schema VERSION string is not compared here: each client compares -- meta.schema_version with the version it was built for. This function only -- reports a missing version row. +-- +-- The function runs with search_path = pg_catalog, so format_type() spells +-- extension types schema-qualified ('public.citext') whatever the caller's +-- search_path is. -- ============================================================================ CREATE OR REPLACE FUNCTION meta.find_schema_problems() RETURNS TABLE(object text, problem text) LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp AS $function$ BEGIN -- Schemas and extension ------------------------------------------------- @@ -48,7 +53,7 @@ BEGIN RETURN QUERY SELECT 'extension ' || e, 'missing' - FROM unnest(ARRAY['pg_trgm']) AS e + FROM unnest(ARRAY['citext', 'pg_trgm']) AS e WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e) ORDER BY 1; @@ -92,12 +97,18 @@ BEGIN ('meta.exchanges', 'currency', 'character varying(3)', false), ('meta.exchanges', 'default_sec_type', 'character varying(18)', false), ('meta.exchanges', 'default_session', 'character varying(40)', false), - ('meta.exchanges', 'default_holiday_calendar', 'character varying(40)', false), + ('meta.exchanges', 'default_holiday_calendar', 'public.citext', false), ('meta.exchanges', 'default_tick_size', 'double precision', false), ('meta.exchanges', 'default_tick_value', 'double precision', false), ('meta.exchanges', 'default_extended_hours', 'boolean', false), ('meta.exchanges', 'country', 'character varying(3)', false), - ('meta.holidays', 'calendar_name', 'character varying(50)', true), + ('meta.holiday_calendars', 'name', 'public.citext', true), + ('meta.holiday_calendars', 'description', 'text', false), + ('meta.holiday_calendars', 'covered_from', 'date', false), + ('meta.holiday_calendars', 'covered_to', 'date', false), + ('meta.holiday_calendars', 'source', 'text', false), + ('meta.holiday_calendars', 'updated_at', 'timestamp with time zone', true), + ('meta.holidays', 'calendar_name', 'public.citext', true), ('meta.holidays', 'date', 'date', true), ('meta.holidays', 'description', 'character varying(255)', false), ('meta.holidays', 'early_close', 'boolean', true), @@ -111,7 +122,7 @@ BEGIN ('meta.instruments', 'strike', 'double precision', false), ('meta.instruments', 'currency', 'character varying(3)', false), ('meta.instruments', 'session', 'character varying(40)', false), - ('meta.instruments', 'holiday_calendar', 'character varying(50)', false), + ('meta.instruments', 'holiday_calendar', 'public.citext', false), ('meta.instruments', 'sec_id', 'character varying(50)', false), ('meta.instruments', 'sec_id_type', 'character varying(20)', false), ('meta.instruments', 'description', 'character varying(255)', false), @@ -150,7 +161,6 @@ BEGIN ('meta.series', 'sec_id', 'character varying(30)', false), ('meta.series', 'sec_id_type', 'character varying(30)', false), ('meta.series', 'create_timestamp', 'timestamp without time zone', false), - ('meta.series', 'num_records', 'bigint', false), ('meta.series', 'first_timestamp', 'timestamp without time zone', false), ('meta.series', 'last_timestamp', 'timestamp without time zone', false), ('meta.series', 'bar_type', 'character varying(30)', false), @@ -218,9 +228,15 @@ BEGIN ('log.pending_changes', 'pending_changes_pkey'), ('log.event_log', 'event_log_pkey'), ('meta.holidays', 'holidays_pkey'), + ('meta.holidays', 'holidays_calendar_fkey'), + ('meta.holiday_calendars', 'holiday_calendars_pkey'), + ('meta.holiday_calendars', 'holiday_calendars_name_check'), + ('meta.holiday_calendars', 'holiday_calendars_coverage_check'), ('log.script_log', 'script_log_pkey'), ('meta.instruments', 'instruments_pkey'), + ('meta.instruments', 'instruments_holiday_calendar_fkey'), ('meta.exchanges', 'exchanges_pkey'), + ('meta.exchanges', 'exchanges_holiday_calendar_fkey'), ('meta.sessions', 'sessions_pkey'), ('meta.series', 'series_pkey'), ('meta.series', 'series_schema_table_key'), diff --git a/sql/10_schema_version.sql b/sql/10_schema_version.sql index 9e94f1e..7f3b9a8 100644 --- a/sql/10_schema_version.sql +++ b/sql/10_schema_version.sql @@ -5,4 +5,4 @@ -- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py). INSERT INTO meta.schema_version (version, description) -VALUES ('1.4.0', 'meta.sessions.json dropped: the definition is the only schedule'); +VALUES ('1.5.0', 'meta.holiday_calendars registry with coverage range; citext calendar names; meta.series.num_records dropped');