From 1f5cfeceeb20da5da3e3aae379a1327442aebee7 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 6 Oct 2026 11:01:28 +0100 Subject: [PATCH] Add meta.find_schema_problems(); every install runs 00-10; drop migrations 09_schema_check.sql adds meta.find_schema_problems(), a read-only check any client (Python, C++, psql) can call: SELECT * FROM meta.find_schema_problems(); -- zero rows = usable One row per problem (object, problem): missing schemas, extension, sequences, tables, columns, wrong column type or NOT NULL, missing constraints, indexes, views, stored-function bodies that differ from the DDL (whitespace-insensitive md5 fingerprint), a missing/disabled/ misdirected event trigger, short reference data, no version row. It mirrors 00-08 and must change in the same commit as they do; the release test (a fresh install returns zero rows) enforces that. 10_schema_version.sql records VERSION in meta.schema_version. Before, only the Python client wrote that row, so databases provisioned by run_setup.py recorded no version at all. run_setup.py now runs 00-10. It skipped 08_script_log.sql, so its databases lacked a table the Python client uses; every fresh install is now identical regardless of provisioner. sql/migrate/ is removed: databases are re-provisioned, not migrated. Co-Authored-By: Claude Opus 5.5 --- README.md | 41 ++- run_setup.py | 13 +- sql/09_schema_check.sql | 316 ++++++++++++++++++ sql/10_schema_version.sql | 8 + sql/migrate/migrate_drop_series_if_exists.sql | 48 --- sql/migrate/migrate_instrument_metadata.sql | 99 ------ sql/migrate/migrate_metadata_identity.sql | 111 ------ 7 files changed, 367 insertions(+), 269 deletions(-) create mode 100644 sql/09_schema_check.sql create mode 100644 sql/10_schema_version.sql delete mode 100644 sql/migrate/migrate_drop_series_if_exists.sql delete mode 100644 sql/migrate/migrate_instrument_metadata.sql delete mode 100644 sql/migrate/migrate_metadata_identity.sql diff --git a/README.md b/README.md index 110af2e..8d06f31 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,15 @@ TSAPI_SERIESDB_SCHEMA/ ├── 04_indexes.sql # indexes + registry constraints ├── 05_procedures.sql # meta.* / log.* stored procedures ├── 06_triggers.sql # prevent_series_drops event trigger - ├── 07_reference_data.sql # timezones + currencies reference data - ├── 08_script_log.sql # auxiliary (not part of a fresh install) - └── migrate/ # one-off migration scripts (run manually) + ├── 07_reference_data.sql # timezones + currencies + security types + ├── 08_script_log.sql # log.script_log table + view + ├── 09_schema_check.sql # meta.find_schema_problems() + └── 10_schema_version.sql # records VERSION in meta.schema_version ``` -`run_setup.py` executes `00`–`07` in order. `08_script_log.sql` and -`sql/migrate/*.sql` are auxiliary and are **not** run by a fresh install. +Every fresh install runs `00`–`10` in order (`run_setup.py`, and the Python +client's `init_postgres_db()`). There are no migration scripts: a database +from an older schema is re-provisioned, not upgraded. --- @@ -74,6 +76,7 @@ database. `log.flush_pending_to_event_log`. - **Event trigger:** `prevent_series_drops` — blocks direct `DROP TABLE` on a registered series (drops must go through `meta.drop_series*`). +- **Self-check:** `meta.find_schema_problems()` — see below. Series data tables have **no `UNIQUE` constraint on `timestamp`** — the index is BRIN. Concurrent writers must serialise per-series with the advisory-lock @@ -82,11 +85,33 @@ the C++ project's `planning/2026-09-02_cpp_locking_interop_plan.md`. --- +## Checking a database + +```sql +SELECT * FROM meta.find_schema_problems(); -- zero rows = complete and usable +``` + +One row per problem, `(object, problem)`, for example +`('table meta.instruments', 'missing columns: sector')`. Read-only. It checks +schemas, the extension, sequences, every table column's type and NOT NULL, +constraints, indexes, views, stored-function bodies (whitespace-insensitive +fingerprint), the event trigger, reference-data row counts and that a version +row exists. If the function itself does not exist, the database is not +initialized or predates it -- re-provision. + +`09_schema_check.sql` mirrors `00`–`08`: **change it in the same commit as any +of them**. The release test -- a freshly provisioned database must return +zero rows -- fails until you do. The file's header shows how to refresh the +function-body fingerprints. + +--- + ## Versioning contract -Bump [`VERSION`](VERSION) whenever `sql/` changes in a way clients must know -about, and have each client read `meta.schema_version` on connect and warn/fail -on mismatch. This turns silent schema drift into a loud error. +Bump [`VERSION`](VERSION) -- and the string in `10_schema_version.sql` -- whenever +`sql/` changes in a way clients must know about. Each client compares +`meta.schema_version` with the version it was built for on connect, and calls +`meta.find_schema_problems()` for everything else. --- diff --git a/run_setup.py b/run_setup.py index 542d49a..6042d16 100644 --- a/run_setup.py +++ b/run_setup.py @@ -8,7 +8,11 @@ into either project -- edit it here. It executes the numbered DDL files in ./sql/ in order: 00_extensions, 01_schemas, 02_sequences, 03_tables, 04_indexes, - 05_procedures, 06_triggers, 07_reference_data + 05_procedures, 06_triggers, 07_reference_data, 08_script_log, + 09_schema_check, 10_schema_version + +Every fresh install runs all of them, so every client sees the same database. +Afterwards `SELECT * FROM meta.find_schema_problems()` must return no rows. Connection settings come from (highest precedence first): 1. command-line flags (--host --port --dbname --user --password) @@ -31,8 +35,8 @@ from typing import Dict SQL_DIR = Path(__file__).parent / "sql" CONFIG_FILE = Path(__file__).parent / "db_config.json" -# Executed in order. 08_script_log.sql and sql/migrate/*.sql are auxiliary and -# are intentionally NOT part of a fresh install. +# Executed in order -- all of them, on every fresh install. 10_schema_version +# runs last, so a database that records a version is complete. SQL_FILES = [ "00_extensions.sql", "01_schemas.sql", @@ -42,6 +46,9 @@ SQL_FILES = [ "05_procedures.sql", "06_triggers.sql", "07_reference_data.sql", + "08_script_log.sql", + "09_schema_check.sql", + "10_schema_version.sql", ] DEFAULTS = { diff --git a/sql/09_schema_check.sql b/sql/09_schema_check.sql new file mode 100644 index 0000000..48a011d --- /dev/null +++ b/sql/09_schema_check.sql @@ -0,0 +1,316 @@ +-- ============================================================================ +-- Schema self-check: meta.find_schema_problems() +-- ============================================================================ +-- Answers one question for every client (Python, C++, psql): +-- is this database complete and as the DDL defines it? +-- +-- SELECT * FROM meta.find_schema_problems(); -- zero rows = usable +-- +-- Returns one row per problem: (object, problem). Read-only. +-- +-- The expectations below mirror 00..08. When any of those files changes, +-- update this file in the same commit. The release tests (a freshly +-- provisioned database must return zero rows) fail until you do. +-- +-- Function bodies are compared by fingerprint: md5 of the body with runs of +-- whitespace collapsed, so line endings and re-indentation do not count as a +-- change. To refresh the fingerprints after editing a function: +-- +-- SELECT n.nspname || '.' || p.proname, +-- md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) +-- FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace +-- WHERE n.nspname IN ('meta', 'log') ORDER BY 1; +-- +-- The schema VERSION string is not compared here: each client compares +-- meta.schema_version with the version it was built for. This function only +-- reports a missing version row. +-- ============================================================================ + +CREATE OR REPLACE FUNCTION meta.find_schema_problems() + RETURNS TABLE(object text, problem text) + LANGUAGE plpgsql + STABLE +AS $function$ +BEGIN + -- Schemas and extension ------------------------------------------------- + RETURN QUERY + SELECT 'schema ' || s, 'missing' + FROM unnest(ARRAY['meta', 'series', 'log']) AS s + WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s); + + RETURN QUERY + SELECT 'extension ' || e, 'missing' + FROM unnest(ARRAY['pg_trgm']) AS e + WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e); + + RETURN QUERY + SELECT 'sequence ' || s, 'missing' + FROM unnest(ARRAY[ + 'meta._table_meta__table_id_seq', + 'log.event_log_id_seq', + 'log.pending_changes_id_seq', + 'log.script_log_id_seq' + ]) AS s + WHERE to_regclass(s) IS NULL; + + -- Tables and columns ---------------------------------------------------- + RETURN QUERY + WITH e(tbl, col, typ, not_null) AS (VALUES + ('log.pending_changes', 'id', 'bigint', true), + ('log.pending_changes', 'timestamp', 'timestamp with time zone', false), + ('log.pending_changes', 'series_name', 'text', true), + ('log.pending_changes', 'db_schema', 'text', true), + ('log.pending_changes', 'op_type', 'text', true), + ('log.pending_changes', 'db_user', 'text', false), + ('log.pending_changes', 'details', 'jsonb', false), + ('log.event_log', 'id', 'bigint', true), + ('log.event_log', 'timestamp', 'timestamp with time zone', false), + ('log.event_log', 'series_name', 'text', true), + ('log.event_log', 'db_schema', 'text', true), + ('log.event_log', 'event_type', 'text', true), + ('log.event_log', 'details', 'jsonb', false), + ('log.event_log', 'source', 'text', false), + ('log.event_log', 'db_user', 'text', false), + ('meta.currencies', 'code', 'character varying(3)', true), + ('meta.currencies', 'name', 'character varying(100)', true), + ('meta.currencies', 'symbol', 'character varying(10)', false), + ('meta.currencies', 'decimal_places', 'integer', false), + ('meta.currencies', 'notes', 'text', false), + ('meta.exchanges', 'id', 'character varying(12)', true), + ('meta.exchanges', 'name', 'character varying(50)', false), + ('meta.exchanges', 'timezone', 'character varying(35)', false), + ('meta.exchanges', 'currency', 'character varying(3)', false), + ('meta.exchanges', 'default_sec_type', 'character varying(18)', false), + ('meta.exchanges', 'default_session', 'character varying(40)', false), + ('meta.exchanges', 'default_holiday_calendar', 'character varying(40)', false), + ('meta.exchanges', 'default_tick_size', 'double precision', false), + ('meta.exchanges', 'default_tick_value', 'double precision', false), + ('meta.exchanges', 'default_extended_hours', 'boolean', false), + ('meta.exchanges', 'country', 'character varying(3)', false), + ('meta.holidays', 'calendar_name', 'character varying(50)', true), + ('meta.holidays', 'date', 'date', true), + ('meta.holidays', 'description', 'character varying(255)', false), + ('meta.holidays', 'early_close', 'boolean', true), + ('meta.instruments', 'symbol', 'character varying(12)', true), + ('meta.instruments', 'exchange', 'character varying(12)', true), + ('meta.instruments', 'name', 'character varying(50)', true), + ('meta.instruments', 'type', 'character varying(10)', false), + ('meta.instruments', 'tick_size', 'double precision', false), + ('meta.instruments', 'tick_value', 'double precision', false), + ('meta.instruments', 'expiry', 'character varying(20)', false), + ('meta.instruments', 'strike', 'double precision', false), + ('meta.instruments', 'currency', 'character varying(3)', false), + ('meta.instruments', 'session', 'character varying(40)', false), + ('meta.instruments', 'holiday_calendar', 'character varying(50)', false), + ('meta.instruments', 'sec_id', 'character varying(50)', false), + ('meta.instruments', 'sec_id_type', 'character varying(20)', false), + ('meta.instruments', 'description', 'character varying(255)', false), + ('meta.instruments', 'country', 'character varying(3)', false), + ('meta.instruments', 'extended_hours', 'boolean', false), + ('meta.instruments', 'sector', 'character varying(50)', false), + ('meta.instruments', 'industry', 'character varying(100)', false), + ('meta.instruments', 'category', 'character varying(50)', false), + ('meta.instruments', 'subcategory', 'character varying(50)', false), + ('meta.instruments', 'asset_class', 'character varying(30)', false), + ('meta.instruments', 'geography', 'character varying(30)', false), + ('meta.instruments', 'underlying', 'character varying(30)', false), + ('meta.security_types', 'code', 'character varying(10)', true), + ('meta.security_types', 'name', 'character varying(50)', true), + ('meta.security_types', 'description', 'text', false), + ('meta.security_types', 'notes', 'text', false), + ('meta.schema_version', 'version', 'character varying(20)', true), + ('meta.schema_version', 'installed_at', 'timestamp with time zone', true), + ('meta.schema_version', 'description', 'text', false), + ('meta.series', 'table_id', 'integer', true), + ('meta.series', 'db_schema', 'character varying(70)', true), + ('meta.series', 'table_name', 'character varying(70)', true), + ('meta.series', 'type', 'character varying(30)', false), + ('meta.series', 'description', 'character varying(255)', false), + ('meta.series', 'column_schema', 'text', false), + ('meta.series', 'symbol', 'character varying(30)', false), + ('meta.series', 'root_symbol', 'character varying(30)', false), + ('meta.series', 'exchange', 'character varying(30)', false), + ('meta.series', 'frequency', 'character varying(30)', false), + ('meta.series', 'session', 'text', false), + ('meta.series', 'sec_type', 'character varying(30)', false), + ('meta.series', 'sec_id', 'character varying(30)', false), + ('meta.series', 'sec_id_type', 'character varying(30)', false), + ('meta.series', 'create_timestamp', 'timestamp without time zone', false), + ('meta.series', 'num_records', 'bigint', false), + ('meta.series', 'first_timestamp', 'timestamp without time zone', false), + ('meta.series', 'last_timestamp', 'timestamp without time zone', false), + ('meta.series', 'bar_type', 'character varying(30)', false), + ('meta.series', 'feed', 'character varying(30)', false), + ('meta.series', 'currency', 'character varying(30)', false), + ('meta.series', 'expiry', 'character varying(30)', false), + ('meta.series', 'roll_type', 'character varying(30)', false), + ('meta.series', 'feed_type', 'character varying(30)', false), + ('meta.series', 'user_attr', 'text', false), + ('meta.series', 'user_attr_2', 'text', false), + ('meta.series', 'script', 'text', false), + ('meta.series', 'notes', 'text', false), + ('meta.series', 'timezone', 'character varying(50)', false), + ('meta.series', 'category', 'character varying(30)', false), + ('meta.series', 'subcategory', 'character varying(30)', false), + ('meta.series', 'json', 'jsonb', false), + ('meta.series', 'needs_reindex', 'boolean', true), + ('meta.sessions', 'name', 'character varying(50)', true), + ('meta.sessions', 'definition', 'text', true), + ('meta.sessions', 'json', 'text', false), + ('meta.sessions', 'exchange_id', 'character varying(12)', true), + ('meta.timezones', 'iana_name', 'character varying(50)', true), + ('meta.timezones', 'display_name', 'character varying(100)', true), + ('meta.timezones', 'description', 'text', false), + ('meta.timezones', 'region', 'character varying(50)', false), + ('meta.timezones', 'utc_offset_hours', 'real', false), + ('meta.timezones', 'notes', 'text', false), + ('log.script_log', 'id', 'integer', true), + ('log.script_log', 'timestamp', 'timestamp with time zone', true), + ('log.script_log', 'level', 'character varying(10)', true), + ('log.script_log', 'message', 'text', true), + ('log.script_log', 'script_name', 'character varying(100)', false), + ('log.script_log', 'series_name', 'character varying(255)', false), + ('log.script_log', 'session_id', 'uuid', false), + ('log.script_log', 'details', 'jsonb', false) + ), + a AS ( + SELECT e.*, x.attname IS NOT NULL AS present, + format_type(x.atttypid, x.atttypmod) AS got_typ, x.attnotnull AS got_not_null + FROM e + LEFT JOIN pg_attribute x ON x.attrelid = to_regclass(e.tbl) AND x.attname = e.col + AND x.attnum > 0 AND NOT x.attisdropped + WHERE to_regclass(e.tbl) IS NOT NULL + ), + found(sort_key, object, problem) AS ( + SELECT 1, 'table ' || tbl, 'missing' + FROM (SELECT DISTINCT tbl FROM e WHERE to_regclass(tbl) IS NULL) m + UNION ALL + SELECT 2, 'table ' || tbl, 'missing columns: ' || string_agg(col, ', ') + FROM a WHERE NOT present GROUP BY tbl + UNION ALL + SELECT 3, 'table ' || tbl, col || ' is ' || got_typ || ', expected ' || typ + FROM a WHERE present AND got_typ <> typ + UNION ALL + SELECT 4, 'table ' || tbl, + col || ' is ' || CASE WHEN got_not_null THEN 'NOT NULL' ELSE 'nullable' END + || ', expected ' || CASE WHEN not_null THEN 'NOT NULL' ELSE 'nullable' END + FROM a WHERE present AND got_not_null <> not_null + ) + SELECT f.object, f.problem FROM found f ORDER BY f.object, f.sort_key, f.problem; + + -- Constraints and indexes (only on tables that exist) ------------------- + RETURN QUERY + SELECT 'constraint ' || c.tbl || '.' || c.name, 'missing' + FROM (VALUES + ('log.pending_changes', 'pending_changes_pkey'), + ('log.event_log', 'event_log_pkey'), + ('meta.holidays', 'holidays_pkey'), + ('log.script_log', 'script_log_pkey'), + ('meta.instruments', 'instruments_pkey'), + ('meta.exchanges', 'exchanges_pkey'), + ('meta.sessions', 'sessions_pkey'), + ('meta.series', 'series_pkey'), + ('meta.series', 'series_schema_table_key'), + ('meta.currencies', 'currencies_pkey'), + ('meta.security_types', 'security_types_pkey'), + ('meta.timezones', 'timezones_pkey') + ) AS c(tbl, name) + WHERE to_regclass(c.tbl) IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM pg_constraint k + WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name); + + RETURN QUERY + SELECT 'index ' || i.idx, 'missing' + FROM (VALUES + ('log.idx_event_log_series_timestamp', 'log.event_log'), + ('log.idx_event_log_timestamp', 'log.event_log'), + ('log.idx_event_log_event_type', 'log.event_log'), + ('meta.idx_series_create_timestamp_desc', 'meta.series'), + ('meta.idx_series_db_schema', 'meta.series'), + ('meta.idx_series_exchange_pattern', 'meta.series'), + ('meta.idx_series_frequency_pattern', 'meta.series'), + ('meta.idx_series_symbol_exchange', 'meta.series'), + ('meta.idx_series_symbol_pattern', 'meta.series'), + ('meta.idx_series_symbol_trgm', 'meta.series'), + ('meta.idx_series_table_name_trgm', 'meta.series'), + ('meta.idx_series_type_pattern', 'meta.series'), + ('log.idx_script_log_timestamp', 'log.script_log'), + ('log.idx_script_log_session', 'log.script_log'), + ('log.idx_script_log_series', 'log.script_log'), + ('log.idx_script_log_level', 'log.script_log') + ) AS i(idx, tbl) + WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL; + + RETURN QUERY + SELECT 'view ' || v, 'missing' + FROM unnest(ARRAY['log.script_log_recent']) AS v + WHERE to_regclass(v) IS NULL; + + -- Functions (compared by whitespace-insensitive body fingerprint) ------- + RETURN QUERY + SELECT 'function ' || f.name, + CASE WHEN NOT EXISTS (SELECT 1 FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = f.name) + THEN 'missing' + ELSE 'differs from the DDL (stale or edited)' END + FROM (VALUES + ('log.flush_pending_to_event_log', 'be02d914fc5e73475eb1ce7ce7a78f29'), + ('log.notify_event_log', '7e5898f1d1d128ef2e44c1d4ae8a7312'), + ('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'), + ('meta.drop_series', '6f2a919de965b87521be68252deee0e2'), + ('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'), + ('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'), + ('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'), + ('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'), + ('meta.read_series_batch_ohlc', '4bdef4d3573caab023ddd12db38b978c'), + ('meta.refresh_series_timestamps', '1cdfd496752142cbde78d9646532d1aa'), + ('meta.update_series_last_timestamp', 'f27394672bc5026456c4625ac3091a62') + ) AS f(name, fingerprint) + WHERE NOT EXISTS (SELECT 1 FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = f.name + AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint); + + -- Drop-protection event trigger ----------------------------------------- + RETURN QUERY + SELECT 'event trigger prevent_series_drops', 'missing' + WHERE NOT EXISTS (SELECT 1 FROM pg_event_trigger WHERE evtname = 'prevent_series_drops'); + + RETURN QUERY + SELECT 'event trigger prevent_series_drops', p + FROM pg_event_trigger t, + LATERAL (VALUES + (CASE WHEN t.evtevent <> 'sql_drop' + THEN 'fires on ' || t.evtevent || ', expected sql_drop' END), + (CASE WHEN t.evtenabled = 'D' THEN 'disabled' END), + (CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop' + THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END) + ) AS x(p) + WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL; + + -- Reference data and version row ---------------------------------------- + RETURN QUERY + SELECT 'reference data ' || r.tbl, + r.actual || ' rows, expected at least ' || r.expected + FROM ( + SELECT e.tbl, e.expected, + (xpath('/row/n/text()', query_to_xml( + format('SELECT count(*) AS n FROM %s', e.tbl), false, true, '')))[1]::text::int + AS actual + FROM (VALUES + ('meta.timezones', 563), + ('meta.currencies', 179), + ('meta.security_types', 13) + ) AS e(tbl, expected) + WHERE to_regclass(e.tbl) IS NOT NULL + ) AS r + WHERE r.actual < r.expected; + + IF to_regclass('meta.schema_version') IS NOT NULL THEN + RETURN QUERY + SELECT 'schema version', 'not recorded in meta.schema_version' + WHERE NOT EXISTS (SELECT 1 FROM meta.schema_version); + END IF; +END; +$function$; diff --git a/sql/10_schema_version.sql b/sql/10_schema_version.sql new file mode 100644 index 0000000..120704a --- /dev/null +++ b/sql/10_schema_version.sql @@ -0,0 +1,8 @@ +-- ============================================================================ +-- Schema version row -- runs last, so a database that has it is complete. +-- ============================================================================ +-- Keep this string equal to ../VERSION. Clients compare it with the version +-- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py). + +INSERT INTO meta.schema_version (version, description) +VALUES ('1.0.0', 'Initial schema'); diff --git a/sql/migrate/migrate_drop_series_if_exists.sql b/sql/migrate/migrate_drop_series_if_exists.sql deleted file mode 100644 index dd39e0b..0000000 --- a/sql/migrate/migrate_drop_series_if_exists.sql +++ /dev/null @@ -1,48 +0,0 @@ --- Migration: repair meta.drop_series_if_exists() --- --- The function dropped a registered series without setting --- application.dropping_series, so the prevent_direct_drop event trigger --- rejected its own DROP: --- --- Table series.X is registered in series and can only be dropped --- using TimeseriesDB.drop_series() --- --- Unlike meta.drop_series() and meta.drop_all_series(), which set the flag --- correctly, this one never did -- so it always failed on exactly the case it --- exists to handle. The Python API does not call it, which is why the defect --- went unnoticed. --- --- New databases pick this up from 05_procedures.sql. Run this against any --- database initialised before the fix. --- --- Safe to run more than once (CREATE OR REPLACE). - -CREATE OR REPLACE FUNCTION meta.drop_series_if_exists(_table_name text, _schema_name text DEFAULT 'series'::text) - RETURNS void - LANGUAGE plpgsql -AS $function$ -BEGIN - -- Check if the series exists in series table - IF EXISTS ( - SELECT 1 FROM meta.series - WHERE "table_name" = _table_name - AND "db_schema" = _schema_name - ) THEN - -- Signal the prevent_direct_drop event trigger that this DROP is - -- coming from a sanctioned procedure. Without it the trigger rejects - -- the statement below, because the table is registered in meta.series. - -- Local to the transaction, so it cannot leak to later statements. - PERFORM set_config('application.dropping_series', 'true', true); - - -- Drop the table - EXECUTE format('DROP TABLE IF EXISTS %I.%I', _schema_name, _table_name); - - -- Remove from series table - DELETE FROM meta.series - WHERE "table_name" = _table_name - AND "db_schema" = _schema_name; - - PERFORM set_config('application.dropping_series', 'false', true); - END IF; -END; -$function$; diff --git a/sql/migrate/migrate_instrument_metadata.sql b/sql/migrate/migrate_instrument_metadata.sql deleted file mode 100644 index f80ef19..0000000 --- a/sql/migrate/migrate_instrument_metadata.sql +++ /dev/null @@ -1,99 +0,0 @@ --- Migration: Add Instrument Metadata Columns --- Date: 2026-02-11 --- Description: Adds classification and trading specification columns to meta.exchanges and meta.instruments - --- ============================================================================ --- 1. Add columns to meta.exchanges --- ============================================================================ - --- Trading specifications -ALTER TABLE meta.exchanges ADD COLUMN IF NOT EXISTS default_tick_size DOUBLE PRECISION; -ALTER TABLE meta.exchanges ADD COLUMN IF NOT EXISTS default_tick_value DOUBLE PRECISION; -ALTER TABLE meta.exchanges ADD COLUMN IF NOT EXISTS default_extended_hours BOOLEAN DEFAULT false; - --- Geographic info -ALTER TABLE meta.exchanges ADD COLUMN IF NOT EXISTS country VARCHAR(3); - -COMMENT ON COLUMN meta.exchanges.default_tick_size IS 'Default minimum price increment (e.g., 0.01 for stocks)'; -COMMENT ON COLUMN meta.exchanges.default_tick_value IS 'Default value per tick in currency (0.01 for stocks = 1 cent per share)'; -COMMENT ON COLUMN meta.exchanges.default_extended_hours IS 'Default for whether instruments support pre/post market trading'; -COMMENT ON COLUMN meta.exchanges.country IS 'ISO 3166-1 alpha-3 country code (USA, GBR, JPN)'; - --- ============================================================================ --- 2. Add columns to meta.instruments --- ============================================================================ - --- Basic info -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS description VARCHAR(255); -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS country VARCHAR(3); - --- Trading specifications -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS extended_hours BOOLEAN; - --- Company/Fund classification -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS sector VARCHAR(50); -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS industry VARCHAR(100); -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS category VARCHAR(50); -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS subcategory VARCHAR(50); -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS asset_class VARCHAR(30); -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS geography VARCHAR(30); - --- ETF-specific -ALTER TABLE meta.instruments ADD COLUMN IF NOT EXISTS underlying VARCHAR(30); - -COMMENT ON COLUMN meta.instruments.description IS 'Full name/description of the instrument'; -COMMENT ON COLUMN meta.instruments.country IS 'ISO 3166-1 alpha-3 country code (overrides exchange default)'; -COMMENT ON COLUMN meta.instruments.extended_hours IS 'Override exchange default for pre/post market trading'; -COMMENT ON COLUMN meta.instruments.sector IS 'GICS sector (e.g., Technology, Financials)'; -COMMENT ON COLUMN meta.instruments.industry IS 'GICS industry (e.g., Software-Application)'; -COMMENT ON COLUMN meta.instruments.category IS 'Custom categorization (e.g., Sector ETF, Broad Market)'; -COMMENT ON COLUMN meta.instruments.subcategory IS 'Custom sub-categorization (e.g., S&P 500, Technology)'; -COMMENT ON COLUMN meta.instruments.asset_class IS 'Asset class (Equity, Fixed Income, Commodity, Currency)'; -COMMENT ON COLUMN meta.instruments.geography IS 'Geographic focus (US, International, Emerging)'; -COMMENT ON COLUMN meta.instruments.underlying IS 'Underlying symbol for ETFs/derivatives (e.g., SPX, NDX)'; - --- ============================================================================ --- 3. Update existing exchange data --- ============================================================================ - --- Update NYSE if it exists -UPDATE meta.exchanges -SET - default_tick_size = 0.01, - default_tick_value = 0.01, - default_extended_hours = true, - country = 'USA' -WHERE id = 'NYSE'; - --- Update CME if it exists -UPDATE meta.exchanges -SET - country = 'USA', - default_extended_hours = true -WHERE id = 'CME'; - --- Add NASDAQ if it doesn't exist -INSERT INTO meta.exchanges ( - id, name, timezone, currency, default_sec_type, - default_tick_size, default_tick_value, default_extended_hours, country -) -SELECT 'NASDAQ', 'NASDAQ Stock Market', 'America/New_York', 'USD', 'STK', - 0.01, 0.01, true, 'USA' -WHERE NOT EXISTS (SELECT 1 FROM meta.exchanges WHERE id = 'NASDAQ'); - --- ============================================================================ --- Migration Complete --- ============================================================================ - --- Verify the changes -SELECT 'meta.exchanges columns:' as info; -SELECT column_name, data_type, character_maximum_length -FROM information_schema.columns -WHERE table_schema = 'meta' AND table_name = 'exchanges' -ORDER BY ordinal_position; - -SELECT 'meta.instruments columns:' as info; -SELECT column_name, data_type, character_maximum_length -FROM information_schema.columns -WHERE table_schema = 'meta' AND table_name = 'instruments' -ORDER BY ordinal_position; diff --git a/sql/migrate/migrate_metadata_identity.sql b/sql/migrate/migrate_metadata_identity.sql deleted file mode 100644 index 938063e..0000000 --- a/sql/migrate/migrate_metadata_identity.sql +++ /dev/null @@ -1,111 +0,0 @@ --- Migration: add identity constraints to the series registry and reference tables --- --- meta.series had no primary key and no unique constraint, so two rows could --- describe the same series -- and copy_series() actively created that, by --- copying the source's table_id and db_schema. meta.currencies, --- meta.security_types and meta.timezones had no key on their natural --- identifier either, while add_currency()/add_security_type()/add_timezone() --- all catch IntegrityError and promise an "already exists" error that no --- constraint could raise. --- --- These are small registries, not timeseries data. A btree key here costs --- almost nothing, and this is unrelated to the deliberate decision NOT to --- constrain series timestamps, where BRIN is used precisely because a btree --- would be prohibitive at millions of rows per series. --- --- New databases get these from 04_indexes.sql. Run this against any database --- initialised before the fix. --- --- Safe to run more than once: each constraint is added only if absent. --- --- IMPORTANT: this ABORTS if duplicate rows already exist, printing what --- collides. It does not guess which row to keep -- that is a data decision. --- Resolve the duplicates, then run it again. - -DO $$ -DECLARE - dup_count integer; - dup_list text; -BEGIN - ---------------------------------------------------------------------- - -- Pre-flight: refuse to run if any natural key already collides. - ---------------------------------------------------------------------- - SELECT count(*), string_agg(format('%s.%s', db_schema, table_name), ', ') - INTO dup_count, dup_list - FROM (SELECT db_schema, table_name FROM meta.series - GROUP BY db_schema, table_name HAVING count(*) > 1) d; - IF dup_count > 0 THEN - RAISE EXCEPTION - 'meta.series has % duplicated (db_schema, table_name): %. ' - 'Remove the extra rows, then re-run this migration.', - dup_count, dup_list; - END IF; - - SELECT count(*), string_agg(table_id::text, ', ') - INTO dup_count, dup_list - FROM (SELECT table_id FROM meta.series - GROUP BY table_id HAVING count(*) > 1) d; - IF dup_count > 0 THEN - RAISE EXCEPTION - 'meta.series has % duplicated table_id: %. Most likely created by ' - 'copy_series() before it was fixed. Reassign with ' - 'UPDATE meta.series SET table_id = nextval(''meta._table_meta__table_id_seq'') ' - 'WHERE ... , then re-run this migration.', - dup_count, dup_list; - END IF; - - SELECT count(*), string_agg(code, ', ') - INTO dup_count, dup_list - FROM (SELECT code FROM meta.currencies GROUP BY code HAVING count(*) > 1) d; - IF dup_count > 0 THEN - RAISE EXCEPTION 'meta.currencies has % duplicated code: %', dup_count, dup_list; - END IF; - - SELECT count(*), string_agg(code, ', ') - INTO dup_count, dup_list - FROM (SELECT code FROM meta.security_types GROUP BY code HAVING count(*) > 1) d; - IF dup_count > 0 THEN - RAISE EXCEPTION 'meta.security_types has % duplicated code: %', dup_count, dup_list; - END IF; - - SELECT count(*), string_agg(iana_name, ', ') - INTO dup_count, dup_list - FROM (SELECT iana_name FROM meta.timezones GROUP BY iana_name HAVING count(*) > 1) d; - IF dup_count > 0 THEN - RAISE EXCEPTION 'meta.timezones has % duplicated iana_name: %', dup_count, dup_list; - END IF; - - ---------------------------------------------------------------------- - -- Add the constraints, skipping any that already exist. - ---------------------------------------------------------------------- - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'series_pkey' - AND conrelid = 'meta.series'::regclass) THEN - ALTER TABLE meta.series ADD CONSTRAINT series_pkey PRIMARY KEY (table_id); - RAISE NOTICE 'added meta.series primary key on (table_id)'; - END IF; - - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'series_schema_table_key' - AND conrelid = 'meta.series'::regclass) THEN - ALTER TABLE meta.series ADD CONSTRAINT series_schema_table_key - UNIQUE (db_schema, table_name); - RAISE NOTICE 'added meta.series unique constraint on (db_schema, table_name)'; - END IF; - - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'currencies_pkey' - AND conrelid = 'meta.currencies'::regclass) THEN - ALTER TABLE meta.currencies ADD CONSTRAINT currencies_pkey PRIMARY KEY (code); - RAISE NOTICE 'added meta.currencies primary key on (code)'; - END IF; - - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'security_types_pkey' - AND conrelid = 'meta.security_types'::regclass) THEN - ALTER TABLE meta.security_types ADD CONSTRAINT security_types_pkey PRIMARY KEY (code); - RAISE NOTICE 'added meta.security_types primary key on (code)'; - END IF; - - IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'timezones_pkey' - AND conrelid = 'meta.timezones'::regclass) THEN - ALTER TABLE meta.timezones ADD CONSTRAINT timezones_pkey PRIMARY KEY (iana_name); - RAISE NOTICE 'added meta.timezones primary key on (iana_name)'; - END IF; -END $$;