peter/tsapi-seriesdb-shared-sql
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
tsapi-seriesdb-shared-sql/sql/09_schema_check.sql
peter 4c067bc4b9 Schema 1.2.0: meta.series_schemas registry; lower-case name checks 
- meta.series_schemas (name, description, created_at, created_by), seeded
  with 'series'. FK series_db_schema_fkey from meta.series.db_schema: a
  series may only live in a registered schema, for every client.
  Unregistered schemas belong to the user and are never touched.
- CHECK series_schemas_name_check and series_table_name_check enforce the
  shared lower-case identifier rule in the database.
- meta.find_schema_problems() covers the registry and reports a registered
  schema that no longer exists; user schemas are not examined.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 11:42:10 +01:00

339 lines
18 KiB
PL/PgSQL
Raw Blame History

-- ============================================================================
-- Schema self-check: meta.find_schema_problems()
-- ============================================================================
-- Answers one question for every client (Python, C++, psql):
-- is this database complete and as the DDL defines it?
--
-- SELECT * FROM meta.find_schema_problems(); -- zero rows = usable
--
-- Returns one row per problem: (object, problem). Read-only.
--
-- The expectations below mirror 00..08. When any of those files changes,
-- update this file in the same commit. The release tests (a freshly
-- provisioned database must return zero rows) fail until you do.
--
-- Function bodies are compared by fingerprint: md5 of the body with runs of
-- whitespace collapsed, so line endings and re-indentation do not count as a
-- change. To refresh the fingerprints after editing a function:
--
-- SELECT n.nspname || '.' || p.proname,
-- md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g')))
-- FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
-- WHERE n.nspname IN ('meta', 'log') ORDER BY 1;
--
-- The schema VERSION string is not compared here: each client compares
-- meta.schema_version with the version it was built for. This function only
-- reports a missing version row.
-- ============================================================================
CREATE OR REPLACE FUNCTION meta.find_schema_problems()
RETURNS TABLE(object text, problem text)
LANGUAGE plpgsql
STABLE
AS $function$
BEGIN
-- Schemas and extension -------------------------------------------------
RETURN QUERY
SELECT 'schema ' || s, 'missing'
FROM unnest(ARRAY['meta', 'series', 'log']) AS s
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s);
RETURN QUERY
SELECT 'extension ' || e, 'missing'
FROM unnest(ARRAY['pg_trgm']) AS e
WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e);
RETURN QUERY
SELECT 'sequence ' || s, 'missing'
FROM unnest(ARRAY[
'meta._table_meta__table_id_seq',
'log.event_log_id_seq',
'log.pending_changes_id_seq',
'log.script_log_id_seq'
]) AS s
WHERE to_regclass(s) IS NULL;
-- Tables and columns ----------------------------------------------------
RETURN QUERY
WITH e(tbl, col, typ, not_null) AS (VALUES
('log.pending_changes', 'id', 'bigint', true),
('log.pending_changes', 'timestamp', 'timestamp with time zone', false),
('log.pending_changes', 'series_name', 'text', true),
('log.pending_changes', 'db_schema', 'text', true),
('log.pending_changes', 'op_type', 'text', true),
('log.pending_changes', 'db_user', 'text', false),
('log.pending_changes', 'details', 'jsonb', false),
('log.event_log', 'id', 'bigint', true),
('log.event_log', 'timestamp', 'timestamp with time zone', false),
('log.event_log', 'series_name', 'text', true),
('log.event_log', 'db_schema', 'text', true),
('log.event_log', 'event_type', 'text', true),
('log.event_log', 'details', 'jsonb', false),
('log.event_log', 'source', 'text', false),
('log.event_log', 'db_user', 'text', false),
('meta.currencies', 'code', 'character varying(3)', true),
('meta.currencies', 'name', 'character varying(100)', true),
('meta.currencies', 'symbol', 'character varying(10)', false),
('meta.currencies', 'decimal_places', 'integer', false),
('meta.currencies', 'notes', 'text', false),
('meta.exchanges', 'id', 'character varying(12)', true),
('meta.exchanges', 'name', 'character varying(50)', false),
('meta.exchanges', 'timezone', 'character varying(35)', false),
('meta.exchanges', 'currency', 'character varying(3)', false),
('meta.exchanges', 'default_sec_type', 'character varying(18)', false),
('meta.exchanges', 'default_session', 'character varying(40)', false),
('meta.exchanges', 'default_holiday_calendar', 'character varying(40)', false),
('meta.exchanges', 'default_tick_size', 'double precision', false),
('meta.exchanges', 'default_tick_value', 'double precision', false),
('meta.exchanges', 'default_extended_hours', 'boolean', false),
('meta.exchanges', 'country', 'character varying(3)', false),
('meta.holidays', 'calendar_name', 'character varying(50)', true),
('meta.holidays', 'date', 'date', true),
('meta.holidays', 'description', 'character varying(255)', false),
('meta.holidays', 'early_close', 'boolean', true),
('meta.instruments', 'symbol', 'character varying(12)', true),
('meta.instruments', 'exchange', 'character varying(12)', true),
('meta.instruments', 'name', 'character varying(50)', true),
('meta.instruments', 'type', 'character varying(10)', false),
('meta.instruments', 'tick_size', 'double precision', false),
('meta.instruments', 'tick_value', 'double precision', false),
('meta.instruments', 'expiry', 'character varying(20)', false),
('meta.instruments', 'strike', 'double precision', false),
('meta.instruments', 'currency', 'character varying(3)', false),
('meta.instruments', 'session', 'character varying(40)', false),
('meta.instruments', 'holiday_calendar', 'character varying(50)', false),
('meta.instruments', 'sec_id', 'character varying(50)', false),
('meta.instruments', 'sec_id_type', 'character varying(20)', false),
('meta.instruments', 'description', 'character varying(255)', false),
('meta.instruments', 'country', 'character varying(3)', false),
('meta.instruments', 'extended_hours', 'boolean', false),
('meta.instruments', 'sector', 'character varying(50)', false),
('meta.instruments', 'industry', 'character varying(100)', false),
('meta.instruments', 'category', 'character varying(50)', false),
('meta.instruments', 'subcategory', 'character varying(50)', false),
('meta.instruments', 'asset_class', 'character varying(30)', false),
('meta.instruments', 'geography', 'character varying(30)', false),
('meta.instruments', 'underlying', 'character varying(30)', false),
('meta.series_schemas', 'name', 'text', true),
('meta.series_schemas', 'description', 'text', false),
('meta.series_schemas', 'created_at', 'timestamp with time zone', true),
('meta.series_schemas', 'created_by', 'text', true),
('meta.security_types', 'code', 'character varying(10)', true),
('meta.security_types', 'name', 'character varying(50)', true),
('meta.security_types', 'description', 'text', false),
('meta.security_types', 'notes', 'text', false),
('meta.schema_version', 'version', 'character varying(20)', true),
('meta.schema_version', 'installed_at', 'timestamp with time zone', true),
('meta.schema_version', 'description', 'text', false),
('meta.series', 'table_id', 'integer', true),
('meta.series', 'db_schema', 'character varying(70)', true),
('meta.series', 'table_name', 'character varying(70)', true),
('meta.series', 'type', 'character varying(30)', false),
('meta.series', 'description', 'character varying(255)', false),
('meta.series', 'column_schema', 'text', false),
('meta.series', 'symbol', 'character varying(30)', false),
('meta.series', 'root_symbol', 'character varying(30)', false),
('meta.series', 'exchange', 'character varying(30)', false),
('meta.series', 'frequency', 'character varying(30)', false),
('meta.series', 'session', 'text', false),
('meta.series', 'sec_type', 'character varying(30)', false),
('meta.series', 'sec_id', 'character varying(30)', false),
('meta.series', 'sec_id_type', 'character varying(30)', false),
('meta.series', 'create_timestamp', 'timestamp without time zone', false),
('meta.series', 'num_records', 'bigint', false),
('meta.series', 'first_timestamp', 'timestamp without time zone', false),
('meta.series', 'last_timestamp', 'timestamp without time zone', false),
('meta.series', 'bar_type', 'character varying(30)', false),
('meta.series', 'feed', 'character varying(30)', false),
('meta.series', 'currency', 'character varying(30)', false),
('meta.series', 'expiry', 'character varying(30)', false),
('meta.series', 'roll_type', 'character varying(30)', false),
('meta.series', 'feed_type', 'character varying(30)', false),
('meta.series', 'user_attr', 'text', false),
('meta.series', 'user_attr_2', 'text', false),
('meta.series', 'script', 'text', false),
('meta.series', 'notes', 'text', false),
('meta.series', 'timezone', 'character varying(50)', false),
('meta.series', 'category', 'character varying(30)', false),
('meta.series', 'subcategory', 'character varying(30)', false),
('meta.series', 'json', 'jsonb', false),
('meta.series', 'needs_reindex', 'boolean', true),
('meta.sessions', 'name', 'character varying(50)', true),
('meta.sessions', 'definition', 'text', true),
('meta.sessions', 'json', 'text', false),
('meta.sessions', 'exchange_id', 'character varying(12)', true),
('meta.timezones', 'iana_name', 'character varying(50)', true),
('meta.timezones', 'display_name', 'character varying(100)', true),
('meta.timezones', 'description', 'text', false),
('meta.timezones', 'region', 'character varying(50)', false),
('meta.timezones', 'utc_offset_hours', 'real', false),
('meta.timezones', 'notes', 'text', false),
('log.script_log', 'id', 'integer', true),
('log.script_log', 'timestamp', 'timestamp with time zone', true),
('log.script_log', 'level', 'character varying(10)', true),
('log.script_log', 'message', 'text', true),
('log.script_log', 'script_name', 'character varying(100)', false),
('log.script_log', 'series_name', 'character varying(255)', false),
('log.script_log', 'session_id', 'uuid', false),
('log.script_log', 'details', 'jsonb', false)
),
a AS (
SELECT e.*, x.attname IS NOT NULL AS present,
format_type(x.atttypid, x.atttypmod) AS got_typ, x.attnotnull AS got_not_null
FROM e
LEFT JOIN pg_attribute x ON x.attrelid = to_regclass(e.tbl) AND x.attname = e.col
AND x.attnum > 0 AND NOT x.attisdropped
WHERE to_regclass(e.tbl) IS NOT NULL
),
found(sort_key, object, problem) AS (
SELECT 1, 'table ' || tbl, 'missing'
FROM (SELECT DISTINCT tbl FROM e WHERE to_regclass(tbl) IS NULL) m
UNION ALL
SELECT 2, 'table ' || tbl, 'missing columns: ' || string_agg(col, ', ')
FROM a WHERE NOT present GROUP BY tbl
UNION ALL
SELECT 3, 'table ' || tbl, col || ' is ' || got_typ || ', expected ' || typ
FROM a WHERE present AND got_typ <> typ
UNION ALL
SELECT 4, 'table ' || tbl,
col || ' is ' || CASE WHEN got_not_null THEN 'NOT NULL' ELSE 'nullable' END
|| ', expected ' || CASE WHEN not_null THEN 'NOT NULL' ELSE 'nullable' END
FROM a WHERE present AND got_not_null <> not_null
)
SELECT f.object, f.problem FROM found f ORDER BY f.object, f.sort_key, f.problem;
-- Constraints and indexes (only on tables that exist) -------------------
RETURN QUERY
SELECT 'constraint ' || c.tbl || '.' || c.name, 'missing'
FROM (VALUES
('log.pending_changes', 'pending_changes_pkey'),
('log.event_log', 'event_log_pkey'),
('meta.holidays', 'holidays_pkey'),
('log.script_log', 'script_log_pkey'),
('meta.instruments', 'instruments_pkey'),
('meta.exchanges', 'exchanges_pkey'),
('meta.sessions', 'sessions_pkey'),
('meta.series', 'series_pkey'),
('meta.series', 'series_schema_table_key'),
('meta.series', 'series_db_schema_fkey'),
('meta.series', 'series_table_name_check'),
('meta.series_schemas', 'series_schemas_pkey'),
('meta.series_schemas', 'series_schemas_name_check'),
('meta.currencies', 'currencies_pkey'),
('meta.security_types', 'security_types_pkey'),
('meta.timezones', 'timezones_pkey')
) AS c(tbl, name)
WHERE to_regclass(c.tbl) IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM pg_constraint k
WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name);
RETURN QUERY
SELECT 'index ' || i.idx, 'missing'
FROM (VALUES
('log.idx_event_log_series_timestamp', 'log.event_log'),
('log.idx_event_log_timestamp', 'log.event_log'),
('log.idx_event_log_event_type', 'log.event_log'),
('meta.idx_series_create_timestamp_desc', 'meta.series'),
('meta.idx_series_db_schema', 'meta.series'),
('meta.idx_series_exchange_pattern', 'meta.series'),
('meta.idx_series_frequency_pattern', 'meta.series'),
('meta.idx_series_symbol_exchange', 'meta.series'),
('meta.idx_series_symbol_pattern', 'meta.series'),
('meta.idx_series_symbol_trgm', 'meta.series'),
('meta.idx_series_table_name_trgm', 'meta.series'),
('meta.idx_series_type_pattern', 'meta.series'),
('log.idx_script_log_timestamp', 'log.script_log'),
('log.idx_script_log_session', 'log.script_log'),
('log.idx_script_log_series', 'log.script_log'),
('log.idx_script_log_level', 'log.script_log')
) AS i(idx, tbl)
WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL;
RETURN QUERY
SELECT 'view ' || v, 'missing'
FROM unnest(ARRAY['log.script_log_recent']) AS v
WHERE to_regclass(v) IS NULL;
-- Functions (compared by whitespace-insensitive body fingerprint) -------
RETURN QUERY
SELECT 'function ' || f.name,
CASE WHEN NOT EXISTS (SELECT 1 FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
WHERE n.nspname || '.' || p.proname = f.name)
THEN 'missing'
ELSE 'differs from the DDL (stale or edited)' END
FROM (VALUES
('log.flush_pending_to_event_log', 'be02d914fc5e73475eb1ce7ce7a78f29'),
('log.notify_event_log', '7e5898f1d1d128ef2e44c1d4ae8a7312'),
('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'),
('meta.drop_series', '6f2a919de965b87521be68252deee0e2'),
('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'),
('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'),
('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'),
('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'),
('meta.read_series_batch_ohlc', '4bdef4d3573caab023ddd12db38b978c'),
('meta.refresh_series_timestamps', '1cdfd496752142cbde78d9646532d1aa'),
('meta.update_series_last_timestamp', 'f27394672bc5026456c4625ac3091a62')
) AS f(name, fingerprint)
WHERE NOT EXISTS (SELECT 1 FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
WHERE n.nspname || '.' || p.proname = f.name
AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint);
-- Drop-protection event trigger -----------------------------------------
RETURN QUERY
SELECT 'event trigger prevent_series_drops', 'missing'
WHERE NOT EXISTS (SELECT 1 FROM pg_event_trigger WHERE evtname = 'prevent_series_drops');
RETURN QUERY
SELECT 'event trigger prevent_series_drops', p
FROM pg_event_trigger t,
LATERAL (VALUES
(CASE WHEN t.evtevent <> 'sql_drop'
THEN 'fires on ' || t.evtevent || ', expected sql_drop' END),
(CASE WHEN t.evtenabled = 'D' THEN 'disabled' END),
(CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop'
THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END)
) AS x(p)
WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL;
-- Reference data and version row ----------------------------------------
RETURN QUERY
SELECT 'reference data ' || r.tbl,
r.actual || ' rows, expected at least ' || r.expected
FROM (
SELECT e.tbl, e.expected,
(xpath('/row/n/text()', query_to_xml(
format('SELECT count(*) AS n FROM %s', e.tbl), false, true, '')))[1]::text::int
AS actual
FROM (VALUES
('meta.timezones', 563),
('meta.currencies', 179),
('meta.security_types', 13),
('meta.series_schemas', 1)
) AS e(tbl, expected)
WHERE to_regclass(e.tbl) IS NOT NULL
) AS r
WHERE r.actual < r.expected;
-- Series schema registry ------------------------------------------------
-- Only REGISTERED schemas are examined; user schemas are never looked at.
IF to_regclass('meta.series_schemas') IS NOT NULL THEN
RETURN QUERY
SELECT 'series schema series', 'not registered in meta.series_schemas'
WHERE NOT EXISTS (SELECT 1 FROM meta.series_schemas WHERE name = 'series');
RETURN QUERY
SELECT 'series schema ' || r.name, 'registered but the schema does not exist'
FROM meta.series_schemas r
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = r.name)
ORDER BY r.name;
END IF;
IF to_regclass('meta.schema_version') IS NOT NULL THEN
RETURN QUERY
SELECT 'schema version', 'not recorded in meta.schema_version'
WHERE NOT EXISTS (SELECT 1 FROM meta.schema_version);
END IF;
END;
$function$;
Reference in a new issue View git blame Copy permalink