From 155fc7c3ddb1291eeccdc33766afde29401afa2e Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 6 Oct 2026 11:20:51 +0100 Subject: [PATCH 1/5] Schema version 1.1.0 meta.find_schema_problems() (09), the version row in the DDL (10), every install running 00-10, and the removal of sql/migrate/ changed what a complete database is. Databases from 1.0.0 are re-provisioned. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- VERSION | 2 +- sql/10_schema_version.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d06f31..9398869 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ timeseries database. Both client libraries target the schema defined here: Neither project should keep its own copy of the DDL. When the schema changes, it changes **here**, and both projects re-provision / re-test against it. -Current schema version: **1.0.0** (see [`VERSION`](VERSION); recorded at runtime +Current schema version: **1.1.0** (see [`VERSION`](VERSION); recorded at runtime in the `meta.schema_version` table). --- diff --git a/VERSION b/VERSION index 3eefcb9..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/sql/10_schema_version.sql b/sql/10_schema_version.sql index 120704a..00c6866 100644 --- a/sql/10_schema_version.sql +++ b/sql/10_schema_version.sql @@ -5,4 +5,4 @@ -- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py). INSERT INTO meta.schema_version (version, description) -VALUES ('1.0.0', 'Initial schema'); +VALUES ('1.1.0', 'meta.find_schema_problems(); every install runs 00-10'); From 4c067bc4b91798330c3c42a90c140121419e75a7 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 6 Oct 2026 11:42:10 +0100 Subject: [PATCH 2/5] Schema 1.2.0: meta.series_schemas registry; lower-case name checks - meta.series_schemas (name, description, created_at, created_by), seeded with 'series'. FK series_db_schema_fkey from meta.series.db_schema: a series may only live in a registered schema, for every client. Unregistered schemas belong to the user and are never touched. - CHECK series_schemas_name_check and series_table_name_check enforce the shared lower-case identifier rule in the database. - meta.find_schema_problems() covers the registry and reports a registered schema that no longer exists; user schemas are not examined. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- VERSION | 2 +- sql/03_tables.sql | 10 ++++++++++ sql/04_indexes.sql | 17 +++++++++++++++++ sql/07_reference_data.sql | 6 +++++- sql/09_schema_check.sql | 25 ++++++++++++++++++++++++- sql/10_schema_version.sql | 2 +- 7 files changed, 59 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9398869..2ca361e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ timeseries database. Both client libraries target the schema defined here: Neither project should keep its own copy of the DDL. When the schema changes, it changes **here**, and both projects re-provision / re-test against it. -Current schema version: **1.1.0** (see [`VERSION`](VERSION); recorded at runtime +Current schema version: **1.2.0** (see [`VERSION`](VERSION); recorded at runtime in the `meta.schema_version` table). --- diff --git a/VERSION b/VERSION index 9084fa2..26aaba0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.2.0 diff --git a/sql/03_tables.sql b/sql/03_tables.sql index f6e5a2b..cc009ed 100644 --- a/sql/03_tables.sql +++ b/sql/03_tables.sql @@ -135,6 +135,16 @@ CREATE TABLE meta.sessions ( exchange_id character varying(12) NOT NULL ); +-- Registry of series schemas. A series may only live in a registered schema +-- (FK from meta.series.db_schema, 04_indexes.sql). Every other schema in the +-- database belongs to the user and is never touched by the clients. +CREATE TABLE meta.series_schemas ( + name text NOT NULL, + description text, + created_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_by text NOT NULL DEFAULT CURRENT_USER +); + CREATE TABLE meta.timezones ( iana_name character varying(50) NOT NULL, display_name character varying(100) NOT NULL, diff --git a/sql/04_indexes.sql b/sql/04_indexes.sql index 20c50c2..6f192df 100644 --- a/sql/04_indexes.sql +++ b/sql/04_indexes.sql @@ -21,6 +21,23 @@ ALTER TABLE meta.currencies ADD CONSTRAINT currencies_pkey PRIMARY KEY (code); ALTER TABLE meta.security_types ADD CONSTRAINT security_types_pkey PRIMARY KEY (code); ALTER TABLE meta.timezones ADD CONSTRAINT timezones_pkey PRIMARY KEY (iana_name); +-- Series schema registry. Names follow the table-name rule shared by both +-- clients: lower case, [a-z_][a-z0-9_]*, at most 63 bytes, never quoted. +ALTER TABLE meta.series_schemas ADD CONSTRAINT series_schemas_pkey PRIMARY KEY (name); +ALTER TABLE meta.series_schemas ADD CONSTRAINT series_schemas_name_check + CHECK (name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(name) <= 63); + +-- A series may only live in a registered schema -- enforced here, so no +-- client can bypass it. RESTRICT: a schema cannot be unregistered while it +-- still holds series. +ALTER TABLE meta.series ADD CONSTRAINT series_db_schema_fkey + FOREIGN KEY (db_schema) REFERENCES meta.series_schemas (name) + ON UPDATE RESTRICT ON DELETE RESTRICT; + +-- Table names follow the same rule (Core contract 6 in both clients). +ALTER TABLE meta.series ADD CONSTRAINT series_table_name_check + CHECK (table_name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(table_name) <= 63); + CREATE INDEX idx_event_log_series_timestamp ON log.event_log USING btree (series_name, "timestamp" DESC); CREATE INDEX idx_event_log_timestamp ON log.event_log USING btree ("timestamp" DESC); diff --git a/sql/07_reference_data.sql b/sql/07_reference_data.sql index 23c1fdd..2a4b73b 100644 --- a/sql/07_reference_data.sql +++ b/sql/07_reference_data.sql @@ -1,4 +1,4 @@ --- Reference data (timezones and currencies) +-- Reference data (series schema registry, timezones, currencies, security types) -- Generated: 2025-10-15T12:24:56.582392 -- Timezones @@ -767,3 +767,7 @@ VALUES ('OPT', 'Option', 'Equity and index options', NULL), ('STK', 'Stock', 'Common and preferred equities', NULL), ('WAR', 'Warrant', 'Equity warrants', NULL); + +-- Series schema registry: the default schema, created in 01_schemas.sql +INSERT INTO meta.series_schemas (name, description) +VALUES ('series', 'Default series schema'); diff --git a/sql/09_schema_check.sql b/sql/09_schema_check.sql index 48a011d..8e6d46f 100644 --- a/sql/09_schema_check.sql +++ b/sql/09_schema_check.sql @@ -114,6 +114,10 @@ BEGIN ('meta.instruments', 'asset_class', 'character varying(30)', false), ('meta.instruments', 'geography', 'character varying(30)', false), ('meta.instruments', 'underlying', 'character varying(30)', false), + ('meta.series_schemas', 'name', 'text', true), + ('meta.series_schemas', 'description', 'text', false), + ('meta.series_schemas', 'created_at', 'timestamp with time zone', true), + ('meta.series_schemas', 'created_by', 'text', true), ('meta.security_types', 'code', 'character varying(10)', true), ('meta.security_types', 'name', 'character varying(50)', true), ('meta.security_types', 'description', 'text', false), @@ -211,6 +215,10 @@ BEGIN ('meta.sessions', 'sessions_pkey'), ('meta.series', 'series_pkey'), ('meta.series', 'series_schema_table_key'), + ('meta.series', 'series_db_schema_fkey'), + ('meta.series', 'series_table_name_check'), + ('meta.series_schemas', 'series_schemas_pkey'), + ('meta.series_schemas', 'series_schemas_name_check'), ('meta.currencies', 'currencies_pkey'), ('meta.security_types', 'security_types_pkey'), ('meta.timezones', 'timezones_pkey') @@ -301,12 +309,27 @@ BEGIN FROM (VALUES ('meta.timezones', 563), ('meta.currencies', 179), - ('meta.security_types', 13) + ('meta.security_types', 13), + ('meta.series_schemas', 1) ) AS e(tbl, expected) WHERE to_regclass(e.tbl) IS NOT NULL ) AS r WHERE r.actual < r.expected; + -- Series schema registry ------------------------------------------------ + -- Only REGISTERED schemas are examined; user schemas are never looked at. + IF to_regclass('meta.series_schemas') IS NOT NULL THEN + RETURN QUERY + SELECT 'series schema series', 'not registered in meta.series_schemas' + WHERE NOT EXISTS (SELECT 1 FROM meta.series_schemas WHERE name = 'series'); + + RETURN QUERY + SELECT 'series schema ' || r.name, 'registered but the schema does not exist' + FROM meta.series_schemas r + WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = r.name) + ORDER BY r.name; + END IF; + IF to_regclass('meta.schema_version') IS NOT NULL THEN RETURN QUERY SELECT 'schema version', 'not recorded in meta.schema_version' diff --git a/sql/10_schema_version.sql b/sql/10_schema_version.sql index 00c6866..c6cd3b4 100644 --- a/sql/10_schema_version.sql +++ b/sql/10_schema_version.sql @@ -5,4 +5,4 @@ -- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py). INSERT INTO meta.schema_version (version, description) -VALUES ('1.1.0', 'meta.find_schema_problems(); every install runs 00-10'); +VALUES ('1.2.0', 'meta.series_schemas registry; lower-case table name check'); From eb0f93e0326870d05a134b17d568ca0c96c9800f Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 6 Oct 2026 11:48:26 +0100 Subject: [PATCH 3/5] Schema 1.3.0: meta.find_series_registry_mismatches() An occasional consistency check (not for connect time) comparing meta.series with the registered series schemas: non-series tables, views, sequences and functions inside a series schema; registered series whose table is missing; series tables without a timestamp column or its BRIN index. Report only; user schemas are never examined. find_schema_problems() fingerprints its body like every other procedure. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- VERSION | 2 +- sql/09_schema_check.sql | 105 ++++++++++++++++++++++++++++++++++++-- sql/10_schema_version.sql | 2 +- 4 files changed, 104 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2ca361e..5d02ae8 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ timeseries database. Both client libraries target the schema defined here: Neither project should keep its own copy of the DDL. When the schema changes, it changes **here**, and both projects re-provision / re-test against it. -Current schema version: **1.2.0** (see [`VERSION`](VERSION); recorded at runtime +Current schema version: **1.3.0** (see [`VERSION`](VERSION); recorded at runtime in the `meta.schema_version` table). --- diff --git a/VERSION b/VERSION index 26aaba0..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/sql/09_schema_check.sql b/sql/09_schema_check.sql index 8e6d46f..951d224 100644 --- a/sql/09_schema_check.sql +++ b/sql/09_schema_check.sql @@ -1,10 +1,16 @@ -- ============================================================================ --- Schema self-check: meta.find_schema_problems() +-- Database self-checks, callable by every client (Python, C++, psql) -- ============================================================================ --- Answers one question for every client (Python, C++, psql): --- is this database complete and as the DDL defines it? -- --- SELECT * FROM meta.find_schema_problems(); -- zero rows = usable +-- meta.find_schema_problems() -- on connect: is the database complete and as +-- the DDL defines it? zero rows = usable +-- +-- meta.find_series_registry_mismatches() -- occasional consistency check: +-- do the series schemas and meta.series agree? (defined at the end) +-- +-- -------------------------------------------------------------------------- +-- meta.find_schema_problems() +-- -------------------------------------------------------------------------- -- -- Returns one row per problem: (object, problem). Read-only. -- @@ -268,6 +274,7 @@ BEGIN ('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'), ('meta.drop_series', '6f2a919de965b87521be68252deee0e2'), ('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'), + ('meta.find_series_registry_mismatches', '6dc9ecd2b7656b91a8b410c78a19fa2b'), ('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'), ('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'), ('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'), @@ -337,3 +344,93 @@ BEGIN END IF; END; $function$; + + +-- -------------------------------------------------------------------------- +-- meta.find_series_registry_mismatches() +-- -------------------------------------------------------------------------- +-- Occasional consistency check -- NOT run on connect. Compares the series +-- registry (meta.series) with what actually exists in the REGISTERED series +-- schemas (meta.series_schemas). User schemas are never examined. +-- +-- SELECT * FROM meta.find_series_registry_mismatches(); -- zero rows = consistent +-- +-- One (object, problem) row per finding: +-- * a table, view, materialized view, foreign table or sequence in a +-- series schema that is not a registered series (an orphaned series table, +-- or a user table in the wrong place) -- harmless to the clients; +-- * a function in a series schema -- harmless; +-- * a series registered in meta.series whose table does not exist -- +-- every operation on that series fails; +-- * a series table without a "timestamp timestamp without time zone NOT +-- NULL" column, or without a BRIN index on it. +-- Read-only. It reports; deciding what to do with a finding is up to the user. + +CREATE OR REPLACE FUNCTION meta.find_series_registry_mismatches() + RETURNS TABLE(object text, problem text) + LANGUAGE plpgsql + STABLE +AS $function$ +BEGIN + -- Relations in series schemas that are not registered series ----------- + RETURN QUERY + SELECT n.nspname || '.' || c.relname, + CASE c.relkind + WHEN 'r' THEN 'table' WHEN 'p' THEN 'table' WHEN 'v' THEN 'view' + WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table' + ELSE 'sequence' END + || ' in a series schema but not registered in meta.series' + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN meta.series_schemas r ON r.name = n.nspname + WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S') + AND NOT EXISTS (SELECT 1 FROM meta.series m + WHERE m.db_schema = n.nspname AND m.table_name = c.relname) + ORDER BY 1; + + RETURN QUERY + SELECT n.nspname || '.' || p.proname, 'function in a series schema' + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + JOIN meta.series_schemas r ON r.name = n.nspname + ORDER BY 1; + + -- Registered series whose table is missing ------------------------------ + RETURN QUERY + SELECT m.db_schema || '.' || m.table_name, + 'registered in meta.series but the table does not exist' + FROM meta.series m + WHERE to_regclass(format('%I.%I', m.db_schema, m.table_name)) IS NULL + ORDER BY 1; + + -- Series tables without a proper timestamp column or its BRIN index ----- + RETURN QUERY + SELECT m.db_schema || '.' || m.table_name, + CASE WHEN a.attname IS NULL THEN 'no "timestamp" column' + ELSE '"timestamp" is ' || format_type(a.atttypid, a.atttypmod) + || CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END + || ', expected timestamp without time zone NOT NULL' END + FROM meta.series m + CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t + LEFT JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp' + AND a.attnum > 0 AND NOT a.attisdropped + WHERE t.rel IS NOT NULL + AND (a.attname IS NULL + OR format_type(a.atttypid, a.atttypmod) <> 'timestamp without time zone' + OR NOT a.attnotnull) + ORDER BY 1; + + RETURN QUERY + SELECT m.db_schema || '.' || m.table_name, 'no BRIN index on "timestamp"' + FROM meta.series m + CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t + JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp' + AND a.attnum > 0 AND NOT a.attisdropped + WHERE NOT EXISTS ( + SELECT 1 FROM pg_index i + JOIN pg_class ic ON ic.oid = i.indexrelid + JOIN pg_am am ON am.oid = ic.relam + WHERE i.indrelid = t.rel AND am.amname = 'brin' AND i.indkey[0] = a.attnum) + ORDER BY 1; +END; +$function$; diff --git a/sql/10_schema_version.sql b/sql/10_schema_version.sql index c6cd3b4..7fee6e6 100644 --- a/sql/10_schema_version.sql +++ b/sql/10_schema_version.sql @@ -5,4 +5,4 @@ -- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py). INSERT INTO meta.schema_version (version, description) -VALUES ('1.2.0', 'meta.series_schemas registry; lower-case table name check'); +VALUES ('1.3.0', 'meta.find_series_registry_mismatches()'); From 9032fd3a458117e89becc70202f57747bdb4b823 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 6 Oct 2026 11:58:46 +0100 Subject: [PATCH 4/5] run_setup: refuse an existing installation; remove --clean-slate - Removed --clean-slate: it dropped every schema in the database, including user data. Nothing used it. - Pre-check (same rule as the Python client's init_postgres_db()): refuse if meta or log exists, series is not empty, or the prevent_series_drops trigger exists, naming the installed version. User schemas and tables are allowed and never touched; an empty series schema is reused. - --drop-existing drops every registered series schema plus meta and log, and refuses if a series schema holds non-series objects. - All files run in one transaction: a failure leaves nothing behind. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 ++++- run_setup.py | 145 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 109 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 5d02ae8..6d2c750 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,18 @@ cp db_config.json.example db_config.json # then edit host/user/password # 2. create the schema python run_setup.py # uses db_config.json / defaults python run_setup.py --host 192.168.1.7 --user claude --password claudepwd -python run_setup.py --drop-existing # drop meta+series+log first -python run_setup.py --clean-slate # drop ALL user objects first +python run_setup.py --drop-existing # drop the existing installation first + # (ALL series data); user schemas untouched ``` +The database may hold the user's own schemas and tables; provisioning never +touches them. It **refuses** a database that already holds the library's +objects -- `meta`, `log`, a non-empty `series` schema, or the +`prevent_series_drops` event trigger -- and reports the installed version. +There are no migrations: re-provision with `--drop-existing` (deletes all +series) or use another database. All files run in one transaction, so a +failure leaves the database untouched. + Requires `psycopg2` and a PostgreSQL user with CREATE privileges on the target database. diff --git a/run_setup.py b/run_setup.py index 6042d16..2a7a2f8 100644 --- a/run_setup.py +++ b/run_setup.py @@ -22,8 +22,12 @@ Connection settings come from (highest precedence first): Usage: python run_setup.py # use db_config.json / defaults python run_setup.py --host 192.168.1.7 --user claude --password claudepwd - python run_setup.py --drop-existing # drop meta+series schemas first - python run_setup.py --clean-slate # drop ALL user objects first + python run_setup.py --drop-existing # drop the existing installation first + # (all series!); user schemas untouched + +A database that already holds the library's objects (meta, log, a non-empty +series schema, the event trigger) is refused -- there are no migrations. +User schemas and tables may exist and are never touched. """ import sys @@ -82,8 +86,7 @@ def execute_sql_file(conn, sql_file: Path): try: sql = sql_file.read_text(encoding="utf-8") with conn.cursor() as cur: - cur.execute(sql) - conn.commit() + cur.execute(sql) # committed by the caller, all files at once return True, f"[OK] Executed {sql_file.name}" except Exception as e: conn.rollback() @@ -122,23 +125,72 @@ def verify_setup(conn) -> None: print(f"[OK] schema_version: {row[0]} ({row[1]})") -def clean_slate(conn) -> None: - print("Performing clean slate...") +# Pre-check, shared rule with the Python client's init_postgres_db(): +# provisioning needs the library's own objects to be ABSENT. User schemas and +# tables are irrelevant and never touched. Never drop-and-recreate: an +# existing installation is refused, because there are no migrations and +# re-provisioning deletes all series. +PRECHECK_SQL = """ +SELECT 'schema ' || nspname || ' exists' +FROM pg_namespace WHERE nspname IN ('meta', 'log') +UNION ALL +SELECT 'schema series is not empty (' || count(*) || ' objects)' +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname = 'series' AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S') +HAVING count(*) > 0 +UNION ALL +SELECT 'event trigger prevent_series_drops exists' +FROM pg_event_trigger WHERE evtname = 'prevent_series_drops' +""" + + +def existing_installation(conn) -> list: + """What would stop a fresh install; [] means go ahead.""" with conn.cursor() as cur: - cur.execute( - "SELECT schema_name FROM information_schema.schemata " - "WHERE schema_name NOT IN ('pg_catalog','information_schema','pg_toast','public') " - "AND schema_name NOT LIKE 'pg_temp_%' AND schema_name NOT LIKE 'pg_toast_temp_%'" - ) - for (schema,) in cur.fetchall(): - print(f" drop schema {schema}") + cur.execute(PRECHECK_SQL) + found = [r[0] for r in cur.fetchall()] + cur.execute("SELECT to_regclass('meta.schema_version') IS NOT NULL") + if cur.fetchone()[0]: + cur.execute("SELECT string_agg(version, ', ') FROM meta.schema_version") + version = cur.fetchone()[0] + found.append(f"installed schema version: {version or 'none recorded'}") + return found + + +def drop_existing(conn) -> None: + """Drop the library's objects: every registered series schema, meta, log, + the event trigger. Refuses if a series schema holds non-series objects -- + those are user data, which this never deletes.""" + with conn.cursor() as cur: + schemas = ['series'] + cur.execute("SELECT to_regclass('meta.series_schemas') IS NOT NULL") + if cur.fetchone()[0]: + cur.execute("SELECT name FROM meta.series_schemas") + schemas = sorted({r[0] for r in cur.fetchall()} | {'series'}) + cur.execute("SELECT to_regclass('meta.series') IS NOT NULL") + has_registry = cur.fetchone()[0] + stray = [] + for schema in schemas: + cur.execute( + "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = %s AND c.relkind IN ('r','p','v','m','f','S')", (schema,)) + names = [r[0] for r in cur.fetchall()] + if has_registry: + cur.execute("SELECT table_name FROM meta.series WHERE db_schema = %s", (schema,)) + names = sorted(set(names) - {r[0] for r in cur.fetchall()}) + stray += [f"{schema}.{n}" for n in names] + if stray: + raise RuntimeError( + "refusing to drop: these objects in series schemas are not series " + f"(user data?): {', '.join(stray)}. Move or drop them first.") + cur.execute("DROP EVENT TRIGGER IF EXISTS prevent_series_drops") + for schema in schemas: cur.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE") - cur.execute("SELECT evtname FROM pg_event_trigger") - for (trig,) in cur.fetchall(): - cur.execute(f"DROP EVENT TRIGGER IF EXISTS {trig}") - cur.execute("CREATE SCHEMA IF NOT EXISTS public") - conn.commit() - print("[OK] clean slate done\n") + print(f" dropped schema {schema}") + cur.execute("DROP SCHEMA IF EXISTS meta CASCADE") + cur.execute("DROP SCHEMA IF EXISTS log CASCADE") + conn.commit() + print("[OK] existing installation dropped\n") def main() -> int: @@ -149,9 +201,8 @@ def main() -> int: p.add_argument("--user") p.add_argument("--password") p.add_argument("--drop-existing", action="store_true", - help="Drop meta and series schemas before creating (destructive)") - p.add_argument("--clean-slate", action="store_true", - help="Drop ALL user objects before creating (very destructive)") + help="Drop the existing installation first: every series schema " + "(ALL series data), meta, log. User schemas are untouched.") p.add_argument("--yes", action="store_true", help="Skip confirmation prompts") args = p.parse_args() @@ -170,8 +221,8 @@ def main() -> int: print(f"Target : {cfg['host']}:{cfg['port']}/{cfg['database']} (user {cfg['user']})") print(f"SQL dir: {SQL_DIR}") - if (args.clean_slate or args.drop_existing) and not args.yes: - what = "ALL user objects" if args.clean_slate else "meta and series schemas" + if args.drop_existing and not args.yes: + what = "the existing installation, including ALL series data" if input(f"This will DROP {what}. Continue? (yes/no): ").lower() != "yes": print("Aborted.") return 1 @@ -186,32 +237,34 @@ def main() -> int: return 1 try: - if args.clean_slate: - clean_slate(conn) - elif args.drop_existing: - with conn.cursor() as cur: - cur.execute("DROP SCHEMA IF EXISTS meta CASCADE") - cur.execute("DROP SCHEMA IF EXISTS series CASCADE") - cur.execute("DROP SCHEMA IF EXISTS log CASCADE") - cur.execute("DROP EVENT TRIGGER IF EXISTS prevent_series_drops") - conn.commit() - print("[OK] existing schemas dropped\n") + if args.drop_existing: + try: + drop_existing(conn) + except RuntimeError as e: + print(f"[FAIL] {e}") + return 1 - failed = [] + existing = existing_installation(conn) + if existing: + print("[FAIL] the database already holds a tsapi installation:") + for line in existing: + print(f" - {line}") + print(" There are no migrations. Re-provision with --drop-existing " + "(deletes all series), or use another database.") + return 1 + + missing = [n for n in SQL_FILES if not (SQL_DIR / n).exists()] + if missing: + print(f"[FAIL] missing {', '.join(missing)}") + return 1 + # One transaction: a failure part-way leaves the database untouched. for name in SQL_FILES: - f = SQL_DIR / name - if not f.exists(): - print(f"[FAIL] missing {name}") - failed.append(name) - continue - ok, msg = execute_sql_file(conn, f) + ok, msg = execute_sql_file(conn, SQL_DIR / name) print(msg) if not ok: - failed.append(name) - - if failed: - print(f"\n[FAIL] {len(failed)} file(s) failed: {', '.join(failed)}") - return 1 + print(f"\n[FAIL] {name} failed; nothing was installed") + return 1 + conn.commit() verify_setup(conn) print("\n[SUCCESS] schema setup complete") return 0 From 738fe0fe2df0accddbd8b4d4a7e0377bd1cb4695 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 6 Oct 2026 12:18:54 +0100 Subject: [PATCH 5/5] find_schema_problems(): report problems in a stable order Several sections had no ORDER BY, so when one section found more than one problem PostgreSQL could return them in any order (found by new break-one-thing tests). Every section is now sorted. Output is otherwise unchanged; no version bump. Co-Authored-By: Claude Opus 5.5 --- sql/09_schema_check.sql | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/sql/09_schema_check.sql b/sql/09_schema_check.sql index 951d224..73ec36c 100644 --- a/sql/09_schema_check.sql +++ b/sql/09_schema_check.sql @@ -12,7 +12,8 @@ -- meta.find_schema_problems() -- -------------------------------------------------------------------------- -- --- Returns one row per problem: (object, problem). Read-only. +-- Returns one row per problem: (object, problem), in a stable order +-- (section by section, each sorted). Read-only. -- -- The expectations below mirror 00..08. When any of those files changes, -- update this file in the same commit. The release tests (a freshly @@ -42,12 +43,14 @@ BEGIN RETURN QUERY SELECT 'schema ' || s, 'missing' FROM unnest(ARRAY['meta', 'series', 'log']) AS s - WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s); + WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s) + ORDER BY 1; RETURN QUERY SELECT 'extension ' || e, 'missing' FROM unnest(ARRAY['pg_trgm']) AS e - WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e); + WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e) + ORDER BY 1; RETURN QUERY SELECT 'sequence ' || s, 'missing' @@ -57,7 +60,8 @@ BEGIN 'log.pending_changes_id_seq', 'log.script_log_id_seq' ]) AS s - WHERE to_regclass(s) IS NULL; + WHERE to_regclass(s) IS NULL + ORDER BY 1; -- Tables and columns ---------------------------------------------------- RETURN QUERY @@ -231,7 +235,8 @@ BEGIN ) AS c(tbl, name) WHERE to_regclass(c.tbl) IS NOT NULL AND NOT EXISTS (SELECT 1 FROM pg_constraint k - WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name); + WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name) + ORDER BY 1; RETURN QUERY SELECT 'index ' || i.idx, 'missing' @@ -253,12 +258,14 @@ BEGIN ('log.idx_script_log_series', 'log.script_log'), ('log.idx_script_log_level', 'log.script_log') ) AS i(idx, tbl) - WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL; + WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL + ORDER BY 1; RETURN QUERY SELECT 'view ' || v, 'missing' FROM unnest(ARRAY['log.script_log_recent']) AS v - WHERE to_regclass(v) IS NULL; + WHERE to_regclass(v) IS NULL + ORDER BY 1; -- Functions (compared by whitespace-insensitive body fingerprint) ------- RETURN QUERY @@ -285,7 +292,8 @@ BEGIN WHERE NOT EXISTS (SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname || '.' || p.proname = f.name - AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint); + AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint) + ORDER BY 1; -- Drop-protection event trigger ----------------------------------------- RETURN QUERY @@ -302,7 +310,8 @@ BEGIN (CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop' THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END) ) AS x(p) - WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL; + WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL + ORDER BY 2; -- Reference data and version row ---------------------------------------- RETURN QUERY @@ -321,7 +330,8 @@ BEGIN ) AS e(tbl, expected) WHERE to_regclass(e.tbl) IS NOT NULL ) AS r - WHERE r.actual < r.expected; + WHERE r.actual < r.expected + ORDER BY 1; -- Series schema registry ------------------------------------------------ -- Only REGISTERED schemas are examined; user schemas are never looked at.