Compare commits

..

No commits in common. "738fe0fe2df0accddbd8b4d4a7e0377bd1cb4695" and "1f5cfeceeb20da5da3e3aae379a1327442aebee7" have entirely different histories.

8 changed files with 67 additions and 289 deletions

View file

@ -9,7 +9,7 @@ timeseries database. Both client libraries target the schema defined here:
Neither project should keep its own copy of the DDL. When the schema changes,
it changes **here**, and both projects re-provision / re-test against it.
Current schema version: **1.3.0** (see [`VERSION`](VERSION); recorded at runtime
Current schema version: **1.0.0** (see [`VERSION`](VERSION); recorded at runtime
in the `meta.schema_version` table).
---
@ -52,18 +52,10 @@ cp db_config.json.example db_config.json # then edit host/user/password
# 2. create the schema
python run_setup.py # uses db_config.json / defaults
python run_setup.py --host 192.168.1.7 --user claude --password claudepwd
python run_setup.py --drop-existing # drop the existing installation first
# (ALL series data); user schemas untouched
python run_setup.py --drop-existing # drop meta+series+log first
python run_setup.py --clean-slate # drop ALL user objects first
```
The database may hold the user's own schemas and tables; provisioning never
touches them. It **refuses** a database that already holds the library's
objects -- `meta`, `log`, a non-empty `series` schema, or the
`prevent_series_drops` event trigger -- and reports the installed version.
There are no migrations: re-provision with `--drop-existing` (deletes all
series) or use another database. All files run in one transaction, so a
failure leaves the database untouched.
Requires `psycopg2` and a PostgreSQL user with CREATE privileges on the target
database.

View file

@ -1 +1 @@
1.3.0
1.0.0

View file

@ -22,12 +22,8 @@ Connection settings come from (highest precedence first):
Usage:
python run_setup.py # use db_config.json / defaults
python run_setup.py --host 192.168.1.7 --user claude --password claudepwd
python run_setup.py --drop-existing # drop the existing installation first
# (all series!); user schemas untouched
A database that already holds the library's objects (meta, log, a non-empty
series schema, the event trigger) is refused -- there are no migrations.
User schemas and tables may exist and are never touched.
python run_setup.py --drop-existing # drop meta+series schemas first
python run_setup.py --clean-slate # drop ALL user objects first
"""
import sys
@ -86,7 +82,8 @@ def execute_sql_file(conn, sql_file: Path):
try:
sql = sql_file.read_text(encoding="utf-8")
with conn.cursor() as cur:
cur.execute(sql) # committed by the caller, all files at once
cur.execute(sql)
conn.commit()
return True, f"[OK] Executed {sql_file.name}"
except Exception as e:
conn.rollback()
@ -125,72 +122,23 @@ def verify_setup(conn) -> None:
print(f"[OK] schema_version: {row[0]} ({row[1]})")
# Pre-check, shared rule with the Python client's init_postgres_db():
# provisioning needs the library's own objects to be ABSENT. User schemas and
# tables are irrelevant and never touched. Never drop-and-recreate: an
# existing installation is refused, because there are no migrations and
# re-provisioning deletes all series.
PRECHECK_SQL = """
SELECT 'schema ' || nspname || ' exists'
FROM pg_namespace WHERE nspname IN ('meta', 'log')
UNION ALL
SELECT 'schema series is not empty (' || count(*) || ' objects)'
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'series' AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
HAVING count(*) > 0
UNION ALL
SELECT 'event trigger prevent_series_drops exists'
FROM pg_event_trigger WHERE evtname = 'prevent_series_drops'
"""
def existing_installation(conn) -> list:
"""What would stop a fresh install; [] means go ahead."""
def clean_slate(conn) -> None:
print("Performing clean slate...")
with conn.cursor() as cur:
cur.execute(PRECHECK_SQL)
found = [r[0] for r in cur.fetchall()]
cur.execute("SELECT to_regclass('meta.schema_version') IS NOT NULL")
if cur.fetchone()[0]:
cur.execute("SELECT string_agg(version, ', ') FROM meta.schema_version")
version = cur.fetchone()[0]
found.append(f"installed schema version: {version or 'none recorded'}")
return found
def drop_existing(conn) -> None:
"""Drop the library's objects: every registered series schema, meta, log,
the event trigger. Refuses if a series schema holds non-series objects --
those are user data, which this never deletes."""
with conn.cursor() as cur:
schemas = ['series']
cur.execute("SELECT to_regclass('meta.series_schemas') IS NOT NULL")
if cur.fetchone()[0]:
cur.execute("SELECT name FROM meta.series_schemas")
schemas = sorted({r[0] for r in cur.fetchall()} | {'series'})
cur.execute("SELECT to_regclass('meta.series') IS NOT NULL")
has_registry = cur.fetchone()[0]
stray = []
for schema in schemas:
cur.execute(
"SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
"WHERE n.nspname = %s AND c.relkind IN ('r','p','v','m','f','S')", (schema,))
names = [r[0] for r in cur.fetchall()]
if has_registry:
cur.execute("SELECT table_name FROM meta.series WHERE db_schema = %s", (schema,))
names = sorted(set(names) - {r[0] for r in cur.fetchall()})
stray += [f"{schema}.{n}" for n in names]
if stray:
raise RuntimeError(
"refusing to drop: these objects in series schemas are not series "
f"(user data?): {', '.join(stray)}. Move or drop them first.")
cur.execute("DROP EVENT TRIGGER IF EXISTS prevent_series_drops")
for schema in schemas:
"SELECT schema_name FROM information_schema.schemata "
"WHERE schema_name NOT IN ('pg_catalog','information_schema','pg_toast','public') "
"AND schema_name NOT LIKE 'pg_temp_%' AND schema_name NOT LIKE 'pg_toast_temp_%'"
)
for (schema,) in cur.fetchall():
print(f" drop schema {schema}")
cur.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
print(f" dropped schema {schema}")
cur.execute("DROP SCHEMA IF EXISTS meta CASCADE")
cur.execute("DROP SCHEMA IF EXISTS log CASCADE")
cur.execute("SELECT evtname FROM pg_event_trigger")
for (trig,) in cur.fetchall():
cur.execute(f"DROP EVENT TRIGGER IF EXISTS {trig}")
cur.execute("CREATE SCHEMA IF NOT EXISTS public")
conn.commit()
print("[OK] existing installation dropped\n")
print("[OK] clean slate done\n")
def main() -> int:
@ -201,8 +149,9 @@ def main() -> int:
p.add_argument("--user")
p.add_argument("--password")
p.add_argument("--drop-existing", action="store_true",
help="Drop the existing installation first: every series schema "
"(ALL series data), meta, log. User schemas are untouched.")
help="Drop meta and series schemas before creating (destructive)")
p.add_argument("--clean-slate", action="store_true",
help="Drop ALL user objects before creating (very destructive)")
p.add_argument("--yes", action="store_true", help="Skip confirmation prompts")
args = p.parse_args()
@ -221,8 +170,8 @@ def main() -> int:
print(f"Target : {cfg['host']}:{cfg['port']}/{cfg['database']} (user {cfg['user']})")
print(f"SQL dir: {SQL_DIR}")
if args.drop_existing and not args.yes:
what = "the existing installation, including ALL series data"
if (args.clean_slate or args.drop_existing) and not args.yes:
what = "ALL user objects" if args.clean_slate else "meta and series schemas"
if input(f"This will DROP {what}. Continue? (yes/no): ").lower() != "yes":
print("Aborted.")
return 1
@ -237,34 +186,32 @@ def main() -> int:
return 1
try:
if args.drop_existing:
try:
drop_existing(conn)
except RuntimeError as e:
print(f"[FAIL] {e}")
return 1
if args.clean_slate:
clean_slate(conn)
elif args.drop_existing:
with conn.cursor() as cur:
cur.execute("DROP SCHEMA IF EXISTS meta CASCADE")
cur.execute("DROP SCHEMA IF EXISTS series CASCADE")
cur.execute("DROP SCHEMA IF EXISTS log CASCADE")
cur.execute("DROP EVENT TRIGGER IF EXISTS prevent_series_drops")
conn.commit()
print("[OK] existing schemas dropped\n")
existing = existing_installation(conn)
if existing:
print("[FAIL] the database already holds a tsapi installation:")
for line in existing:
print(f" - {line}")
print(" There are no migrations. Re-provision with --drop-existing "
"(deletes all series), or use another database.")
return 1
missing = [n for n in SQL_FILES if not (SQL_DIR / n).exists()]
if missing:
print(f"[FAIL] missing {', '.join(missing)}")
return 1
# One transaction: a failure part-way leaves the database untouched.
failed = []
for name in SQL_FILES:
ok, msg = execute_sql_file(conn, SQL_DIR / name)
f = SQL_DIR / name
if not f.exists():
print(f"[FAIL] missing {name}")
failed.append(name)
continue
ok, msg = execute_sql_file(conn, f)
print(msg)
if not ok:
print(f"\n[FAIL] {name} failed; nothing was installed")
failed.append(name)
if failed:
print(f"\n[FAIL] {len(failed)} file(s) failed: {', '.join(failed)}")
return 1
conn.commit()
verify_setup(conn)
print("\n[SUCCESS] schema setup complete")
return 0

View file

@ -135,16 +135,6 @@ CREATE TABLE meta.sessions (
exchange_id character varying(12) NOT NULL
);
-- Registry of series schemas. A series may only live in a registered schema
-- (FK from meta.series.db_schema, 04_indexes.sql). Every other schema in the
-- database belongs to the user and is never touched by the clients.
CREATE TABLE meta.series_schemas (
name text NOT NULL,
description text,
created_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
created_by text NOT NULL DEFAULT CURRENT_USER
);
CREATE TABLE meta.timezones (
iana_name character varying(50) NOT NULL,
display_name character varying(100) NOT NULL,

View file

@ -21,23 +21,6 @@ ALTER TABLE meta.currencies ADD CONSTRAINT currencies_pkey PRIMARY KEY (code);
ALTER TABLE meta.security_types ADD CONSTRAINT security_types_pkey PRIMARY KEY (code);
ALTER TABLE meta.timezones ADD CONSTRAINT timezones_pkey PRIMARY KEY (iana_name);
-- Series schema registry. Names follow the table-name rule shared by both
-- clients: lower case, [a-z_][a-z0-9_]*, at most 63 bytes, never quoted.
ALTER TABLE meta.series_schemas ADD CONSTRAINT series_schemas_pkey PRIMARY KEY (name);
ALTER TABLE meta.series_schemas ADD CONSTRAINT series_schemas_name_check
CHECK (name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(name) <= 63);
-- A series may only live in a registered schema -- enforced here, so no
-- client can bypass it. RESTRICT: a schema cannot be unregistered while it
-- still holds series.
ALTER TABLE meta.series ADD CONSTRAINT series_db_schema_fkey
FOREIGN KEY (db_schema) REFERENCES meta.series_schemas (name)
ON UPDATE RESTRICT ON DELETE RESTRICT;
-- Table names follow the same rule (Core contract 6 in both clients).
ALTER TABLE meta.series ADD CONSTRAINT series_table_name_check
CHECK (table_name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(table_name) <= 63);
CREATE INDEX idx_event_log_series_timestamp ON log.event_log USING btree (series_name, "timestamp" DESC);
CREATE INDEX idx_event_log_timestamp ON log.event_log USING btree ("timestamp" DESC);

View file

@ -1,4 +1,4 @@
-- Reference data (series schema registry, timezones, currencies, security types)
-- Reference data (timezones and currencies)
-- Generated: 2025-10-15T12:24:56.582392
-- Timezones
@ -767,7 +767,3 @@ VALUES
('OPT', 'Option', 'Equity and index options', NULL),
('STK', 'Stock', 'Common and preferred equities', NULL),
('WAR', 'Warrant', 'Equity warrants', NULL);
-- Series schema registry: the default schema, created in 01_schemas.sql
INSERT INTO meta.series_schemas (name, description)
VALUES ('series', 'Default series schema');

View file

@ -1,19 +1,12 @@
-- ============================================================================
-- Database self-checks, callable by every client (Python, C++, psql)
-- Schema self-check: meta.find_schema_problems()
-- ============================================================================
-- Answers one question for every client (Python, C++, psql):
-- is this database complete and as the DDL defines it?
--
-- meta.find_schema_problems() -- on connect: is the database complete and as
-- the DDL defines it? zero rows = usable
-- SELECT * FROM meta.find_schema_problems(); -- zero rows = usable
--
-- meta.find_series_registry_mismatches() -- occasional consistency check:
-- do the series schemas and meta.series agree? (defined at the end)
--
-- --------------------------------------------------------------------------
-- meta.find_schema_problems()
-- --------------------------------------------------------------------------
--
-- Returns one row per problem: (object, problem), in a stable order
-- (section by section, each sorted). Read-only.
-- Returns one row per problem: (object, problem). Read-only.
--
-- The expectations below mirror 00..08. When any of those files changes,
-- update this file in the same commit. The release tests (a freshly
@ -43,14 +36,12 @@ BEGIN
RETURN QUERY
SELECT 'schema ' || s, 'missing'
FROM unnest(ARRAY['meta', 'series', 'log']) AS s
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s)
ORDER BY 1;
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s);
RETURN QUERY
SELECT 'extension ' || e, 'missing'
FROM unnest(ARRAY['pg_trgm']) AS e
WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e)
ORDER BY 1;
WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e);
RETURN QUERY
SELECT 'sequence ' || s, 'missing'
@ -60,8 +51,7 @@ BEGIN
'log.pending_changes_id_seq',
'log.script_log_id_seq'
]) AS s
WHERE to_regclass(s) IS NULL
ORDER BY 1;
WHERE to_regclass(s) IS NULL;
-- Tables and columns ----------------------------------------------------
RETURN QUERY
@ -124,10 +114,6 @@ BEGIN
('meta.instruments', 'asset_class', 'character varying(30)', false),
('meta.instruments', 'geography', 'character varying(30)', false),
('meta.instruments', 'underlying', 'character varying(30)', false),
('meta.series_schemas', 'name', 'text', true),
('meta.series_schemas', 'description', 'text', false),
('meta.series_schemas', 'created_at', 'timestamp with time zone', true),
('meta.series_schemas', 'created_by', 'text', true),
('meta.security_types', 'code', 'character varying(10)', true),
('meta.security_types', 'name', 'character varying(50)', true),
('meta.security_types', 'description', 'text', false),
@ -225,18 +211,13 @@ BEGIN
('meta.sessions', 'sessions_pkey'),
('meta.series', 'series_pkey'),
('meta.series', 'series_schema_table_key'),
('meta.series', 'series_db_schema_fkey'),
('meta.series', 'series_table_name_check'),
('meta.series_schemas', 'series_schemas_pkey'),
('meta.series_schemas', 'series_schemas_name_check'),
('meta.currencies', 'currencies_pkey'),
('meta.security_types', 'security_types_pkey'),
('meta.timezones', 'timezones_pkey')
) AS c(tbl, name)
WHERE to_regclass(c.tbl) IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM pg_constraint k
WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name)
ORDER BY 1;
WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name);
RETURN QUERY
SELECT 'index ' || i.idx, 'missing'
@ -258,14 +239,12 @@ BEGIN
('log.idx_script_log_series', 'log.script_log'),
('log.idx_script_log_level', 'log.script_log')
) AS i(idx, tbl)
WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL
ORDER BY 1;
WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL;
RETURN QUERY
SELECT 'view ' || v, 'missing'
FROM unnest(ARRAY['log.script_log_recent']) AS v
WHERE to_regclass(v) IS NULL
ORDER BY 1;
WHERE to_regclass(v) IS NULL;
-- Functions (compared by whitespace-insensitive body fingerprint) -------
RETURN QUERY
@ -281,7 +260,6 @@ BEGIN
('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'),
('meta.drop_series', '6f2a919de965b87521be68252deee0e2'),
('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'),
('meta.find_series_registry_mismatches', '6dc9ecd2b7656b91a8b410c78a19fa2b'),
('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'),
('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'),
('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'),
@ -292,8 +270,7 @@ BEGIN
WHERE NOT EXISTS (SELECT 1 FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
WHERE n.nspname || '.' || p.proname = f.name
AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint)
ORDER BY 1;
AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint);
-- Drop-protection event trigger -----------------------------------------
RETURN QUERY
@ -310,8 +287,7 @@ BEGIN
(CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop'
THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END)
) AS x(p)
WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL
ORDER BY 2;
WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL;
-- Reference data and version row ----------------------------------------
RETURN QUERY
@ -325,27 +301,11 @@ BEGIN
FROM (VALUES
('meta.timezones', 563),
('meta.currencies', 179),
('meta.security_types', 13),
('meta.series_schemas', 1)
('meta.security_types', 13)
) AS e(tbl, expected)
WHERE to_regclass(e.tbl) IS NOT NULL
) AS r
WHERE r.actual < r.expected
ORDER BY 1;
-- Series schema registry ------------------------------------------------
-- Only REGISTERED schemas are examined; user schemas are never looked at.
IF to_regclass('meta.series_schemas') IS NOT NULL THEN
RETURN QUERY
SELECT 'series schema series', 'not registered in meta.series_schemas'
WHERE NOT EXISTS (SELECT 1 FROM meta.series_schemas WHERE name = 'series');
RETURN QUERY
SELECT 'series schema ' || r.name, 'registered but the schema does not exist'
FROM meta.series_schemas r
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = r.name)
ORDER BY r.name;
END IF;
WHERE r.actual < r.expected;
IF to_regclass('meta.schema_version') IS NOT NULL THEN
RETURN QUERY
@ -354,93 +314,3 @@ BEGIN
END IF;
END;
$function$;
-- --------------------------------------------------------------------------
-- meta.find_series_registry_mismatches()
-- --------------------------------------------------------------------------
-- Occasional consistency check -- NOT run on connect. Compares the series
-- registry (meta.series) with what actually exists in the REGISTERED series
-- schemas (meta.series_schemas). User schemas are never examined.
--
-- SELECT * FROM meta.find_series_registry_mismatches(); -- zero rows = consistent
--
-- One (object, problem) row per finding:
-- * a table, view, materialized view, foreign table or sequence in a
-- series schema that is not a registered series (an orphaned series table,
-- or a user table in the wrong place) -- harmless to the clients;
-- * a function in a series schema -- harmless;
-- * a series registered in meta.series whose table does not exist --
-- every operation on that series fails;
-- * a series table without a "timestamp timestamp without time zone NOT
-- NULL" column, or without a BRIN index on it.
-- Read-only. It reports; deciding what to do with a finding is up to the user.
CREATE OR REPLACE FUNCTION meta.find_series_registry_mismatches()
RETURNS TABLE(object text, problem text)
LANGUAGE plpgsql
STABLE
AS $function$
BEGIN
-- Relations in series schemas that are not registered series -----------
RETURN QUERY
SELECT n.nspname || '.' || c.relname,
CASE c.relkind
WHEN 'r' THEN 'table' WHEN 'p' THEN 'table' WHEN 'v' THEN 'view'
WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table'
ELSE 'sequence' END
|| ' in a series schema but not registered in meta.series'
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN meta.series_schemas r ON r.name = n.nspname
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
AND NOT EXISTS (SELECT 1 FROM meta.series m
WHERE m.db_schema = n.nspname AND m.table_name = c.relname)
ORDER BY 1;
RETURN QUERY
SELECT n.nspname || '.' || p.proname, 'function in a series schema'
FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
JOIN meta.series_schemas r ON r.name = n.nspname
ORDER BY 1;
-- Registered series whose table is missing ------------------------------
RETURN QUERY
SELECT m.db_schema || '.' || m.table_name,
'registered in meta.series but the table does not exist'
FROM meta.series m
WHERE to_regclass(format('%I.%I', m.db_schema, m.table_name)) IS NULL
ORDER BY 1;
-- Series tables without a proper timestamp column or its BRIN index -----
RETURN QUERY
SELECT m.db_schema || '.' || m.table_name,
CASE WHEN a.attname IS NULL THEN 'no "timestamp" column'
ELSE '"timestamp" is ' || format_type(a.atttypid, a.atttypmod)
|| CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END
|| ', expected timestamp without time zone NOT NULL' END
FROM meta.series m
CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t
LEFT JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp'
AND a.attnum > 0 AND NOT a.attisdropped
WHERE t.rel IS NOT NULL
AND (a.attname IS NULL
OR format_type(a.atttypid, a.atttypmod) <> 'timestamp without time zone'
OR NOT a.attnotnull)
ORDER BY 1;
RETURN QUERY
SELECT m.db_schema || '.' || m.table_name, 'no BRIN index on "timestamp"'
FROM meta.series m
CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t
JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp'
AND a.attnum > 0 AND NOT a.attisdropped
WHERE NOT EXISTS (
SELECT 1 FROM pg_index i
JOIN pg_class ic ON ic.oid = i.indexrelid
JOIN pg_am am ON am.oid = ic.relam
WHERE i.indrelid = t.rel AND am.amname = 'brin' AND i.indkey[0] = a.attnum)
ORDER BY 1;
END;
$function$;

View file

@ -5,4 +5,4 @@
-- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py).
INSERT INTO meta.schema_version (version, description)
VALUES ('1.3.0', 'meta.find_series_registry_mismatches()');
VALUES ('1.0.0', 'Initial schema');