Compare commits
No commits in common. "738fe0fe2df0accddbd8b4d4a7e0377bd1cb4695" and "1f5cfeceeb20da5da3e3aae379a1327442aebee7" have entirely different histories.
738fe0fe2d
...
1f5cfeceeb
8 changed files with 67 additions and 289 deletions
14
README.md
14
README.md
|
|
@ -9,7 +9,7 @@ timeseries database. Both client libraries target the schema defined here:
|
||||||
Neither project should keep its own copy of the DDL. When the schema changes,
|
Neither project should keep its own copy of the DDL. When the schema changes,
|
||||||
it changes **here**, and both projects re-provision / re-test against it.
|
it changes **here**, and both projects re-provision / re-test against it.
|
||||||
|
|
||||||
Current schema version: **1.3.0** (see [`VERSION`](VERSION); recorded at runtime
|
Current schema version: **1.0.0** (see [`VERSION`](VERSION); recorded at runtime
|
||||||
in the `meta.schema_version` table).
|
in the `meta.schema_version` table).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
@ -52,18 +52,10 @@ cp db_config.json.example db_config.json # then edit host/user/password
|
||||||
# 2. create the schema
|
# 2. create the schema
|
||||||
python run_setup.py # uses db_config.json / defaults
|
python run_setup.py # uses db_config.json / defaults
|
||||||
python run_setup.py --host 192.168.1.7 --user claude --password claudepwd
|
python run_setup.py --host 192.168.1.7 --user claude --password claudepwd
|
||||||
python run_setup.py --drop-existing # drop the existing installation first
|
python run_setup.py --drop-existing # drop meta+series+log first
|
||||||
# (ALL series data); user schemas untouched
|
python run_setup.py --clean-slate # drop ALL user objects first
|
||||||
```
|
```
|
||||||
|
|
||||||
The database may hold the user's own schemas and tables; provisioning never
|
|
||||||
touches them. It **refuses** a database that already holds the library's
|
|
||||||
objects -- `meta`, `log`, a non-empty `series` schema, or the
|
|
||||||
`prevent_series_drops` event trigger -- and reports the installed version.
|
|
||||||
There are no migrations: re-provision with `--drop-existing` (deletes all
|
|
||||||
series) or use another database. All files run in one transaction, so a
|
|
||||||
failure leaves the database untouched.
|
|
||||||
|
|
||||||
Requires `psycopg2` and a PostgreSQL user with CREATE privileges on the target
|
Requires `psycopg2` and a PostgreSQL user with CREATE privileges on the target
|
||||||
database.
|
database.
|
||||||
|
|
||||||
|
|
|
||||||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
||||||
1.3.0
|
1.0.0
|
||||||
|
|
|
||||||
145
run_setup.py
145
run_setup.py
|
|
@ -22,12 +22,8 @@ Connection settings come from (highest precedence first):
|
||||||
Usage:
|
Usage:
|
||||||
python run_setup.py # use db_config.json / defaults
|
python run_setup.py # use db_config.json / defaults
|
||||||
python run_setup.py --host 192.168.1.7 --user claude --password claudepwd
|
python run_setup.py --host 192.168.1.7 --user claude --password claudepwd
|
||||||
python run_setup.py --drop-existing # drop the existing installation first
|
python run_setup.py --drop-existing # drop meta+series schemas first
|
||||||
# (all series!); user schemas untouched
|
python run_setup.py --clean-slate # drop ALL user objects first
|
||||||
|
|
||||||
A database that already holds the library's objects (meta, log, a non-empty
|
|
||||||
series schema, the event trigger) is refused -- there are no migrations.
|
|
||||||
User schemas and tables may exist and are never touched.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|
@ -86,7 +82,8 @@ def execute_sql_file(conn, sql_file: Path):
|
||||||
try:
|
try:
|
||||||
sql = sql_file.read_text(encoding="utf-8")
|
sql = sql_file.read_text(encoding="utf-8")
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(sql) # committed by the caller, all files at once
|
cur.execute(sql)
|
||||||
|
conn.commit()
|
||||||
return True, f"[OK] Executed {sql_file.name}"
|
return True, f"[OK] Executed {sql_file.name}"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
conn.rollback()
|
conn.rollback()
|
||||||
|
|
@ -125,72 +122,23 @@ def verify_setup(conn) -> None:
|
||||||
print(f"[OK] schema_version: {row[0]} ({row[1]})")
|
print(f"[OK] schema_version: {row[0]} ({row[1]})")
|
||||||
|
|
||||||
|
|
||||||
# Pre-check, shared rule with the Python client's init_postgres_db():
|
def clean_slate(conn) -> None:
|
||||||
# provisioning needs the library's own objects to be ABSENT. User schemas and
|
print("Performing clean slate...")
|
||||||
# tables are irrelevant and never touched. Never drop-and-recreate: an
|
|
||||||
# existing installation is refused, because there are no migrations and
|
|
||||||
# re-provisioning deletes all series.
|
|
||||||
PRECHECK_SQL = """
|
|
||||||
SELECT 'schema ' || nspname || ' exists'
|
|
||||||
FROM pg_namespace WHERE nspname IN ('meta', 'log')
|
|
||||||
UNION ALL
|
|
||||||
SELECT 'schema series is not empty (' || count(*) || ' objects)'
|
|
||||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
||||||
WHERE n.nspname = 'series' AND c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
|
|
||||||
HAVING count(*) > 0
|
|
||||||
UNION ALL
|
|
||||||
SELECT 'event trigger prevent_series_drops exists'
|
|
||||||
FROM pg_event_trigger WHERE evtname = 'prevent_series_drops'
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def existing_installation(conn) -> list:
|
|
||||||
"""What would stop a fresh install; [] means go ahead."""
|
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
cur.execute(PRECHECK_SQL)
|
cur.execute(
|
||||||
found = [r[0] for r in cur.fetchall()]
|
"SELECT schema_name FROM information_schema.schemata "
|
||||||
cur.execute("SELECT to_regclass('meta.schema_version') IS NOT NULL")
|
"WHERE schema_name NOT IN ('pg_catalog','information_schema','pg_toast','public') "
|
||||||
if cur.fetchone()[0]:
|
"AND schema_name NOT LIKE 'pg_temp_%' AND schema_name NOT LIKE 'pg_toast_temp_%'"
|
||||||
cur.execute("SELECT string_agg(version, ', ') FROM meta.schema_version")
|
)
|
||||||
version = cur.fetchone()[0]
|
for (schema,) in cur.fetchall():
|
||||||
found.append(f"installed schema version: {version or 'none recorded'}")
|
print(f" drop schema {schema}")
|
||||||
return found
|
|
||||||
|
|
||||||
|
|
||||||
def drop_existing(conn) -> None:
|
|
||||||
"""Drop the library's objects: every registered series schema, meta, log,
|
|
||||||
the event trigger. Refuses if a series schema holds non-series objects --
|
|
||||||
those are user data, which this never deletes."""
|
|
||||||
with conn.cursor() as cur:
|
|
||||||
schemas = ['series']
|
|
||||||
cur.execute("SELECT to_regclass('meta.series_schemas') IS NOT NULL")
|
|
||||||
if cur.fetchone()[0]:
|
|
||||||
cur.execute("SELECT name FROM meta.series_schemas")
|
|
||||||
schemas = sorted({r[0] for r in cur.fetchall()} | {'series'})
|
|
||||||
cur.execute("SELECT to_regclass('meta.series') IS NOT NULL")
|
|
||||||
has_registry = cur.fetchone()[0]
|
|
||||||
stray = []
|
|
||||||
for schema in schemas:
|
|
||||||
cur.execute(
|
|
||||||
"SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
|
|
||||||
"WHERE n.nspname = %s AND c.relkind IN ('r','p','v','m','f','S')", (schema,))
|
|
||||||
names = [r[0] for r in cur.fetchall()]
|
|
||||||
if has_registry:
|
|
||||||
cur.execute("SELECT table_name FROM meta.series WHERE db_schema = %s", (schema,))
|
|
||||||
names = sorted(set(names) - {r[0] for r in cur.fetchall()})
|
|
||||||
stray += [f"{schema}.{n}" for n in names]
|
|
||||||
if stray:
|
|
||||||
raise RuntimeError(
|
|
||||||
"refusing to drop: these objects in series schemas are not series "
|
|
||||||
f"(user data?): {', '.join(stray)}. Move or drop them first.")
|
|
||||||
cur.execute("DROP EVENT TRIGGER IF EXISTS prevent_series_drops")
|
|
||||||
for schema in schemas:
|
|
||||||
cur.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
|
cur.execute(f"DROP SCHEMA IF EXISTS {schema} CASCADE")
|
||||||
print(f" dropped schema {schema}")
|
cur.execute("SELECT evtname FROM pg_event_trigger")
|
||||||
cur.execute("DROP SCHEMA IF EXISTS meta CASCADE")
|
for (trig,) in cur.fetchall():
|
||||||
cur.execute("DROP SCHEMA IF EXISTS log CASCADE")
|
cur.execute(f"DROP EVENT TRIGGER IF EXISTS {trig}")
|
||||||
conn.commit()
|
cur.execute("CREATE SCHEMA IF NOT EXISTS public")
|
||||||
print("[OK] existing installation dropped\n")
|
conn.commit()
|
||||||
|
print("[OK] clean slate done\n")
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
|
|
@ -201,8 +149,9 @@ def main() -> int:
|
||||||
p.add_argument("--user")
|
p.add_argument("--user")
|
||||||
p.add_argument("--password")
|
p.add_argument("--password")
|
||||||
p.add_argument("--drop-existing", action="store_true",
|
p.add_argument("--drop-existing", action="store_true",
|
||||||
help="Drop the existing installation first: every series schema "
|
help="Drop meta and series schemas before creating (destructive)")
|
||||||
"(ALL series data), meta, log. User schemas are untouched.")
|
p.add_argument("--clean-slate", action="store_true",
|
||||||
|
help="Drop ALL user objects before creating (very destructive)")
|
||||||
p.add_argument("--yes", action="store_true", help="Skip confirmation prompts")
|
p.add_argument("--yes", action="store_true", help="Skip confirmation prompts")
|
||||||
args = p.parse_args()
|
args = p.parse_args()
|
||||||
|
|
||||||
|
|
@ -221,8 +170,8 @@ def main() -> int:
|
||||||
print(f"Target : {cfg['host']}:{cfg['port']}/{cfg['database']} (user {cfg['user']})")
|
print(f"Target : {cfg['host']}:{cfg['port']}/{cfg['database']} (user {cfg['user']})")
|
||||||
print(f"SQL dir: {SQL_DIR}")
|
print(f"SQL dir: {SQL_DIR}")
|
||||||
|
|
||||||
if args.drop_existing and not args.yes:
|
if (args.clean_slate or args.drop_existing) and not args.yes:
|
||||||
what = "the existing installation, including ALL series data"
|
what = "ALL user objects" if args.clean_slate else "meta and series schemas"
|
||||||
if input(f"This will DROP {what}. Continue? (yes/no): ").lower() != "yes":
|
if input(f"This will DROP {what}. Continue? (yes/no): ").lower() != "yes":
|
||||||
print("Aborted.")
|
print("Aborted.")
|
||||||
return 1
|
return 1
|
||||||
|
|
@ -237,34 +186,32 @@ def main() -> int:
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if args.drop_existing:
|
if args.clean_slate:
|
||||||
try:
|
clean_slate(conn)
|
||||||
drop_existing(conn)
|
elif args.drop_existing:
|
||||||
except RuntimeError as e:
|
with conn.cursor() as cur:
|
||||||
print(f"[FAIL] {e}")
|
cur.execute("DROP SCHEMA IF EXISTS meta CASCADE")
|
||||||
return 1
|
cur.execute("DROP SCHEMA IF EXISTS series CASCADE")
|
||||||
|
cur.execute("DROP SCHEMA IF EXISTS log CASCADE")
|
||||||
|
cur.execute("DROP EVENT TRIGGER IF EXISTS prevent_series_drops")
|
||||||
|
conn.commit()
|
||||||
|
print("[OK] existing schemas dropped\n")
|
||||||
|
|
||||||
existing = existing_installation(conn)
|
failed = []
|
||||||
if existing:
|
|
||||||
print("[FAIL] the database already holds a tsapi installation:")
|
|
||||||
for line in existing:
|
|
||||||
print(f" - {line}")
|
|
||||||
print(" There are no migrations. Re-provision with --drop-existing "
|
|
||||||
"(deletes all series), or use another database.")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
missing = [n for n in SQL_FILES if not (SQL_DIR / n).exists()]
|
|
||||||
if missing:
|
|
||||||
print(f"[FAIL] missing {', '.join(missing)}")
|
|
||||||
return 1
|
|
||||||
# One transaction: a failure part-way leaves the database untouched.
|
|
||||||
for name in SQL_FILES:
|
for name in SQL_FILES:
|
||||||
ok, msg = execute_sql_file(conn, SQL_DIR / name)
|
f = SQL_DIR / name
|
||||||
|
if not f.exists():
|
||||||
|
print(f"[FAIL] missing {name}")
|
||||||
|
failed.append(name)
|
||||||
|
continue
|
||||||
|
ok, msg = execute_sql_file(conn, f)
|
||||||
print(msg)
|
print(msg)
|
||||||
if not ok:
|
if not ok:
|
||||||
print(f"\n[FAIL] {name} failed; nothing was installed")
|
failed.append(name)
|
||||||
return 1
|
|
||||||
conn.commit()
|
if failed:
|
||||||
|
print(f"\n[FAIL] {len(failed)} file(s) failed: {', '.join(failed)}")
|
||||||
|
return 1
|
||||||
verify_setup(conn)
|
verify_setup(conn)
|
||||||
print("\n[SUCCESS] schema setup complete")
|
print("\n[SUCCESS] schema setup complete")
|
||||||
return 0
|
return 0
|
||||||
|
|
|
||||||
|
|
@ -135,16 +135,6 @@ CREATE TABLE meta.sessions (
|
||||||
exchange_id character varying(12) NOT NULL
|
exchange_id character varying(12) NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Registry of series schemas. A series may only live in a registered schema
|
|
||||||
-- (FK from meta.series.db_schema, 04_indexes.sql). Every other schema in the
|
|
||||||
-- database belongs to the user and is never touched by the clients.
|
|
||||||
CREATE TABLE meta.series_schemas (
|
|
||||||
name text NOT NULL,
|
|
||||||
description text,
|
|
||||||
created_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
created_by text NOT NULL DEFAULT CURRENT_USER
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE meta.timezones (
|
CREATE TABLE meta.timezones (
|
||||||
iana_name character varying(50) NOT NULL,
|
iana_name character varying(50) NOT NULL,
|
||||||
display_name character varying(100) NOT NULL,
|
display_name character varying(100) NOT NULL,
|
||||||
|
|
|
||||||
|
|
@ -21,23 +21,6 @@ ALTER TABLE meta.currencies ADD CONSTRAINT currencies_pkey PRIMARY KEY (code);
|
||||||
ALTER TABLE meta.security_types ADD CONSTRAINT security_types_pkey PRIMARY KEY (code);
|
ALTER TABLE meta.security_types ADD CONSTRAINT security_types_pkey PRIMARY KEY (code);
|
||||||
ALTER TABLE meta.timezones ADD CONSTRAINT timezones_pkey PRIMARY KEY (iana_name);
|
ALTER TABLE meta.timezones ADD CONSTRAINT timezones_pkey PRIMARY KEY (iana_name);
|
||||||
|
|
||||||
-- Series schema registry. Names follow the table-name rule shared by both
|
|
||||||
-- clients: lower case, [a-z_][a-z0-9_]*, at most 63 bytes, never quoted.
|
|
||||||
ALTER TABLE meta.series_schemas ADD CONSTRAINT series_schemas_pkey PRIMARY KEY (name);
|
|
||||||
ALTER TABLE meta.series_schemas ADD CONSTRAINT series_schemas_name_check
|
|
||||||
CHECK (name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(name) <= 63);
|
|
||||||
|
|
||||||
-- A series may only live in a registered schema -- enforced here, so no
|
|
||||||
-- client can bypass it. RESTRICT: a schema cannot be unregistered while it
|
|
||||||
-- still holds series.
|
|
||||||
ALTER TABLE meta.series ADD CONSTRAINT series_db_schema_fkey
|
|
||||||
FOREIGN KEY (db_schema) REFERENCES meta.series_schemas (name)
|
|
||||||
ON UPDATE RESTRICT ON DELETE RESTRICT;
|
|
||||||
|
|
||||||
-- Table names follow the same rule (Core contract 6 in both clients).
|
|
||||||
ALTER TABLE meta.series ADD CONSTRAINT series_table_name_check
|
|
||||||
CHECK (table_name ~ '^[a-z_][a-z0-9_]*$' AND octet_length(table_name) <= 63);
|
|
||||||
|
|
||||||
CREATE INDEX idx_event_log_series_timestamp ON log.event_log USING btree (series_name, "timestamp" DESC);
|
CREATE INDEX idx_event_log_series_timestamp ON log.event_log USING btree (series_name, "timestamp" DESC);
|
||||||
|
|
||||||
CREATE INDEX idx_event_log_timestamp ON log.event_log USING btree ("timestamp" DESC);
|
CREATE INDEX idx_event_log_timestamp ON log.event_log USING btree ("timestamp" DESC);
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
-- Reference data (series schema registry, timezones, currencies, security types)
|
-- Reference data (timezones and currencies)
|
||||||
-- Generated: 2025-10-15T12:24:56.582392
|
-- Generated: 2025-10-15T12:24:56.582392
|
||||||
|
|
||||||
-- Timezones
|
-- Timezones
|
||||||
|
|
@ -767,7 +767,3 @@ VALUES
|
||||||
('OPT', 'Option', 'Equity and index options', NULL),
|
('OPT', 'Option', 'Equity and index options', NULL),
|
||||||
('STK', 'Stock', 'Common and preferred equities', NULL),
|
('STK', 'Stock', 'Common and preferred equities', NULL),
|
||||||
('WAR', 'Warrant', 'Equity warrants', NULL);
|
('WAR', 'Warrant', 'Equity warrants', NULL);
|
||||||
|
|
||||||
-- Series schema registry: the default schema, created in 01_schemas.sql
|
|
||||||
INSERT INTO meta.series_schemas (name, description)
|
|
||||||
VALUES ('series', 'Default series schema');
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,12 @@
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
-- Database self-checks, callable by every client (Python, C++, psql)
|
-- Schema self-check: meta.find_schema_problems()
|
||||||
-- ============================================================================
|
-- ============================================================================
|
||||||
|
-- Answers one question for every client (Python, C++, psql):
|
||||||
|
-- is this database complete and as the DDL defines it?
|
||||||
--
|
--
|
||||||
-- meta.find_schema_problems() -- on connect: is the database complete and as
|
-- SELECT * FROM meta.find_schema_problems(); -- zero rows = usable
|
||||||
-- the DDL defines it? zero rows = usable
|
|
||||||
--
|
--
|
||||||
-- meta.find_series_registry_mismatches() -- occasional consistency check:
|
-- Returns one row per problem: (object, problem). Read-only.
|
||||||
-- do the series schemas and meta.series agree? (defined at the end)
|
|
||||||
--
|
|
||||||
-- --------------------------------------------------------------------------
|
|
||||||
-- meta.find_schema_problems()
|
|
||||||
-- --------------------------------------------------------------------------
|
|
||||||
--
|
|
||||||
-- Returns one row per problem: (object, problem), in a stable order
|
|
||||||
-- (section by section, each sorted). Read-only.
|
|
||||||
--
|
--
|
||||||
-- The expectations below mirror 00..08. When any of those files changes,
|
-- The expectations below mirror 00..08. When any of those files changes,
|
||||||
-- update this file in the same commit. The release tests (a freshly
|
-- update this file in the same commit. The release tests (a freshly
|
||||||
|
|
@ -43,14 +36,12 @@ BEGIN
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
SELECT 'schema ' || s, 'missing'
|
SELECT 'schema ' || s, 'missing'
|
||||||
FROM unnest(ARRAY['meta', 'series', 'log']) AS s
|
FROM unnest(ARRAY['meta', 'series', 'log']) AS s
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s)
|
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s);
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
SELECT 'extension ' || e, 'missing'
|
SELECT 'extension ' || e, 'missing'
|
||||||
FROM unnest(ARRAY['pg_trgm']) AS e
|
FROM unnest(ARRAY['pg_trgm']) AS e
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e)
|
WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e);
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
SELECT 'sequence ' || s, 'missing'
|
SELECT 'sequence ' || s, 'missing'
|
||||||
|
|
@ -60,8 +51,7 @@ BEGIN
|
||||||
'log.pending_changes_id_seq',
|
'log.pending_changes_id_seq',
|
||||||
'log.script_log_id_seq'
|
'log.script_log_id_seq'
|
||||||
]) AS s
|
]) AS s
|
||||||
WHERE to_regclass(s) IS NULL
|
WHERE to_regclass(s) IS NULL;
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
-- Tables and columns ----------------------------------------------------
|
-- Tables and columns ----------------------------------------------------
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
|
|
@ -124,10 +114,6 @@ BEGIN
|
||||||
('meta.instruments', 'asset_class', 'character varying(30)', false),
|
('meta.instruments', 'asset_class', 'character varying(30)', false),
|
||||||
('meta.instruments', 'geography', 'character varying(30)', false),
|
('meta.instruments', 'geography', 'character varying(30)', false),
|
||||||
('meta.instruments', 'underlying', 'character varying(30)', false),
|
('meta.instruments', 'underlying', 'character varying(30)', false),
|
||||||
('meta.series_schemas', 'name', 'text', true),
|
|
||||||
('meta.series_schemas', 'description', 'text', false),
|
|
||||||
('meta.series_schemas', 'created_at', 'timestamp with time zone', true),
|
|
||||||
('meta.series_schemas', 'created_by', 'text', true),
|
|
||||||
('meta.security_types', 'code', 'character varying(10)', true),
|
('meta.security_types', 'code', 'character varying(10)', true),
|
||||||
('meta.security_types', 'name', 'character varying(50)', true),
|
('meta.security_types', 'name', 'character varying(50)', true),
|
||||||
('meta.security_types', 'description', 'text', false),
|
('meta.security_types', 'description', 'text', false),
|
||||||
|
|
@ -225,18 +211,13 @@ BEGIN
|
||||||
('meta.sessions', 'sessions_pkey'),
|
('meta.sessions', 'sessions_pkey'),
|
||||||
('meta.series', 'series_pkey'),
|
('meta.series', 'series_pkey'),
|
||||||
('meta.series', 'series_schema_table_key'),
|
('meta.series', 'series_schema_table_key'),
|
||||||
('meta.series', 'series_db_schema_fkey'),
|
|
||||||
('meta.series', 'series_table_name_check'),
|
|
||||||
('meta.series_schemas', 'series_schemas_pkey'),
|
|
||||||
('meta.series_schemas', 'series_schemas_name_check'),
|
|
||||||
('meta.currencies', 'currencies_pkey'),
|
('meta.currencies', 'currencies_pkey'),
|
||||||
('meta.security_types', 'security_types_pkey'),
|
('meta.security_types', 'security_types_pkey'),
|
||||||
('meta.timezones', 'timezones_pkey')
|
('meta.timezones', 'timezones_pkey')
|
||||||
) AS c(tbl, name)
|
) AS c(tbl, name)
|
||||||
WHERE to_regclass(c.tbl) IS NOT NULL
|
WHERE to_regclass(c.tbl) IS NOT NULL
|
||||||
AND NOT EXISTS (SELECT 1 FROM pg_constraint k
|
AND NOT EXISTS (SELECT 1 FROM pg_constraint k
|
||||||
WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name)
|
WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name);
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
SELECT 'index ' || i.idx, 'missing'
|
SELECT 'index ' || i.idx, 'missing'
|
||||||
|
|
@ -258,14 +239,12 @@ BEGIN
|
||||||
('log.idx_script_log_series', 'log.script_log'),
|
('log.idx_script_log_series', 'log.script_log'),
|
||||||
('log.idx_script_log_level', 'log.script_log')
|
('log.idx_script_log_level', 'log.script_log')
|
||||||
) AS i(idx, tbl)
|
) AS i(idx, tbl)
|
||||||
WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL
|
WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL;
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
SELECT 'view ' || v, 'missing'
|
SELECT 'view ' || v, 'missing'
|
||||||
FROM unnest(ARRAY['log.script_log_recent']) AS v
|
FROM unnest(ARRAY['log.script_log_recent']) AS v
|
||||||
WHERE to_regclass(v) IS NULL
|
WHERE to_regclass(v) IS NULL;
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
-- Functions (compared by whitespace-insensitive body fingerprint) -------
|
-- Functions (compared by whitespace-insensitive body fingerprint) -------
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
|
|
@ -281,7 +260,6 @@ BEGIN
|
||||||
('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'),
|
('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'),
|
||||||
('meta.drop_series', '6f2a919de965b87521be68252deee0e2'),
|
('meta.drop_series', '6f2a919de965b87521be68252deee0e2'),
|
||||||
('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'),
|
('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'),
|
||||||
('meta.find_series_registry_mismatches', '6dc9ecd2b7656b91a8b410c78a19fa2b'),
|
|
||||||
('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'),
|
('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'),
|
||||||
('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'),
|
('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'),
|
||||||
('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'),
|
('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'),
|
||||||
|
|
@ -292,8 +270,7 @@ BEGIN
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM pg_proc p
|
WHERE NOT EXISTS (SELECT 1 FROM pg_proc p
|
||||||
JOIN pg_namespace n ON n.oid = p.pronamespace
|
JOIN pg_namespace n ON n.oid = p.pronamespace
|
||||||
WHERE n.nspname || '.' || p.proname = f.name
|
WHERE n.nspname || '.' || p.proname = f.name
|
||||||
AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint)
|
AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint);
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
-- Drop-protection event trigger -----------------------------------------
|
-- Drop-protection event trigger -----------------------------------------
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
|
|
@ -310,8 +287,7 @@ BEGIN
|
||||||
(CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop'
|
(CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop'
|
||||||
THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END)
|
THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END)
|
||||||
) AS x(p)
|
) AS x(p)
|
||||||
WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL
|
WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL;
|
||||||
ORDER BY 2;
|
|
||||||
|
|
||||||
-- Reference data and version row ----------------------------------------
|
-- Reference data and version row ----------------------------------------
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
|
|
@ -325,27 +301,11 @@ BEGIN
|
||||||
FROM (VALUES
|
FROM (VALUES
|
||||||
('meta.timezones', 563),
|
('meta.timezones', 563),
|
||||||
('meta.currencies', 179),
|
('meta.currencies', 179),
|
||||||
('meta.security_types', 13),
|
('meta.security_types', 13)
|
||||||
('meta.series_schemas', 1)
|
|
||||||
) AS e(tbl, expected)
|
) AS e(tbl, expected)
|
||||||
WHERE to_regclass(e.tbl) IS NOT NULL
|
WHERE to_regclass(e.tbl) IS NOT NULL
|
||||||
) AS r
|
) AS r
|
||||||
WHERE r.actual < r.expected
|
WHERE r.actual < r.expected;
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
-- Series schema registry ------------------------------------------------
|
|
||||||
-- Only REGISTERED schemas are examined; user schemas are never looked at.
|
|
||||||
IF to_regclass('meta.series_schemas') IS NOT NULL THEN
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT 'series schema series', 'not registered in meta.series_schemas'
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM meta.series_schemas WHERE name = 'series');
|
|
||||||
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT 'series schema ' || r.name, 'registered but the schema does not exist'
|
|
||||||
FROM meta.series_schemas r
|
|
||||||
WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = r.name)
|
|
||||||
ORDER BY r.name;
|
|
||||||
END IF;
|
|
||||||
|
|
||||||
IF to_regclass('meta.schema_version') IS NOT NULL THEN
|
IF to_regclass('meta.schema_version') IS NOT NULL THEN
|
||||||
RETURN QUERY
|
RETURN QUERY
|
||||||
|
|
@ -354,93 +314,3 @@ BEGIN
|
||||||
END IF;
|
END IF;
|
||||||
END;
|
END;
|
||||||
$function$;
|
$function$;
|
||||||
|
|
||||||
|
|
||||||
-- --------------------------------------------------------------------------
|
|
||||||
-- meta.find_series_registry_mismatches()
|
|
||||||
-- --------------------------------------------------------------------------
|
|
||||||
-- Occasional consistency check -- NOT run on connect. Compares the series
|
|
||||||
-- registry (meta.series) with what actually exists in the REGISTERED series
|
|
||||||
-- schemas (meta.series_schemas). User schemas are never examined.
|
|
||||||
--
|
|
||||||
-- SELECT * FROM meta.find_series_registry_mismatches(); -- zero rows = consistent
|
|
||||||
--
|
|
||||||
-- One (object, problem) row per finding:
|
|
||||||
-- * a table, view, materialized view, foreign table or sequence in a
|
|
||||||
-- series schema that is not a registered series (an orphaned series table,
|
|
||||||
-- or a user table in the wrong place) -- harmless to the clients;
|
|
||||||
-- * a function in a series schema -- harmless;
|
|
||||||
-- * a series registered in meta.series whose table does not exist --
|
|
||||||
-- every operation on that series fails;
|
|
||||||
-- * a series table without a "timestamp timestamp without time zone NOT
|
|
||||||
-- NULL" column, or without a BRIN index on it.
|
|
||||||
-- Read-only. It reports; deciding what to do with a finding is up to the user.
|
|
||||||
|
|
||||||
CREATE OR REPLACE FUNCTION meta.find_series_registry_mismatches()
|
|
||||||
RETURNS TABLE(object text, problem text)
|
|
||||||
LANGUAGE plpgsql
|
|
||||||
STABLE
|
|
||||||
AS $function$
|
|
||||||
BEGIN
|
|
||||||
-- Relations in series schemas that are not registered series -----------
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT n.nspname || '.' || c.relname,
|
|
||||||
CASE c.relkind
|
|
||||||
WHEN 'r' THEN 'table' WHEN 'p' THEN 'table' WHEN 'v' THEN 'view'
|
|
||||||
WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table'
|
|
||||||
ELSE 'sequence' END
|
|
||||||
|| ' in a series schema but not registered in meta.series'
|
|
||||||
FROM pg_class c
|
|
||||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
||||||
JOIN meta.series_schemas r ON r.name = n.nspname
|
|
||||||
WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
|
|
||||||
AND NOT EXISTS (SELECT 1 FROM meta.series m
|
|
||||||
WHERE m.db_schema = n.nspname AND m.table_name = c.relname)
|
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT n.nspname || '.' || p.proname, 'function in a series schema'
|
|
||||||
FROM pg_proc p
|
|
||||||
JOIN pg_namespace n ON n.oid = p.pronamespace
|
|
||||||
JOIN meta.series_schemas r ON r.name = n.nspname
|
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
-- Registered series whose table is missing ------------------------------
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT m.db_schema || '.' || m.table_name,
|
|
||||||
'registered in meta.series but the table does not exist'
|
|
||||||
FROM meta.series m
|
|
||||||
WHERE to_regclass(format('%I.%I', m.db_schema, m.table_name)) IS NULL
|
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
-- Series tables without a proper timestamp column or its BRIN index -----
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT m.db_schema || '.' || m.table_name,
|
|
||||||
CASE WHEN a.attname IS NULL THEN 'no "timestamp" column'
|
|
||||||
ELSE '"timestamp" is ' || format_type(a.atttypid, a.atttypmod)
|
|
||||||
|| CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END
|
|
||||||
|| ', expected timestamp without time zone NOT NULL' END
|
|
||||||
FROM meta.series m
|
|
||||||
CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t
|
|
||||||
LEFT JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp'
|
|
||||||
AND a.attnum > 0 AND NOT a.attisdropped
|
|
||||||
WHERE t.rel IS NOT NULL
|
|
||||||
AND (a.attname IS NULL
|
|
||||||
OR format_type(a.atttypid, a.atttypmod) <> 'timestamp without time zone'
|
|
||||||
OR NOT a.attnotnull)
|
|
||||||
ORDER BY 1;
|
|
||||||
|
|
||||||
RETURN QUERY
|
|
||||||
SELECT m.db_schema || '.' || m.table_name, 'no BRIN index on "timestamp"'
|
|
||||||
FROM meta.series m
|
|
||||||
CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t
|
|
||||||
JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp'
|
|
||||||
AND a.attnum > 0 AND NOT a.attisdropped
|
|
||||||
WHERE NOT EXISTS (
|
|
||||||
SELECT 1 FROM pg_index i
|
|
||||||
JOIN pg_class ic ON ic.oid = i.indexrelid
|
|
||||||
JOIN pg_am am ON am.oid = ic.relam
|
|
||||||
WHERE i.indrelid = t.rel AND am.amname = 'brin' AND i.indkey[0] = a.attnum)
|
|
||||||
ORDER BY 1;
|
|
||||||
END;
|
|
||||||
$function$;
|
|
||||||
|
|
|
||||||
|
|
@ -5,4 +5,4 @@
|
||||||
-- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py).
|
-- they were built for (Python: DB_SCHEMA_VERSION in tsapi_pgdb/_version.py).
|
||||||
|
|
||||||
INSERT INTO meta.schema_version (version, description)
|
INSERT INTO meta.schema_version (version, description)
|
||||||
VALUES ('1.3.0', 'meta.find_series_registry_mismatches()');
|
VALUES ('1.0.0', 'Initial schema');
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue