-- ============================================================================
-- Database self-checks, callable by every client (Python, C++, psql)
-- ============================================================================
--
-- meta.find_schema_problems()  -- on connect: is the database complete and as
--                                 the DDL defines it?   zero rows = usable
--
-- meta.find_series_registry_mismatches()  -- occasional consistency check:
--     do the series schemas and meta.series agree? (defined at the end)
--
-- --------------------------------------------------------------------------
-- meta.find_schema_problems()
-- --------------------------------------------------------------------------
--
-- Returns one row per problem: (object, problem), in a stable order
-- (section by section, each sorted). Read-only.
--
-- The expectations below mirror 00..08. When any of those files changes,
-- update this file in the same commit. The release tests (a freshly
-- provisioned database must return zero rows) fail until you do.
--
-- Function bodies are compared by fingerprint: md5 of the body with runs of
-- whitespace collapsed, so line endings and re-indentation do not count as a
-- change. To refresh the fingerprints after editing a function:
--
--     SELECT n.nspname || '.' || p.proname,
--            md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g')))
--     FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
--     WHERE n.nspname IN ('meta', 'log') ORDER BY 1;
--
-- The schema VERSION string is not compared here: each client compares
-- meta.schema_version with the version it was built for. This function only
-- reports a missing version row.
-- ============================================================================

CREATE OR REPLACE FUNCTION meta.find_schema_problems()
 RETURNS TABLE(object text, problem text)
 LANGUAGE plpgsql
 STABLE
AS $function$
BEGIN
    -- Schemas and extension -------------------------------------------------
    RETURN QUERY
    SELECT 'schema ' || s, 'missing'
    FROM unnest(ARRAY['meta', 'series', 'log']) AS s
    WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = s)
    ORDER BY 1;

    RETURN QUERY
    SELECT 'extension ' || e, 'missing'
    FROM unnest(ARRAY['pg_trgm']) AS e
    WHERE NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = e)
    ORDER BY 1;

    RETURN QUERY
    SELECT 'sequence ' || s, 'missing'
    FROM unnest(ARRAY[
        'meta._table_meta__table_id_seq',
        'log.event_log_id_seq',
        'log.pending_changes_id_seq',
        'log.script_log_id_seq'
    ]) AS s
    WHERE to_regclass(s) IS NULL
    ORDER BY 1;

    -- Tables and columns ----------------------------------------------------
    RETURN QUERY
    WITH e(tbl, col, typ, not_null) AS (VALUES
        ('log.pending_changes', 'id', 'bigint', true),
        ('log.pending_changes', 'timestamp', 'timestamp with time zone', false),
        ('log.pending_changes', 'series_name', 'text', true),
        ('log.pending_changes', 'db_schema', 'text', true),
        ('log.pending_changes', 'op_type', 'text', true),
        ('log.pending_changes', 'db_user', 'text', false),
        ('log.pending_changes', 'details', 'jsonb', false),
        ('log.event_log', 'id', 'bigint', true),
        ('log.event_log', 'timestamp', 'timestamp with time zone', false),
        ('log.event_log', 'series_name', 'text', true),
        ('log.event_log', 'db_schema', 'text', true),
        ('log.event_log', 'event_type', 'text', true),
        ('log.event_log', 'details', 'jsonb', false),
        ('log.event_log', 'source', 'text', false),
        ('log.event_log', 'db_user', 'text', false),
        ('meta.currencies', 'code', 'character varying(3)', true),
        ('meta.currencies', 'name', 'character varying(100)', true),
        ('meta.currencies', 'symbol', 'character varying(10)', false),
        ('meta.currencies', 'decimal_places', 'integer', false),
        ('meta.currencies', 'notes', 'text', false),
        ('meta.exchanges', 'id', 'character varying(12)', true),
        ('meta.exchanges', 'name', 'character varying(50)', false),
        ('meta.exchanges', 'timezone', 'character varying(35)', false),
        ('meta.exchanges', 'currency', 'character varying(3)', false),
        ('meta.exchanges', 'default_sec_type', 'character varying(18)', false),
        ('meta.exchanges', 'default_session', 'character varying(40)', false),
        ('meta.exchanges', 'default_holiday_calendar', 'character varying(40)', false),
        ('meta.exchanges', 'default_tick_size', 'double precision', false),
        ('meta.exchanges', 'default_tick_value', 'double precision', false),
        ('meta.exchanges', 'default_extended_hours', 'boolean', false),
        ('meta.exchanges', 'country', 'character varying(3)', false),
        ('meta.holidays', 'calendar_name', 'character varying(50)', true),
        ('meta.holidays', 'date', 'date', true),
        ('meta.holidays', 'description', 'character varying(255)', false),
        ('meta.holidays', 'early_close', 'boolean', true),
        ('meta.instruments', 'symbol', 'character varying(12)', true),
        ('meta.instruments', 'exchange', 'character varying(12)', true),
        ('meta.instruments', 'name', 'character varying(50)', true),
        ('meta.instruments', 'type', 'character varying(10)', false),
        ('meta.instruments', 'tick_size', 'double precision', false),
        ('meta.instruments', 'tick_value', 'double precision', false),
        ('meta.instruments', 'expiry', 'character varying(20)', false),
        ('meta.instruments', 'strike', 'double precision', false),
        ('meta.instruments', 'currency', 'character varying(3)', false),
        ('meta.instruments', 'session', 'character varying(40)', false),
        ('meta.instruments', 'holiday_calendar', 'character varying(50)', false),
        ('meta.instruments', 'sec_id', 'character varying(50)', false),
        ('meta.instruments', 'sec_id_type', 'character varying(20)', false),
        ('meta.instruments', 'description', 'character varying(255)', false),
        ('meta.instruments', 'country', 'character varying(3)', false),
        ('meta.instruments', 'extended_hours', 'boolean', false),
        ('meta.instruments', 'sector', 'character varying(50)', false),
        ('meta.instruments', 'industry', 'character varying(100)', false),
        ('meta.instruments', 'category', 'character varying(50)', false),
        ('meta.instruments', 'subcategory', 'character varying(50)', false),
        ('meta.instruments', 'asset_class', 'character varying(30)', false),
        ('meta.instruments', 'geography', 'character varying(30)', false),
        ('meta.instruments', 'underlying', 'character varying(30)', false),
        ('meta.series_schemas', 'name', 'text', true),
        ('meta.series_schemas', 'description', 'text', false),
        ('meta.series_schemas', 'created_at', 'timestamp with time zone', true),
        ('meta.series_schemas', 'created_by', 'text', true),
        ('meta.security_types', 'code', 'character varying(10)', true),
        ('meta.security_types', 'name', 'character varying(50)', true),
        ('meta.security_types', 'description', 'text', false),
        ('meta.security_types', 'notes', 'text', false),
        ('meta.schema_version', 'version', 'character varying(20)', true),
        ('meta.schema_version', 'installed_at', 'timestamp with time zone', true),
        ('meta.schema_version', 'description', 'text', false),
        ('meta.series', 'table_id', 'integer', true),
        ('meta.series', 'db_schema', 'character varying(70)', true),
        ('meta.series', 'table_name', 'character varying(70)', true),
        ('meta.series', 'type', 'character varying(30)', false),
        ('meta.series', 'description', 'character varying(255)', false),
        ('meta.series', 'column_schema', 'text', false),
        ('meta.series', 'symbol', 'character varying(30)', false),
        ('meta.series', 'root_symbol', 'character varying(30)', false),
        ('meta.series', 'exchange', 'character varying(30)', false),
        ('meta.series', 'frequency', 'character varying(30)', false),
        ('meta.series', 'session', 'text', false),
        ('meta.series', 'sec_type', 'character varying(30)', false),
        ('meta.series', 'sec_id', 'character varying(30)', false),
        ('meta.series', 'sec_id_type', 'character varying(30)', false),
        ('meta.series', 'create_timestamp', 'timestamp without time zone', false),
        ('meta.series', 'num_records', 'bigint', false),
        ('meta.series', 'first_timestamp', 'timestamp without time zone', false),
        ('meta.series', 'last_timestamp', 'timestamp without time zone', false),
        ('meta.series', 'bar_type', 'character varying(30)', false),
        ('meta.series', 'feed', 'character varying(30)', false),
        ('meta.series', 'currency', 'character varying(30)', false),
        ('meta.series', 'expiry', 'character varying(30)', false),
        ('meta.series', 'roll_type', 'character varying(30)', false),
        ('meta.series', 'feed_type', 'character varying(30)', false),
        ('meta.series', 'user_attr', 'text', false),
        ('meta.series', 'user_attr_2', 'text', false),
        ('meta.series', 'script', 'text', false),
        ('meta.series', 'notes', 'text', false),
        ('meta.series', 'timezone', 'character varying(50)', false),
        ('meta.series', 'category', 'character varying(30)', false),
        ('meta.series', 'subcategory', 'character varying(30)', false),
        ('meta.series', 'json', 'jsonb', false),
        ('meta.series', 'needs_reindex', 'boolean', true),
        ('meta.sessions', 'name', 'character varying(50)', true),
        ('meta.sessions', 'definition', 'text', true),
        ('meta.sessions', 'json', 'text', false),
        ('meta.sessions', 'exchange_id', 'character varying(12)', true),
        ('meta.timezones', 'iana_name', 'character varying(50)', true),
        ('meta.timezones', 'display_name', 'character varying(100)', true),
        ('meta.timezones', 'description', 'text', false),
        ('meta.timezones', 'region', 'character varying(50)', false),
        ('meta.timezones', 'utc_offset_hours', 'real', false),
        ('meta.timezones', 'notes', 'text', false),
        ('log.script_log', 'id', 'integer', true),
        ('log.script_log', 'timestamp', 'timestamp with time zone', true),
        ('log.script_log', 'level', 'character varying(10)', true),
        ('log.script_log', 'message', 'text', true),
        ('log.script_log', 'script_name', 'character varying(100)', false),
        ('log.script_log', 'series_name', 'character varying(255)', false),
        ('log.script_log', 'session_id', 'uuid', false),
        ('log.script_log', 'details', 'jsonb', false)
    ),
    a AS (
        SELECT e.*, x.attname IS NOT NULL AS present,
               format_type(x.atttypid, x.atttypmod) AS got_typ, x.attnotnull AS got_not_null
        FROM e
        LEFT JOIN pg_attribute x ON x.attrelid = to_regclass(e.tbl) AND x.attname = e.col
                                AND x.attnum > 0 AND NOT x.attisdropped
        WHERE to_regclass(e.tbl) IS NOT NULL
    ),
    found(sort_key, object, problem) AS (
        SELECT 1, 'table ' || tbl, 'missing'
        FROM (SELECT DISTINCT tbl FROM e WHERE to_regclass(tbl) IS NULL) m
        UNION ALL
        SELECT 2, 'table ' || tbl, 'missing columns: ' || string_agg(col, ', ')
        FROM a WHERE NOT present GROUP BY tbl
        UNION ALL
        SELECT 3, 'table ' || tbl, col || ' is ' || got_typ || ', expected ' || typ
        FROM a WHERE present AND got_typ <> typ
        UNION ALL
        SELECT 4, 'table ' || tbl,
               col || ' is ' || CASE WHEN got_not_null THEN 'NOT NULL' ELSE 'nullable' END
                   || ', expected ' || CASE WHEN not_null THEN 'NOT NULL' ELSE 'nullable' END
        FROM a WHERE present AND got_not_null <> not_null
    )
    SELECT f.object, f.problem FROM found f ORDER BY f.object, f.sort_key, f.problem;

    -- Constraints and indexes (only on tables that exist) -------------------
    RETURN QUERY
    SELECT 'constraint ' || c.tbl || '.' || c.name, 'missing'
    FROM (VALUES
        ('log.pending_changes', 'pending_changes_pkey'),
        ('log.event_log', 'event_log_pkey'),
        ('meta.holidays', 'holidays_pkey'),
        ('log.script_log', 'script_log_pkey'),
        ('meta.instruments', 'instruments_pkey'),
        ('meta.exchanges', 'exchanges_pkey'),
        ('meta.sessions', 'sessions_pkey'),
        ('meta.series', 'series_pkey'),
        ('meta.series', 'series_schema_table_key'),
        ('meta.series', 'series_db_schema_fkey'),
        ('meta.series', 'series_table_name_check'),
        ('meta.series_schemas', 'series_schemas_pkey'),
        ('meta.series_schemas', 'series_schemas_name_check'),
        ('meta.currencies', 'currencies_pkey'),
        ('meta.security_types', 'security_types_pkey'),
        ('meta.timezones', 'timezones_pkey')
    ) AS c(tbl, name)
    WHERE to_regclass(c.tbl) IS NOT NULL
      AND NOT EXISTS (SELECT 1 FROM pg_constraint k
                      WHERE k.conrelid = to_regclass(c.tbl) AND k.conname = c.name)
    ORDER BY 1;

    RETURN QUERY
    SELECT 'index ' || i.idx, 'missing'
    FROM (VALUES
        ('log.idx_event_log_series_timestamp', 'log.event_log'),
        ('log.idx_event_log_timestamp', 'log.event_log'),
        ('log.idx_event_log_event_type', 'log.event_log'),
        ('meta.idx_series_create_timestamp_desc', 'meta.series'),
        ('meta.idx_series_db_schema', 'meta.series'),
        ('meta.idx_series_exchange_pattern', 'meta.series'),
        ('meta.idx_series_frequency_pattern', 'meta.series'),
        ('meta.idx_series_symbol_exchange', 'meta.series'),
        ('meta.idx_series_symbol_pattern', 'meta.series'),
        ('meta.idx_series_symbol_trgm', 'meta.series'),
        ('meta.idx_series_table_name_trgm', 'meta.series'),
        ('meta.idx_series_type_pattern', 'meta.series'),
        ('log.idx_script_log_timestamp', 'log.script_log'),
        ('log.idx_script_log_session', 'log.script_log'),
        ('log.idx_script_log_series', 'log.script_log'),
        ('log.idx_script_log_level', 'log.script_log')
    ) AS i(idx, tbl)
    WHERE to_regclass(i.tbl) IS NOT NULL AND to_regclass(i.idx) IS NULL
    ORDER BY 1;

    RETURN QUERY
    SELECT 'view ' || v, 'missing'
    FROM unnest(ARRAY['log.script_log_recent']) AS v
    WHERE to_regclass(v) IS NULL
    ORDER BY 1;

    -- Functions (compared by whitespace-insensitive body fingerprint) -------
    RETURN QUERY
    SELECT 'function ' || f.name,
           CASE WHEN NOT EXISTS (SELECT 1 FROM pg_proc p
                                 JOIN pg_namespace n ON n.oid = p.pronamespace
                                 WHERE n.nspname || '.' || p.proname = f.name)
                THEN 'missing'
                ELSE 'differs from the DDL (stale or edited)' END
    FROM (VALUES
        ('log.flush_pending_to_event_log', 'be02d914fc5e73475eb1ce7ce7a78f29'),
        ('log.notify_event_log', '7e5898f1d1d128ef2e44c1d4ae8a7312'),
        ('meta.drop_all_series', 'fc404a746d91578860bea0d89b166bc7'),
        ('meta.drop_series', '6f2a919de965b87521be68252deee0e2'),
        ('meta.drop_series_if_exists', 'aa1b6ab64d2d3b9ca19ed9033f129524'),
        ('meta.find_series_registry_mismatches', '6dc9ecd2b7656b91a8b410c78a19fa2b'),
        ('meta.prevent_direct_drop', '1d38cbad0caba72a5104da42e3030929'),
        ('meta.read_series_batch', '93ef12652fd22b14d210971b537de736'),
        ('meta.read_series_batch_cursor', '2ed7d17f545fdfb9d6af577a051699e1'),
        ('meta.read_series_batch_ohlc', '4bdef4d3573caab023ddd12db38b978c'),
        ('meta.refresh_series_timestamps', '1cdfd496752142cbde78d9646532d1aa'),
        ('meta.update_series_last_timestamp', 'f27394672bc5026456c4625ac3091a62')
    ) AS f(name, fingerprint)
    WHERE NOT EXISTS (SELECT 1 FROM pg_proc p
                      JOIN pg_namespace n ON n.oid = p.pronamespace
                      WHERE n.nspname || '.' || p.proname = f.name
                        AND md5(btrim(regexp_replace(p.prosrc, '\s+', ' ', 'g'))) = f.fingerprint)
    ORDER BY 1;

    -- Drop-protection event trigger -----------------------------------------
    RETURN QUERY
    SELECT 'event trigger prevent_series_drops', 'missing'
    WHERE NOT EXISTS (SELECT 1 FROM pg_event_trigger WHERE evtname = 'prevent_series_drops');

    RETURN QUERY
    SELECT 'event trigger prevent_series_drops', p
    FROM pg_event_trigger t,
         LATERAL (VALUES
             (CASE WHEN t.evtevent <> 'sql_drop'
                   THEN 'fires on ' || t.evtevent || ', expected sql_drop' END),
             (CASE WHEN t.evtenabled = 'D' THEN 'disabled' END),
             (CASE WHEN t.evtfoid::regproc::text <> 'meta.prevent_direct_drop'
                   THEN 'calls ' || t.evtfoid::regproc::text || ', expected meta.prevent_direct_drop' END)
         ) AS x(p)
    WHERE t.evtname = 'prevent_series_drops' AND p IS NOT NULL
    ORDER BY 2;

    -- Reference data and version row ----------------------------------------
    RETURN QUERY
    SELECT 'reference data ' || r.tbl,
           r.actual || ' rows, expected at least ' || r.expected
    FROM (
        SELECT e.tbl, e.expected,
               (xpath('/row/n/text()', query_to_xml(
                   format('SELECT count(*) AS n FROM %s', e.tbl), false, true, '')))[1]::text::int
                   AS actual
        FROM (VALUES
        ('meta.timezones', 563),
        ('meta.currencies', 179),
        ('meta.security_types', 13),
        ('meta.series_schemas', 1)
        ) AS e(tbl, expected)
        WHERE to_regclass(e.tbl) IS NOT NULL
    ) AS r
    WHERE r.actual < r.expected
    ORDER BY 1;

    -- Series schema registry ------------------------------------------------
    -- Only REGISTERED schemas are examined; user schemas are never looked at.
    IF to_regclass('meta.series_schemas') IS NOT NULL THEN
        RETURN QUERY
        SELECT 'series schema series', 'not registered in meta.series_schemas'
        WHERE NOT EXISTS (SELECT 1 FROM meta.series_schemas WHERE name = 'series');

        RETURN QUERY
        SELECT 'series schema ' || r.name, 'registered but the schema does not exist'
        FROM meta.series_schemas r
        WHERE NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = r.name)
        ORDER BY r.name;
    END IF;

    IF to_regclass('meta.schema_version') IS NOT NULL THEN
        RETURN QUERY
        SELECT 'schema version', 'not recorded in meta.schema_version'
        WHERE NOT EXISTS (SELECT 1 FROM meta.schema_version);
    END IF;
END;
$function$;


-- --------------------------------------------------------------------------
-- meta.find_series_registry_mismatches()
-- --------------------------------------------------------------------------
-- Occasional consistency check -- NOT run on connect. Compares the series
-- registry (meta.series) with what actually exists in the REGISTERED series
-- schemas (meta.series_schemas). User schemas are never examined.
--
--     SELECT * FROM meta.find_series_registry_mismatches();  -- zero rows = consistent
--
-- One (object, problem) row per finding:
--   * a table, view, materialized view, foreign table or sequence in a
--     series schema that is not a registered series (an orphaned series table,
--     or a user table in the wrong place) -- harmless to the clients;
--   * a function in a series schema -- harmless;
--   * a series registered in meta.series whose table does not exist --
--     every operation on that series fails;
--   * a series table without a "timestamp timestamp without time zone NOT
--     NULL" column, or without a BRIN index on it.
-- Read-only. It reports; deciding what to do with a finding is up to the user.

CREATE OR REPLACE FUNCTION meta.find_series_registry_mismatches()
 RETURNS TABLE(object text, problem text)
 LANGUAGE plpgsql
 STABLE
AS $function$
BEGIN
    -- Relations in series schemas that are not registered series -----------
    RETURN QUERY
    SELECT n.nspname || '.' || c.relname,
           CASE c.relkind
               WHEN 'r' THEN 'table' WHEN 'p' THEN 'table' WHEN 'v' THEN 'view'
               WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table'
               ELSE 'sequence' END
           || ' in a series schema but not registered in meta.series'
    FROM pg_class c
    JOIN pg_namespace n ON n.oid = c.relnamespace
    JOIN meta.series_schemas r ON r.name = n.nspname
    WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
      AND NOT EXISTS (SELECT 1 FROM meta.series m
                      WHERE m.db_schema = n.nspname AND m.table_name = c.relname)
    ORDER BY 1;

    RETURN QUERY
    SELECT n.nspname || '.' || p.proname, 'function in a series schema'
    FROM pg_proc p
    JOIN pg_namespace n ON n.oid = p.pronamespace
    JOIN meta.series_schemas r ON r.name = n.nspname
    ORDER BY 1;

    -- Registered series whose table is missing ------------------------------
    RETURN QUERY
    SELECT m.db_schema || '.' || m.table_name,
           'registered in meta.series but the table does not exist'
    FROM meta.series m
    WHERE to_regclass(format('%I.%I', m.db_schema, m.table_name)) IS NULL
    ORDER BY 1;

    -- Series tables without a proper timestamp column or its BRIN index -----
    RETURN QUERY
    SELECT m.db_schema || '.' || m.table_name,
           CASE WHEN a.attname IS NULL THEN 'no "timestamp" column'
                ELSE '"timestamp" is ' || format_type(a.atttypid, a.atttypmod)
                     || CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END
                     || ', expected timestamp without time zone NOT NULL' END
    FROM meta.series m
    CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t
    LEFT JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp'
                            AND a.attnum > 0 AND NOT a.attisdropped
    WHERE t.rel IS NOT NULL
      AND (a.attname IS NULL
           OR format_type(a.atttypid, a.atttypmod) <> 'timestamp without time zone'
           OR NOT a.attnotnull)
    ORDER BY 1;

    RETURN QUERY
    SELECT m.db_schema || '.' || m.table_name, 'no BRIN index on "timestamp"'
    FROM meta.series m
    CROSS JOIN LATERAL (SELECT to_regclass(format('%I.%I', m.db_schema, m.table_name)) AS rel) t
    JOIN pg_attribute a ON a.attrelid = t.rel AND a.attname = 'timestamp'
                       AND a.attnum > 0 AND NOT a.attisdropped
    WHERE NOT EXISTS (
        SELECT 1 FROM pg_index i
        JOIN pg_class ic ON ic.oid = i.indexrelid
        JOIN pg_am am ON am.oid = ic.relam
        WHERE i.indrelid = t.rel AND am.amname = 'brin' AND i.indkey[0] = a.attnum)
    ORDER BY 1;
END;
$function$;