From 6c68c510444a48a81ef9914ef54e0d94a242333c Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Mon, 7 Aug 2023 15:12:33 -0700 Subject: [PATCH 001/109] Recalculate search_path after ALTER ROLE. Renaming a role can affect the meaning of the special string $user, so must cause search_path to be recalculated. Discussion: https://postgr.es/m/186761d32c0255debbdf50b6310b581b9c973e6c.camel@j-davis.com Reviewed-by: Nathan Bossart, Michael Paquier Backpatch-through: 11 --- src/backend/catalog/namespace.c | 6 +- .../isolation/expected/search-path-inval.out | 97 +++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../isolation/specs/search-path-inval.spec | 59 +++++++++++ 4 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 src/test/isolation/expected/search-path-inval.out create mode 100644 src/test/isolation/specs/search-path-inval.spec diff --git a/src/backend/catalog/namespace.c b/src/backend/catalog/namespace.c index be09847022b..7271a410706 100644 --- a/src/backend/catalog/namespace.c +++ b/src/backend/catalog/namespace.c @@ -4777,11 +4777,15 @@ InitializeSearchPath(void) { /* * In normal mode, arrange for a callback on any syscache invalidation - * of pg_namespace rows. + * of pg_namespace or pg_authid rows. (Changing a role name may affect + * the meaning of the special string $user.) */ CacheRegisterSyscacheCallback(NAMESPACEOID, NamespaceCallback, (Datum) 0); + CacheRegisterSyscacheCallback(AUTHOID, + NamespaceCallback, + (Datum) 0); /* Force search path to be recomputed on next use */ baseSearchPathValid = false; } diff --git a/src/test/isolation/expected/search-path-inval.out b/src/test/isolation/expected/search-path-inval.out new file mode 100644 index 00000000000..e0173afdb2d --- /dev/null +++ b/src/test/isolation/expected/search-path-inval.out @@ -0,0 +1,97 @@ +Parsed test spec with 3 sessions + +starting permutation: s1a s2a s1a s2b +step s1a: + SELECT CURRENT_USER; + SHOW search_path; + SELECT t FROM x; + +current_user +---------------- +regress_sp_user1 +(1 row) + +search_path +-------------------------- +"$user", regress_sp_public +(1 row) + +t +-------------------------- +data in regress_sp_user1.x +(1 row) + +step s2a: + ALTER ROLE regress_sp_user1 RENAME TO regress_sp_user2; + +step s1a: + SELECT CURRENT_USER; + SHOW search_path; + SELECT t FROM x; + +current_user +---------------- +regress_sp_user2 +(1 row) + +search_path +-------------------------- +"$user", regress_sp_public +(1 row) + +t +--------------------------- +data in regress_sp_public.x +(1 row) + +step s2b: + ALTER ROLE regress_sp_user2 RENAME TO regress_sp_user1; + + +starting permutation: s1a s3a s1a s3b +step s1a: + SELECT CURRENT_USER; + SHOW search_path; + SELECT t FROM x; + +current_user +---------------- +regress_sp_user1 +(1 row) + +search_path +-------------------------- +"$user", regress_sp_public +(1 row) + +t +-------------------------- +data in regress_sp_user1.x +(1 row) + +step s3a: + ALTER SCHEMA regress_sp_user1 RENAME TO regress_sp_user2; + +step s1a: + SELECT CURRENT_USER; + SHOW search_path; + SELECT t FROM x; + +current_user +---------------- +regress_sp_user1 +(1 row) + +search_path +-------------------------- +"$user", regress_sp_public +(1 row) + +t +--------------------------- +data in regress_sp_public.x +(1 row) + +step s3b: + ALTER SCHEMA regress_sp_user2 RENAME TO regress_sp_user1; + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 3f12f923c02..ee787b6f74b 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -157,3 +157,4 @@ test: ao-repeatable-read-vacuum test: ao-insert-eof create_index_hot udf-insert-deadlock heap-repeatable-read ao-repeatable-read test: vacuum-ao-concurrent-drop +test: search-path-inval diff --git a/src/test/isolation/specs/search-path-inval.spec b/src/test/isolation/specs/search-path-inval.spec new file mode 100644 index 00000000000..08b1bba2fc9 --- /dev/null +++ b/src/test/isolation/specs/search-path-inval.spec @@ -0,0 +1,59 @@ +# Test search_path invalidation. + +setup +{ + CREATE USER regress_sp_user1; + CREATE SCHEMA regress_sp_user1 AUTHORIZATION regress_sp_user1; + CREATE SCHEMA regress_sp_public; + GRANT ALL PRIVILEGES ON SCHEMA regress_sp_public TO regress_sp_user1; +} + +teardown +{ + DROP SCHEMA regress_sp_public CASCADE; + DROP SCHEMA regress_sp_user1 CASCADE; + DROP USER regress_sp_user1; +} + +session s1 +setup +{ + SET search_path = "$user", regress_sp_public; + SET SESSION AUTHORIZATION regress_sp_user1; + CREATE TABLE regress_sp_user1.x(t) AS SELECT 'data in regress_sp_user1.x'; + CREATE TABLE regress_sp_public.x(t) AS SELECT 'data in regress_sp_public.x'; +} +step s1a +{ + SELECT CURRENT_USER; + SHOW search_path; + SELECT t FROM x; +} + +session s2 +step s2a +{ + ALTER ROLE regress_sp_user1 RENAME TO regress_sp_user2; +} +step s2b +{ + ALTER ROLE regress_sp_user2 RENAME TO regress_sp_user1; +} + +session s3 +step s3a +{ + ALTER SCHEMA regress_sp_user1 RENAME TO regress_sp_user2; +} +step s3b +{ + ALTER SCHEMA regress_sp_user2 RENAME TO regress_sp_user1; +} + +# s1's search_path is invalidated by role name change in s2a, and +# falls back to regress_sp_public.x +permutation s1a s2a s1a s2b + +# s1's search_path is invalidated by schema name change in s2b, and +# falls back to regress_sp_public.x +permutation s1a s3a s1a s3b From 8836f7110dbc61ddf5ad61409b22fbee4ac67884 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Thu, 10 Aug 2023 10:16:59 -0700 Subject: [PATCH 002/109] Remove test from commit fa2e874946. The fix itself is fine, but the test revealed other problems related to parallel query that are not easily fixable. Remove the test for now to fix the buildfarm. Discussion: https://postgr.es/m/88825.1691665432@sss.pgh.pa.us Backpatch-through: 11 --- .../isolation/expected/search-path-inval.out | 97 ------------------- src/test/isolation/isolation_schedule | 1 - .../isolation/specs/search-path-inval.spec | 59 ----------- 3 files changed, 157 deletions(-) delete mode 100644 src/test/isolation/expected/search-path-inval.out delete mode 100644 src/test/isolation/specs/search-path-inval.spec diff --git a/src/test/isolation/expected/search-path-inval.out b/src/test/isolation/expected/search-path-inval.out deleted file mode 100644 index e0173afdb2d..00000000000 --- a/src/test/isolation/expected/search-path-inval.out +++ /dev/null @@ -1,97 +0,0 @@ -Parsed test spec with 3 sessions - -starting permutation: s1a s2a s1a s2b -step s1a: - SELECT CURRENT_USER; - SHOW search_path; - SELECT t FROM x; - -current_user ----------------- -regress_sp_user1 -(1 row) - -search_path --------------------------- -"$user", regress_sp_public -(1 row) - -t --------------------------- -data in regress_sp_user1.x -(1 row) - -step s2a: - ALTER ROLE regress_sp_user1 RENAME TO regress_sp_user2; - -step s1a: - SELECT CURRENT_USER; - SHOW search_path; - SELECT t FROM x; - -current_user ----------------- -regress_sp_user2 -(1 row) - -search_path --------------------------- -"$user", regress_sp_public -(1 row) - -t ---------------------------- -data in regress_sp_public.x -(1 row) - -step s2b: - ALTER ROLE regress_sp_user2 RENAME TO regress_sp_user1; - - -starting permutation: s1a s3a s1a s3b -step s1a: - SELECT CURRENT_USER; - SHOW search_path; - SELECT t FROM x; - -current_user ----------------- -regress_sp_user1 -(1 row) - -search_path --------------------------- -"$user", regress_sp_public -(1 row) - -t --------------------------- -data in regress_sp_user1.x -(1 row) - -step s3a: - ALTER SCHEMA regress_sp_user1 RENAME TO regress_sp_user2; - -step s1a: - SELECT CURRENT_USER; - SHOW search_path; - SELECT t FROM x; - -current_user ----------------- -regress_sp_user1 -(1 row) - -search_path --------------------------- -"$user", regress_sp_public -(1 row) - -t ---------------------------- -data in regress_sp_public.x -(1 row) - -step s3b: - ALTER SCHEMA regress_sp_user2 RENAME TO regress_sp_user1; - diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index ee787b6f74b..3f12f923c02 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -157,4 +157,3 @@ test: ao-repeatable-read-vacuum test: ao-insert-eof create_index_hot udf-insert-deadlock heap-repeatable-read ao-repeatable-read test: vacuum-ao-concurrent-drop -test: search-path-inval diff --git a/src/test/isolation/specs/search-path-inval.spec b/src/test/isolation/specs/search-path-inval.spec deleted file mode 100644 index 08b1bba2fc9..00000000000 --- a/src/test/isolation/specs/search-path-inval.spec +++ /dev/null @@ -1,59 +0,0 @@ -# Test search_path invalidation. - -setup -{ - CREATE USER regress_sp_user1; - CREATE SCHEMA regress_sp_user1 AUTHORIZATION regress_sp_user1; - CREATE SCHEMA regress_sp_public; - GRANT ALL PRIVILEGES ON SCHEMA regress_sp_public TO regress_sp_user1; -} - -teardown -{ - DROP SCHEMA regress_sp_public CASCADE; - DROP SCHEMA regress_sp_user1 CASCADE; - DROP USER regress_sp_user1; -} - -session s1 -setup -{ - SET search_path = "$user", regress_sp_public; - SET SESSION AUTHORIZATION regress_sp_user1; - CREATE TABLE regress_sp_user1.x(t) AS SELECT 'data in regress_sp_user1.x'; - CREATE TABLE regress_sp_public.x(t) AS SELECT 'data in regress_sp_public.x'; -} -step s1a -{ - SELECT CURRENT_USER; - SHOW search_path; - SELECT t FROM x; -} - -session s2 -step s2a -{ - ALTER ROLE regress_sp_user1 RENAME TO regress_sp_user2; -} -step s2b -{ - ALTER ROLE regress_sp_user2 RENAME TO regress_sp_user1; -} - -session s3 -step s3a -{ - ALTER SCHEMA regress_sp_user1 RENAME TO regress_sp_user2; -} -step s3b -{ - ALTER SCHEMA regress_sp_user2 RENAME TO regress_sp_user1; -} - -# s1's search_path is invalidated by role name change in s2a, and -# falls back to regress_sp_public.x -permutation s1a s2a s1a s2b - -# s1's search_path is invalidated by schema name change in s2b, and -# falls back to regress_sp_public.x -permutation s1a s3a s1a s3b From 87fc7a947c943119fe0dfa1a03548791c036ce4d Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Tue, 22 Aug 2023 11:57:08 -0400 Subject: [PATCH 003/109] Cache by-reference missing values in a long lived context Attribute missing values might be needed past the lifetime of the tuple descriptors from which they are extracted. To avoid possibly using pointers for by-reference values which might thus be left dangling, we cache a datumCopy'd version of the datum in the TopMemoryContext. Since we first search for the value this only needs to be done once per session for any such value. Original complaint from Tom Lane, idea for mitigation by Andrew Dunstan, tweaked by Tom Lane. Backpatch to version 11 where missing values were introduced. Discussion: https://postgr.es/m/1306569.1687978174@sss.pgh.pa.us --- src/backend/access/common/heaptuple.c | 90 ++++++++++++++++++++++++++- src/tools/pgindent/typedefs.list | 1 + 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index 66aee71c706..c4ad0d6c3d6 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -62,9 +62,13 @@ #include "access/heaptoast.h" #include "access/sysattr.h" #include "access/tupdesc_details.h" +#include "common/hashfn.h" #include "executor/tuptable.h" #include "foreign/foreign.h" +#include "utils/datum.h" #include "utils/expandeddatum.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" #include "catalog/pg_type.h" #include "cdb/cdbvars.h" /* GpIdentity.segindex */ @@ -76,6 +80,57 @@ #define VARLENA_ATT_IS_PACKABLE(att) \ ((att)->attstorage != TYPSTORAGE_PLAIN) +/* + * Setup for cacheing pass-by-ref missing attributes in a way that survives + * tupleDesc destruction. + */ + +typedef struct +{ + int len; + Datum value; +} missing_cache_key; + +static HTAB *missing_cache = NULL; + +static uint32 +missing_hash(const void *key, Size keysize) +{ + const missing_cache_key *entry = (missing_cache_key *) key; + + return hash_bytes((const unsigned char *) entry->value, entry->len); +} + +static int +missing_match(const void *key1, const void *key2, Size keysize) +{ + const missing_cache_key *entry1 = (missing_cache_key *) key1; + const missing_cache_key *entry2 = (missing_cache_key *) key2; + + if (entry1->len != entry2->len) + return entry1->len > entry2->len ? 1 : -1; + + return memcmp(DatumGetPointer(entry1->value), + DatumGetPointer(entry2->value), + entry1->len); +} + +static void +init_missing_cache() +{ + HASHCTL hash_ctl; + + hash_ctl.keysize = sizeof(missing_cache_key); + hash_ctl.entrysize = sizeof(missing_cache_key); + hash_ctl.hcxt = TopMemoryContext; + hash_ctl.hash = missing_hash; + hash_ctl.match = missing_match; + missing_cache = + hash_create("Missing Values Cache", + 32, + &hash_ctl, + HASH_ELEM | HASH_CONTEXT | HASH_FUNCTION | HASH_COMPARE); +} /* ---------------------------------------------------------------- * misc support routines @@ -107,8 +162,41 @@ getmissingattr(TupleDesc tupleDesc, if (attrmiss->am_present) { + missing_cache_key key; + missing_cache_key *entry; + bool found; + MemoryContext oldctx; + *isnull = false; - return attrmiss->am_value; + + /* no need to cache by-value attributes */ + if (att->attbyval) + return attrmiss->am_value; + + /* set up cache if required */ + if (missing_cache == NULL) + init_missing_cache(); + + /* check if there's a cache entry */ + Assert(att->attlen > 0 || att->attlen == -1); + if (att->attlen > 0) + key.len = att->attlen; + else + key.len = VARSIZE_ANY(attrmiss->am_value); + key.value = attrmiss->am_value; + + entry = hash_search(missing_cache, &key, HASH_ENTER, &found); + + if (!found) + { + /* cache miss, so we need a non-transient copy of the datum */ + oldctx = MemoryContextSwitchTo(TopMemoryContext); + entry->value = + datumCopy(attrmiss->am_value, false, att->attlen); + MemoryContextSwitchTo(oldctx); + } + + return entry->value; } } diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index fe36db36936..fe934d1f61b 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -3293,6 +3293,7 @@ mbstr_verifier memoize_hash memoize_iterator metastring +missing_cache_key mix_data_t mixedStruct mode_t From e55b93698e02a6b4f817d22aa7b725b0a184110b Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Wed, 23 Aug 2023 14:13:07 +0200 Subject: [PATCH 004/109] doc: Replace list of drivers and PLs with wiki link The list of external language drivers and procedural languages was never complete or exhaustive, and rather than attempting to manage it the content has migrated to the wiki. This replaces the tables altogether with links to the wiki as we regularly get requests for adding various projects, which we reject without any clear policy for why or how the content should be managed. The threads linked to below are the most recent discussions about this, the archives contain many more. Backpatch to all supported branches since the list on the wiki applies to all branches. Author: Jonathan Katz Discussion: https://postgr.es/m/169165415312.635.10247434927885764880@wrigleys.postgresql.org Discussion: https://postgr.es/m/169177958824.635.11087800083040275266@wrigleys.postgresql.org Backpatch-through: v11 --- doc/src/sgml/external-projects.sgml | 158 ++++------------------------ 1 file changed, 18 insertions(+), 140 deletions(-) diff --git a/doc/src/sgml/external-projects.sgml b/doc/src/sgml/external-projects.sgml index bf590aba5d9..50872dfd88e 100644 --- a/doc/src/sgml/external-projects.sgml +++ b/doc/src/sgml/external-projects.sgml @@ -40,99 +40,17 @@ All other language interfaces are external projects and are distributed - separately. includes a list of - some of these projects. Note that some of these packages might not be - released under the same license as PostgreSQL. For more - information on each language interface, including licensing terms, refer to - its website and documentation. + separately. A + list of language interfaces + is maintained on the PostgreSQL wiki. Note that some of these packages are + not released under the same license as PostgreSQL. + For more information on each language interface, including licensing terms, + refer to its website and documentation. - - Externally Maintained Client Interfaces - - - - - Name - Language - Comments - Website - - - - - - DBD::Pg - Perl - Perl DBI driver - - - - - JDBC - Java - Type 4 JDBC driver - - - - - libpqxx - C++ - C++ interface - - - - - node-postgres - JavaScript - Node.js driver - - - - - Npgsql - .NET - .NET data provider - - - - - pgtcl - Tcl - - - - - - pgtclng - Tcl - - - - - - pq - Go - Pure Go driver for Go's database/sql - - - - - psqlODBC - ODBC - ODBC driver - - - - - psycopg - Python - DB API 2.0-compliant - - - - -
+ + + @@ -170,58 +88,18 @@ In addition, there are a number of procedural languages that are developed and maintained outside the core PostgreSQL - distribution. lists some of these - packages. Note that some of these projects might not be released under the same - license as PostgreSQL. For more information on each - procedural language, including licensing information, refer to its website + distribution. A list of + procedural languages + is maintained on the PostgreSQL wiki. Note that some of these projects are + not released under the same license as PostgreSQL. + For more information on each procedural language, including licensing + information, refer to its website and documentation. - - Externally Maintained Procedural Languages - - - - - Name - Language - Website - - - - - - PL/Java - Java - - - - - PL/Lua - Lua - - - - - PL/R - R - - - - - PL/sh - Unix shell - - - - - PL/v8 - JavaScript - - - - -
+ + + From 11ae6746ed030e7d6a3cc1654191e720d9709386 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 24 Aug 2023 12:02:40 -0400 Subject: [PATCH 005/109] Avoid unnecessary plancache revalidation of utility statements. Revalidation of a plancache entry (after a cache invalidation event) requires acquiring a snapshot. Normally that is harmless, but not if the cached statement is one that needs to run without acquiring a snapshot. We were already aware of that for TransactionStmts, but for some reason hadn't extrapolated to the other statements that PlannedStmtRequiresSnapshot() knows mustn't set a snapshot. This can lead to unexpected failures of commands such as SET TRANSACTION ISOLATION LEVEL. We can fix it in the same way, by excluding those command types from revalidation. However, we can do even better than that: there is no need to revalidate for any statement type for which parse analysis, rewrite, and plan steps do nothing interesting, which is nearly all utility commands. To mechanize this, invent a parser function stmt_requires_parse_analysis() that tells whether parse analysis does anything beyond wrapping a CMD_UTILITY Query around the raw parse tree. If that's what it does, then rewrite and plan will just skip the Query, so that it is not possible for the same raw parse tree to produce a different plan tree after cache invalidation. stmt_requires_parse_analysis() is basically equivalent to the existing function analyze_requires_snapshot(), except that for obscure reasons that function omits ReturnStmt and CallStmt. It is unclear whether those were oversights or intentional. I have not been able to demonstrate a bug from not acquiring a snapshot while analyzing these commands, but at best it seems mighty fragile. It seems safer to acquire a snapshot for parse analysis of these commands too, which allows making stmt_requires_parse_analysis and analyze_requires_snapshot equivalent. In passing this fixes a second bug, which is that ResetPlanCache would exclude ReturnStmts and CallStmts from revalidation. That's surely *not* safe, since they contain parsable expressions. Per bug #18059 from Pavel Kulakov. Back-patch to all supported branches. Discussion: https://postgr.es/m/18059-79c692f036b25346@postgresql.org --- src/backend/parser/analyze.c | 52 +++++++++++++-- src/backend/utils/cache/plancache.c | 69 ++++++++------------ src/include/parser/analyze.h | 1 + src/pl/plpgsql/src/expected/plpgsql_call.out | 17 +++++ src/pl/plpgsql/src/sql/plpgsql_call.sql | 18 +++++ 5 files changed, 108 insertions(+), 49 deletions(-) diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c index d8091c6c4f4..714588167fd 100644 --- a/src/backend/parser/analyze.c +++ b/src/backend/parser/analyze.c @@ -404,6 +404,11 @@ transformStmt(ParseState *pstate, Node *parseTree) } #endif /* RAW_EXPRESSION_COVERAGE_TEST */ + /* + * Caution: when changing the set of statement types that have non-default + * processing here, see also stmt_requires_parse_analysis() and + * analyze_requires_snapshot(). + */ switch (nodeTag(parseTree)) { /* @@ -489,14 +494,22 @@ transformStmt(ParseState *pstate, Node *parseTree) } /* - * analyze_requires_snapshot - * Returns true if a snapshot must be set before doing parse analysis - * on the given raw parse tree. + * stmt_requires_parse_analysis + * Returns true if parse analysis will do anything non-trivial + * with the given raw parse tree. + * + * Generally, this should return true for any statement type for which + * transformStmt() does more than wrap a CMD_UTILITY Query around it. + * When it returns false, the caller can assume that there is no situation + * in which parse analysis of the raw statement could need to be re-done. * - * Classification here should match transformStmt(). + * Currently, since the rewriter and planner do nothing for CMD_UTILITY + * Queries, a false result means that the entire parse analysis/rewrite/plan + * pipeline will never need to be re-done. If that ever changes, callers + * will likely need adjustment. */ bool -analyze_requires_snapshot(RawStmt *parseTree) +stmt_requires_parse_analysis(RawStmt *parseTree) { bool result; @@ -509,6 +522,7 @@ analyze_requires_snapshot(RawStmt *parseTree) case T_DeleteStmt: case T_UpdateStmt: case T_SelectStmt: + case T_ReturnStmt: case T_PLAssignStmt: result = true; break; @@ -519,12 +533,12 @@ analyze_requires_snapshot(RawStmt *parseTree) case T_DeclareCursorStmt: case T_ExplainStmt: case T_CreateTableAsStmt: - /* yes, because we must analyze the contained statement */ + case T_CallStmt: result = true; break; default: - /* other utility statements don't have any real parse analysis */ + /* all other statements just get wrapped in a CMD_UTILITY Query */ result = false; break; } @@ -532,6 +546,30 @@ analyze_requires_snapshot(RawStmt *parseTree) return result; } +/* + * analyze_requires_snapshot + * Returns true if a snapshot must be set before doing parse analysis + * on the given raw parse tree. + */ +bool +analyze_requires_snapshot(RawStmt *parseTree) +{ + /* + * Currently, this should return true in exactly the same cases that + * stmt_requires_parse_analysis() does, so we just invoke that function + * rather than duplicating it. We keep the two entry points separate for + * clarity of callers, since from the callers' standpoint these are + * different conditions. + * + * While there may someday be a statement type for which transformStmt() + * does something nontrivial and yet no snapshot is needed for that + * processing, it seems likely that making such a choice would be fragile. + * If you want to install an exception, document the reasoning for it in a + * comment. + */ + return stmt_requires_parse_analysis(parseTree); +} + /* * transformDeleteStmt - * transforms a Delete Statement diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index 50089f61905..7a1bef9f210 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -78,13 +78,15 @@ /* * We must skip "overhead" operations that involve database access when the - * cached plan's subject statement is a transaction control command. - * For the convenience of postgres.c, treat empty statements as control - * commands too. + * cached plan's subject statement is a transaction control command or one + * that requires a snapshot not to be set yet (such as SET or LOCK). More + * generally, statements that do not require parse analysis/rewrite/plan + * activity never need to be revalidated, so we can treat them all like that. + * For the convenience of postgres.c, treat empty statements that way too. */ -#define IsTransactionStmtPlan(plansource) \ - ((plansource)->raw_parse_tree == NULL || \ - IsA((plansource)->raw_parse_tree->stmt, TransactionStmt)) +#define StmtPlanRequiresRevalidation(plansource) \ + ((plansource)->raw_parse_tree != NULL && \ + stmt_requires_parse_analysis((plansource)->raw_parse_tree)) /* * This is the head of the backend's list of "saved" CachedPlanSources (i.e., @@ -390,13 +392,13 @@ CompleteCachedPlan(CachedPlanSource *plansource, plansource->query_context = querytree_context; plansource->query_list = querytree_list; - if (!plansource->is_oneshot && !IsTransactionStmtPlan(plansource)) + if (!plansource->is_oneshot && StmtPlanRequiresRevalidation(plansource)) { /* * Use the planner machinery to extract dependencies. Data is saved * in query_context. (We assume that not a lot of extra cruft is * created by this call.) We can skip this for one-shot plans, and - * transaction control commands have no such dependencies anyway. + * plans not needing revalidation have no such dependencies anyway. */ extract_query_dependencies((Node *) querytree_list, &plansource->relationOids, @@ -579,11 +581,11 @@ RevalidateCachedQuery(CachedPlanSource *plansource, /* * For one-shot plans, we do not support revalidation checking; it's * assumed the query is parsed, planned, and executed in one transaction, - * so that no lock re-acquisition is necessary. Also, there is never any - * need to revalidate plans for transaction control commands (and we - * mustn't risk any catalog accesses when handling those). + * so that no lock re-acquisition is necessary. Also, if the statement + * type can't require revalidation, we needn't do anything (and we mustn't + * risk catalog accesses when handling, eg, transaction control commands). */ - if (plansource->is_oneshot || IsTransactionStmtPlan(plansource)) + if (plansource->is_oneshot || !StmtPlanRequiresRevalidation(plansource)) { Assert(plansource->is_valid); return NIL; @@ -1076,8 +1078,8 @@ choose_custom_plan(CachedPlanSource *plansource, ParamListInfo boundParams, Into /* Otherwise, never any point in a custom plan if there's no parameters */ if (boundParams == NULL) return false; - /* ... nor for transaction control statements */ - if (IsTransactionStmtPlan(plansource)) + /* ... nor when planning would be a no-op */ + if (!StmtPlanRequiresRevalidation(plansource)) return false; /* Let settings force the decision */ @@ -2072,8 +2074,8 @@ PlanCacheRelCallback(Datum arg, Oid relid) if (!plansource->is_valid) continue; - /* Never invalidate transaction control commands */ - if (IsTransactionStmtPlan(plansource)) + /* Never invalidate if parse/plan would be a no-op anyway */ + if (!StmtPlanRequiresRevalidation(plansource)) continue; /* @@ -2157,8 +2159,8 @@ PlanCacheObjectCallback(Datum arg, int cacheid, uint32 hashvalue) if (!plansource->is_valid) continue; - /* Never invalidate transaction control commands */ - if (IsTransactionStmtPlan(plansource)) + /* Never invalidate if parse/plan would be a no-op anyway */ + if (!StmtPlanRequiresRevalidation(plansource)) continue; /* @@ -2267,7 +2269,6 @@ ResetPlanCache(void) { CachedPlanSource *plansource = dlist_container(CachedPlanSource, node, iter.cur); - ListCell *lc; Assert(plansource->magic == CACHEDPLANSOURCE_MAGIC); @@ -2279,32 +2280,16 @@ ResetPlanCache(void) * We *must not* mark transaction control statements as invalid, * particularly not ROLLBACK, because they may need to be executed in * aborted transactions when we can't revalidate them (cf bug #5269). + * In general there's no point in invalidating statements for which a + * new parse analysis/rewrite/plan cycle would certainly give the same + * results. */ - if (IsTransactionStmtPlan(plansource)) + if (!StmtPlanRequiresRevalidation(plansource)) continue; - /* - * In general there is no point in invalidating utility statements - * since they have no plans anyway. So invalidate it only if it - * contains at least one non-utility statement, or contains a utility - * statement that contains a pre-analyzed query (which could have - * dependencies.) - */ - foreach(lc, plansource->query_list) - { - Query *query = lfirst_node(Query, lc); - - if (query->commandType != CMD_UTILITY || - UtilityContainsQuery(query->utilityStmt)) - { - /* non-utility statement, so invalidate */ - plansource->is_valid = false; - if (plansource->gplan) - plansource->gplan->is_valid = false; - /* no need to look further */ - break; - } - } + plansource->is_valid = false; + if (plansource->gplan) + plansource->gplan->is_valid = false; } /* Likewise invalidate cached expressions */ diff --git a/src/include/parser/analyze.h b/src/include/parser/analyze.h index 13b7d59bf46..d39a7c20a1a 100644 --- a/src/include/parser/analyze.h +++ b/src/include/parser/analyze.h @@ -40,6 +40,7 @@ extern Query *parse_sub_analyze(Node *parseTree, ParseState *parentParseState, extern Query *transformTopLevelStmt(ParseState *pstate, RawStmt *parseTree); extern Query *transformStmt(ParseState *pstate, Node *parseTree); +extern bool stmt_requires_parse_analysis(RawStmt *parseTree); extern bool analyze_requires_snapshot(RawStmt *parseTree); extern const char *LCS_asString(LockClauseStrength strength); diff --git a/src/pl/plpgsql/src/expected/plpgsql_call.out b/src/pl/plpgsql/src/expected/plpgsql_call.out index db83bc0e7ac..f4481ce66db 100644 --- a/src/pl/plpgsql/src/expected/plpgsql_call.out +++ b/src/pl/plpgsql/src/expected/plpgsql_call.out @@ -35,6 +35,23 @@ SELECT * FROM test1; 55 (1 row) +-- Check that plan revalidation doesn't prevent setting transaction properties +-- (bug #18059). This test must include the first temp-object creation in +-- this script, or it won't exercise the bug scenario. Hence we put it early. +CREATE PROCEDURE test_proc3a() +LANGUAGE plpgsql +AS $$ +BEGIN + COMMIT; + SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; + RAISE NOTICE 'done'; +END; +$$; +CALL test_proc3a(); +NOTICE: done +CREATE TEMP TABLE tt1(f1 int); +CALL test_proc3a(); +NOTICE: done -- nested CALL TRUNCATE TABLE test1; CREATE PROCEDURE test_proc4(y int) diff --git a/src/pl/plpgsql/src/sql/plpgsql_call.sql b/src/pl/plpgsql/src/sql/plpgsql_call.sql index 8108d050609..f93f28435fa 100644 --- a/src/pl/plpgsql/src/sql/plpgsql_call.sql +++ b/src/pl/plpgsql/src/sql/plpgsql_call.sql @@ -38,6 +38,24 @@ CALL test_proc3(55); SELECT * FROM test1; +-- Check that plan revalidation doesn't prevent setting transaction properties +-- (bug #18059). This test must include the first temp-object creation in +-- this script, or it won't exercise the bug scenario. Hence we put it early. +CREATE PROCEDURE test_proc3a() +LANGUAGE plpgsql +AS $$ +BEGIN + COMMIT; + SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; + RAISE NOTICE 'done'; +END; +$$; + +CALL test_proc3a(); +CREATE TEMP TABLE tt1(f1 int); +CALL test_proc3a(); + + -- nested CALL TRUNCATE TABLE test1; From 61bd06b421a7f2c28a7e4279e3ed77af3bbbb7f7 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 29 Aug 2023 09:09:40 +0300 Subject: [PATCH 006/109] Initialize ListenSocket array earlier. After commit b0bea38705, syslogger prints 63 warnings about failing to close a listen socket at postmaster startup. That's because the syslogger process forks before the ListenSockets array is initialized, so ClosePostmasterPorts() calls "close(0)" 64 times. The first call succeeds, because fd 0 is stdin. This has been like this since commit 9a86f03b4e in version 13, which moved the SysLogger_Start() call to before initializing ListenSockets. We just didn't notice until commit b0bea38705 added the LOG message. Reported by Michael Paquier and Jeff Janes. Author: Michael Paquier Discussion: https://www.postgresql.org/message-id/ZOvvuQe0rdj2slA9%40paquier.xyz Discussion: https://www.postgresql.org/message-id/ZO0fgDwVw2SUJiZx@paquier.xyz#482670177eb4eaf4c9f03c1eed963e5f Backpatch-through: 13 --- src/backend/postmaster/postmaster.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 164ed03ad6e..4970153d001 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -1348,6 +1348,17 @@ PostmasterMain(int argc, char *argv[]) errmsg("could not remove file \"%s\": %m", LOG_METAINFO_DATAFILE))); + /* + * Initialize input sockets. + * + * Mark them all closed, and set up an on_proc_exit function that's + * charged with closing the sockets again at postmaster shutdown. + */ + for (i = 0; i < MAXLISTEN; i++) + ListenSocket[i] = PGINVALID_SOCKET; + + on_proc_exit(CloseServerPorts, 0); + /* * If enabled, start up syslogger collection subprocess */ @@ -1382,15 +1393,7 @@ PostmasterMain(int argc, char *argv[]) /* * Establish input sockets. - * - * First, mark them all closed, and set up an on_proc_exit function that's - * charged with closing the sockets again at postmaster shutdown. */ - for (i = 0; i < MAXLISTEN; i++) - ListenSocket[i] = PGINVALID_SOCKET; - - on_proc_exit(CloseServerPorts, 0); - if (ListenAddresses) { char *rawstring; From bc112bc48a6ec70b52d3bbf9602255534b3afb93 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 30 Aug 2023 08:03:52 +0900 Subject: [PATCH 007/109] Avoid possible overflow with ltsGetFreeBlock() in logtape.c nFreeBlocks, defined as a long, stores the number of free blocks in a logical tape. ltsGetFreeBlock() has been using an int to store the value of nFreeBlocks, which could lead to overflows on platforms where long and int are not the same size (in short everything except Windows where long is 4 bytes). The problematic intermediate variable is switched to be a long instead of an int. Issue introduced by c02fdc9223015, so backpatch down to 13. Author: Ranier vilela Reviewed-by: Peter Geoghegan, David Rowley Discussion: https://postgr.es/m/CAEudQApLDWCBR_xmwNjGBrDo+f+S4E87x3s7-+hoaKqYdtC4JQ@mail.gmail.com Backpatch-through: 13 --- src/backend/utils/sort/logtape.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index 815c9371bcf..cd85ef97944 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -396,7 +396,7 @@ ltsGetFreeBlock(LogicalTapeSet *lts) { long *heap = lts->freeBlocks; long blocknum; - int heapsize; + long heapsize; unsigned long pos; /* freelist empty; allocate a new block */ From 1faec64172eb4ca6bcf0896e794949274989e58e Mon Sep 17 00:00:00 2001 From: Etsuro Fujita Date: Wed, 30 Aug 2023 17:15:05 +0900 Subject: [PATCH 008/109] postgres_fdw: Fix test for parameterized foreign scan. Commit e4106b252 should have updated this test, but did not; back-patch to all supported branches. Reviewed by Richard Guo. Discussion: http://postgr.es/m/CAPmGK15nR0NXLSCKQAcqbZbTzrzd5MozowWnTnGfPkayndF43Q%40mail.gmail.com --- contrib/postgres_fdw/expected/postgres_fdw.out | 8 ++++---- contrib/postgres_fdw/sql/postgres_fdw.sql | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 6de00516fed..caa361a8df5 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -836,10 +836,10 @@ EXPLAIN (VERBOSE, COSTS OFF) Optimizer: Postgres query optimizer (18 rows) -SELECT * FROM ft2 a, ft2 b WHERE a.c1 = 47 AND b.c1 = a.c2; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 -----+----+-------+------------------------------+--------------------------+----+------------+-----+----+----+-------+------------------------------+--------------------------+----+------------+----- - 47 | 7 | 00047 | Tue Feb 17 00:00:00 1970 PST | Tue Feb 17 00:00:00 1970 | 7 | 7 | foo | 7 | 7 | 00007 | Thu Jan 08 00:00:00 1970 PST | Thu Jan 08 00:00:00 1970 | 7 | 7 | foo +SELECT * FROM "S 1"."T 1" a, ft2 b WHERE a."C 1" = 47 AND b.c1 = a.c2; + C 1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 +-----+----+-------+------------------------------+--------------------------+----+------------+-----+----+----+-------+------------------------------+--------------------------+----+------------+----- + 47 | 7 | 00047 | Tue Feb 17 00:00:00 1970 PST | Tue Feb 17 00:00:00 1970 | 7 | 7 | foo | 7 | 7 | 00007 | Thu Jan 08 00:00:00 1970 PST | Thu Jan 08 00:00:00 1970 | 7 | 7 | foo (1 row) -- check both safe and unsafe join conditions diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 0b33965a6ac..9a77a47f4e7 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -346,7 +346,7 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM ft1 t1 WHERE c8 = 'foo'; -- can't be -- parameterized remote path for foreign table EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM "S 1"."T 1" a, ft2 b WHERE a."C 1" = 47 AND b.c1 = a.c2; -SELECT * FROM ft2 a, ft2 b WHERE a.c1 = 47 AND b.c1 = a.c2; +SELECT * FROM "S 1"."T 1" a, ft2 b WHERE a."C 1" = 47 AND b.c1 = a.c2; -- check both safe and unsafe join conditions EXPLAIN (VERBOSE, COSTS OFF) From 76e39bc14ef2e6027ab986513aa6461ecb42d0c3 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 4 Sep 2023 14:55:53 +0900 Subject: [PATCH 009/109] Fix out-of-bound read in gtsvector_picksplit() This could lead to an imprecise choice when splitting an index page of a GiST index on a tsvector, deciding which entries should remain on the old page and which entries should move to a new page. This is wrong since tsearch2 has been moved into core with commit 140d4ebcb46e, so backpatch all the way down. This error has been spotted by valgrind. Author: Alexander Lakhin Discussion: https://postgr.es/m/17950-6c80a8d2b94ec695@postgresql.org Backpatch-through: 11 --- src/backend/utils/adt/tsgistidx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/tsgistidx.c b/src/backend/utils/adt/tsgistidx.c index c09eefdda23..157cc4536bd 100644 --- a/src/backend/utils/adt/tsgistidx.c +++ b/src/backend/utils/adt/tsgistidx.c @@ -728,7 +728,7 @@ gtsvector_picksplit(PG_FUNCTION_ARGS) size_alpha = SIGLENBIT(siglen) - sizebitvec((cache[j].allistrue) ? GETSIGN(datum_l) : - GETSIGN(cache[j].sign), + cache[j].sign, siglen); } else @@ -742,7 +742,7 @@ gtsvector_picksplit(PG_FUNCTION_ARGS) size_beta = SIGLENBIT(siglen) - sizebitvec((cache[j].allistrue) ? GETSIGN(datum_r) : - GETSIGN(cache[j].sign), + cache[j].sign, siglen); } else From cf8604a9d725fcafdff004787c17b25a0f3bbd7b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 5 Sep 2023 13:05:27 -0400 Subject: [PATCH 010/109] doc: mention libpq regression tests Reported-by: Ryo Matsumura Discussion: https://postgr.es/m/TYCPR01MB11316B3FB56EE54D70BF0CEF6E8E4A@TYCPR01MB11316.jpnprd01.prod.outlook.com Backpatch-through: 11 --- doc/src/sgml/regress.sgml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index 724ef566e76..98a702ef1af 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -194,8 +194,9 @@ make check-world -j8 >/dev/null - Regression tests for the ECPG interface library, - located in src/interfaces/ecpg/test. + Regression tests for the interface libraries, + located in src/interfaces/libpq/test and + src/interfaces/ecpg/test. From 9b02a25cdd732f5daecfef4a9c92b418bab0d9e8 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 6 Sep 2023 16:52:24 -0400 Subject: [PATCH 011/109] doc: mention that to_char() values are rounded Reported-by: barsikdacat@gmail.com Diagnosed-by: Laurenz Albe Discussion: https://postgr.es/m/168991536429.626.9957835774751337210@wrigleys.postgresql.org Author: Laurenz Albe Backpatch-through: 11 --- doc/src/sgml/func.sgml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index cb4f44e7e7f..8578c4d81aa 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -8034,6 +8034,14 @@ SELECT regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}'); + + + If the format provides fewer fractional digits than the number being + formatted, to_char() will round the number to + the specified number of fractional digits. + + + The pattern characters S, L, D, From 1fa06b090941c71b274386aa83e6f8e4ecf31dc8 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 7 Sep 2023 14:12:31 +0900 Subject: [PATCH 012/109] pg_basebackup: Generate valid temporary slot names under PQbackendPID() pgbouncer can cause PQbackendPID() to return negative values due to it filling be_pid with random bytes (even these days pid_max can only be set up to 2^22 on 64b machines on Linux, for example, so this cannot happen with normal PID numbers). When this happens, pg_basebackup may generate a temporary slot name that may not be accepted by the parser, leading to spurious failures, like: pg_basebackup: error: could not send replication command ERROR: replication slot name "pg_basebackup_-1201966863" contains invalid character This commit fixes that problem by formatting the result from PQbackendPID() as an unsigned integer when creating the temporary replication slot name, so as the invalid character is gone and the command can be parsed. Author: Jelte Fennema Reviewed-by: Daniel Gustafsson, Nishant Sharma Discussion: https://postgr.es/m/CAGECzQQOGvYfp8ziF4fWQ_o8s2K7ppaoWBQnTmdakn3s-4Z=5g@mail.gmail.com Backpatch-through: 11 --- src/bin/pg_basebackup/pg_basebackup.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index c7dbe5da069..f68cfd0b123 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -671,7 +671,8 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) * Create replication slot if requested */ if (temp_replication_slot && !replication_slot) - replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn)); + replication_slot = psprintf("pg_basebackup_%u", + (unsigned int) PQbackendPID(param->bgconn)); if (temp_replication_slot || create_slot) { if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL, From 33524f98b0dfb4e2a7d5698cde6323b74e9cb03a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 8 Sep 2023 17:25:15 -0400 Subject: [PATCH 013/109] doc: remove mention of backslash doubling in strings Reported-by: Laurenz Albe Discussion: https://postgr.es/m/0b03f91a875fb44182f5bed9e1d404ed6d138066.camel@cybertec.at Author: Laurenz Albe Backpatch-through: 11 --- doc/src/sgml/syntax.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml index d66560b587c..bcb67b74592 100644 --- a/doc/src/sgml/syntax.sgml +++ b/doc/src/sgml/syntax.sgml @@ -555,7 +555,7 @@ U&'d!0061t!+000061' UESCAPE '!' While the standard syntax for specifying string constants is usually convenient, it can be difficult to understand when the desired string - contains many single quotes or backslashes, since each of those must + contains many single quotes, since each of those must be doubled. To allow more readable queries in such situations, PostgreSQL provides another way, called dollar quoting, to write string constants. From 43e1733ab79245208dba3aae6e8961a7640c80ea Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 6 Oct 2021 00:16:03 +0900 Subject: [PATCH 014/109] Make recovery report error message when invalid page header is found. Commit 0668719801 changed XLogPageRead() so that it validated the page header, if invalid page header was found reset the error message and retried reading the page, to fix the scenario where streaming standby got stuck at a continuation record. This change hid the error message about invalid page header, which would make it harder for users to investigate what the actual issue was found in WAL. To fix the issue, this commit makes XLogPageRead() report the error message when invalid page header is found. When not in standby mode, an invalid page header should cause recovery to end, not retry reading the page, so XLogPageRead() doesn't need to validate the page header for the retry. Instead, ReadPageInternal() should be responsible for the validation in that case. Therefore this commit changes XLogPageRead() so that if not in standby mode it doesn't validate the page header for the retry. This commit has been originally pushed as of 68601985e699 for 15 and newer versions, but not to the older branches. A recent investigation related to WAL replay failures has showed up that the lack of this patch in 12~14 is an issue, as we want to be able to improve the WAL reader to make a correct distinction between the end-of-wal and OOM cases when validating record headers. REL_11_STABLE is left out as it will be EOL'd soon. Reported-by: Yugo Nagata Author: Yugo Nagata, Kyotaro Horiguchi Reviewed-by: Ranier Vilela, Fujii Masao Discussion: https://postgr.es/m/20210718045505.32f463ed6c227111038d8ae4@sraoss.co.jp Discussion: https://postgr.es/m/17928-aa92416a70ff44a2@postgresql.org Backpatch-through: 12 --- src/backend/access/transam/xlog.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index cc465f296d8..69cbb5ee1c4 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -13287,7 +13287,7 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, /* * Check the page header immediately, so that we can retry immediately if - * it's not valid. This may seem unnecessary, because XLogReadRecord() + * it's not valid. This may seem unnecessary, because ReadPageInternal() * validates the page header anyway, and would propagate the failure up to * ReadRecord(), which would retry. However, there's a corner case with * continuation records, if a record is split across two pages such that @@ -13311,9 +13311,23 @@ XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen, * * Validating the page header is cheap enough that doing it twice * shouldn't be a big deal from a performance point of view. + * + * When not in standby mode, an invalid page header should cause recovery + * to end, not retry reading the page, so we don't need to validate the + * page header here for the retry. Instead, ReadPageInternal() is + * responsible for the validation. */ - if (!XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) + if (StandbyMode && + !XLogReaderValidatePageHeader(xlogreader, targetPagePtr, readBuf)) { + /* + * Emit this error right now then retry this page immediately. Use + * errmsg_internal() because the message was already translated. + */ + if (xlogreader->errormsg_buf[0]) + ereport(emode_for_corrupt_record(emode, EndRecPtr), + (errmsg_internal("%s", xlogreader->errormsg_buf))); + /* reset any error XLogReaderValidatePageHeader() might have set */ xlogreader->errormsg_buf[0] = '\0'; goto next_record_is_invalid; From e795abe47540e8c8e3c5d64f59cf91be621cee3f Mon Sep 17 00:00:00 2001 From: Amit Kapila Date: Tue, 12 Sep 2023 10:12:51 +0530 Subject: [PATCH 015/109] Fix uninitialized access to InitialRunningXacts during decoding after ERROR. The transactions and subtransactions array that was allocated under snapshot builder memory context and recorded during decoding was not cleared in case of errors. This can result in an assertion failure if we attempt to retry logical decoding within the same session. To address this issue, we register a callback function under the snapshot builder memory context to clear the recorded transactions and subtransactions array along with the context. This problem doesn't exist in PG16 and HEAD as instead of using InitialRunningXacts, we added the list of transaction IDs and sub-transaction IDs, that have modified catalogs and are running during snapshot serialization, to the serialized snapshot (see commit 7f13ac8123). Author: Hou Zhijie Reviewed-by: Amit Kapila Backpatch-through: 11 Discussion: http://postgr.es/m/18055-ab3beed9f4b7b7d6@postgresql.org --- src/backend/replication/logical/snapbuild.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index 1df82bc9ddc..a5388563bea 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -300,6 +300,17 @@ static void SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutof static void SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn); static bool SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn); +/* + * Memory context reset callback for clearing the array of running transactions + * and subtransactions. + */ +static void +SnapBuildResetRunningXactsCallback(void *arg) +{ + NInitialRunningXacts = 0; + InitialRunningXacts = NULL; +} + /* * Allocate a new snapshot builder. * @@ -316,6 +327,7 @@ AllocateSnapshotBuilder(ReorderBuffer *reorder, MemoryContext context; MemoryContext oldcontext; SnapBuild *builder; + MemoryContextCallback *mcallback; /* allocate memory in own context, to have better accountability */ context = AllocSetContextCreate(CurrentMemoryContext, @@ -341,6 +353,10 @@ AllocateSnapshotBuilder(ReorderBuffer *reorder, builder->building_full_snapshot = need_full_snapshot; builder->initial_consistent_point = initial_consistent_point; + mcallback = palloc0(sizeof(MemoryContextCallback)); + mcallback->func = SnapBuildResetRunningXactsCallback; + MemoryContextRegisterResetCallback(CurrentMemoryContext, mcallback); + MemoryContextSwitchTo(oldcontext); /* The initial running transactions array must be empty. */ @@ -366,10 +382,6 @@ FreeSnapshotBuilder(SnapBuild *builder) /* other resources are deallocated via memory context reset */ MemoryContextDelete(context); - - /* InitialRunningXacts is freed along with the context */ - NInitialRunningXacts = 0; - InitialRunningXacts = NULL; } /* From c515a0dcf83a238fbfbbc47e230ea74551a3125f Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 14 Sep 2023 11:27:43 +1200 Subject: [PATCH 016/109] Fix incorrect logic in plan dependency recording Both 50e17ad28 and 29f45e299 mistakenly tried to record a plan dependency on a function but mistakenly inverted the OidIsValid test. This meant that we'd record a dependency only when the function's Oid was InvalidOid. Clearly this was meant to *not* record the dependency in that case. 50e17ad28 made this mistake first, then in v15 29f45e299 copied the same mistake. Reported-by: Tom Lane Backpatch-through: 14, where 50e17ad28 first made this mistake Discussion: https://postgr.es/m/2277537.1694301772@sss.pgh.pa.us --- src/backend/optimizer/plan/setrefs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 5ed6f9dacab..776f239ed4f 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -2190,7 +2190,7 @@ fix_expr_common(PlannerInfo *root, Node *node) set_sa_opfuncid(saop); record_plan_function_dependency(root, saop->opfuncid); - if (!OidIsValid(saop->hashfuncid)) + if (OidIsValid(saop->hashfuncid)) record_plan_function_dependency(root, saop->hashfuncid); } else if (IsA(node, Const)) From b9846b06117fe279492d2c65ea6b8482cfebce16 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 14 Sep 2023 10:30:30 +0900 Subject: [PATCH 017/109] Improve error message on snapshot import in snapmgr.c When a snapshot file fails to be read in ImportSnapshot(), it would issue an ERROR as "invalid snapshot identifier" when opening a stream for it in read-only mode. This error message is reworded to be the same as all the other messages used in this case on failure, which is useful when debugging this area. Thinko introduced by bb446b689b66 where snapshot imports have been added. A backpatch down to 11 is done as this can improve any work related to snapshot imports in older branches. Author: Bharath Rupireddy Reviewed-by: Daniel Gustafsson Discussion: https://postgr.es/m/CALj2ACWmr=3KdxDkm8h7Zn1XxBoF6hdzq8WQyMn2y1OL5RYFrg@mail.gmail.com Backpatch-through: 11 --- src/backend/utils/time/snapmgr.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index 34f45d5b59b..de6d6d19597 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -1611,8 +1611,9 @@ ImportSnapshot(const char *idstr) f = AllocateFile(path, PG_BINARY_R); if (!f) ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("invalid snapshot identifier: \"%s\"", idstr))); + (errcode_for_file_access(), + errmsg("could not open file \"%s\" for reading: %m", + path))); /* get the size of the file so that we know how much memory we need */ if (fstat(fileno(f), &stat_buf)) From 44f96ed6217d9bae04f66c750e4c8873bc46079f Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 14 Sep 2023 16:00:41 +0900 Subject: [PATCH 018/109] Revert "Improve error message on snapshot import in snapmgr.c" This reverts commit a0d87bcd9b57, following a remark from Andres Frend that the new error can be triggered with an incorrect SET TRANSACTION SNAPSHOT command without being really helpful for the user as it uses the internal file name. Discussion: https://postgr.es/m/20230914020724.hlks7vunitvtbbz4@awork3.anarazel.de Backpatch-through: 11 --- src/backend/utils/time/snapmgr.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index de6d6d19597..34f45d5b59b 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -1611,9 +1611,8 @@ ImportSnapshot(const char *idstr) f = AllocateFile(path, PG_BINARY_R); if (!f) ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not open file \"%s\" for reading: %m", - path))); + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid snapshot identifier: \"%s\"", idstr))); /* get the size of the file so that we know how much memory we need */ if (fstat(fileno(f), &stat_buf)) From b364840a4efa24e3aac717096c3a1aa1aebd3073 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 15 Sep 2023 17:01:26 -0400 Subject: [PATCH 019/109] Track nesting depth correctly when drilling down into RECORD Vars. expandRecordVariable() failed to adjust the parse nesting structure correctly when recursing to inspect an outer-level Var. This could result in assertion failures or core dumps in corner cases. Likewise, get_name_for_var_field() failed to adjust the deparse namespace stack correctly when recursing to inspect an outer-level Var. In this case the likely result was a "bogus varno" error while deparsing a view. Per bug #18077 from Jingzhou Fu. Back-patch to all supported branches. Richard Guo, with some adjustments by me Discussion: https://postgr.es/m/18077-b9db97c6e0ab45d8@postgresql.org --- src/backend/parser/parse_target.c | 20 ++++++--- src/backend/utils/adt/ruleutils.c | 37 +++++++++------- src/test/regress/expected/rowtypes.out | 60 ++++++++++++++++++++++++++ src/test/regress/sql/rowtypes.sql | 25 +++++++++++ 4 files changed, 120 insertions(+), 22 deletions(-) diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c index 039952aa8ae..274b38d2643 100644 --- a/src/backend/parser/parse_target.c +++ b/src/backend/parser/parse_target.c @@ -1507,7 +1507,8 @@ ExpandRowReference(ParseState *pstate, Node *expr, * drill down to find the ultimate defining expression and attempt to infer * the tupdesc from it. We ereport if we can't determine the tupdesc. * - * levelsup is an extra offset to interpret the Var's varlevelsup correctly. + * levelsup is an extra offset to interpret the Var's varlevelsup correctly + * when recursing. Outside callers should pass zero. */ TupleDesc expandRecordVariable(ParseState *pstate, Var *var, int levelsup) @@ -1595,11 +1596,17 @@ expandRecordVariable(ParseState *pstate, Var *var, int levelsup) /* * Recurse into the sub-select to see what its Var refers * to. We have to build an additional level of ParseState - * to keep in step with varlevelsup in the subselect. + * to keep in step with varlevelsup in the subselect; + * furthermore, the subquery RTE might be from an outer + * query level, in which case the ParseState for the + * subselect must have that outer level as parent. */ - ParseState mypstate; + ParseState mypstate = {0}; + Index levelsup; - MemSet(&mypstate, 0, sizeof(mypstate)); + /* this loop must work, since GetRTEByRangeTablePosn did */ + for (levelsup = 0; levelsup < netlevelsup; levelsup++) + pstate = pstate->parentParseState; mypstate.parentParseState = pstate; mypstate.p_rtable = rte->subquery->rtable; /* don't bother filling the rest of the fake pstate */ @@ -1651,12 +1658,11 @@ expandRecordVariable(ParseState *pstate, Var *var, int levelsup) * Recurse into the CTE to see what its Var refers to. We * have to build an additional level of ParseState to keep * in step with varlevelsup in the CTE; furthermore it - * could be an outer CTE. + * could be an outer CTE (compare SUBQUERY case above). */ - ParseState mypstate; + ParseState mypstate = {0}; Index levelsup; - MemSet(&mypstate, 0, sizeof(mypstate)); /* this loop must work, since GetCTEForRTE did */ for (levelsup = 0; levelsup < rte->ctelevelsup + netlevelsup; diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 5a25c76c775..f34013add7d 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -7727,22 +7727,28 @@ get_name_for_var_field(Var *var, int fieldno, * Recurse into the sub-select to see what its Var * refers to. We have to build an additional level of * namespace to keep in step with varlevelsup in the - * subselect. + * subselect; furthermore, the subquery RTE might be + * from an outer query level, in which case the + * namespace for the subselect must have that outer + * level as parent namespace. */ + List *save_nslist = context->namespaces; + List *parent_namespaces; deparse_namespace mydpns; const char *result; + parent_namespaces = list_copy_tail(context->namespaces, + netlevelsup); + set_deparse_for_query(&mydpns, rte->subquery, - context->namespaces); + parent_namespaces); - context->namespaces = lcons(&mydpns, - context->namespaces); + context->namespaces = lcons(&mydpns, parent_namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); - context->namespaces = - list_delete_first(context->namespaces); + context->namespaces = save_nslist; return result; } @@ -7835,7 +7841,7 @@ get_name_for_var_field(Var *var, int fieldno, attnum); if (ste == NULL || ste->resjunk) - elog(ERROR, "subquery %s does not have attribute %d", + elog(ERROR, "CTE %s does not have attribute %d", rte->eref->aliasname, attnum); expr = (Node *) ste->expr; if (IsA(expr, Var)) @@ -7843,21 +7849,22 @@ get_name_for_var_field(Var *var, int fieldno, /* * Recurse into the CTE to see what its Var refers to. * We have to build an additional level of namespace - * to keep in step with varlevelsup in the CTE. - * Furthermore it could be an outer CTE, so we may - * have to delete some levels of namespace. + * to keep in step with varlevelsup in the CTE; + * furthermore it could be an outer CTE (compare + * SUBQUERY case above). */ List *save_nslist = context->namespaces; - List *new_nslist; + List *parent_namespaces; deparse_namespace mydpns; const char *result; + parent_namespaces = list_copy_tail(context->namespaces, + ctelevelsup); + set_deparse_for_query(&mydpns, ctequery, - context->namespaces); + parent_namespaces); - new_nslist = list_copy_tail(context->namespaces, - ctelevelsup); - context->namespaces = lcons(&mydpns, new_nslist); + context->namespaces = lcons(&mydpns, parent_namespaces); result = get_name_for_var_field((Var *) expr, fieldno, 0, context); diff --git a/src/test/regress/expected/rowtypes.out b/src/test/regress/expected/rowtypes.out index 03dc2ab3a79..2c85583cce0 100644 --- a/src/test/regress/expected/rowtypes.out +++ b/src/test/regress/expected/rowtypes.out @@ -1234,6 +1234,66 @@ select r, r is null as isnull, r is not null as isnotnull from r; (,) | t | f (6 rows) +-- +-- Check parsing of indirect references to composite values (bug #18077) +-- +explain (verbose, costs off) +with cte(c) as materialized (select row(1, 2)), + cte2(c) as (select * from cte) +select * from cte2 as t +where (select * from (select c as c1) s + where (select (c1).f1 > 0)) is not null; + QUERY PLAN +-------------------------------------------- + CTE Scan on cte + Output: cte.c + Filter: ((SubPlan 3) IS NOT NULL) + CTE cte + -> Result + Output: '(1,2)'::record + SubPlan 3 + -> Result + Output: cte.c + One-Time Filter: $2 + InitPlan 2 (returns $2) + -> Result + Output: ((cte.c).f1 > 0) +(13 rows) + +with cte(c) as materialized (select row(1, 2)), + cte2(c) as (select * from cte) +select * from cte2 as t +where (select * from (select c as c1) s + where (select (c1).f1 > 0)) is not null; + c +------- + (1,2) +(1 row) + +-- Also check deparsing of such cases +create view composite_v as +with cte(c) as materialized (select row(1, 2)), + cte2(c) as (select * from cte) +select 1 as one from cte2 as t +where (select * from (select c as c1) s + where (select (c1).f1 > 0)) is not null; +select pg_get_viewdef('composite_v', true); + pg_get_viewdef +-------------------------------------------------------- + WITH cte(c) AS MATERIALIZED ( + + SELECT ROW(1, 2) AS "row" + + ), cte2(c) AS ( + + SELECT cte.c + + FROM cte + + ) + + SELECT 1 AS one + + FROM cte2 t + + WHERE (( SELECT s.c1 + + FROM ( SELECT t.c AS c1) s + + WHERE ( SELECT (s.c1).f1 > 0))) IS NOT NULL; +(1 row) + +drop view composite_v; -- -- Tests for component access / FieldSelect -- diff --git a/src/test/regress/sql/rowtypes.sql b/src/test/regress/sql/rowtypes.sql index 54f338af1f8..50e0ca6cd6b 100644 --- a/src/test/regress/sql/rowtypes.sql +++ b/src/test/regress/sql/rowtypes.sql @@ -501,6 +501,31 @@ with r(a,b) as materialized (null,row(1,2)), (null,row(null,null)), (null,null) ) select r, r is null as isnull, r is not null as isnotnull from r; +-- +-- Check parsing of indirect references to composite values (bug #18077) +-- +explain (verbose, costs off) +with cte(c) as materialized (select row(1, 2)), + cte2(c) as (select * from cte) +select * from cte2 as t +where (select * from (select c as c1) s + where (select (c1).f1 > 0)) is not null; + +with cte(c) as materialized (select row(1, 2)), + cte2(c) as (select * from cte) +select * from cte2 as t +where (select * from (select c as c1) s + where (select (c1).f1 > 0)) is not null; + +-- Also check deparsing of such cases +create view composite_v as +with cte(c) as materialized (select row(1, 2)), + cte2(c) as (select * from cte) +select 1 as one from cte2 as t +where (select * from (select c as c1) s + where (select (c1).f1 > 0)) is not null; +select pg_get_viewdef('composite_v', true); +drop view composite_v; -- -- Tests for component access / FieldSelect From 2e32fa4d5ace388c63341f04e27760c67219ea56 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 18 Sep 2023 14:27:47 -0400 Subject: [PATCH 020/109] Don't crash if cursor_to_xmlschema is used on a non-data-returning Portal. cursor_to_xmlschema() assumed that any Portal must have a tupDesc, which is not so. Add a defensive check. It's plausible that this mistake occurred because of the rather poorly chosen name of the lookup function SPI_cursor_find(), which in such cases is returning something that isn't very much like a cursor. Add some documentation to try to forestall future errors of the same ilk. Report and patch by Boyu Yang (docs changes by me). Back-patch to all supported branches. Discussion: https://postgr.es/m/dd343010-c637-434c-a8cb-418f53bda3b8.yangboyu.yby@alibaba-inc.com --- doc/src/sgml/spi.sgml | 13 +++++++++++++ src/backend/utils/adt/xml.c | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/doc/src/sgml/spi.sgml b/doc/src/sgml/spi.sgml index 6c5ee2123aa..5f466d2f0b0 100644 --- a/doc/src/sgml/spi.sgml +++ b/doc/src/sgml/spi.sgml @@ -2718,6 +2718,19 @@ Portal SPI_cursor_find(const char * name) NULL if none was found + + + Notes + + + Beware that this function can return a Portal object + that does not have cursor-like properties; for example it might not + return tuples. If you simply pass the Portal pointer + to other SPI functions, they can defend themselves against such + cases, but caution is appropriate when directly inspecting + the Portal. + + diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index aafa6203b93..06562ec930c 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -2786,6 +2786,10 @@ cursor_to_xmlschema(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_CURSOR), errmsg("cursor \"%s\" does not exist", name))); + if (portal->tupDesc == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_CURSOR_STATE), + errmsg("portal \"%s\" does not return tuples", name))); xmlschema = _SPI_strdup(map_sql_table_to_xmlschema(portal->tupDesc, InvalidOid, nulls, From b106a091274899dbff02ba52f0b60e58ea6e2c6e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 19 Sep 2023 11:53:51 +0300 Subject: [PATCH 021/109] Fix GiST README's explanation of the NSN cross-check. The text got the condition backwards, it's "NSN > LSN", not "NSN < LSN". While we're at it, expand it a little for clarity. Reviewed-by: Daniel Gustafsson Discussion: https://www.postgresql.org/message-id/4cb46e18-e688-524a-0f73-b1f03ed5d6ee@iki.fi --- src/backend/access/gist/README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README index 4558a5668ba..db92b852d3c 100644 --- a/src/backend/access/gist/README +++ b/src/backend/access/gist/README @@ -270,10 +270,10 @@ should be visited too. When split inserts the downlink to the parent, it clears the F_FOLLOW_RIGHT flag in the child, and sets the NSN field in the child page header to match the LSN of the insertion on the parent. If the F_FOLLOW_RIGHT flag is not set, a scan compares the NSN on the child and the -LSN it saw in the parent. If NSN < LSN, the scan looked at the parent page -before the downlink was inserted, so it should follow the rightlink. Otherwise -the scan saw the downlink in the parent page, and will/did follow that as -usual. +LSN it saw in the parent. If the child's NSN is greater than the LSN seen on +the parent, the scan looked at the parent page before the downlink was +inserted, so it should follow the rightlink. Otherwise the scan saw the +downlink in the parent page, and will/did follow that as usual. A scan can't normally see a page with the F_FOLLOW_RIGHT flag set, because a page split keeps the child pages locked until the downlink has been inserted From f64386e0843b2bc76036a89553a59f0c419a1ee6 Mon Sep 17 00:00:00 2001 From: Etsuro Fujita Date: Thu, 21 Sep 2023 19:45:05 +0900 Subject: [PATCH 022/109] Update comment about set_join_pathlist_hook(). The comment introduced by commit e7cb7ee14 was a bit too terse, which could lead to extensions doing different things within the hook function than we intend to allow. Extend the comment to explain what they can do within the hook function. Back-patch to all supported branches. In passing, I rephrased a nearby comment that I recently added to the back branches. Reviewed by David Rowley and Andrei Lepikhov. Discussion: https://postgr.es/m/CAPmGK15SBPA1nr3Aqsdm%2BYyS-ay0Ayo2BRYQ8_A2To9eLqwopQ%40mail.gmail.com --- src/backend/optimizer/path/joinpath.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index b5e7ef3b60c..5defbdd3613 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -367,6 +367,18 @@ add_paths_to_join_relation(PlannerInfo *root, hash_inner_and_outer(root, joinrel, outerrel, innerrel, jointype, &extra); + /* + * createplan.c does not currently support handling of pseudoconstant + * clauses assigned to joins pushed down by extensions; check if the + * restrictlist has such clauses, and if not, allow them to consider + * pushing down joins. + */ + if ((joinrel->fdwroutine && + joinrel->fdwroutine->GetForeignJoinPaths) || + set_join_pathlist_hook) + consider_join_pushdown = !has_pseudoconstant_clauses(root, + restrictlist); + /* * 5. If inner and outer relations are foreign tables (or joins) belonging * to the same server and assigned to the same user to check access @@ -424,7 +436,10 @@ add_paths_to_join_relation(PlannerInfo *root, } /* - * 6. Finally, give extensions a chance to manipulate the path list. + * 6. Finally, give extensions a chance to manipulate the path list. They + * could add new paths (such as CustomPaths) by calling add_path(), or + * add_partial_path() if parallel aware. They could also delete or modify + * paths added by the core code. */ if (set_join_pathlist_hook) set_join_pathlist_hook(root, joinrel, outerrel, innerrel, From 1c6d1fad2dd1d1f674fe8dba0d4b148d99f69f14 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 21 Sep 2023 23:11:31 -0400 Subject: [PATCH 023/109] Fix COMMIT/ROLLBACK AND CHAIN in the presence of subtransactions. In older branches, COMMIT/ROLLBACK AND CHAIN failed to propagate the current transaction's properties to the new transaction if there was any open subtransaction (unreleased savepoint). Instead, some previous transaction's properties would be restored. This is because the "if (s->chain)" check in CommitTransactionCommand examined the wrong instance of the "chain" flag and falsely concluded that it didn't need to save transaction properties. Our regression tests would have noticed this, except they used identical transaction properties for multiple tests in a row, so that the faulty behavior was not distinguishable from correct behavior. Commit 12d768e70 fixed the problem in v15 and later, but only rather accidentally, because I removed the "if (s->chain)" test to avoid a compiler warning, while not realizing that the warning was flagging a real bug. In v14 and before, remove the if-test and save transaction properties unconditionally; just as in the newer branches, that's not expensive enough to justify thinking harder. Add the comment and extra regression test to v15 and later to forestall any future recurrence, but there's no live bug in those branches. Patch by me, per bug #18118 from Liu Xiang. Back-patch to v12 where the AND CHAIN feature was added. Discussion: https://postgr.es/m/18118-4b72fcbb903aace6@postgresql.org --- src/backend/access/transam/xact.c | 4 +-- src/test/regress/expected/transactions.out | 40 ++++++++++++++++++++++ src/test/regress/sql/transactions.sql | 11 ++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 62ddae7a649..3aaf2ef9cb5 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -3952,8 +3952,8 @@ CommitTransactionCommand(void) elog(DEBUG1,"CommitTransactionCommand: called as segment Reader in state %s", BlockStateAsString(s->blockState)); - if (s->chain) - SaveTransactionCharacteristics(); + /* Must save in case we need to restore below */ + SaveTransactionCharacteristics(); switch (s->blockState) { diff --git a/src/test/regress/expected/transactions.out b/src/test/regress/expected/transactions.out index 5dcd21665d7..8771f85dc01 100644 --- a/src/test/regress/expected/transactions.out +++ b/src/test/regress/expected/transactions.out @@ -834,6 +834,46 @@ SHOW transaction_deferrable; on (1 row) +COMMIT; +START TRANSACTION ISOLATION LEVEL READ COMMITTED, READ WRITE, DEFERRABLE; +SHOW transaction_isolation; + transaction_isolation +----------------------- + read committed +(1 row) + +SHOW transaction_read_only; + transaction_read_only +----------------------- + off +(1 row) + +SHOW transaction_deferrable; + transaction_deferrable +------------------------ + on +(1 row) + +SAVEPOINT x; +COMMIT AND CHAIN; -- TBLOCK_SUBCOMMIT +SHOW transaction_isolation; + transaction_isolation +----------------------- + read committed +(1 row) + +SHOW transaction_read_only; + transaction_read_only +----------------------- + off +(1 row) + +SHOW transaction_deferrable; + transaction_deferrable +------------------------ + on +(1 row) + COMMIT; -- different mix of options just for fun START TRANSACTION ISOLATION LEVEL SERIALIZABLE, READ WRITE, NOT DEFERRABLE; diff --git a/src/test/regress/sql/transactions.sql b/src/test/regress/sql/transactions.sql index 5fd3c85ed95..54e72a67841 100644 --- a/src/test/regress/sql/transactions.sql +++ b/src/test/regress/sql/transactions.sql @@ -497,6 +497,17 @@ SHOW transaction_read_only; SHOW transaction_deferrable; COMMIT; +START TRANSACTION ISOLATION LEVEL READ COMMITTED, READ WRITE, DEFERRABLE; +SHOW transaction_isolation; +SHOW transaction_read_only; +SHOW transaction_deferrable; +SAVEPOINT x; +COMMIT AND CHAIN; -- TBLOCK_SUBCOMMIT +SHOW transaction_isolation; +SHOW transaction_read_only; +SHOW transaction_deferrable; +COMMIT; + -- different mix of options just for fun START TRANSACTION ISOLATION LEVEL SERIALIZABLE, READ WRITE, NOT DEFERRABLE; SHOW transaction_isolation; From 704b692d2fa804a90cb037a76029b5a9311b94db Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 22 Sep 2023 14:52:36 -0400 Subject: [PATCH 024/109] Doc: copy-edit the introductory para for the pg_class catalog. The previous wording had a faint archaic whiff to it, and more importantly used "catalogs" as a verb, which while cutely self-referential seems likely to provoke confusion in this particular context. Also consistently use "kind" not "type" to refer to the different kinds of relations distinguished by relkind. Per gripe from Martin Nash. Back-patch to supported versions. Discussion: https://postgr.es/m/169518739902.3727338.4793815593763320945@wrigleys.postgresql.org --- doc/src/sgml/catalogs.sgml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index b8b82208a40..d160d223ac5 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1852,8 +1852,8 @@ SCRAM-SHA-256$<iteration count>:&l - The catalog pg_class catalogs tables and most - everything else that has columns or is otherwise similar to a + The catalog pg_class describes tables and + other objects that have columns or are otherwise similar to a table. This includes indexes (but see also pg_index), sequences (but see also <iteration count>:&l views, materialized views, composite types, and TOAST tables; see relkind. Below, when we mean all of these kinds of objects we speak of - relations. Not all columns are meaningful for all relation - types. + relations. Not all of pg_class's + columns are meaningful for all relation kinds. From 1a354d620ce81729d3c17001a89d4e2bb8d2edac Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sat, 23 Sep 2023 10:26:24 +1200 Subject: [PATCH 025/109] Don't trust unvalidated xl_tot_len. xl_tot_len comes first in a WAL record. Usually we don't trust it to be the true length until we've validated the record header. If the record header was split across two pages, previously we wouldn't do the validation until after we'd already tried to allocate enough memory to hold the record, which was bad because it might actually be garbage bytes from a recycled WAL file, so we could try to allocate a lot of memory. Release 15 made it worse. Since 70b4f82a4b5, we'd at least generate an end-of-WAL condition if the garbage 4 byte value happened to be > 1GB, but we'd still try to allocate up to 1GB of memory bogusly otherwise. That was an improvement, but unfortunately release 15 tries to allocate another object before that, so you could get a FATAL error and recovery could fail. We can fix both variants of the problem more fundamentally using pre-existing page-level validation, if we just re-order some logic. The new order of operations in the split-header case defers all memory allocation based on xl_tot_len until we've read the following page. At that point we know that its first few bytes are not recycled data, by checking its xlp_pageaddr, and that its xlp_rem_len agrees with xl_tot_len on the preceding page. That is strong evidence that xl_tot_len was truly the start of a record that was logged. This problem was most likely to occur on a standby, because walreceiver.c recycles WAL files without zeroing out trailing regions of each page. We could fix that too, but it wouldn't protect us from rare crash scenarios where the trailing zeroes don't make it to disk. With reliable xl_tot_len validation in place, the ancient policy of considering malloc failure to indicate corruption at end-of-WAL seems quite surprising, but changing that is left for later work. Also included is a new TAP test to exercise various cases of end-of-WAL detection by writing contrived data into the WAL from Perl. Back-patch to 12. We decided not to put this change into the final release of 11. Author: Thomas Munro Author: Michael Paquier Reported-by: Alexander Lakhin Reviewed-by: Noah Misch (the idea, not the code) Reviewed-by: Michael Paquier Reviewed-by: Sergei Kornilov Reviewed-by: Alexander Lakhin Discussion: https://postgr.es/m/17928-aa92416a70ff44a2%40postgresql.org --- src/backend/access/transam/xlogreader.c | 75 ++-- src/test/perl/TestLib.pm | 41 +++ src/test/recovery/t/039_end_of_wal.pl | 466 ++++++++++++++++++++++++ 3 files changed, 544 insertions(+), 38 deletions(-) create mode 100644 src/test/recovery/t/039_end_of_wal.pl diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index ba34d5ca6fa..67e6b9cf717 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -176,6 +176,9 @@ XLogReaderFree(XLogReaderState *state) * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start * with. (That is enough for all "normal" records, but very large commit or * abort records might need more space.) + * + * Note: This routine should *never* be called for xl_tot_len until the header + * of the record has been fully validated. */ static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength) @@ -185,25 +188,6 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); -#ifndef FRONTEND - - /* - * Note that in much unlucky circumstances, the random data read from a - * recycled segment can cause this routine to be called with a size - * causing a hard failure at allocation. For a standby, this would cause - * the instance to stop suddenly with a hard failure, preventing it to - * retry fetching WAL from one of its sources which could allow it to move - * on with replay without a manual restart. If the data comes from a past - * recycled segment and is still valid, then the allocation may succeed - * but record checks are going to fail so this would be short-lived. If - * the allocation fails because of a memory shortage, then this is not a - * hard failure either per the guarantee given by MCXT_ALLOC_NO_OOM. - */ - if (!AllocSizeIsValid(newSize)) - return false; - -#endif - if (state->readRecordBuf) pfree(state->readRecordBuf); state->readRecordBuf = @@ -404,15 +388,7 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) } else { - /* XXX: more validation should be done here */ - if (total_len < SizeOfXLogRecord) - { - report_invalid_record(state, - "invalid record length at %X/%X: wanted %u, got %u", - LSN_FORMAT_ARGS(RecPtr), - (uint32) SizeOfXLogRecord, total_len); - goto err; - } + /* We'll validate the header once we have the next page. */ gotheader = false; } @@ -428,16 +404,11 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) assembled = true; /* - * Enlarge readRecordBuf as needed. + * We always have space for a couple of pages, enough to validate a + * boundary-spanning record header. */ - if (total_len > state->readRecordBufSize && - !allocate_recordbuf(state, total_len)) - { - /* We treat this as a "bogus data" condition */ - report_invalid_record(state, "record length %u at %X/%X too long", - total_len, LSN_FORMAT_ARGS(RecPtr)); - goto err; - } + Assert(state->readRecordBufSize >= XLOG_BLCKSZ * 2); + Assert(state->readRecordBufSize >= len); /* Copy the first fragment of the record from the first page. */ memcpy(state->readRecordBuf, @@ -533,8 +504,36 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) goto err; gotheader = true; } - } while (gotlen < total_len); + /* + * We might need a bigger buffer. We have validated the record + * header, in the case that it split over a page boundary. We've + * also cross-checked total_len against xlp_rem_len on the second + * page, and verified xlp_pageaddr on both. + */ + Assert(gotheader); + if (total_len > state->readRecordBufSize) + { + char save_copy[XLOG_BLCKSZ * 2]; + + /* + * Save and restore the data we already had. It can't be more + * than two pages. + */ + Assert(gotlen <= lengthof(save_copy)); + Assert(gotlen <= state->readRecordBufSize); + memcpy(save_copy, state->readRecordBuf, gotlen); + if (!allocate_recordbuf(state, total_len)) + { + /* We treat this as a "bogus data" condition */ + report_invalid_record(state, "record length %u at %X/%X too long", + total_len, LSN_FORMAT_ARGS(RecPtr)); + goto err; + } + memcpy(state->readRecordBuf, save_copy, gotlen); + buffer = state->readRecordBuf + gotlen; + } + } while (gotlen < total_len); Assert(gotheader); record = (XLogRecord *) state->readRecordBuf; diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index 7860425b543..1aed8d4c213 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -70,6 +70,7 @@ our @EXPORT = qw( chmod_recursive check_pg_config dir_symlink + scan_server_header system_or_bail system_log run_log @@ -648,6 +649,46 @@ sub chmod_recursive =pod +=item scan_server_header(header_path, regexp) + +Returns an array that stores all the matches of the given regular expression +within the PostgreSQL installation's C. This can be used to +retrieve specific value patterns from the installation's header files. + +=cut + +sub scan_server_header +{ + my ($header_path, $regexp) = @_; + + my ($stdout, $stderr); + my $result = IPC::Run::run [ 'pg_config', '--includedir-server' ], '>', + \$stdout, '2>', \$stderr + or die "could not execute pg_config"; + chomp($stdout); + $stdout =~ s/\r$//; + + open my $header_h, '<', "$stdout/$header_path" or die "$!"; + + my @match = undef; + while (<$header_h>) + { + my $line = $_; + + if (@match = $line =~ /^$regexp/) + { + last; + } + } + + close $header_h; + die "could not find match in header $header_path\n" + unless @match; + return @match; +} + +=pod + =item check_pg_config(regexp) Return the number of matches of the given regular expression diff --git a/src/test/recovery/t/039_end_of_wal.pl b/src/test/recovery/t/039_end_of_wal.pl new file mode 100644 index 00000000000..00a669c77d9 --- /dev/null +++ b/src/test/recovery/t/039_end_of_wal.pl @@ -0,0 +1,466 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group +# +# Test detecting end-of-WAL conditions. This test suite generates +# fake defective page and record headers to trigger various failure +# scenarios. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Fcntl qw(SEEK_SET); + +use integer; # causes / operator to use integer math + +# Header size of record header. +my $RECORD_HEADER_SIZE = 24; + +# Fields retrieved from code headers. +my @scan_result = scan_server_header('access/xlog_internal.h', + '#define\s+XLOG_PAGE_MAGIC\s+(\w+)'); +my $XLP_PAGE_MAGIC = hex($scan_result[0]); +@scan_result = scan_server_header('access/xlog_internal.h', + '#define\s+XLP_FIRST_IS_CONTRECORD\s+(\w+)'); +my $XLP_FIRST_IS_CONTRECORD = hex($scan_result[0]); + +# Values queried from the server +my $WAL_SEGMENT_SIZE; +my $WAL_BLOCK_SIZE; +my $TLI; + +# Build path of a WAL segment. +sub wal_segment_path +{ + my $node = shift; + my $tli = shift; + my $segment = shift; + my $wal_path = + sprintf("%s/pg_wal/%08X%08X%08X", $node->data_dir, $tli, 0, $segment); + return $wal_path; +} + +# Calculate from a LSN (in bytes) its segment number and its offset. +sub lsn_to_segment_and_offset +{ + my $lsn = shift; + return ($lsn / $WAL_SEGMENT_SIZE, $lsn % $WAL_SEGMENT_SIZE); +} + +# Write some arbitrary data in WAL for the given segment at LSN. +# This should be called while the cluster is not running. +sub write_wal +{ + my $node = shift; + my $tli = shift; + my $lsn = shift; + my $data = shift; + + my ($segment, $offset) = lsn_to_segment_and_offset($lsn); + my $path = wal_segment_path($node, $tli, $segment); + + open my $fh, "+<:raw", $path or die; + seek($fh, $offset, SEEK_SET) or die; + print $fh $data; + close $fh; +} + +sub format_lsn +{ + my $lsn = shift; + return sprintf("%X/%X", $lsn >> 32, $lsn & 0xffffffff); +} + +# Emit a WAL record of arbitrary size. Returns the end LSN of the +# record inserted, in bytes. +sub emit_message +{ + my $node = shift; + my $size = shift; + return int( + $node->safe_psql( + 'postgres', + "SELECT pg_logical_emit_message(true, '', repeat('a', $size)) - '0/0'" + )); +} + +# Get the current insert LSN of a node, in bytes. +sub get_insert_lsn +{ + my $node = shift; + return int( + $node->safe_psql( + 'postgres', "SELECT pg_current_wal_insert_lsn() - '0/0'")); +} + +# Get GUC value, converted to an int. +sub get_int_setting +{ + my $node = shift; + my $name = shift; + return int( + $node->safe_psql( + 'postgres', + "SELECT setting FROM pg_settings WHERE name = '$name'")); +} + +sub start_of_page +{ + my $lsn = shift; + return $lsn & ~($WAL_BLOCK_SIZE - 1); +} + +sub start_of_next_page +{ + my $lsn = shift; + return start_of_page($lsn) + $WAL_BLOCK_SIZE; +} + +# Build a fake WAL record header based on the data given by the caller. +# This needs to follow the format of the C structure XLogRecord. To +# be inserted with write_wal(). +sub build_record_header +{ + my $xl_tot_len = shift; + my $xl_xid = shift || 0; + my $xl_prev = shift || 0; + my $xl_info = shift || 0; + my $xl_rmid = shift || 0; + my $xl_crc = shift || 0; + + # This needs to follow the structure XLogRecord: + # I for xl_tot_len + # I for xl_xid + # Q for xl_prev + # C for xl_info + # C for xl_rmid + # BB for two bytes of padding + # I for xl_crc + return pack("IIQCCBBI", + $xl_tot_len, $xl_xid, $xl_prev, $xl_info, $xl_rmid, 0, 0, $xl_crc); +} + +# Build a fake WAL page header, based on the data given by the caller +# This needs to follow the format of the C structure XLogPageHeaderData. +# To be inserted with write_wal(). +sub build_page_header +{ + my $xlp_magic = shift; + my $xlp_info = shift || 0; + my $xlp_tli = shift || 0; + my $xlp_pageaddr = shift || 0; + my $xlp_rem_len = shift || 0; + + # This needs to follow the structure XLogPageHeaderData: + # S for xlp_magic + # S for xlp_info + # I for xlp_tli + # Q for xlp_pageaddr + # I for xlp_rem_len + return pack("SSIQI", + $xlp_magic, $xlp_info, $xlp_tli, $xlp_pageaddr, $xlp_rem_len); +} + +# Make sure we are far away enough from the end of a page that we could insert +# a couple of small records. This inserts a few records of a fixed size, until +# the threshold gets close enough to the end of the WAL page inserting records +# to. +sub advance_out_of_record_splitting_zone +{ + my $node = shift; + + my $page_threshold = 2000; + my $end_lsn = get_insert_lsn($node); + my $page_offset = $end_lsn % $WAL_BLOCK_SIZE; + while ($page_offset >= $WAL_BLOCK_SIZE - $page_threshold) + { + emit_message($node, $page_threshold); + $end_lsn = get_insert_lsn($node); + $page_offset = $end_lsn % $WAL_BLOCK_SIZE; + } + return $end_lsn; +} + +# Advance so close to the end of a page that an XLogRecordHeader would not +# fit on it. +sub advance_to_record_splitting_zone +{ + my $node = shift; + + my $end_lsn = get_insert_lsn($node); + my $page_offset = $end_lsn % $WAL_BLOCK_SIZE; + + # Get fairly close to the end of a page in big steps + while ($page_offset <= $WAL_BLOCK_SIZE - 512) + { + emit_message($node, $WAL_BLOCK_SIZE - $page_offset - 256); + $end_lsn = get_insert_lsn($node); + $page_offset = $end_lsn % $WAL_BLOCK_SIZE; + } + + # Calibrate our message size so that we can get closer 8 bytes at + # a time. + my $message_size = $WAL_BLOCK_SIZE - 80; + while ($page_offset <= $WAL_BLOCK_SIZE - $RECORD_HEADER_SIZE) + { + emit_message($node, $message_size); + $end_lsn = get_insert_lsn($node); + + my $old_offset = $page_offset; + $page_offset = $end_lsn % $WAL_BLOCK_SIZE; + + # Adjust the message size until it causes 8 bytes changes in + # offset, enough to be able to split a record header. + my $delta = $page_offset - $old_offset; + if ($delta > 8) + { + $message_size -= 8; + } + elsif ($delta <= 0) + { + $message_size += 8; + } + } + return $end_lsn; +} + +# Setup a new node. The configuration chosen here minimizes the number +# of arbitrary records that could get generated in a cluster. Enlarging +# checkpoint_timeout avoids noise with checkpoint activity. wal_level +# set to "minimal" avoids random standby snapshot records. Autovacuum +# could also trigger randomly, generating random WAL activity of its own. +my $node = PostgreSQL::Test::Cluster->new("node"); +$node->init; +$node->append_conf( + 'postgresql.conf', + q[wal_level = minimal + autovacuum = off + checkpoint_timeout = '30min' +]); +$node->start; +$node->safe_psql('postgres', "CREATE TABLE t AS SELECT 42"); + +$WAL_SEGMENT_SIZE = get_int_setting($node, 'wal_segment_size'); +$WAL_BLOCK_SIZE = get_int_setting($node, 'wal_block_size'); +$TLI = $node->safe_psql('postgres', + "SELECT timeline_id FROM pg_control_checkpoint();"); + +my $end_lsn; +my $prev_lsn; + +########################################################################### +note "Single-page end-of-WAL detection"; +########################################################################### + +# xl_tot_len is 0 (a common case, we hit trailing zeroes). +emit_message($node, 0); +$end_lsn = advance_out_of_record_splitting_zone($node); +$node->stop('immediate'); +my $log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "invalid record length at .*: wanted 24, got 0", $log_size + ), + "xl_tot_len zero"); + +# xl_tot_len is < 24 (presumably recycled garbage). +emit_message($node, 0); +$end_lsn = advance_out_of_record_splitting_zone($node); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, build_record_header(23)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "invalid record length at .*: wanted 24, got 23", + $log_size), + "xl_tot_len short"); + +# Need more pages, but xl_prev check fails first. +emit_message($node, 0); +$end_lsn = advance_out_of_record_splitting_zone($node); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "record with incorrect prev-link 0/DEADBEEF at .*", $log_size), + "xl_prev bad"); + +# xl_crc check fails. +emit_message($node, 0); +advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 10); +$node->stop('immediate'); +# Corrupt a byte in that record, breaking its CRC. +write_wal($node, $TLI, $end_lsn - 8, '!'); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "incorrect resource manager data checksum in record at .*", $log_size + ), + "xl_crc bad"); + + +########################################################################### +note "Multi-page end-of-WAL detection, header is not split"; +########################################################################### + +# This series of tests requires a valid xl_prev set in the record header +# written to WAL. + +# Good xl_prev, we hit zero page next (zero magic). +emit_message($node, 0); +$prev_lsn = advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 0); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn)); +$log_size = -s $node->logfile; +$node->start; +ok($node->log_contains("invalid magic number 0000 ", $log_size), + "xlp_magic zero"); + +# Good xl_prev, we hit garbage page next (bad magic). +emit_message($node, 0); +$prev_lsn = advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 0); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header(0xcafe, 0, 1, 0)); +$log_size = -s $node->logfile; +$node->start; +ok($node->log_contains("invalid magic number CAFE ", $log_size), + "xlp_magic bad"); + +# Good xl_prev, we hit typical recycled page (good xlp_magic, bad +# xlp_pageaddr). +emit_message($node, 0); +$prev_lsn = advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 0); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, $prev_lsn)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header($XLP_PAGE_MAGIC, 0, 1, 0xbaaaaaad)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "unexpected pageaddr 0/BAAAAAAD ", $log_size), + "xlp_pageaddr bad"); + +# Good xl_prev, xlp_magic, xlp_pageaddr, but bogus xlp_info. +emit_message($node, 0); +$prev_lsn = advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 0); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header( + $XLP_PAGE_MAGIC, 0x1234, 1, start_of_next_page($end_lsn))); +$log_size = -s $node->logfile; +$node->start; +ok($node->log_contains("invalid info bits 1234 ", $log_size), + "xlp_info bad"); + +# Good xl_prev, xlp_magic, xlp_pageaddr, but xlp_info doesn't mention +# continuation record. +emit_message($node, 0); +$prev_lsn = advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 0); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header($XLP_PAGE_MAGIC, 0, 1, start_of_next_page($end_lsn))); +$log_size = -s $node->logfile; +$node->start; +ok($node->log_contains("there is no contrecord flag at .*", $log_size), + "xlp_info lacks XLP_FIRST_IS_CONTRECORD"); + +# Good xl_prev, xlp_magic, xlp_pageaddr, xlp_info but xlp_rem_len doesn't add +# up. +emit_message($node, 0); +$prev_lsn = advance_out_of_record_splitting_zone($node); +$end_lsn = emit_message($node, 0); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 42, $prev_lsn)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header( + $XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD, + 1, start_of_next_page($end_lsn), + 123456)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "invalid contrecord length 123456 .* at .*", $log_size), + "xlp_rem_len bad"); + + +########################################################################### +note "Multi-page, but header is split, so page checks are done first"; +########################################################################### + +# xl_prev is bad and xl_tot_len is too big, but we'll check xlp_magic first. +emit_message($node, 0); +$end_lsn = advance_to_record_splitting_zone($node); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef)); +$log_size = -s $node->logfile; +$node->start; +ok($node->log_contains("invalid magic number 0000 ", $log_size), + "xlp_magic zero (split record header)"); + +# And we'll also check xlp_pageaddr before any header checks. +emit_message($node, 0); +$end_lsn = advance_to_record_splitting_zone($node); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header( + $XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD, 1, 0xbaaaaaad)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "unexpected pageaddr 0/BAAAAAAD ", $log_size), + "xlp_pageaddr bad (split record header)"); + +# We'll also discover that xlp_rem_len doesn't add up before any +# header checks, +emit_message($node, 0); +$end_lsn = advance_to_record_splitting_zone($node); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, + build_record_header(2 * 1024 * 1024 * 1024, 0, 0xdeadbeef)); +write_wal( + $node, $TLI, + start_of_next_page($end_lsn), + build_page_header( + $XLP_PAGE_MAGIC, $XLP_FIRST_IS_CONTRECORD, + 1, start_of_next_page($end_lsn), + 123456)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "invalid contrecord length 123456 .* at .*", $log_size), + "xlp_rem_len bad (split record header)"); + +done_testing(); From d138da91b75e19f7c906ba1dc5df16730895d54b Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sat, 23 Sep 2023 14:13:06 +1200 Subject: [PATCH 026/109] Don't use Perl pack('Q') in 039_end_of_wal.pl. 'Q' for 64 bit integers turns out not to work on 32 bit Perl, as revealed by the build farm. Use 'II' instead, and deal with endianness. Back-patch to 12, like bae868ca. Discussion: https://postgr.es/m/ZQ4r1vHcryBsSi_V%40paquier.xyz --- src/test/recovery/t/039_end_of_wal.pl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/test/recovery/t/039_end_of_wal.pl b/src/test/recovery/t/039_end_of_wal.pl index 00a669c77d9..1eb65c07654 100644 --- a/src/test/recovery/t/039_end_of_wal.pl +++ b/src/test/recovery/t/039_end_of_wal.pl @@ -13,6 +13,13 @@ use integer; # causes / operator to use integer math +# Is this a big-endian system ("network" byte order)? We can't use 'Q' in +# pack() calls because it's not available in some perl builds, so we need to +# break 64 bit LSN values into two 'I' values. Fortunately we don't need to +# deal with high values, so we can just write 0 for the high order 32 bits, but +# we need to know the endianness to do that. +my $BIG_ENDIAN = pack("L", 0x12345678) eq pack("N", 0x12345678); + # Header size of record header. my $RECORD_HEADER_SIZE = 24; @@ -131,13 +138,16 @@ sub build_record_header # This needs to follow the structure XLogRecord: # I for xl_tot_len # I for xl_xid - # Q for xl_prev + # II for xl_prev # C for xl_info # C for xl_rmid # BB for two bytes of padding # I for xl_crc - return pack("IIQCCBBI", - $xl_tot_len, $xl_xid, $xl_prev, $xl_info, $xl_rmid, 0, 0, $xl_crc); + return pack("IIIICCBBI", + $xl_tot_len, $xl_xid, + $BIG_ENDIAN ? 0 : $xl_prev, + $BIG_ENDIAN ? $xl_prev : 0, + $xl_info, $xl_rmid, 0, 0, $xl_crc); } # Build a fake WAL page header, based on the data given by the caller @@ -155,10 +165,12 @@ sub build_page_header # S for xlp_magic # S for xlp_info # I for xlp_tli - # Q for xlp_pageaddr + # II for xlp_pageaddr # I for xlp_rem_len - return pack("SSIQI", - $xlp_magic, $xlp_info, $xlp_tli, $xlp_pageaddr, $xlp_rem_len); + return pack("SSIIII", + $xlp_magic, $xlp_info, $xlp_tli, + $BIG_ENDIAN ? 0 : $xlp_pageaddr, + $BIG_ENDIAN ? $xlp_pageaddr : 0, $xlp_rem_len); } # Make sure we are far away enough from the end of a page that we could insert From ba5316e082c6edb10b66b468991506525e0a5fd9 Mon Sep 17 00:00:00 2001 From: Alvaro Herrera Date: Mon, 25 Sep 2023 14:34:06 +0200 Subject: [PATCH 027/109] pg_upgrade: check for types removed in pg12 Commit cda6a8d01d39 removed a few datatypes, but didn't update pg_upgrade --check to throw error if these types are used. So the users find that pg_upgrade --check tells them that everything is fine, only to fail when the real upgrade is attempted. Reviewed-by: Tristan Partin Reviewed-by: Suraj Kharage Discussion: https://postgr.es/m/202309201654.ng4ksea25mti@alvherre.pgsql --- src/bin/pg_upgrade/check.c | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 456d13c28a3..80a8f251126 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -131,6 +131,16 @@ check_and_dump_old_cluster(bool live_check, char **sequence_script_file_name) /* GPDB 7 removed support for SHA-256 hashed passwords */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) old_GPDB6_check_for_unsupported_sha256_password_hashes(); + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) + { + check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); + check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); + check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); + } + /* * PG 14 changed the function signature of encoding conversion functions. * Conversions from older versions cannot be upgraded automatically @@ -1551,6 +1561,40 @@ check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, check_ok(); } +/* + * check_for_removed_data_type_usage + * + * Check for in-core data types that have been removed. Callers know + * the exact list. + */ +static void +check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, + const char *datatype) +{ + char output_path[MAXPGPATH]; + char typename[NAMEDATALEN]; + + prep_status("Checking for removed \"%s\" data type in user tables", + datatype); + + snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", + datatype); + snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); + + if (check_for_data_type_usage(cluster, typename, output_path)) + { + pg_log(PG_REPORT, "fatal"); + pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" + "The \"%s\" type has been removed in PostgreSQL version %s,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n" + " %s\n\n", datatype, datatype, version, output_path); + } + else + check_ok(); +} + /* * check_for_jsonb_9_4_usage() From b8941cb324bc6a3be57f6653da9c1471c4c7d82d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 25 Sep 2023 11:50:28 -0400 Subject: [PATCH 028/109] Limit to_tsvector_byid's initial array allocation to something sane. The initial estimate of the number of distinct ParsedWords is just that: an estimate. Don't let it exceed what palloc is willing to allocate. If in fact we need more entries, we'll eventually fail trying to enlarge the array. But if we don't, this allows success on inputs that currently draw "invalid memory alloc request size". Per bug #18080 from Uwe Binder. Back-patch to all supported branches. Discussion: https://postgr.es/m/18080-d5c5e58fef8c99b7@postgresql.org --- src/backend/tsearch/to_tsany.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/tsearch/to_tsany.c b/src/backend/tsearch/to_tsany.c index f4ddfc01059..15d754564e8 100644 --- a/src/backend/tsearch/to_tsany.c +++ b/src/backend/tsearch/to_tsany.c @@ -252,6 +252,8 @@ to_tsvector_byid(PG_FUNCTION_ARGS) * number */ if (prs.lenwords < 2) prs.lenwords = 2; + else if (prs.lenwords > MaxAllocSize / sizeof(ParsedWord)) + prs.lenwords = MaxAllocSize / sizeof(ParsedWord); prs.curwords = 0; prs.pos = 0; prs.words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs.lenwords); From 41709698b342cb2ec3fc0ea502112105503f7c38 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 25 Sep 2023 14:41:57 -0400 Subject: [PATCH 029/109] Collect dependency information for parsed CallStmts. Parse analysis of a CallStmt will inject mutable information, for instance the OID of the called procedure, so that subsequent DDL may create a need to re-parse the CALL. We failed to detect this for CALLs in plpgsql routines, because no dependency information was collected when putting a CallStmt into the plan cache. That could lead to misbehavior or strange errors such as "cache lookup failed". Before commit ee895a655, the issue would only manifest for CALLs appearing in atomic contexts, because we re-planned non-atomic CALLs every time through anyway. It is now apparent that extract_query_dependencies() probably needs a special case for every utility statement type for which stmt_requires_parse_analysis() returns true. I wanted to add something like Assert(!stmt_requires_parse_analysis(...)) when falling out of extract_query_dependencies_walker without doing anything, but there are API issues as well as a more fundamental point: stmt_requires_parse_analysis is supposed to be applied to raw parser output, so it'd be cheating to assume it will give the correct answer for post-parse-analysis trees. I contented myself with adding a comment. Per bug #18131 from Christian Stork. Back-patch to all supported branches. Discussion: https://postgr.es/m/18131-576854e79c5cd264@postgresql.org --- src/backend/optimizer/plan/setrefs.c | 23 +++++++++- src/pl/plpgsql/src/expected/plpgsql_call.out | 47 ++++++++++++++++++++ src/pl/plpgsql/src/sql/plpgsql_call.sql | 38 ++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 776f239ed4f..fd271d68ffe 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -3847,8 +3847,27 @@ extract_query_dependencies_walker(Node *node, PlannerInfo *context) if (query->commandType == CMD_UTILITY) { /* - * Ignore utility statements, except those (such as EXPLAIN) that - * contain a parsed-but-not-planned query. + * This logic must handle any utility command for which parse + * analysis was nontrivial (cf. stmt_requires_parse_analysis). + * + * Notably, CALL requires its own processing. + */ + if (IsA(query->utilityStmt, CallStmt)) + { + CallStmt *callstmt = (CallStmt *) query->utilityStmt; + + /* We need not examine funccall, just the transformed exprs */ + (void) extract_query_dependencies_walker((Node *) callstmt->funcexpr, + context); + (void) extract_query_dependencies_walker((Node *) callstmt->outargs, + context); + return false; + } + + /* + * Ignore other utility statements, except those (such as EXPLAIN) + * that contain a parsed-but-not-planned query. For those, we + * just need to transfer our attention to the contained query. */ query = UtilityContainsQuery(query->utilityStmt); if (query == NULL) diff --git a/src/pl/plpgsql/src/expected/plpgsql_call.out b/src/pl/plpgsql/src/expected/plpgsql_call.out index f4481ce66db..892e0780b18 100644 --- a/src/pl/plpgsql/src/expected/plpgsql_call.out +++ b/src/pl/plpgsql/src/expected/plpgsql_call.out @@ -472,3 +472,50 @@ BEGIN END; $$; NOTICE: +-- check that we detect change of dependencies in CALL +-- atomic and non-atomic call sites used to do this differently, so check both +CREATE PROCEDURE inner_p (f1 int) +AS $$ +BEGIN + RAISE NOTICE 'inner_p(%)', f1; +END +$$ LANGUAGE plpgsql; +CREATE FUNCTION f(int) RETURNS int AS $$ SELECT $1 + 1 $$ LANGUAGE sql; +CREATE PROCEDURE outer_p (f1 int) +AS $$ +BEGIN + RAISE NOTICE 'outer_p(%)', f1; + CALL inner_p(f(f1)); +END +$$ LANGUAGE plpgsql; +CREATE FUNCTION outer_f (f1 int) RETURNS void +AS $$ +BEGIN + RAISE NOTICE 'outer_f(%)', f1; + CALL inner_p(f(f1)); +END +$$ LANGUAGE plpgsql; +CALL outer_p(42); +NOTICE: outer_p(42) +NOTICE: inner_p(43) +SELECT outer_f(42); +NOTICE: outer_f(42) +NOTICE: inner_p(43) + outer_f +--------- + +(1 row) + +DROP FUNCTION f(int); +CREATE FUNCTION f(int) RETURNS int AS $$ SELECT $1 + 2 $$ LANGUAGE sql; +CALL outer_p(42); +NOTICE: outer_p(42) +NOTICE: inner_p(44) +SELECT outer_f(42); +NOTICE: outer_f(42) +NOTICE: inner_p(44) + outer_f +--------- + +(1 row) + diff --git a/src/pl/plpgsql/src/sql/plpgsql_call.sql b/src/pl/plpgsql/src/sql/plpgsql_call.sql index f93f28435fa..d80a5d2819e 100644 --- a/src/pl/plpgsql/src/sql/plpgsql_call.sql +++ b/src/pl/plpgsql/src/sql/plpgsql_call.sql @@ -442,3 +442,41 @@ BEGIN RAISE NOTICE '%', v_Text; END; $$; + + +-- check that we detect change of dependencies in CALL +-- atomic and non-atomic call sites used to do this differently, so check both + +CREATE PROCEDURE inner_p (f1 int) +AS $$ +BEGIN + RAISE NOTICE 'inner_p(%)', f1; +END +$$ LANGUAGE plpgsql; + +CREATE FUNCTION f(int) RETURNS int AS $$ SELECT $1 + 1 $$ LANGUAGE sql; + +CREATE PROCEDURE outer_p (f1 int) +AS $$ +BEGIN + RAISE NOTICE 'outer_p(%)', f1; + CALL inner_p(f(f1)); +END +$$ LANGUAGE plpgsql; + +CREATE FUNCTION outer_f (f1 int) RETURNS void +AS $$ +BEGIN + RAISE NOTICE 'outer_f(%)', f1; + CALL inner_p(f(f1)); +END +$$ LANGUAGE plpgsql; + +CALL outer_p(42); +SELECT outer_f(42); + +DROP FUNCTION f(int); +CREATE FUNCTION f(int) RETURNS int AS $$ SELECT $1 + 2 $$ LANGUAGE sql; + +CALL outer_p(42); +SELECT outer_f(42); From 972bcc9c0403798cfcc616ed73ed64c3250fed33 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Mon, 25 Sep 2023 11:50:02 -0700 Subject: [PATCH 030/109] pg_dump: tests: Correct test condition for invalid databases For some reason I used not_like = { pg_dumpall_dbprivs => 1, } in the test condition of one of the tests added in in c66a7d75e65. That doesn't make sense for two reasons: 1) not_like isn't a valid test condition 2) the database should not be dumped in any of the tests. Due to 1), the test achieved its goal, but clearly the formulation is confusing. Instead use like => {}, with a comment explaining why. Reported-by: Peter Eisentraut Discussion: https://postgr.es/m/3ddf79f2-8b7b-a093-11d2-5c739bc64f86@eisentraut.org Backpatch: 11-, like c66a7d75e65 --- src/bin/pg_dump/t/002_pg_dump.pl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index c472d5d5cee..ec9bb8b8656 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -1596,6 +1596,16 @@ regexp => qr/\n--[^\n]*\nattack/s, like => {}, }, + 'CREATE DATABASE regression_invalid...' => { + create_order => 1, + create_sql => q( + CREATE DATABASE regression_invalid; + UPDATE pg_database SET datconnlimit = -2 WHERE datname = 'regression_invalid'), + regexp => qr/^CREATE DATABASE regression_invalid/m, + + # invalid databases should never be dumped + like => {}, + }, 'CREATE ACCESS METHOD gist2' => { create_order => 52, From fc83cd7d88920c78e059452422347335ed9c6228 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Tue, 26 Sep 2023 09:07:26 +1300 Subject: [PATCH 031/109] Fix edge-case for xl_tot_len broken by bae868ca. bae868ca removed a check that was still needed. If you had an xl_tot_len at the end of a page that was too small for a record header, but not big enough to span onto the next page, we'd immediately perform the CRC check using a bogus large length. Because of arbitrary coding differences between the CRC implementations on different platforms, nothing very bad happened on common modern systems. On systems using the _sb8.c fallback we could segfault. Restore that check, add a new assertion and supply a test for that case. Back-patch to 12, like bae868ca. Tested-by: Tom Lane Tested-by: Alexander Lakhin Discussion: https://postgr.es/m/CA%2BhUKGLCkTT7zYjzOxuLGahBdQ%3DMcF%3Dz5ZvrjSOnW4EDhVjT-g%40mail.gmail.com --- src/backend/access/transam/xlogreader.c | 11 +++++++++++ src/test/recovery/t/039_end_of_wal.pl | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 67e6b9cf717..af5e8508bf1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -388,6 +388,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) } else { + /* There may be no next page if it's too small. */ + if (total_len < SizeOfXLogRecord) + { + report_invalid_record(state, + "invalid record length at %X/%X: wanted %u, got %u", + LSN_FORMAT_ARGS(RecPtr), + (uint32) SizeOfXLogRecord, total_len); + goto err; + } /* We'll validate the header once we have the next page. */ gotheader = false; } @@ -800,6 +809,8 @@ ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr) { pg_crc32c crc; + Assert(record->xl_tot_len >= SizeOfXLogRecord); + /* Calculate the CRC */ INIT_CRC32C(crc); COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord); diff --git a/src/test/recovery/t/039_end_of_wal.pl b/src/test/recovery/t/039_end_of_wal.pl index 1eb65c07654..1d1d883be6a 100644 --- a/src/test/recovery/t/039_end_of_wal.pl +++ b/src/test/recovery/t/039_end_of_wal.pl @@ -287,6 +287,19 @@ sub advance_to_record_splitting_zone $log_size), "xl_tot_len short"); +# xl_tot_len in final position, not big enough to span into a new page but +# also not eligible for regular record header validation +emit_message($node, 0); +$end_lsn = advance_to_record_splitting_zone($node); +$node->stop('immediate'); +write_wal($node, $TLI, $end_lsn, build_record_header(1)); +$log_size = -s $node->logfile; +$node->start; +ok( $node->log_contains( + "invalid record length at .*: wanted 24, got 1", $log_size + ), + "xl_tot_len short at end-of-page"); + # Need more pages, but xl_prev check fails first. emit_message($node, 0); $end_lsn = advance_out_of_record_splitting_zone($node); From ccf51763fd3cf7313f03fe302777cb58f7c7c0f2 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 26 Sep 2023 08:16:44 +0900 Subject: [PATCH 032/109] doc: Tell about "vcregress taptest" for regression tests on Windows There was no mention of this command in the documentation, and it is useful to run the TAP tests of a target source directory. Author: Yugo Nagata Discussion: https://postgr.es/m/20230925153204.926d685d347ee1c8f527090c@sraoss.co.jp Backpatch-through: 11 --- doc/src/sgml/install-windows.sgml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 176f15dea09..e4e1abdff39 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -462,6 +462,7 @@ $ENV{CONFIG}="Debug"; vcregress isolationcheck vcregress bincheck vcregress recoverycheck +vcregress taptest vcregress upgradecheck @@ -469,6 +470,12 @@ $ENV{CONFIG}="Debug"; command line like: vcregress check serial + + + vcregress taptest can be used to run the TAP tests + of a target directory, like: + +vcregress taptest src\bin\initdb\ For more information about the regression tests, see @@ -476,9 +483,10 @@ $ENV{CONFIG}="Debug"; - Running the regression tests on client programs, with - vcregress bincheck, or on recovery tests, with - vcregress recoverycheck, requires an additional Perl module + Running the regression tests on client programs with + vcregress bincheck, on recovery tests with + vcregress recoverycheck, or TAP tests specified with + vcregress taptest requires an additional Perl module to be installed: From ebe216dd021203451bdb774230d422cdbc8203b1 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 26 Sep 2023 14:14:49 +0300 Subject: [PATCH 033/109] Fix another bug in parent page splitting during GiST index build. Yet another bug in the ilk of commits a7ee7c851 and 741b88435. In 741b88435, we took care to clear the memorized location of the downlink when we split the parent page, because splitting the parent page can move the downlink. But we missed that even *updating* a tuple on the parent can move it, because updating a tuple on a gist page is implemented as a delete+insert, so the updated tuple gets moved to the end of the page. This commit fixes the bug in two different ways (belt and suspenders): 1. Clear the downlink when we update a tuple on the parent page, even if it's not split. This the same approach as in commits a7ee7c851 and 741b88435. I also noticed that gistFindCorrectParent did not clear the 'downlinkoffnum' when it stepped to the right sibling. Fix that too, as it seems like a clear bug even though I haven't been able to find a test case to hit that. 2. Change gistFindCorrectParent so that it treats 'downlinkoffnum' merely as a hint. It now always first checks if the downlink is still at that location, and if not, it scans the page like before. That's more robust if there are still more cases where we fail to clear 'downlinkoffnum' that we haven't yet uncovered. With this, it's no longer necessary to meticulously clear 'downlinkoffnum', so this makes the previous fixes unnecessary, but I didn't revert them because it still seems nice to clear it when we know that the downlink has moved. Also add the test case using the same test data that Alexander posted. I tried to reduce it to a smaller test, and I also tried to reproduce this with different test data, but I was not able to, so let's just include what we have. Backpatch to v12, like the previous fixes. Reported-by: Alexander Lakhin Discussion: https://www.postgresql.org/message-id/18129-caca016eaf0c3702@postgresql.org --- contrib/intarray/expected/_int.out | 91 +++++++++++++++ contrib/intarray/sql/_int.sql | 35 ++++++ src/backend/access/gist/gist.c | 180 ++++++++++++++++------------- 3 files changed, 226 insertions(+), 80 deletions(-) diff --git a/contrib/intarray/expected/_int.out b/contrib/intarray/expected/_int.out index b78175b87a0..755432255f5 100644 --- a/contrib/intarray/expected/_int.out +++ b/contrib/intarray/expected/_int.out @@ -862,4 +862,95 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6343 (1 row) +DROP INDEX text_idx; +-- Repeat the same queries with an extended data set. The data set is the +-- same that we used before, except that each element in the array is +-- repeated three times, offset by 1000 and 2000. For example, {1, 5} +-- becomes {1, 1001, 2001, 5, 1005, 2005}. +-- +-- That has proven to be unreasonably effective at exercising codepaths in +-- core GiST code related to splitting parent pages, which is not covered by +-- other tests. This is a bit out-of-place as the point is to test core GiST +-- code rather than this extension, but there is no suitable GiST opclass in +-- core that would reach the same codepaths. +CREATE TABLE more__int AS SELECT + -- Leave alone NULLs, empty arrays and the one row that we use to test + -- equality + CASE WHEN a IS NULL OR a = '{}' OR a = '{73,23,20}' THEN a ELSE + (select array_agg(u) || array_agg(u + 1000) || array_agg(u + 2000) from (select unnest(a) u) x) + END AS a, a as b + FROM test__int; +CREATE INDEX ON more__int using gist (a gist__int_ops(numranges = 252)); +SELECT count(*) from more__int WHERE a && '{23,50}'; + count +------- + 403 +(1 row) + +SELECT count(*) from more__int WHERE a @@ '23|50'; + count +------- + 403 +(1 row) + +SELECT count(*) from more__int WHERE a @> '{23,50}'; + count +------- + 12 +(1 row) + +SELECT count(*) from more__int WHERE a @@ '23&50'; + count +------- + 12 +(1 row) + +SELECT count(*) from more__int WHERE a @> '{20,23}'; + count +------- + 12 +(1 row) + +SELECT count(*) from more__int WHERE a <@ '{73,23,20}'; + count +------- + 10 +(1 row) + +SELECT count(*) from more__int WHERE a = '{73,23,20}'; + count +------- + 1 +(1 row) + +SELECT count(*) from more__int WHERE a @@ '50&68'; + count +------- + 9 +(1 row) + +SELECT count(*) from more__int WHERE a @> '{20,23}' or a @> '{50,68}'; + count +------- + 21 +(1 row) + +SELECT count(*) from more__int WHERE a @@ '(20&23)|(50&68)'; + count +------- + 21 +(1 row) + +SELECT count(*) from more__int WHERE a @@ '20 | !21'; + count +------- + 6566 +(1 row) + +SELECT count(*) from more__int WHERE a @@ '!20 & !21'; + count +------- + 6343 +(1 row) + RESET enable_seqscan; diff --git a/contrib/intarray/sql/_int.sql b/contrib/intarray/sql/_int.sql index 4994188c2e4..4c4ef29b48f 100644 --- a/contrib/intarray/sql/_int.sql +++ b/contrib/intarray/sql/_int.sql @@ -182,4 +182,39 @@ SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +DROP INDEX text_idx; + +-- Repeat the same queries with an extended data set. The data set is the +-- same that we used before, except that each element in the array is +-- repeated three times, offset by 1000 and 2000. For example, {1, 5} +-- becomes {1, 1001, 2001, 5, 1005, 2005}. +-- +-- That has proven to be unreasonably effective at exercising codepaths in +-- core GiST code related to splitting parent pages, which is not covered by +-- other tests. This is a bit out-of-place as the point is to test core GiST +-- code rather than this extension, but there is no suitable GiST opclass in +-- core that would reach the same codepaths. +CREATE TABLE more__int AS SELECT + -- Leave alone NULLs, empty arrays and the one row that we use to test + -- equality + CASE WHEN a IS NULL OR a = '{}' OR a = '{73,23,20}' THEN a ELSE + (select array_agg(u) || array_agg(u + 1000) || array_agg(u + 2000) from (select unnest(a) u) x) + END AS a, a as b + FROM test__int; +CREATE INDEX ON more__int using gist (a gist__int_ops(numranges = 252)); + +SELECT count(*) from more__int WHERE a && '{23,50}'; +SELECT count(*) from more__int WHERE a @@ '23|50'; +SELECT count(*) from more__int WHERE a @> '{23,50}'; +SELECT count(*) from more__int WHERE a @@ '23&50'; +SELECT count(*) from more__int WHERE a @> '{20,23}'; +SELECT count(*) from more__int WHERE a <@ '{73,23,20}'; +SELECT count(*) from more__int WHERE a = '{73,23,20}'; +SELECT count(*) from more__int WHERE a @@ '50&68'; +SELECT count(*) from more__int WHERE a @> '{20,23}' or a @> '{50,68}'; +SELECT count(*) from more__int WHERE a @@ '(20&23)|(50&68)'; +SELECT count(*) from more__int WHERE a @@ '20 | !21'; +SELECT count(*) from more__int WHERE a @@ '!20 & !21'; + + RESET enable_seqscan; diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index 3e021ac2527..42568c03829 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -1017,87 +1017,106 @@ gistFindPath(Relation r, BlockNumber child, OffsetNumber *downlinkoffnum) * remain so at exit, but it might not be the same page anymore. */ static void -gistFindCorrectParent(Relation r, GISTInsertStack *child) +gistFindCorrectParent(Relation r, GISTInsertStack *child, bool is_build) { GISTInsertStack *parent = child->parent; + ItemId iid; + IndexTuple idxtuple; + OffsetNumber maxoff; + GISTInsertStack *ptr; gistcheckpage(r, parent->buffer); parent->page = (Page) BufferGetPage(parent->buffer); + maxoff = PageGetMaxOffsetNumber(parent->page); - /* here we don't need to distinguish between split and page update */ - if (child->downlinkoffnum == InvalidOffsetNumber || - parent->lsn != PageGetLSN(parent->page)) + /* Check if the downlink is still where it was before */ + if (child->downlinkoffnum != InvalidOffsetNumber && child->downlinkoffnum <= maxoff) { - /* parent is changed, look child in right links until found */ - OffsetNumber i, - maxoff; - ItemId iid; - IndexTuple idxtuple; - GISTInsertStack *ptr; + iid = PageGetItemId(parent->page, child->downlinkoffnum); + idxtuple = (IndexTuple) PageGetItem(parent->page, iid); + if (ItemPointerGetBlockNumber(&(idxtuple->t_tid)) == child->blkno) + return; /* still there */ + } - while (true) - { - maxoff = PageGetMaxOffsetNumber(parent->page); - for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) - { - iid = PageGetItemId(parent->page, i); - idxtuple = (IndexTuple) PageGetItem(parent->page, iid); - if (ItemPointerGetBlockNumber(&(idxtuple->t_tid)) == child->blkno) - { - /* yes!!, found */ - child->downlinkoffnum = i; - return; - } - } + /* + * The page has changed since we looked. During normal operation, every + * update of a page changes its LSN, so the LSN we memorized should have + * changed too. During index build, however, we don't WAL-log the changes + * until we have built the index, so the LSN doesn't change. There is no + * concurrent activity during index build, but we might have changed the + * parent ourselves. + */ + Assert(parent->lsn != PageGetLSN(parent->page) || is_build); + + /* + * Scan the page to re-find the downlink. If the page was split, it might + * have moved to a different page, so follow the right links until we find + * it. + */ + while (true) + { + OffsetNumber i; - parent->blkno = GistPageGetOpaque(parent->page)->rightlink; - UnlockReleaseBuffer(parent->buffer); - if (parent->blkno == InvalidBlockNumber) + maxoff = PageGetMaxOffsetNumber(parent->page); + for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) + { + iid = PageGetItemId(parent->page, i); + idxtuple = (IndexTuple) PageGetItem(parent->page, iid); + if (ItemPointerGetBlockNumber(&(idxtuple->t_tid)) == child->blkno) { - /* - * End of chain and still didn't find parent. It's a very-very - * rare situation when root splitted. - */ - break; + /* yes!!, found */ + child->downlinkoffnum = i; + return; } - parent->buffer = ReadBuffer(r, parent->blkno); - LockBuffer(parent->buffer, GIST_EXCLUSIVE); - gistcheckpage(r, parent->buffer); - parent->page = (Page) BufferGetPage(parent->buffer); } - /* - * awful!!, we need search tree to find parent ... , but before we - * should release all old parent - */ - - ptr = child->parent->parent; /* child->parent already released - * above */ - while (ptr) + parent->blkno = GistPageGetOpaque(parent->page)->rightlink; + parent->downlinkoffnum = InvalidOffsetNumber; + UnlockReleaseBuffer(parent->buffer); + if (parent->blkno == InvalidBlockNumber) { - ReleaseBuffer(ptr->buffer); - ptr = ptr->parent; + /* + * End of chain and still didn't find parent. It's a very-very + * rare situation when root splitted. + */ + break; } + parent->buffer = ReadBuffer(r, parent->blkno); + LockBuffer(parent->buffer, GIST_EXCLUSIVE); + gistcheckpage(r, parent->buffer); + parent->page = (Page) BufferGetPage(parent->buffer); + } - /* ok, find new path */ - ptr = parent = gistFindPath(r, child->blkno, &child->downlinkoffnum); + /* + * awful!!, we need search tree to find parent ... , but before we should + * release all old parent + */ - /* read all buffers as expected by caller */ - /* note we don't lock them or gistcheckpage them here! */ - while (ptr) - { - ptr->buffer = ReadBuffer(r, ptr->blkno); - ptr->page = (Page) BufferGetPage(ptr->buffer); - ptr = ptr->parent; - } + ptr = child->parent->parent; /* child->parent already released above */ + while (ptr) + { + ReleaseBuffer(ptr->buffer); + ptr = ptr->parent; + } - /* install new chain of parents to stack */ - child->parent = parent; + /* ok, find new path */ + ptr = parent = gistFindPath(r, child->blkno, &child->downlinkoffnum); - /* make recursive call to normal processing */ - LockBuffer(child->parent->buffer, GIST_EXCLUSIVE); - gistFindCorrectParent(r, child); + /* read all buffers as expected by caller */ + /* note we don't lock them or gistcheckpage them here! */ + while (ptr) + { + ptr->buffer = ReadBuffer(r, ptr->blkno); + ptr->page = (Page) BufferGetPage(ptr->buffer); + ptr = ptr->parent; } + + /* install new chain of parents to stack */ + child->parent = parent; + + /* make recursive call to normal processing */ + LockBuffer(child->parent->buffer, GIST_EXCLUSIVE); + gistFindCorrectParent(r, child, is_build); } /* @@ -1105,7 +1124,7 @@ gistFindCorrectParent(Relation r, GISTInsertStack *child) */ static IndexTuple gistformdownlink(Relation rel, Buffer buf, GISTSTATE *giststate, - GISTInsertStack *stack) + GISTInsertStack *stack, bool is_build) { Page page = BufferGetPage(buf); OffsetNumber maxoff; @@ -1146,7 +1165,7 @@ gistformdownlink(Relation rel, Buffer buf, GISTSTATE *giststate, ItemId iid; LockBuffer(stack->parent->buffer, GIST_EXCLUSIVE); - gistFindCorrectParent(rel, stack); + gistFindCorrectParent(rel, stack, is_build); iid = PageGetItemId(stack->parent->page, stack->downlinkoffnum); downlink = (IndexTuple) PageGetItem(stack->parent->page, iid); downlink = CopyIndexTuple(downlink); @@ -1192,7 +1211,7 @@ gistfixsplit(GISTInsertState *state, GISTSTATE *giststate) page = BufferGetPage(buf); /* Form the new downlink tuples to insert to parent */ - downlink = gistformdownlink(state->r, buf, giststate, stack); + downlink = gistformdownlink(state->r, buf, giststate, stack, state->is_build); si->buf = buf; si->downlink = downlink; @@ -1346,7 +1365,7 @@ gistfinishsplit(GISTInsertState *state, GISTInsertStack *stack, right = (GISTPageSplitInfo *) list_nth(splitinfo, pos); left = (GISTPageSplitInfo *) list_nth(splitinfo, pos - 1); - gistFindCorrectParent(state->r, stack); + gistFindCorrectParent(state->r, stack, state->is_build); if (gistinserttuples(state, stack->parent, giststate, &right->downlink, 1, InvalidOffsetNumber, @@ -1371,21 +1390,22 @@ gistfinishsplit(GISTInsertState *state, GISTInsertStack *stack, */ tuples[0] = left->downlink; tuples[1] = right->downlink; - gistFindCorrectParent(state->r, stack); - if (gistinserttuples(state, stack->parent, giststate, - tuples, 2, - stack->downlinkoffnum, - left->buf, right->buf, - true, /* Unlock parent */ - unlockbuf /* Unlock stack->buffer if caller wants - * that */ - )) - { - /* - * If the parent page was split, the downlink might have moved. - */ - stack->downlinkoffnum = InvalidOffsetNumber; - } + gistFindCorrectParent(state->r, stack, state->is_build); + (void) gistinserttuples(state, stack->parent, giststate, + tuples, 2, + stack->downlinkoffnum, + left->buf, right->buf, + true, /* Unlock parent */ + unlockbuf /* Unlock stack->buffer if caller + * wants that */ + ); + + /* + * The downlink might have moved when we updated it. Even if the page + * wasn't split, because gistinserttuples() implements updating the old + * tuple by removing and re-inserting it! + */ + stack->downlinkoffnum = InvalidOffsetNumber; Assert(left->buf == stack->buffer); From 9faea9cfe73ce6320a73a1f17f9b6d1d1ae62876 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 26 Sep 2023 17:31:06 -0400 Subject: [PATCH 034/109] doc: mention GROUP BY columns can reference target col numbers Reported-by: hape Discussion: https://postgr.es/m/168871536004.379168.9352636188330923805@wrigleys.postgresql.org Backpatch-through: 11 --- doc/src/sgml/ref/select.sgml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/src/sgml/ref/select.sgml b/doc/src/sgml/ref/select.sgml index 7b8b24b9af8..27e4d12534d 100644 --- a/doc/src/sgml/ref/select.sgml +++ b/doc/src/sgml/ref/select.sgml @@ -131,6 +131,9 @@ TABLE [ ONLY ] table_name [ * ] eliminates groups that do not satisfy the given condition. (See and below.) + Although query output columns are nominally computed in the next + step, they can also be referenced (by name or ordinal number) + in the GROUP BY clause. From 87c7aef21fa324295b7532d0b20d6ec789ec5b49 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 26 Sep 2023 18:54:10 -0400 Subject: [PATCH 035/109] doc: pg_upgrade, clarify standby servers must remain running Also mention that mismatching primary/standby LSNs should never happen. Reported-by: Nikolay Samokhvalov Discussion: https://postgr.es/m/CAM527d8heqkjG5VrvjU3Xjsqxg41ufUyabD9QZccdAxnpbRH-Q@mail.gmail.com Backpatch-through: 11 --- doc/src/sgml/ref/pgupgrade.sgml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml index 81cd4135eba..fc91ec31d9b 100644 --- a/doc/src/sgml/ref/pgupgrade.sgml +++ b/doc/src/sgml/ref/pgupgrade.sgml @@ -373,8 +373,8 @@ NET STOP postgresql-&majorversion; - Streaming replication and log-shipping standby servers can - remain running until a later step. + Streaming replication and log-shipping standby servers must be + running during this shutdown so they receive all changes. @@ -387,8 +387,6 @@ NET STOP postgresql-&majorversion; servers are caught up by running pg_controldata against the old primary and standby clusters. Verify that the Latest checkpoint location values match in all clusters. - (There will be a mismatch if old standby servers were shut down - before the old primary or if the old standby servers are still running.) Also, make sure wal_level is not set to minimal in the postgresql.conf file on the new primary cluster. From ad82c42bc0dca84b7a89cb9b46f61e48048ca930 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 26 Sep 2023 19:02:18 -0400 Subject: [PATCH 036/109] doc: clarify the behavior of unopenable listen_addresses Reported-by: Gurjeet Singh Discussion: https://postgr.es/m/CABwTF4WYPD9ov-kcSq1+J+ZJ5wYDQLXquY6Lu2cvb-Y7pTpSGA@mail.gmail.com Backpatch-through: 11 --- doc/src/sgml/config.sgml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index ec623af937f..80f9c35d771 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -653,10 +653,15 @@ include_dir 'conf.d' :: allows listening for all IPv6 addresses. If the list is empty, the server does not listen on any IP interface at all, in which case only Unix-domain sockets can be used to connect - to it. + to it. If the list is not empty, the server will start if it + can listen on at least one TCP/IP address. A warning will be + emitted for any TCP/IP address which cannot be opened. The default value is localhost, which allows only local TCP/IP loopback connections to be - made. While client authentication ( + + While client authentication () allows fine-grained control over who can access the server, listen_addresses controls which interfaces accept connection attempts, which From bb3c2b89d022dc1b255c22f65c61bf5266d7e3c7 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 26 Sep 2023 19:23:59 -0400 Subject: [PATCH 037/109] doc: clarify handling of time zones with "time with time zone" Reported-by: davecramer@postgres.rocks Discussion: https://postgr.es/m/168451942371.714.9173574930845904336@wrigleys.postgresql.org Backpatch-through: 11 --- doc/src/sgml/datatype.sgml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml index 692b6fe2c43..c17f3976b38 100644 --- a/doc/src/sgml/datatype.sgml +++ b/doc/src/sgml/datatype.sgml @@ -1986,7 +1986,8 @@ MINUTE TO SECOND America/New_York. In this case specifying the date is required in order to determine whether standard or daylight-savings time applies. The appropriate time zone offset is recorded in the - time with time zone value. + time with time zone value and is output as stored; + it is not adjusted to the active time zone.
From 79b3814eb9f9aea78252ac3af343089771418a7c Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 26 Sep 2023 19:44:21 -0400 Subject: [PATCH 038/109] doc: clarify the effect of concurrent work_mem allocations Reported-by: Sami Imseih Discussion: https://postgr.es/m/66590882-F48C-4A25-83E3-73792CF8C51F@amazon.com Backpatch-through: 11 --- doc/src/sgml/config.sgml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 80f9c35d771..edbd5968ec7 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1829,9 +1829,10 @@ include_dir 'conf.d' (such as a sort or hash table) before writing to temporary disk files. If this value is specified without units, it is taken as kilobytes. The default value is four megabytes (4MB). - Note that for a complex query, several sort or hash operations might be - running in parallel; each operation will generally be allowed - to use as much memory as this value specifies before it starts + Note that a complex query might perform several sort and hash + operations at the same time, with each operation generally being + allowed to use as much memory as this value specifies before + it starts to write data into temporary files. Also, several running sessions could be doing such operations concurrently. Therefore, the total memory used could be many times the value @@ -1845,7 +1846,7 @@ include_dir 'conf.d' Hash-based operations are generally more sensitive to memory availability than equivalent sort-based operations. The - memory available for hash tables is computed by multiplying + memory limit for a hash table is computed by multiplying work_mem by hash_mem_multiplier. This makes it possible for hash-based operations to use an amount of memory From c99c0f6c87350bbc27d06c712173276613374bf8 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 26 Sep 2023 21:06:21 -0400 Subject: [PATCH 039/109] Stop using "-multiply_defined suppress" on macOS. We started to use this linker switch in commit 9df308697 of 2004-07-13, which was in the OS X 10.3 era. Apparently it's been a no-op since around OS X 10.9. Apple's most recent toolchain version actively complains about it, so it's time to get rid of it. Discussion: https://postgr.es/m/467042.1695766998@sss.pgh.pa.us --- src/Makefile.shlib | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile.shlib b/src/Makefile.shlib index 50214226de6..4eb25cf6550 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -122,13 +122,13 @@ ifeq ($(PORTNAME), darwin) ifneq ($(SO_MAJOR_VERSION), 0) version_link = -compatibility_version $(SO_MAJOR_VERSION) -current_version $(SO_MAJOR_VERSION).$(SO_MINOR_VERSION) endif - LINK.shared = $(COMPILER) -dynamiclib -install_name '$(libdir)/lib$(NAME).$(SO_MAJOR_VERSION)$(DLSUFFIX)' $(version_link) $(exported_symbols_list) -multiply_defined suppress + LINK.shared = $(COMPILER) -dynamiclib -install_name '$(libdir)/lib$(NAME).$(SO_MAJOR_VERSION)$(DLSUFFIX)' $(version_link) $(exported_symbols_list) shlib = lib$(NAME).$(SO_MAJOR_VERSION).$(SO_MINOR_VERSION)$(DLSUFFIX) shlib_major = lib$(NAME).$(SO_MAJOR_VERSION)$(DLSUFFIX) else # loadable module DLSUFFIX = .so - LINK.shared = $(COMPILER) -bundle -multiply_defined suppress + LINK.shared = $(COMPILER) -bundle endif BUILD.exports = $(AWK) '/^[^\043]/ {printf "_%s\n",$$1}' $< >$@ exports_file = $(SHLIB_EXPORTS:%.txt=%.list) From d0a6c30a61f24fc0d2af4048eb03ee6a6461b780 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 27 Sep 2023 14:41:23 +0900 Subject: [PATCH 040/109] unaccent: Tweak value of PYTHON when building without Python support As coded, the module's Makefile would fail to set a value for PYTHON as it checked if the variable is defined. When compiling without --with-python, PYTHON is defined and set to an empty value, so the existing check is not able to do its work. This commit switches the rule to check if the value is empty rather than defined, allowing the generation of unaccent.rules even if --with-python is not used as long as "python" exists. BISON and FLEX do the same in pgxs.mk, for instance. Thinko in f85a485f89e2. Author: Japin Li Discussion: https://postgr.es/m/MEYP282MB1669F86C0DC7B4DC48489CB0B6C3A@MEYP282MB1669.AUSP282.PROD.OUTLOOK.COM Backpatch-through: 13 --- contrib/unaccent/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/unaccent/Makefile b/contrib/unaccent/Makefile index b8307d1601e..9e42f3e638a 100644 --- a/contrib/unaccent/Makefile +++ b/contrib/unaccent/Makefile @@ -30,7 +30,9 @@ endif update-unicode: unaccent.rules # Allow running this even without --with-python -PYTHON ?= python +ifeq ($(PYTHON),) +PYTHON = python +endif unaccent.rules: generate_unaccent_rules.py ../../src/common/unicode/UnicodeData.txt Latin-ASCII.xml $(PYTHON) $< --unicode-data-file $(word 2,$^) --latin-ascii-file $(word 3,$^) >$@ From 5699e5b875df5f774633fb5d06b1c2d8fcdbaeaf Mon Sep 17 00:00:00 2001 From: Etsuro Fujita Date: Thu, 28 Sep 2023 19:45:05 +0900 Subject: [PATCH 041/109] Fix typo in src/backend/access/transam/README. --- src/backend/access/transam/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README index efac0cb505e..fdfa7155ffd 100644 --- a/src/backend/access/transam/README +++ b/src/backend/access/transam/README @@ -324,7 +324,7 @@ changes much more often than its xid, having GetSnapshotData look at xmins can lead to a lot of unnecessary cacheline ping-pong. Instead GetSnapshotData updates approximate thresholds (one that guarantees that all deleted rows older than it can be removed, another determining that deleted -rows newer than it can not be removed). GlobalVisTest* uses those threshold +rows newer than it can not be removed). GlobalVisTest* uses those thresholds to make invisibility decision, falling back to ComputeXidHorizons if necessary. From 8850050bf071f28ca838a22dd94e5a02914457ca Mon Sep 17 00:00:00 2001 From: David Rowley Date: Fri, 29 Sep 2023 00:04:03 +1300 Subject: [PATCH 042/109] Add missing TidRangePath handling in print_path() Tid Range scans were added back in bb437f995. That commit forgot to add handling for TidRangePaths in print_path(). Only people building with OPTIMIZER_DEBUG might have noticed this, which likely is the reason it's taken 4 years for anyone to notice. Author: Andrey Lepikhov Reported-by: Andrey Lepikhov Discussion: https://postgr.es/m/379082d6-1b6a-4cd6-9ecf-7157d8c08635@postgrespro.ru Backpatch-through: 14, where bb437f995 was introduced --- src/backend/optimizer/path/allpaths.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 10e41de8cd9..daa9b9e998b 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -5022,6 +5022,9 @@ print_path(PlannerInfo *root, Path *path, int indent) case T_TidPath: ptype = "TidScan"; break; + case T_TidRangePath: + ptype = "TidRangePath"; + break; case T_SubqueryScanPath: ptype = "SubqueryScan"; break; From b0bdfa75e7a635376d3a0cb2d33ac00af5753cf2 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 28 Sep 2023 14:05:25 -0400 Subject: [PATCH 043/109] Fix checking of index expressions in CompareIndexInfo(). This code was sloppy about comparison of index columns that are expressions. It didn't reliably reject cases where one index has an expression where the other has a plain column, and it could index off the start of the attmap array, leading to a Valgrind complaint (though an actual crash seems unlikely). I'm not sure that the expression-vs-column sloppiness leads to any visible problem in practice, because the subsequent comparison of the two expression lists would reject cases where the indexes have different numbers of expressions overall. Maybe we could falsely match indexes having the same expressions in different column positions, but it'd require unlucky contents of the word before the attmap array. It's not too surprising that no problem has been reported from the field. Nonetheless, this code is clearly wrong. Per bug #18135 from Alexander Lakhin. Back-patch to all supported branches. Discussion: https://postgr.es/m/18135-532f4a755e71e4d2@postgresql.org --- src/backend/catalog/index.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 2a1a93f5aa7..7035c4a74b2 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -2709,7 +2709,7 @@ CompareIndexInfo(IndexInfo *info1, IndexInfo *info2, /* * and columns match through the attribute map (actual attribute numbers - * might differ!) Note that this implies that index columns that are + * might differ!) Note that this checks that index columns that are * expressions appear in the same positions. We will next compare the * expressions themselves. */ @@ -2718,13 +2718,22 @@ CompareIndexInfo(IndexInfo *info1, IndexInfo *info2, if (attmap->maplen < info2->ii_IndexAttrNumbers[i]) elog(ERROR, "incorrect attribute map"); - /* ignore expressions at this stage */ - if ((info1->ii_IndexAttrNumbers[i] != InvalidAttrNumber) && - (attmap->attnums[info2->ii_IndexAttrNumbers[i] - 1] != - info1->ii_IndexAttrNumbers[i])) - return false; + /* ignore expressions for now (but check their collation/opfamily) */ + if (!(info1->ii_IndexAttrNumbers[i] == InvalidAttrNumber && + info2->ii_IndexAttrNumbers[i] == InvalidAttrNumber)) + { + /* fail if just one index has an expression in this column */ + if (info1->ii_IndexAttrNumbers[i] == InvalidAttrNumber || + info2->ii_IndexAttrNumbers[i] == InvalidAttrNumber) + return false; + + /* both are columns, so check for match after mapping */ + if (attmap->attnums[info2->ii_IndexAttrNumbers[i] - 1] != + info1->ii_IndexAttrNumbers[i]) + return false; + } - /* collation and opfamily is not valid for including columns */ + /* collation and opfamily are not valid for included columns */ if (i >= info1->ii_NumIndexKeyAttrs) continue; From cb38938c71bd2c9ee9a486a2010abc94bd02843c Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Thu, 28 Sep 2023 16:29:29 -0700 Subject: [PATCH 044/109] Fix btmarkpos/btrestrpos array key wraparound bug. nbtree's mark/restore processing failed to correctly handle an edge case involving array key advancement and related search-type scan key state. Scans with ScalarArrayScalarArrayOpExpr quals requiring mark/restore processing (for a merge join) could incorrectly conclude that an affected array/scan key must not have advanced during the time between marking and restoring the scan's position. As a result of all this, array key handling within btrestrpos could skip a required call to _bt_preprocess_keys(). This confusion allowed later primitive index scans to overlook tuples matching the true current array keys. The scan's search-type scan keys would still have spurious values corresponding to the final array element(s) -- not values matching the first/now-current array element(s). To fix, remember that "array key wraparound" has taken place during the ongoing btrescan in a flag variable stored in the scan's state, and use that information at the point where btrestrpos decides if another call to _bt_preprocess_keys is required. Oversight in commit 70bc5833, which taught nbtree to handle array keys during mark/restore processing, but missed this subtlety. That commit was itself a bug fix for an issue in commit 9e8da0f7, which taught nbtree to handle ScalarArrayOpExpr quals natively. Author: Peter Geoghegan Discussion: https://postgr.es/m/CAH2-WzkgP3DDRJxw6DgjCxo-cu-DKrvjEv_ArkP2ctBJatDCYg@mail.gmail.com Backpatch: 11- (all supported branches). --- src/backend/access/nbtree/nbtree.c | 1 + src/backend/access/nbtree/nbtutils.c | 17 ++++++++++++++++- src/include/access/nbtree.h | 4 +++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index 8edf4694401..8d4a587899c 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -463,6 +463,7 @@ btbeginscan(Relation rel, int nkeys, int norderbys) so->keyData = NULL; so->arrayKeyData = NULL; /* assume no array keys for now */ + so->arraysStarted = false; so->numArrayKeys = 0; so->arrayKeys = NULL; so->arrayContext = NULL; diff --git a/src/backend/access/nbtree/nbtutils.c b/src/backend/access/nbtree/nbtutils.c index 733a97a76df..cee04bf0d1b 100644 --- a/src/backend/access/nbtree/nbtutils.c +++ b/src/backend/access/nbtree/nbtutils.c @@ -532,6 +532,8 @@ _bt_start_array_keys(IndexScanDesc scan, ScanDirection dir) curArrayKey->cur_elem = 0; skey->sk_argument = curArrayKey->elem_values[curArrayKey->cur_elem]; } + + so->arraysStarted = true; } /* @@ -591,6 +593,14 @@ _bt_advance_array_keys(IndexScanDesc scan, ScanDirection dir) if (scan->parallel_scan != NULL) _bt_parallel_advance_array_keys(scan); + /* + * When no new array keys were found, the scan is "past the end" of the + * array keys. _bt_start_array_keys can still "restart" the array keys if + * a rescan is required. + */ + if (!found) + so->arraysStarted = false; + return found; } @@ -644,8 +654,13 @@ _bt_restore_array_keys(IndexScanDesc scan) * If we changed any keys, we must redo _bt_preprocess_keys. That might * sound like overkill, but in cases with multiple keys per index column * it seems necessary to do the full set of pushups. + * + * Also do this whenever the scan's set of array keys "wrapped around" at + * the end of the last primitive index scan. There won't have been a call + * to _bt_preprocess_keys from some other place following wrap around, so + * we do it for ourselves. */ - if (changed) + if (changed || !so->arraysStarted) { _bt_preprocess_keys(scan); /* The mark should have been set on a consistent set of keys... */ diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index 816a0fe761d..8d7d972f715 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -1029,8 +1029,10 @@ typedef struct BTArrayKeyInfo typedef struct BTScanOpaqueData { - /* these fields are set by _bt_preprocess_keys(): */ + /* all fields (except arraysStarted) are set by _bt_preprocess_keys(): */ bool qual_ok; /* false if qual can never be satisfied */ + bool arraysStarted; /* Started array keys, but have yet to "reach + * past the end" of all arrays? */ int numberOfKeys; /* number of preprocessed scan keys */ ScanKey keyData; /* array of preprocessed scan keys */ From 699954b36d50c173435ceace29e8f9a6d3543963 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Fri, 29 Sep 2023 15:55:37 +0200 Subject: [PATCH 045/109] doc: Change statistics function xref to the right target Commit 7d3b7011b added a link to the statistics functions, which at the time were anchored under the section for statistics views. aebe989477a added a separate section for statistics functions, but the link was not updated to point to the new anchor. Fix by changing the xref. Backpatch to all supported branches. Author: Peter Smith Discussion: https://postgr.es/m/CAHut+Ptr0jKzNNtWnssLq+3jNhbyaBseqf6NPrWHk08mQFRoTg@mail.gmail.com Backpatch-through: 11 --- doc/src/sgml/func.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 8578c4d81aa..a30c79abbbd 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -21767,7 +21767,7 @@ SELECT * FROM pg_ls_dir('.') WITH ORDINALITY AS t(ls,n); In addition to the functions listed in this section, there are a number of functions related to the statistics system that also provide system - information. See for more + information. See for more information. From 1f86340f0ef9607ce0ae1c29e0ea47ba59cd4f92 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 29 Sep 2023 13:13:54 -0400 Subject: [PATCH 046/109] Doc: improve description of dump/restore's --clean and --if-exists. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try to make these option descriptions a little clearer for novices. Per gripe from Attila Gulyás. Discussion: https://postgr.es/m/169590536647.3727336.11070254203649648453@wrigleys.postgresql.org --- doc/src/sgml/ref/pg_dump.sgml | 17 ++++++++++------- doc/src/sgml/ref/pg_dumpall.sgml | 17 +++++++++++------ doc/src/sgml/ref/pg_restore.sgml | 18 +++++++++++------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 6a2ad26a17d..b317138cbef 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -180,11 +180,12 @@ PostgreSQL documentation - Output commands to clean (drop) + Output commands to DROP all the dumped database objects prior to outputting the commands for creating them. - (Unless is also specified, - restore might generate some harmless error messages, if any objects - were not present in the destination database.) + This option is useful when the restore is to overwrite an existing + database. If any of the objects do not exist in the destination + database, ignorable error messages will be reported during + restore, unless is also specified. @@ -830,9 +831,11 @@ PostgreSQL documentation - Use conditional commands (i.e., add an IF EXISTS - clause) when cleaning database objects. This option is not valid - unless is also specified. + Use DROP ... IF EXISTS commands to drop objects + in mode. This suppresses does not + exist errors that might otherwise be reported. This + option is not valid unless is also + specified. diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index c025c4cf27f..f407aac497f 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -100,9 +100,12 @@ PostgreSQL documentation - Include SQL commands to clean (drop) databases before - recreating them. DROP commands for roles and - tablespaces are added as well. + Emit SQL commands to DROP all the dumped + databases, roles, and tablespaces before recreating them. + This option is useful when the restore is to overwrite an existing + cluster. If any of the objects do not exist in the destination + cluster, ignorable error messages will be reported during + restore, unless is also specified. @@ -368,9 +371,11 @@ PostgreSQL documentation - Use conditional commands (i.e., add an IF EXISTS - clause) to drop databases and other objects. This option is not valid - unless is also specified. + Use DROP ... IF EXISTS commands to drop objects + in mode. This suppresses does not + exist errors that might otherwise be reported. This + option is not valid unless is also + specified. diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index 40f136670cb..71ea95a9c7b 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -123,10 +123,12 @@ PostgreSQL documentation - Clean (drop) database objects before recreating them. - (Unless is used, - this might generate some harmless error messages, if any objects - were not present in the destination database.) + Before restoring database objects, issue commands + to DROP all the objects that will be restored. + This option is useful for overwriting an existing database. + If any of the objects do not exist in the destination database, + ignorable error messages will be reported, + unless is also specified. @@ -592,9 +594,11 @@ PostgreSQL documentation - Use conditional commands (i.e., add an IF EXISTS - clause) to drop database objects. This option is not valid - unless is also specified. + Use DROP ... IF EXISTS commands to drop objects + in mode. This suppresses does not + exist errors that might otherwise be reported. This + option is not valid unless is also + specified. From 640e1ba63826e6ba0604453aa21a3bfc7f2593dc Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 29 Sep 2023 14:07:30 -0400 Subject: [PATCH 047/109] Suppress macOS warnings about duplicate libraries in link commands. As of Xcode 15 (macOS Sonoma), the linker complains about duplicate references to the same library. We see warnings about libpgport and libpgcommon being duplicated in many client executables. This is a consequence of the hack introduced in commit 6b7ef076b to list libpgport before libpq while not removing it from $(LIBS). (Commit 8396447cd later applied the same rule to libpgcommon.) The concern in 6b7ef076b was to ensure that the client executable wouldn't unintentionally depend on pgport functions from libpq. That concern is obsolete on any platform for which we can do symbol export control, because if we can then the pgport functions in libpq won't be exposed anyway. Hence, we can fix this problem by just removing libpgport and libpgcommon from $(libpq_pgport), and letting clients depend on the occurrences in $(LIBS). In the back branches, do that only on macOS (which we know has symbol export control). In HEAD, let's be more aggressive and remove the extra libraries everywhere. The only still-supported platforms that lack export control are MinGW/Cygwin, and it doesn't seem worth sweating over ABI stability details for those (or if somebody does care, it'd probably be possible to perform symbol export control for those too). As well as being simpler, this might give some microscopic improvement in build time. The meson build system is not changed here, as it doesn't have this particular disease, though it does have some related issues that we'll fix separately. Discussion: https://postgr.es/m/467042.1695766998@sss.pgh.pa.us --- src/Makefile.global.in | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 457a4a0944e..5b58f241964 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -663,19 +663,32 @@ endif libpq = -L$(libpq_builddir) -lpq # libpq_pgport is for use by client executables (not libraries) that use libpq. -# We force clients to pull symbols from the non-shared libraries libpgport +# We want clients to pull symbols from the non-shared libraries libpgport # and libpgcommon rather than pulling some libpgport symbols from libpq just # because libpq uses those functions too. This makes applications less -# dependent on changes in libpq's usage of pgport (on platforms where we -# don't have symbol export control for libpq). To do this we link to +# dependent on changes in libpq's usage of pgport. To do this we link to # pgport before libpq. This does cause duplicate -lpgport's to appear -# on client link lines, since that also appears in $(LIBS). +# on client link lines, since that also appears in $(LIBS). On platforms +# where we have symbol export control for libpq, the whole exercise is +# unnecessary because libpq won't expose any of these symbols. Currently, +# only macOS warns about duplicate library references, so we only suppress +# the duplicates on macOS. +ifeq ($(PORTNAME),darwin) +libpq_pgport = $(libpq) +else ifdef PGXS +libpq_pgport = -L$(libdir) -lpgcommon -lpgport $(libpq) +else +libpq_pgport = -L$(top_builddir)/src/common -lpgcommon -L$(top_builddir)/src/port -lpgport $(libpq) +endif + # libpq_pgport_shlib is the same idea, but for use in client shared libraries. +# We need those clients to use the shlib variants. (Ideally, users of this +# macro would strip libpgport and libpgcommon from $(LIBS), but no harm is +# done if they don't, since they will have satisfied all their references +# from these libraries.) ifdef PGXS -libpq_pgport = -L$(libdir) -lpgcommon -lpgport $(libpq) libpq_pgport_shlib = -L$(libdir) -lpgcommon_shlib -lpgport_shlib $(libpq) else -libpq_pgport = -L$(top_builddir)/src/common -lpgcommon -L$(top_builddir)/src/port -lpgport $(libpq) libpq_pgport_shlib = -L$(top_builddir)/src/common -lpgcommon_shlib -L$(top_builddir)/src/port -lpgport_shlib $(libpq) endif From 3e4117299fbd16689623d2466ddcebfc683edb5b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 29 Sep 2023 20:20:57 -0400 Subject: [PATCH 048/109] Remove environment sensitivity in pl/tcl regression test. Add "-gmt 1" to our test invocations of the Tcl "clock" command, so that they do not consult the timezone environment. While it doesn't really matter which timezone is used here, it does matter that the command not fall over entirely. We've now discovered that at least on FreeBSD, "clock scan" will fail if /etc/localtime is missing. It seems worth making the test insensitive to that. Per Tomas Vondras' buildfarm animal dikkop. Thanks to Thomas Munro for the diagnosis. Discussion: https://postgr.es/m/316d304a-1dcd-cea1-3d6c-27f794727a06@enterprisedb.com --- src/pl/tcl/expected/pltcl_setup.out | 2 +- src/pl/tcl/sql/pltcl_setup.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pl/tcl/expected/pltcl_setup.out b/src/pl/tcl/expected/pltcl_setup.out index ed809f02bfb..a8fdcf31256 100644 --- a/src/pl/tcl/expected/pltcl_setup.out +++ b/src/pl/tcl/expected/pltcl_setup.out @@ -119,7 +119,7 @@ CREATE OPERATOR CLASS tcl_int4_ops -- for initialization problems. -- create function tcl_date_week(int4,int4,int4) returns text as $$ - return [clock format [clock scan "$2/$3/$1"] -format "%U"] + return [clock format [clock scan "$2/$3/$1" -gmt 1] -format "%U" -gmt 1] $$ language pltcl immutable; select tcl_date_week(2010,1,26); tcl_date_week diff --git a/src/pl/tcl/sql/pltcl_setup.sql b/src/pl/tcl/sql/pltcl_setup.sql index e9f59989b5b..b9892ea4f76 100644 --- a/src/pl/tcl/sql/pltcl_setup.sql +++ b/src/pl/tcl/sql/pltcl_setup.sql @@ -142,7 +142,7 @@ CREATE OPERATOR CLASS tcl_int4_ops -- for initialization problems. -- create function tcl_date_week(int4,int4,int4) returns text as $$ - return [clock format [clock scan "$2/$3/$1"] -format "%U"] + return [clock format [clock scan "$2/$3/$1" -gmt 1] -format "%U" -gmt 1] $$ language pltcl immutable; select tcl_date_week(2010,1,26); From 049b262f88e784f6fc805748d9b2d9cbb00b08a9 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sat, 30 Sep 2023 17:03:50 +0300 Subject: [PATCH 049/109] Fix briefly showing old progress stats for ANALYZE on inherited tables. ANALYZE on a table with inheritance children analyzes all the child tables in a loop. When stepping to next child table, it updated the child rel ID value in the command progress stats, but did not reset the 'sample_blks_total' and 'sample_blks_scanned' counters. acquire_sample_rows() updates 'sample_blks_total' as soon as the scan starts and 'sample_blks_scanned' after processing the first block, but until then, pg_stat_progress_analyze would display a bogus combination of the new child table relid with old counter values from the previously processed child table. Fix by resetting 'sample_blks_total' and 'sample_blks_scanned' to zero at the same time that 'current_child_table_relid' is updated. Backpatch to v13, where pg_stat_progress_analyze view was introduced. Reported-by: Justin Pryzby Discussion: https://www.postgresql.org/message-id/20230122162345.GP13860%40telsasoft.com --- src/backend/commands/analyze.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index a7d5025124e..fbd26f2ee9c 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -2168,8 +2168,25 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, AcquireSampleRowsFunc acquirefunc = acquirefuncs[i]; double childblocks = relblocks[i]; - pgstat_progress_update_param(PROGRESS_ANALYZE_CURRENT_CHILD_TABLE_RELID, - RelationGetRelid(childrel)); + /* + * Report progress. The sampling function will normally report blocks + * done/total, but we need to reset them to 0 here, so that they don't + * show an old value until that. + */ + { + const int progress_index[] = { + PROGRESS_ANALYZE_CURRENT_CHILD_TABLE_RELID, + PROGRESS_ANALYZE_BLOCKS_DONE, + PROGRESS_ANALYZE_BLOCKS_TOTAL + }; + const int64 progress_vals[] = { + RelationGetRelid(childrel), + 0, + 0, + }; + + pgstat_progress_update_multi_param(3, progress_index, progress_vals); + } if (childblocks > 0) { From e82968b87b9444edfb686dfdbdf49f777c5e7b27 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 1 Oct 2023 12:09:26 -0400 Subject: [PATCH 050/109] In COPY FROM, fail cleanly when unsupported encoding conversion is needed. In recent releases, such cases fail with "cache lookup failed for function 0" rather than complaining that the conversion function doesn't exist as prior versions did. Seems to be a consequence of sloppy refactoring in commit f82de5c46. Add the missing error check. Per report from Pierre Fortin. Back-patch to v14 where the oversight crept in. Discussion: https://postgr.es/m/20230929163739.3bea46e5.pfortin@pfortin.com --- src/backend/commands/copyfrom.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 1fcd9643db2..a60266b8d1d 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -2731,6 +2731,12 @@ BeginCopyFrom(ParseState *pstate, { cstate->need_transcoding = true; cstate->conversion_proc = FindDefaultConversionProc(cstate->file_encoding, GetDatabaseEncoding()); + if (!OidIsValid(cstate->conversion_proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(cstate->file_encoding), + pg_encoding_to_char(GetDatabaseEncoding())))); } /* From f15dd61a7bbe78018f0c7055f0f4a705ae1d3954 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 1 Oct 2023 13:16:47 -0400 Subject: [PATCH 051/109] Fix datalen calculation in tsvectorrecv(). After receiving position data for a lexeme, tsvectorrecv() advanced its "datalen" value by (npos+1)*sizeof(WordEntry) where the correct calculation is (npos+1)*sizeof(WordEntryPos). This accidentally failed to render the constructed tsvector invalid, but it did result in leaving some wasted space approximately equal to the space consumed by the position data. That could have several bad effects: * Disk space is wasted if the received tsvector is stored into a table as-is. * A legal tsvector could get rejected with "maximum total lexeme length exceeded" if the extra space pushes it over the MAXSTRPOS limit. * In edge cases, the finished tsvector could be assigned a length larger than the allocated size of its palloc chunk, conceivably leading to SIGSEGV when the tsvector gets copied somewhere else. The odds of a field failure of this sort seem low, though valgrind testing could probably have found this. While we're here, let's express the calculation as "sizeof(uint16) + npos * sizeof(WordEntryPos)" to avoid the type pun implicit in the "npos + 1" formulation. It's not wrong given that WordEntryPos had better be 2 bytes to avoid padding problems, but it seems clearer this way. Report and patch by Denis Erokhin. Back-patch to all supported versions. Discussion: https://postgr.es/m/009801d9f2d9$f29730c0$d7c59240$@datagile.ru --- src/backend/utils/adt/tsvector.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/utils/adt/tsvector.c b/src/backend/utils/adt/tsvector.c index b02fecc0811..62f0926d864 100644 --- a/src/backend/utils/adt/tsvector.c +++ b/src/backend/utils/adt/tsvector.c @@ -491,7 +491,7 @@ tsvectorrecv(PG_FUNCTION_ARGS) * But make sure the buffer is large enough first. */ while (hdrlen + SHORTALIGN(datalen + lex_len) + - (npos + 1) * sizeof(WordEntryPos) >= len) + sizeof(uint16) + npos * sizeof(WordEntryPos) >= len) { len *= 2; vec = (TSVector) repalloc(vec, len); @@ -537,7 +537,7 @@ tsvectorrecv(PG_FUNCTION_ARGS) elog(ERROR, "position information is misordered"); } - datalen += (npos + 1) * sizeof(WordEntry); + datalen += sizeof(uint16) + npos * sizeof(WordEntryPos); } } From 552241cd0a654230e75dfae119a91c29b003cd7b Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 2 Oct 2023 12:39:35 +0300 Subject: [PATCH 052/109] Flush WAL stats in bgwriter bgwriter can write out WAL, but did not flush the WAL pgstat counters, so the writes were not seen in pg_stat_wal. Back-patch to v14, where pg_stat_wal was introduced. Author: Nazir Bilal Yavuz Reviewed-by: Matthias van de Meent, Kyotaro Horiguchi Discussion: https://www.postgresql.org/message-id/CAN55FZ2FPYngovZstr%3D3w1KSEHe6toiZwrurbhspfkXe5UDocg%40mail.gmail.com --- src/backend/postmaster/bgwriter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c index d59f3d284ae..dfe4e6dc53f 100644 --- a/src/backend/postmaster/bgwriter.c +++ b/src/backend/postmaster/bgwriter.c @@ -250,6 +250,7 @@ BackgroundWriterMain(void) * Send off activity statistics to the stats collector */ pgstat_send_bgwriter(); + pgstat_send_wal(true); if (FirstCallSinceLastCheckpoint()) { From b5e4a8e7de2fbbab05d72d99bfbd7c254ab4a1ca Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 3 Oct 2023 10:25:15 +0900 Subject: [PATCH 053/109] Fail hard on out-of-memory failures in xlogreader.c This commit changes the WAL reader routines so as a FATAL for the backend or exit(FAILURE) for the frontend is triggered if an allocation for a WAL record decode fails in walreader.c, rather than treating this case as bogus data, which would be equivalent to the end of WAL. The key is to avoid palloc_extended(MCXT_ALLOC_NO_OOM) in walreader.c, relying on plain palloc() calls. The previous behavior could make WAL replay finish too early than it should. For example, crash recovery finishing earlier may corrupt clusters because not all the WAL available locally was replayed to ensure a consistent state. Out-of-memory failures would show up randomly depending on the memory pressure on the host, but one simple case would be to generate a large record, then replay this record after downsizing a host, as Ethan Mertz originally reported. This relies on bae868caf222, as the WAL reader routines now do the memory allocation required for a record only once its header has been fully read and validated, making xl_tot_len trustable. Making the WAL reader react differently on out-of-memory or bogus record data would require ABI changes, so this is the safest choice for stable branches. Also, it is worth noting that 3f1ce973467a has been using a plain palloc() in this code for some time now. Thanks to Noah Misch and Thomas Munro for the discussion. Like the other commit, backpatch down to 12, leaving out v11 that will be EOL'd soon. The behavior of considering a failed allocation as bogus data comes originally from 0ffe11abd3a0, where the record length retrieved from its header was not entirely trustable. Reported-by: Ethan Mertz Discussion: https://postgr.es/m/ZRKKdI5-RRlta3aF@paquier.xyz Backpatch-through: 12 --- src/backend/access/transam/xlogreader.c | 31 ++++--------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index af5e8508bf1..9ea662c5be8 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -40,7 +40,7 @@ static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); -static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); +static void allocate_recordbuf(XLogReaderState *state, uint32 reclength); static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen); static void XLogReaderInvalReadState(XLogReaderState *state); @@ -132,14 +132,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir, * Allocate an initial readRecordBuf of minimal size, which can later be * enlarged if necessary. */ - if (!allocate_recordbuf(state, 0)) - { - pfree(state->errormsg_buf); - pfree(state->readBuf); - pfree(state); - return NULL; - } - + allocate_recordbuf(state, 0); return state; } @@ -168,7 +161,6 @@ XLogReaderFree(XLogReaderState *state) /* * Allocate readRecordBuf to fit a record of at least the given length. - * Returns true if successful, false if out of memory. * * readRecordBufSize is set to the new buffer size. * @@ -180,7 +172,7 @@ XLogReaderFree(XLogReaderState *state) * Note: This routine should *never* be called for xl_tot_len until the header * of the record has been fully validated. */ -static bool +static void allocate_recordbuf(XLogReaderState *state, uint32 reclength) { uint32 newSize = reclength; @@ -190,15 +182,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength) if (state->readRecordBuf) pfree(state->readRecordBuf); - state->readRecordBuf = - (char *) palloc_extended(newSize, MCXT_ALLOC_NO_OOM); - if (state->readRecordBuf == NULL) - { - state->readRecordBufSize = 0; - return false; - } + state->readRecordBuf = (char *) palloc(newSize); state->readRecordBufSize = newSize; - return true; } /* @@ -532,13 +517,7 @@ XLogReadRecord(XLogReaderState *state, char **errormsg) Assert(gotlen <= lengthof(save_copy)); Assert(gotlen <= state->readRecordBufSize); memcpy(save_copy, state->readRecordBuf, gotlen); - if (!allocate_recordbuf(state, total_len)) - { - /* We treat this as a "bogus data" condition */ - report_invalid_record(state, "record length %u at %X/%X too long", - total_len, LSN_FORMAT_ARGS(RecPtr)); - goto err; - } + allocate_recordbuf(state, total_len); memcpy(state->readRecordBuf, save_copy, gotlen); buffer = state->readRecordBuf + gotlen; } From 76201ae07bef7a460fc3b9f587ea4ffaad762e82 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 3 Oct 2023 15:37:21 +0900 Subject: [PATCH 054/109] Avoid memory size overflow when allocating backend activity buffer The code in charge of copying the contents of PgBackendStatus to local memory could fail on memory allocation because of an overflow on the amount of memory to use. The overflow can happen when combining a high value track_activity_query_size (max at 1MB) with a large max_connections, when both multiplied get higher than INT32_MAX as both parameters treated as signed integers. This could for example trigger with the following functions, all calling pgstat_read_current_status(): - pg_stat_get_backend_subxact() - pg_stat_get_backend_idset() - pg_stat_get_progress_info() - pg_stat_get_activity() - pg_stat_get_db_numbackends() The change to use MemoryContextAllocHuge() has been introduced in 8d0ddccec636, so backpatch down to 12. Author: Jakub Wartak Discussion: https://postgr.es/m/CAKZiRmw8QSNVw2qNK-dznsatQqz+9DkCquxP0GHbbv1jMkGHMA@mail.gmail.com Backpatch-through: 12 --- src/backend/utils/activity/backend_status.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 217483c1c61..2c60349b808 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -777,7 +777,8 @@ pgstat_read_current_status(void) NAMEDATALEN * NumBackendStatSlots); localactivity = (char *) MemoryContextAllocHuge(backendStatusSnapContext, - pgstat_track_activity_query_size * NumBackendStatSlots); + (Size) pgstat_track_activity_query_size * + (Size) NumBackendStatSlots); #ifdef USE_SSL localsslstatus = (PgBackendSSLStatus *) MemoryContextAlloc(backendStatusSnapContext, From f3857e4eadc7cac5ef48550f595b2f0a190e97c6 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 5 Oct 2023 20:32:14 +1300 Subject: [PATCH 055/109] Fix memory leak in Memoize code Ensure we switch to the per-tuple memory context to prevent any memory leaks of detoasted Datums in MemoizeHash_hash() and MemoizeHash_equal(). Reported-by: Orlov Aleksej Author: Orlov Aleksej, David Rowley Discussion: https://postgr.es/m/83281eed63c74e4f940317186372abfd%40cft.ru Backpatch-through: 14, where Memoize was added --- src/backend/executor/nodeMemoize.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/backend/executor/nodeMemoize.c b/src/backend/executor/nodeMemoize.c index c078b68740b..6a387ba8fc5 100644 --- a/src/backend/executor/nodeMemoize.c +++ b/src/backend/executor/nodeMemoize.c @@ -158,10 +158,14 @@ static uint32 MemoizeHash_hash(struct memoize_hash *tb, const MemoizeKey *key) { MemoizeState *mstate = (MemoizeState *) tb->private_data; + ExprContext *econtext = mstate->ss.ps.ps_ExprContext; + MemoryContext oldcontext; TupleTableSlot *pslot = mstate->probeslot; uint32 hashkey = 0; int numkeys = mstate->nkeys; + oldcontext = MemoryContextSwitchTo(econtext->ecxt_per_tuple_memory); + if (mstate->binary_mode) { for (int i = 0; i < numkeys; i++) @@ -203,6 +207,8 @@ MemoizeHash_hash(struct memoize_hash *tb, const MemoizeKey *key) } } + ResetExprContext(econtext); + MemoryContextSwitchTo(oldcontext); return murmurhash32(hashkey); } @@ -226,7 +232,11 @@ MemoizeHash_equal(struct memoize_hash *tb, const MemoizeKey *key1, if (mstate->binary_mode) { + MemoryContext oldcontext; int numkeys = mstate->nkeys; + bool match = true; + + oldcontext = MemoryContextSwitchTo(econtext->ecxt_per_tuple_memory); slot_getallattrs(tslot); slot_getallattrs(pslot); @@ -236,7 +246,10 @@ MemoizeHash_equal(struct memoize_hash *tb, const MemoizeKey *key1, FormData_pg_attribute *attr; if (tslot->tts_isnull[i] != pslot->tts_isnull[i]) - return false; + { + match = false; + break; + } /* both NULL? they're equal */ if (tslot->tts_isnull[i]) @@ -246,9 +259,15 @@ MemoizeHash_equal(struct memoize_hash *tb, const MemoizeKey *key1, attr = &tslot->tts_tupleDescriptor->attrs[i]; if (!datum_image_eq(tslot->tts_values[i], pslot->tts_values[i], attr->attbyval, attr->attlen)) - return false; + { + match = false; + break; + } } - return true; + + ResetExprContext(econtext); + MemoryContextSwitchTo(oldcontext); + return match; } else { From 2424c136430397d7fe7d932b3da1bc248e0cff61 Mon Sep 17 00:00:00 2001 From: Etsuro Fujita Date: Fri, 6 Oct 2023 18:30:05 +0900 Subject: [PATCH 056/109] Remove extra parenthesis from comment. --- src/backend/storage/ipc/procarray.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 8eea86a16ad..09afe6c5378 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -2869,7 +2869,7 @@ GetSnapshotDataReuse(Snapshot snapshot) * GetSnapshotData() cannot change while ProcArrayLock is held. Snapshot * contents only depend on transactions with xids and xactCompletionCount * is incremented whenever a transaction with an xid finishes (while - * holding ProcArrayLock) exclusively). Thus the xactCompletionCount check + * holding ProcArrayLock exclusively). Thus the xactCompletionCount check * ensures we would detect if the snapshot would have changed. * * As the snapshot contents are the same as it was before, it is safe to From 94382fb038f1130186a33792fe23185360e986f4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 9 Oct 2023 11:29:21 -0400 Subject: [PATCH 057/109] Doc: use CURRENT_USER not USER in plpgsql trigger examples. While these two built-in functions do exactly the same thing, CURRENT_USER seems preferable to use in documentation examples. It's easier to look up if the reader is unsure what it is. Also, this puts these examples in sync with an adjacent example that already used CURRENT_USER. Per question from Kirk Parker. Discussion: https://postgr.es/m/CANwZ8rmN_Eb0h0hoMRS8Feftaik0z89PxVsKg+cP+PctuOq=Qg@mail.gmail.com --- doc/src/sgml/plpgsql.sgml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index 22fa317f7b5..8017ba392a4 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -4300,11 +4300,11 @@ CREATE OR REPLACE FUNCTION process_emp_audit() RETURNS TRIGGER AS $emp_audit$ -- making use of the special variable TG_OP to work out the operation. -- IF (TG_OP = 'DELETE') THEN - INSERT INTO emp_audit SELECT 'D', now(), user, OLD.*; + INSERT INTO emp_audit SELECT 'D', now(), current_user, OLD.*; ELSIF (TG_OP = 'UPDATE') THEN - INSERT INTO emp_audit SELECT 'U', now(), user, NEW.*; + INSERT INTO emp_audit SELECT 'U', now(), current_user, NEW.*; ELSIF (TG_OP = 'INSERT') THEN - INSERT INTO emp_audit SELECT 'I', now(), user, NEW.*; + INSERT INTO emp_audit SELECT 'I', now(), current_user, NEW.*; END IF; RETURN NULL; -- result is ignored since this is an AFTER trigger END; @@ -4370,20 +4370,20 @@ CREATE OR REPLACE FUNCTION update_emp_view() RETURNS TRIGGER AS $$ IF NOT FOUND THEN RETURN NULL; END IF; OLD.last_updated = now(); - INSERT INTO emp_audit VALUES('D', user, OLD.*); + INSERT INTO emp_audit VALUES('D', current_user, OLD.*); RETURN OLD; ELSIF (TG_OP = 'UPDATE') THEN UPDATE emp SET salary = NEW.salary WHERE empname = OLD.empname; IF NOT FOUND THEN RETURN NULL; END IF; NEW.last_updated = now(); - INSERT INTO emp_audit VALUES('U', user, NEW.*); + INSERT INTO emp_audit VALUES('U', current_user, NEW.*); RETURN NEW; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO emp VALUES(NEW.empname, NEW.salary); NEW.last_updated = now(); - INSERT INTO emp_audit VALUES('I', user, NEW.*); + INSERT INTO emp_audit VALUES('I', current_user, NEW.*); RETURN NEW; END IF; END; @@ -4598,13 +4598,13 @@ CREATE OR REPLACE FUNCTION process_emp_audit() RETURNS TRIGGER AS $emp_audit$ -- IF (TG_OP = 'DELETE') THEN INSERT INTO emp_audit - SELECT 'D', now(), user, o.* FROM old_table o; + SELECT 'D', now(), current_user, o.* FROM old_table o; ELSIF (TG_OP = 'UPDATE') THEN INSERT INTO emp_audit - SELECT 'U', now(), user, n.* FROM new_table n; + SELECT 'U', now(), current_user, n.* FROM new_table n; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO emp_audit - SELECT 'I', now(), user, n.* FROM new_table n; + SELECT 'I', now(), current_user, n.* FROM new_table n; END IF; RETURN NULL; -- result is ignored since this is an AFTER trigger END; From ff43e1c6ba33788c01f0f51515980676682ad181 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Tue, 10 Oct 2023 11:01:13 -0700 Subject: [PATCH 058/109] Fix bug in GenericXLogFinish(). Mark the buffers dirty before writing WAL. Discussion: https://postgr.es/m/25104133-7df8-cae3-b9a2-1c0aaa1c094a@iki.fi Reviewed-by: Heikki Linnakangas Backpatch-through: 11 --- src/backend/access/transam/generic_xlog.c | 56 +++++++++++------------ 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/backend/access/transam/generic_xlog.c b/src/backend/access/transam/generic_xlog.c index 63301a1ab16..c6759fd59eb 100644 --- a/src/backend/access/transam/generic_xlog.c +++ b/src/backend/access/transam/generic_xlog.c @@ -342,6 +342,10 @@ GenericXLogFinish(GenericXLogState *state) START_CRIT_SECTION(); + /* + * Compute deltas if necessary, write changes to buffers, mark + * buffers dirty, and register changes. + */ for (i = 0; i < MAX_GENERIC_XLOG_PAGES; i++) { PageData *pageData = &state->pages[i]; @@ -354,41 +358,34 @@ GenericXLogFinish(GenericXLogState *state) page = BufferGetPage(pageData->buffer); pageHeader = (PageHeader) pageData->image; + /* + * Compute delta while we still have both the unmodified page and + * the new image. Not needed if we are logging the full image. + */ + if (!(pageData->flags & GENERIC_XLOG_FULL_IMAGE)) + computeDelta(pageData, page, (Page) pageData->image); + + /* + * Apply the image, being careful to zero the "hole" between + * pd_lower and pd_upper in order to avoid divergence between + * actual page state and what replay would produce. + */ + memcpy(page, pageData->image, pageHeader->pd_lower); + memset(page + pageHeader->pd_lower, 0, + pageHeader->pd_upper - pageHeader->pd_lower); + memcpy(page + pageHeader->pd_upper, + pageData->image + pageHeader->pd_upper, + BLCKSZ - pageHeader->pd_upper); + + MarkBufferDirty(pageData->buffer); + if (pageData->flags & GENERIC_XLOG_FULL_IMAGE) { - /* - * A full-page image does not require us to supply any xlog - * data. Just apply the image, being careful to zero the - * "hole" between pd_lower and pd_upper in order to avoid - * divergence between actual page state and what replay would - * produce. - */ - memcpy(page, pageData->image, pageHeader->pd_lower); - memset(page + pageHeader->pd_lower, 0, - pageHeader->pd_upper - pageHeader->pd_lower); - memcpy(page + pageHeader->pd_upper, - pageData->image + pageHeader->pd_upper, - BLCKSZ - pageHeader->pd_upper); - XLogRegisterBuffer(i, pageData->buffer, REGBUF_FORCE_IMAGE | REGBUF_STANDARD); } else { - /* - * In normal mode, calculate delta and write it as xlog data - * associated with this page. - */ - computeDelta(pageData, page, (Page) pageData->image); - - /* Apply the image, with zeroed "hole" as above */ - memcpy(page, pageData->image, pageHeader->pd_lower); - memset(page + pageHeader->pd_lower, 0, - pageHeader->pd_upper - pageHeader->pd_lower); - memcpy(page + pageHeader->pd_upper, - pageData->image + pageHeader->pd_upper, - BLCKSZ - pageHeader->pd_upper); - XLogRegisterBuffer(i, pageData->buffer, REGBUF_STANDARD); XLogRegisterBufData(i, pageData->delta, pageData->deltaLen); } @@ -397,7 +394,7 @@ GenericXLogFinish(GenericXLogState *state) /* Insert xlog record */ lsn = XLogInsert(RM_GENERIC_ID, 0); - /* Set LSN and mark buffers dirty */ + /* Set LSN */ for (i = 0; i < MAX_GENERIC_XLOG_PAGES; i++) { PageData *pageData = &state->pages[i]; @@ -405,7 +402,6 @@ GenericXLogFinish(GenericXLogState *state) if (BufferIsInvalid(pageData->buffer)) continue; PageSetLSN(BufferGetPage(pageData->buffer), lsn); - MarkBufferDirty(pageData->buffer); } END_CRIT_SECTION(); } From cd32d36b08fb1a535db449e1a81a4e3ee63c3483 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 10 Oct 2023 15:14:18 -0400 Subject: [PATCH 059/109] doc: document the need to analyze partitioned tables Autovacuum does not do it. Reported-by: Justin Pryzby Discussion: https://postgr.es/m/20210913035409.GA10647@telsasoft.com Backpatch-through: 11 --- doc/src/sgml/maintenance.sgml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index eaffb5a6592..0b5860ebbbd 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -841,10 +841,15 @@ analyze threshold = analyze base threshold + analyze scale factor * number of tu - Partitioned tables are not processed by autovacuum. Statistics - should be collected by running a manual ANALYZE when it is - first populated, and again whenever the distribution of data in its - partitions changes significantly. + Partitioned tables do not directly store tuples and consequently + are not processed by autovacuum. (Autovacuum does process table + partitions just like other tables.) Unfortunately, this means that + autovacuum does not run ANALYZE on partitioned + tables, and this can cause suboptimal plans for queries that reference + partitioned table statistics. You can work around this problem by + manually running ANALYZE on partitioned tables + when they are first populated, and again whenever the distribution + of data in their partitions changes significantly. From 278edcb46fed869a60ae7cc5c6b46bd8d63de5d7 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 10 Oct 2023 15:54:28 -0400 Subject: [PATCH 060/109] doc: add SSL configuration section reference Reported-by: Steve Atkins Discussion: https://postgr.es/m/B82E80DD-1452-4175-B19C-564FE46705BA@blighty.com Backpatch-through: 11 --- doc/src/sgml/client-auth.sgml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index eb5e9f48db1..e99bb5ae59f 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -2048,7 +2048,8 @@ host ... radius radiusservers="server1,server2" radiussecrets="""secret one"","" This authentication method uses SSL client certificates to perform - authentication. It is therefore only available for SSL connections. + authentication. It is therefore only available for SSL connections; + see for SSL configuration instructions. When using this authentication method, the server will require that the client provide a valid, trusted certificate. No password prompt will be sent to the client. The cn (Common Name) From 91247e925748ab8f02e6ba44ca07091270655891 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 10 Oct 2023 16:04:56 -0400 Subject: [PATCH 061/109] doc: foreign servers with pushdown need matching collation Reported-by: Pete Storer Discussion: https://postgr.es/m/BL0PR05MB66283C57D72E321591AE4EB1F3CE9@BL0PR05MB6628.namprd05.prod.outlook.com Backpatch-through: 11 --- doc/src/sgml/ref/create_server.sgml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/src/sgml/ref/create_server.sgml b/doc/src/sgml/ref/create_server.sgml index ca5be066f96..fe3098a079c 100644 --- a/doc/src/sgml/ref/create_server.sgml +++ b/doc/src/sgml/ref/create_server.sgml @@ -148,6 +148,11 @@ CREATE SERVER [ IF NOT EXISTS ] server_nameUSAGE privilege on the foreign server to be able to use it in this way. + + + If the foreign server supports sort pushdown, it is necessary for it + to have the same sort ordering as the local server. + From 7ec28b89939fb836ca5cbbfcf7a0aee46255fd3b Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 10 Oct 2023 16:51:08 -0400 Subject: [PATCH 062/109] doc: clarify that SSPI and GSSAPI are interchangeable Reported-by: tpo_deb@sourcepole.ch Discussion: https://postgr.es/m/167846222574.1803490.15815104179136215862@wrigleys.postgresql.org Backpatch-through: 11 --- doc/src/sgml/client-auth.sgml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index e99bb5ae59f..44f8fd02b0c 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -1388,10 +1388,12 @@ omicron bryanh guest1 negotiate mode, which will use Kerberos when possible and automatically fall back to NTLM in other cases. - SSPI authentication only works when both - server and client are running Windows, - or, on non-Windows platforms, when GSSAPI - is available. + SSPI and GSSAPI + interoperate as clients and servers, e.g., an + SSPI client can authenticate to an + GSSAPI server. It is recommended to use + SSPI on Windows clients and servers and + GSSAPI on non-Windows platforms. From c700612d428ce506c1cf2e87fcfadbc64e9e5889 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 12 Oct 2023 19:52:31 +1300 Subject: [PATCH 063/109] Fix incorrect step generation in HASH partition pruning get_steps_using_prefix_recurse() incorrectly assumed that it could stop recursive processing of the 'prefix' list when cur_keyno was one before the step_lastkeyno. Since hash partition pruning can prune using IS NULL quals, and these IS NULL quals are not present in the 'prefix' list, then that logic could cause more levels of recursion than what is needed and lead to there being no more items in the 'prefix' list to process. This would manifest itself as a crash in some code that expected the 'start' ListCell not to be NULL. Here we adjust the logic so that instead of stopping recursion at 1 key before the step_lastkeyno, we just look at the llast(prefix) item and ensure we only recursively process up until just before whichever the last key is. This effectively allows keys to be missing in the 'prefix' list. This change does mean that step_lastkeyno is no longer needed, so we remove that from the static functions. I also spent quite some time reading this code and testing it to try to convince myself that there are no other issues. That resulted in the irresistible temptation of rewriting some comments, many of which were just not true or inconcise. Reported-by: Sergei Glukhov Reviewed-by: Sergei Glukhov, tender wang Discussion: https://postgr.es/m/2f09ce72-315e-2a33-589a-8519ada8df61@postgrespro.ru Backpatch-through: 11, where partition pruning was introduced. --- src/backend/partitioning/partprune.c | 103 ++++---- src/test/regress/expected/partition_prune.out | 221 ++++++++++++++++-- src/test/regress/sql/partition_prune.sql | 55 ++++- 3 files changed, 315 insertions(+), 64 deletions(-) diff --git a/src/backend/partitioning/partprune.c b/src/backend/partitioning/partprune.c index a9ff52941ca..6d7c2a5f53d 100644 --- a/src/backend/partitioning/partprune.c +++ b/src/backend/partitioning/partprune.c @@ -172,7 +172,6 @@ static List *get_steps_using_prefix(GeneratePruningStepsContext *context, bool step_op_is_ne, Expr *step_lastexpr, Oid step_lastcmpfn, - int step_lastkeyno, Bitmapset *step_nullkeys, List *prefix); static List *get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, @@ -180,7 +179,6 @@ static List *get_steps_using_prefix_recurse(GeneratePruningStepsContext *context bool step_op_is_ne, Expr *step_lastexpr, Oid step_lastcmpfn, - int step_lastkeyno, Bitmapset *step_nullkeys, List *prefix, ListCell *start, @@ -1557,7 +1555,6 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context, pc->op_is_ne, pc->expr, pc->cmpfn, - 0, NULL, NIL); opsteps = list_concat(opsteps, pc_steps); @@ -1683,7 +1680,6 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context, pc->op_is_ne, pc->expr, pc->cmpfn, - pc->keyno, NULL, prefix); opsteps = list_concat(opsteps, pc_steps); @@ -1757,7 +1753,6 @@ gen_prune_steps_from_opexps(GeneratePruningStepsContext *context, false, pc->expr, pc->cmpfn, - pc->keyno, nullkeys, prefix); opsteps = list_concat(opsteps, pc_steps); @@ -2380,25 +2375,31 @@ match_clause_to_partition_key(GeneratePruningStepsContext *context, /* * get_steps_using_prefix - * Generate list of PartitionPruneStepOp steps each consisting of given - * opstrategy - * - * To generate steps, step_lastexpr and step_lastcmpfn are appended to - * expressions and cmpfns, respectively, extracted from the clauses in - * 'prefix'. Actually, since 'prefix' may contain multiple clauses for the - * same partition key column, we must generate steps for various combinations - * of the clauses of different keys. - * - * For list/range partitioning, callers must ensure that step_nullkeys is - * NULL, and that prefix contains at least one clause for each of the - * partition keys earlier than one specified in step_lastkeyno if it's - * greater than zero. For hash partitioning, step_nullkeys is allowed to be - * non-NULL, but they must ensure that prefix contains at least one clause - * for each of the partition keys other than those specified in step_nullkeys - * and step_lastkeyno. - * - * For both cases, callers must also ensure that clauses in prefix are sorted - * in ascending order of their partition key numbers. + * Generate a list of PartitionPruneStepOps based on the given input. + * + * 'step_lastexpr' and 'step_lastcmpfn' are the Expr and comparison function + * belonging to the final partition key that we have a clause for. 'prefix' + * is a list of PartClauseInfos for partition key numbers prior to the given + * 'step_lastexpr' and 'step_lastcmpfn'. 'prefix' may contain multiple + * PartClauseInfos belonging to a single partition key. We will generate a + * PartitionPruneStepOp for each combination of the given PartClauseInfos + * using, at most, one PartClauseInfo per partition key. + * + * For LIST and RANGE partitioned tables, callers must ensure that + * step_nullkeys is NULL, and that prefix contains at least one clause for + * each of the partition keys prior to the key that 'step_lastexpr' and + * 'step_lastcmpfn'belong to. + * + * For HASH partitioned tables, callers must ensure that 'prefix' contains at + * least one clause for each of the partition keys apart from the final key + * (the expr and comparison function for the final key are in 'step_lastexpr' + * and 'step_lastcmpfn'). A bit set in step_nullkeys can substitute clauses + * in the 'prefix' list for any given key. If a bit is set in 'step_nullkeys' + * for a given key, then there must be no PartClauseInfo for that key in the + * 'prefix' list. + * + * For each of the above cases, callers must ensure that PartClauseInfos in + * 'prefix' are sorted in ascending order of keyno. */ static List * get_steps_using_prefix(GeneratePruningStepsContext *context, @@ -2406,14 +2407,17 @@ get_steps_using_prefix(GeneratePruningStepsContext *context, bool step_op_is_ne, Expr *step_lastexpr, Oid step_lastcmpfn, - int step_lastkeyno, Bitmapset *step_nullkeys, List *prefix) { + /* step_nullkeys must be empty for RANGE and LIST partitioned tables */ Assert(step_nullkeys == NULL || context->rel->part_scheme->strategy == PARTITION_STRATEGY_HASH); - /* Quick exit if there are no values to prefix with. */ + /* + * No recursive processing is required when 'prefix' is an empty list. This + * occurs when there is only 1 partition key column. + */ if (list_length(prefix) == 0) { PartitionPruneStep *step; @@ -2427,13 +2431,12 @@ get_steps_using_prefix(GeneratePruningStepsContext *context, return list_make1(step); } - /* Recurse to generate steps for various combinations. */ + /* Recurse to generate steps for every combination of clauses. */ return get_steps_using_prefix_recurse(context, step_opstrategy, step_op_is_ne, step_lastexpr, step_lastcmpfn, - step_lastkeyno, step_nullkeys, prefix, list_head(prefix), @@ -2442,13 +2445,17 @@ get_steps_using_prefix(GeneratePruningStepsContext *context, /* * get_steps_using_prefix_recurse - * Recursively generate combinations of clauses for different partition - * keys and start generating steps upon reaching clauses for the greatest - * column that is less than the one for which we're currently generating - * steps (that is, step_lastkeyno) + * Generate and return a list of PartitionPruneStepOps using the 'prefix' + * list of PartClauseInfos starting at the 'start' cell. + * + * When 'prefix' contains multiple PartClauseInfos for a single partition key + * we create a PartitionPruneStepOp for each combination of duplicated + * PartClauseInfos. The returned list will contain a PartitionPruneStepOp + * for each unique combination of input PartClauseInfos containing at most one + * PartClauseInfo per partition key. * - * 'prefix' is the list of PartClauseInfos. - * 'start' is where we should start iterating for the current invocation. + * 'prefix' is the input list of PartClauseInfos sorted by keyno. + * 'start' marks the cell that searching the 'prefix' list should start from. * 'step_exprs' and 'step_cmpfns' each contains the expressions and cmpfns * we've generated so far from the clauses for the previous part keys. */ @@ -2458,7 +2465,6 @@ get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, bool step_op_is_ne, Expr *step_lastexpr, Oid step_lastcmpfn, - int step_lastkeyno, Bitmapset *step_nullkeys, List *prefix, ListCell *start, @@ -2468,23 +2474,25 @@ get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, List *result = NIL; ListCell *lc; int cur_keyno; + int final_keyno; /* Actually, recursion would be limited by PARTITION_MAX_KEYS. */ check_stack_depth(); - /* Check if we need to recurse. */ Assert(start != NULL); cur_keyno = ((PartClauseInfo *) lfirst(start))->keyno; - if (cur_keyno < step_lastkeyno - 1) + final_keyno = ((PartClauseInfo *) llast(prefix))->keyno; + + /* Check if we need to recurse. */ + if (cur_keyno < final_keyno) { PartClauseInfo *pc; ListCell *next_start; /* - * For each clause with cur_keyno, add its expr and cmpfn to - * step_exprs and step_cmpfns, respectively, and recurse after setting - * next_start to the ListCell of the first clause for the next - * partition key. + * Find the first PartClauseInfo belonging to the next partition key, the + * next recursive call must start iteration of the prefix list from that + * point. */ for_each_cell(lc, prefix, start) { @@ -2493,8 +2501,15 @@ get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, if (pc->keyno > cur_keyno) break; } + + /* record where to start iterating in the next recursive call */ next_start = lc; + /* + * For each PartClauseInfo with keyno set to cur_keyno, add its expr and + * cmpfn to step_exprs and step_cmpfns, respectively, and recurse using + * 'next_start' as the starting point in the 'prefix' list. + */ for_each_cell(lc, prefix, start) { List *moresteps; @@ -2514,6 +2529,7 @@ get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, } else { + /* check the 'prefix' list is sorted correctly */ Assert(pc->keyno > cur_keyno); break; } @@ -2523,7 +2539,6 @@ get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, step_op_is_ne, step_lastexpr, step_lastcmpfn, - step_lastkeyno, step_nullkeys, prefix, next_start, @@ -2542,8 +2557,8 @@ get_steps_using_prefix_recurse(GeneratePruningStepsContext *context, * each clause with cur_keyno, which is all clauses from here onward * till the end of the list. Note that for hash partitioning, * step_nullkeys is allowed to be non-empty, in which case step_exprs - * would only contain expressions for the earlier partition keys that - * are not specified in step_nullkeys. + * would only contain expressions for the partition keys that are not + * specified in step_nullkeys. */ Assert(list_length(step_exprs) == cur_keyno || !bms_is_empty(step_nullkeys)); diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index 75e646374be..566779d21a1 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -4359,22 +4359,217 @@ explain (costs off) select * from rp_prefix_test3 where a >= 1 and b >= 1 and b Optimizer: Postgres query optimizer (4 rows) -create table hp_prefix_test (a int, b int, c int, d int) partition by hash (a part_test_int4_ops, b part_test_int4_ops, c part_test_int4_ops, d part_test_int4_ops); -create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 2, remainder 0); -create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 2, remainder 1); --- Test that get_steps_using_prefix() handles non-NULL step_nullkeys -explain (costs off) select * from hp_prefix_test where a = 1 and b is null and c = 1 and d = 1; - QUERY PLAN -------------------------------------------------------------------- - Gather Motion 1:1 (slice1; segments: 1) - -> Seq Scan on hp_prefix_test_p1 hp_prefix_test - Filter: ((b IS NULL) AND (a = 1) AND (c = 1) AND (d = 1)) - Optimizer: Postgres query optimizer -(4 rows) - drop table rp_prefix_test1; drop table rp_prefix_test2; drop table rp_prefix_test3; +-- +-- Test that get_steps_using_prefix() handles IS NULL clauses correctly +-- +create table hp_prefix_test (a int, b int, c int, d int) + partition by hash (a part_test_int4_ops, b part_test_int4_ops, c part_test_int4_ops, d part_test_int4_ops); +-- create 8 partitions +select 'create table hp_prefix_test_p' || x::text || ' partition of hp_prefix_test for values with (modulus 8, remainder ' || x::text || ');' +from generate_Series(0,7) x; + ?column? +------------------------------------------------------------------------------------------------------ + create table hp_prefix_test_p0 partition of hp_prefix_test for values with (modulus 8, remainder 0); + create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 8, remainder 1); + create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 8, remainder 2); + create table hp_prefix_test_p3 partition of hp_prefix_test for values with (modulus 8, remainder 3); + create table hp_prefix_test_p4 partition of hp_prefix_test for values with (modulus 8, remainder 4); + create table hp_prefix_test_p5 partition of hp_prefix_test for values with (modulus 8, remainder 5); + create table hp_prefix_test_p6 partition of hp_prefix_test for values with (modulus 8, remainder 6); + create table hp_prefix_test_p7 partition of hp_prefix_test for values with (modulus 8, remainder 7); +(8 rows) + +\gexec +create table hp_prefix_test_p0 partition of hp_prefix_test for values with (modulus 8, remainder 0); +create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 8, remainder 1); +create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 8, remainder 2); +create table hp_prefix_test_p3 partition of hp_prefix_test for values with (modulus 8, remainder 3); +create table hp_prefix_test_p4 partition of hp_prefix_test for values with (modulus 8, remainder 4); +create table hp_prefix_test_p5 partition of hp_prefix_test for values with (modulus 8, remainder 5); +create table hp_prefix_test_p6 partition of hp_prefix_test for values with (modulus 8, remainder 6); +create table hp_prefix_test_p7 partition of hp_prefix_test for values with (modulus 8, remainder 7); +-- insert 16 rows, one row for each test to perform. +insert into hp_prefix_test +select + case a when 0 then null else 1 end, + case b when 0 then null else 2 end, + case c when 0 then null else 3 end, + case d when 0 then null else 4 end +from + generate_series(0,1) a, + generate_series(0,1) b, + generate_Series(0,1) c, + generate_Series(0,1) d; +-- Ensure partition pruning works correctly for each combination of IS NULL +-- and equality quals. This may seem a little excessive, but there have been +-- a number of bugs in this area over the years. We make use of row only +-- output to reduce the size of the expected results. +\t on +select + 'explain (costs off) select tableoid::regclass,* from hp_prefix_test where ' || + string_agg(c.colname || case when g.s & (1 << c.colpos) = 0 then ' is null' else ' = ' || (colpos+1)::text end, ' and ' order by c.colpos) +from (values('a',0),('b',1),('c',2),('d',3)) c(colname, colpos), generate_Series(0,15) g(s) +group by g.s +order by g.s; + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + +\gexec +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + Seq Scan on hp_prefix_test_p0 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d IS NULL)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + Seq Scan on hp_prefix_test_p1 hp_prefix_test + Filter: ((b IS NULL) AND (c IS NULL) AND (d IS NULL) AND (a = 1)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + Seq Scan on hp_prefix_test_p2 hp_prefix_test + Filter: ((a IS NULL) AND (c IS NULL) AND (d IS NULL) AND (b = 2)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((c IS NULL) AND (d IS NULL) AND (a = 1) AND (b = 2)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + Seq Scan on hp_prefix_test_p3 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (d IS NULL) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + Seq Scan on hp_prefix_test_p7 hp_prefix_test + Filter: ((b IS NULL) AND (d IS NULL) AND (a = 1) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (d IS NULL) AND (b = 2) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((d IS NULL) AND (a = 1) AND (b = 2) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((b IS NULL) AND (c IS NULL) AND (a = 1) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((a IS NULL) AND (c IS NULL) AND (b = 2) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((c IS NULL) AND (a = 1) AND (b = 2) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c = 3) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((b IS NULL) AND (a = 1) AND (c = 3) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((a IS NULL) AND (b = 2) AND (c = 3) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a = 1) AND (b = 2) AND (c = 3) AND (d = 4)) + +-- And ensure we get exactly 1 row from each. Again, all 16 possible combinations. +select + 'select tableoid::regclass,* from hp_prefix_test where ' || + string_agg(c.colname || case when g.s & (1 << c.colpos) = 0 then ' is null' else ' = ' || (colpos+1)::text end, ' and ' order by c.colpos) +from (values('a',0),('b',1),('c',2),('d',3)) c(colname, colpos), generate_Series(0,15) g(s) +group by g.s +order by g.s; + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + +\gexec +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + hp_prefix_test_p0 | | | | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + hp_prefix_test_p1 | 1 | | | + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + hp_prefix_test_p2 | | 2 | | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + hp_prefix_test_p4 | 1 | 2 | | + +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + hp_prefix_test_p3 | | | 3 | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + hp_prefix_test_p7 | 1 | | 3 | + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + hp_prefix_test_p4 | | 2 | 3 | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + hp_prefix_test_p5 | 1 | 2 | 3 | + +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + hp_prefix_test_p4 | | | | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + hp_prefix_test_p6 | 1 | | | 4 + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + hp_prefix_test_p5 | | 2 | | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + hp_prefix_test_p6 | 1 | 2 | | 4 + +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + hp_prefix_test_p4 | | | 3 | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + hp_prefix_test_p5 | 1 | | 3 | 4 + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + hp_prefix_test_p6 | | 2 | 3 | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + hp_prefix_test_p4 | 1 | 2 | 3 | 4 + +\t off drop table hp_prefix_test; -- -- Check that gen_partprune_steps() detects self-contradiction from clauses diff --git a/src/test/regress/sql/partition_prune.sql b/src/test/regress/sql/partition_prune.sql index f9e3b1f2013..0dca83079e4 100644 --- a/src/test/regress/sql/partition_prune.sql +++ b/src/test/regress/sql/partition_prune.sql @@ -1191,16 +1191,57 @@ explain (costs off) select * from rp_prefix_test3 where a >= 1 and b >= 1 and b -- that the caller arranges clauses in that prefix in the required order) explain (costs off) select * from rp_prefix_test3 where a >= 1 and b >= 1 and b = 2 and c = 2 and d >= 0; -create table hp_prefix_test (a int, b int, c int, d int) partition by hash (a part_test_int4_ops, b part_test_int4_ops, c part_test_int4_ops, d part_test_int4_ops); -create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 2, remainder 0); -create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 2, remainder 1); - --- Test that get_steps_using_prefix() handles non-NULL step_nullkeys -explain (costs off) select * from hp_prefix_test where a = 1 and b is null and c = 1 and d = 1; - drop table rp_prefix_test1; drop table rp_prefix_test2; drop table rp_prefix_test3; + +-- +-- Test that get_steps_using_prefix() handles IS NULL clauses correctly +-- +create table hp_prefix_test (a int, b int, c int, d int) + partition by hash (a part_test_int4_ops, b part_test_int4_ops, c part_test_int4_ops, d part_test_int4_ops); + +-- create 8 partitions +select 'create table hp_prefix_test_p' || x::text || ' partition of hp_prefix_test for values with (modulus 8, remainder ' || x::text || ');' +from generate_Series(0,7) x; +\gexec + +-- insert 16 rows, one row for each test to perform. +insert into hp_prefix_test +select + case a when 0 then null else 1 end, + case b when 0 then null else 2 end, + case c when 0 then null else 3 end, + case d when 0 then null else 4 end +from + generate_series(0,1) a, + generate_series(0,1) b, + generate_Series(0,1) c, + generate_Series(0,1) d; + +-- Ensure partition pruning works correctly for each combination of IS NULL +-- and equality quals. This may seem a little excessive, but there have been +-- a number of bugs in this area over the years. We make use of row only +-- output to reduce the size of the expected results. +\t on +select + 'explain (costs off) select tableoid::regclass,* from hp_prefix_test where ' || + string_agg(c.colname || case when g.s & (1 << c.colpos) = 0 then ' is null' else ' = ' || (colpos+1)::text end, ' and ' order by c.colpos) +from (values('a',0),('b',1),('c',2),('d',3)) c(colname, colpos), generate_Series(0,15) g(s) +group by g.s +order by g.s; +\gexec + +-- And ensure we get exactly 1 row from each. Again, all 16 possible combinations. +select + 'select tableoid::regclass,* from hp_prefix_test where ' || + string_agg(c.colname || case when g.s & (1 << c.colpos) = 0 then ' is null' else ' = ' || (colpos+1)::text end, ' and ' order by c.colpos) +from (values('a',0),('b',1),('c',2),('d',3)) c(colname, colpos), generate_Series(0,15) g(s) +group by g.s +order by g.s; +\gexec +\t off + drop table hp_prefix_test; -- From adc97457b869c54a864cc068945ed1430440e735 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Thu, 12 Oct 2023 21:16:43 +1300 Subject: [PATCH 064/109] Doc: fix grammatical errors for enable_partitionwise_aggregate Author: Andrew Atkinson Reviewed-by: Ashutosh Bapat Discussion: https://postgr.es/m/CAG6XLEnC%3DEgq0YHRic2kWWDs4xwQnQ_kBA6qhhzAq1-pO_9Tfw%40mail.gmail.com Backpatch-through: 11, where enable_partitionwise_aggregate was added --- doc/src/sgml/config.sgml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index edbd5968ec7..0396704f3d8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5175,13 +5175,14 @@ ANY num_sync ( fds.fd_count + 1 >= FD_SETSIZE) + { + pg_log_fatal("too many concurrent database clients for this platform: %d", + sa->fds.fd_count + 1); + exit(1); + } +#else if (fd < 0 || fd >= FD_SETSIZE) { - /* - * Doing a hard exit here is a bit grotty, but it doesn't seem worth - * complicating the API to make it less grotty. - */ - pg_log_fatal("too many client connections for select()"); + pg_log_fatal("socket file descriptor out of range for select(): %d", + fd); exit(1); } +#endif FD_SET(fd, &sa->fds); if (fd > sa->maxfd) sa->maxfd = fd; diff --git a/src/fe_utils/parallel_slot.c b/src/fe_utils/parallel_slot.c index dcdad9e30c5..8d558064343 100644 --- a/src/fe_utils/parallel_slot.c +++ b/src/fe_utils/parallel_slot.c @@ -297,11 +297,40 @@ connect_slot(ParallelSlotArray *sa, int slotno, const char *dbname) slot->connection = connectDatabase(sa->cparams, sa->progname, sa->echo, false, true); sa->cparams->override_dbname = old_override; - if (PQsocket(slot->connection) >= FD_SETSIZE) + /* + * POSIX defines FD_SETSIZE as the highest file descriptor acceptable to + * FD_SET() and allied macros. Windows defines it as a ceiling on the + * count of file descriptors in the set, not a ceiling on the value of + * each file descriptor; see + * https://learn.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-select + * and + * https://learn.microsoft.com/en-us/windows/win32/api/winsock/ns-winsock-fd_set. + * We can't ignore that, because Windows starts file descriptors at a + * higher value, delays reuse, and skips values. With less than ten + * concurrent file descriptors, opened and closed rapidly, one can reach + * file descriptor 1024. + * + * Doing a hard exit here is a bit grotty, but it doesn't seem worth + * complicating the API to make it less grotty. + */ +#ifdef WIN32 + if (slotno >= FD_SETSIZE) { - pg_log_fatal("too many jobs for this platform"); + pg_log_fatal("too many jobs for this platform: %d", slotno); exit(1); } +#else + { + int fd = PQsocket(slot->connection); + + if (fd >= FD_SETSIZE) + { + pg_log_fatal("socket file descriptor out of range for select(): %d", + fd); + exit(1); + } + } +#endif /* Setup the connection using the supplied command, if any. */ if (sa->initcmd) From f461aaf26e8d81ba42754b2487d74f3b6082c963 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sat, 14 Oct 2023 16:33:51 -0700 Subject: [PATCH 067/109] Dissociate btequalimage() from interval_ops, ending its deduplication. Under interval_ops, some equal values are distinguishable. One such pair is '24:00:00' and '1 day'. With that being so, btequalimage() breaches the documented contract for the "equalimage" btree support function. This can cause incorrect results from index-only scans. Users should REINDEX any btree indexes having interval-type columns. After updating, pg_amcheck will report an error for almost all such indexes. This fix makes interval_ops simply omit the support function, like numeric_ops does. Back-pack to v13, where btequalimage() first appeared. In back branches, for the benefit of old catalog content, btequalimage() code will return false for type "interval". Going forward, back-branch initdb will include the catalog change. Reviewed by Peter Geoghegan. Discussion: https://postgr.es/m/20231011013317.22.nmisch@google.com --- contrib/amcheck/verify_nbtree.c | 13 ++++++++++++- src/backend/utils/adt/datum.c | 14 ++++++-------- src/include/catalog/pg_amproc.dat | 2 -- src/include/catalog/pg_opfamily.dat | 2 +- src/test/regress/expected/opr_sanity.out | 1 + 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index 7f8231a6815..6c3b3c27ccc 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -31,6 +31,7 @@ #include "access/xact.h" #include "catalog/index.h" #include "catalog/pg_am.h" +#include "catalog/pg_opfamily_d.h" #include "commands/tablecmds.h" #include "lib/bloomfilter.h" #include "miscadmin.h" @@ -337,10 +338,20 @@ bt_index_check_internal(Oid indrelid, bool parentcheck, bool heapallindexed, errmsg("index \"%s\" metapage has equalimage field set on unsupported nbtree version", RelationGetRelationName(indrel)))); if (allequalimage && !_bt_allequalimage(indrel, false)) + { + bool has_interval_ops = false; + + for (int i = 0; i < IndexRelationGetNumberOfKeyAttributes(indrel); i++) + if (indrel->rd_opfamily[i] == INTERVAL_BTREE_FAM_OID) + has_interval_ops = true; ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("index \"%s\" metapage incorrectly indicates that deduplication is safe", - RelationGetRelationName(indrel)))); + RelationGetRelationName(indrel)), + has_interval_ops + ? errhint("This is known of \"interval\" indexes last built on a version predating 2023-11.") + : 0)); + } /* Check index, possibly against table it is an index on */ bt_check_every_level(indrel, heaprel, heapkeyspace, parentcheck, diff --git a/src/backend/utils/adt/datum.c b/src/backend/utils/adt/datum.c index eed6ae262f2..788b7d61bcc 100644 --- a/src/backend/utils/adt/datum.c +++ b/src/backend/utils/adt/datum.c @@ -43,6 +43,7 @@ #include "postgres.h" #include "access/detoast.h" +#include "catalog/pg_type_d.h" #include "common/hashfn.h" #include "fmgr.h" #include "utils/builtins.h" @@ -388,20 +389,17 @@ datum_image_hash(Datum value, bool typByVal, int typLen) * datum_image_eq() in all cases can use this as their "equalimage" support * function. * - * Currently, we unconditionally assume that any B-Tree operator class that - * registers btequalimage as its support function 4 must be able to safely use - * optimizations like deduplication (i.e. we return true unconditionally). If - * it ever proved necessary to rescind support for an operator class, we could - * do that in a targeted fashion by doing something with the opcintype - * argument. + * Earlier minor releases erroneously associated this function with + * interval_ops. Detect that case to rescind deduplication support, without + * requiring initdb. *------------------------------------------------------------------------- */ Datum btequalimage(PG_FUNCTION_ARGS) { - /* Oid opcintype = PG_GETARG_OID(0); */ + Oid opcintype = PG_GETARG_OID(0); - PG_RETURN_BOOL(true); + PG_RETURN_BOOL(opcintype != INTERVALOID); } /*------------------------------------------------------------------------- diff --git a/src/include/catalog/pg_amproc.dat b/src/include/catalog/pg_amproc.dat index 8ba4de4f868..5ec2f50c9fa 100644 --- a/src/include/catalog/pg_amproc.dat +++ b/src/include/catalog/pg_amproc.dat @@ -172,8 +172,6 @@ { amprocfamily => 'btree/interval_ops', amproclefttype => 'interval', amprocrighttype => 'interval', amprocnum => '3', amproc => 'in_range(interval,interval,interval,bool,bool)' }, -{ amprocfamily => 'btree/interval_ops', amproclefttype => 'interval', - amprocrighttype => 'interval', amprocnum => '4', amproc => 'btequalimage' }, { amprocfamily => 'btree/macaddr_ops', amproclefttype => 'macaddr', amprocrighttype => 'macaddr', amprocnum => '1', amproc => 'macaddr_cmp' }, { amprocfamily => 'btree/macaddr_ops', amproclefttype => 'macaddr', diff --git a/src/include/catalog/pg_opfamily.dat b/src/include/catalog/pg_opfamily.dat index 8cb8ce386d2..0e73063ac3f 100644 --- a/src/include/catalog/pg_opfamily.dat +++ b/src/include/catalog/pg_opfamily.dat @@ -50,7 +50,7 @@ opfmethod => 'btree', opfname => 'integer_ops' }, { oid => '1977', opfmethod => 'hash', opfname => 'integer_ops' }, -{ oid => '1982', +{ oid => '1982', oid_symbol => 'INTERVAL_BTREE_FAM_OID', opfmethod => 'btree', opfname => 'interval_ops' }, { oid => '1983', opfmethod => 'hash', opfname => 'interval_ops' }, diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 2fb7ba06584..4ab11e74d6e 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -2231,6 +2231,7 @@ ORDER BY 1, 2, 3; | complex_ops | complex_ops | complex | float_ops | float4_ops | real | float_ops | float8_ops | double precision + | interval_ops | interval_ops | interval | jsonb_ops | jsonb_ops | jsonb | multirange_ops | multirange_ops | anymultirange | numeric_ops | numeric_ops | numeric From ba3e1f81ad70ebdd90fbe52003abf380df60d22b Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 16 Oct 2023 10:43:47 +1300 Subject: [PATCH 068/109] Acquire ControlFileLock in relevant SQL functions. Commit dc7d70ea added functions that read the control file, but didn't acquire ControlFileLock. With unlucky timing, file systems that have weak interlocking like ext4 and ntfs could expose partially overwritten contents, and the checksum would fail. Back-patch to all supported releases. Reviewed-by: David Steele Reviewed-by: Anton A. Melnikov Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/20221123014224.xisi44byq3cf5psi%40awork3.anarazel.de --- src/backend/utils/misc/pg_controldata.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c index 908179cdf8b..6b695064e82 100644 --- a/src/backend/utils/misc/pg_controldata.c +++ b/src/backend/utils/misc/pg_controldata.c @@ -24,6 +24,7 @@ #include "common/controldata_utils.h" #include "funcapi.h" #include "miscadmin.h" +#include "storage/lwlock.h" #include "utils/builtins.h" #include "utils/pg_lsn.h" #include "utils/timestamp.h" @@ -56,7 +57,9 @@ pg_control_system(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* read the control file */ + LWLockAcquire(ControlFileLock, LW_SHARED); ControlFile = get_controlfile(DataDir, &crc_ok); + LWLockRelease(ControlFileLock); if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); @@ -134,7 +137,9 @@ pg_control_checkpoint(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* Read the control file. */ + LWLockAcquire(ControlFileLock, LW_SHARED); ControlFile = get_controlfile(DataDir, &crc_ok); + LWLockRelease(ControlFileLock); if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); @@ -237,7 +242,9 @@ pg_control_recovery(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* read the control file */ + LWLockAcquire(ControlFileLock, LW_SHARED); ControlFile = get_controlfile(DataDir, &crc_ok); + LWLockRelease(ControlFileLock); if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); @@ -304,7 +311,9 @@ pg_control_init(PG_FUNCTION_ARGS) tupdesc = BlessTupleDesc(tupdesc); /* read the control file */ + LWLockAcquire(ControlFileLock, LW_SHARED); ControlFile = get_controlfile(DataDir, &crc_ok); + LWLockRelease(ControlFileLock); if (!crc_ok) ereport(ERROR, (errmsg("calculated CRC checksum does not match value stored in file"))); From 727cbd5d4d96b339433a2275bc6d0d5349cd37c9 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Mon, 16 Oct 2023 17:10:13 +1300 Subject: [PATCH 069/109] Try to handle torn reads of pg_control in frontend. Some of our src/bin tools read the control file without any kind of interlocking against concurrent writes from the server. At least ext4 and ntfs can expose partially modified contents when you do that. For now, we'll try to tolerate this by retrying up to 10 times if the checksum doesn't match, until we get two reads in a row with the same bad checksum. This is not guaranteed to reach the right conclusion, but it seems very likely to. Thanks to Tom Lane for this suggestion. Various ideas for interlocking or atomicity were considered too complicated, unportable or expensive given the lack of field reports, but remain open for future reconsideration. Back-patch as far as 12. It doesn't seem like a good idea to put a heuristic change for a very rare problem into the final release of 11. Reviewed-by: Anton A. Melnikov Reviewed-by: David Steele Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/20221123014224.xisi44byq3cf5psi%40awork3.anarazel.de --- src/common/controldata_utils.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/common/controldata_utils.c b/src/common/controldata_utils.c index 7d4af7881ea..c6f133fe0c0 100644 --- a/src/common/controldata_utils.c +++ b/src/common/controldata_utils.c @@ -55,12 +55,22 @@ get_controlfile(const char *DataDir, bool *crc_ok_p) char ControlFilePath[MAXPGPATH]; pg_crc32c crc; int r; +#ifdef FRONTEND + pg_crc32c last_crc; + int retries = 0; +#endif AssertArg(crc_ok_p); ControlFile = palloc(sizeof(ControlFileData)); snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", DataDir); +#ifdef FRONTEND + INIT_CRC32C(last_crc); + +retry: +#endif + #ifndef FRONTEND if ((fd = OpenTransientFile(ControlFilePath, O_RDONLY | PG_BINARY)) == -1) ereport(ERROR, @@ -128,6 +138,26 @@ get_controlfile(const char *DataDir, bool *crc_ok_p) *crc_ok_p = EQ_CRC32C(crc, ControlFile->crc); +#ifdef FRONTEND + + /* + * If the server was writing at the same time, it is possible that we read + * partially updated contents on some systems. If the CRC doesn't match, + * retry a limited number of times until we compute the same bad CRC twice + * in a row with a short sleep in between. Then the failure is unlikely + * to be due to a concurrent write. + */ + if (!*crc_ok_p && + (retries == 0 || !EQ_CRC32C(crc, last_crc)) && + retries < 10) + { + retries++; + last_crc = crc; + pg_usleep(10000); + goto retry; + } +#endif + /* Make sure the control file is valid byte order. */ if (ControlFile->pg_control_version % 65536 == 0 && ControlFile->pg_control_version / 65536 != 0) From ec3a8f4bd41c57f6d36dc3bb1b39d13f4431ba5a Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 16 Oct 2023 12:57:39 -0400 Subject: [PATCH 070/109] Update the documentation on recovering from (M)XID exhaustion. The old documentation encourages entering single-user mode for no reason, which is a bad plan in most cases. Instead, discourage users from doing that, and explain the limited cases in which it may be desirable. The old documentation claims that running VACUUM as anyone but the superuser can't possibly work, which is not really true, because it might be that some other user has enough permissions to VACUUM all the tables that matter. Weaken the language just a bit. The old documentation claims that you can't run any commands when near XID exhaustion, which is false because you can still run commands that don't require an XID, like a SELECT without a locking clause. The old documentation doesn't clearly explain that it's a good idea to get rid of prepared transactons, long-running transactions, and replication slots that are preventing (M)XID horizon advancement. Spell out the steps to do that. Also, discourage the use of VACUUM FULL and VACUUM FREEZE in this type of scenario. Back-patch to v14. Much of this is good advice on all supported versions, but before 60f1f09ff44308667ef6c72fbafd68235e55ae27 the chances of VACUUM failing in multi-user mode were much higher. Alexander Alekseev, John Naylor, Robert Haas, reviewed at various times by Peter Geoghegan, Hannu Krosing, and Andres Freund. Discussion: http://postgr.es/m/CA+TgmoYtsUDrzaHcmjFhLzTk1VEv29mO_u-MT+XWHrBJ_4nD8A@mail.gmail.com --- doc/src/sgml/maintenance.sgml | 112 +++++++++++++++++++++++++++++----- 1 file changed, 97 insertions(+), 15 deletions(-) diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 0b5860ebbbd..4cd243f361e 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -641,29 +641,79 @@ HINT: To avoid a database shutdown, execute a database-wide VACUUM in that data (A manual VACUUM should fix the problem, as suggested by the - hint; but note that the VACUUM must be performed by a - superuser, else it will fail to process system catalogs and thus not - be able to advance the database's datfrozenxid.) - If these warnings are - ignored, the system will shut down and refuse to start any new - transactions once there are fewer than three million transactions left - until wraparound: + hint; but note that the VACUUM should be performed by a + superuser, else it will fail to process system catalogs, which prevent it from + being able to advance the database's datfrozenxid.) + If these warnings are ignored, the system will refuse to assign new XIDs once + there are fewer than three million transactions left until wraparound: ERROR: database is not accepting commands to avoid wraparound data loss in database "mydb" HINT: Stop the postmaster and vacuum that database in single-user mode. - The three-million-transaction safety margin exists to let the - administrator recover without data loss, by manually executing the - required VACUUM commands. However, since the system will not - execute commands once it has gone into the safety shutdown mode, - the only way to do this is to stop the server and start the server in single-user - mode to execute VACUUM. The shutdown mode is not enforced - in single-user mode. See the reference - page for details about using single-user mode. + In this condition any transactions already in progress can continue, + but only read-only transactions can be started. Operations that + modify database records or truncate relations will fail. + The VACUUM command can still be run normally. + Contrary to what the hint states, it is not necessary or desirable to stop the + postmaster or enter single user-mode in order to restore normal operation. + Instead, follow these steps: + + + + Resolve old prepared transactions. You can find these by checking + pg_prepared_xacts for rows where + age(transactionid) is large. Such transactions should be + committed or rolled back. + + + End long-running open transactions. You can find these by checking + pg_stat_activity for rows where + age(backend_xid) or age(backend_xmin) is + large. Such transactions should be committed or rolled back, or the session + can be terminated using pg_terminate_backend. + + + Drop any old replication slots. Use + pg_stat_replication to + find slots where age(xmin) or age(catalog_xmin) + is large. In many cases, such slots were created for replication to servers that no + longer exist, or that have been down for a long time. If you drop a slot for a server + that still exists and might still try to connect to that slot, that replica may + need to be rebuilt. + + + Execute VACUUM in the target database. A database-wide + VACUUM is simplest; to reduce the time required, it as also possible + to issue manual VACUUM commands on the tables where + relminxid is oldest. Do not use VACUUM FULL + in this scenario, because it requires an XID and will therefore fail, except in super-user + mode, where it will instead consume an XID and thus increase the risk of transaction ID + wraparound. Do not use VACUUM FREEZE either, because it will do + more than the minimum amount of work required to restore normal operation. + + + Once normal operation is restored, ensure that autovacuum is properly configured + in the target database in order to avoid future problems. + + + + + In earlier versions, it was sometimes necessary to stop the postmaster and + VACUUM the database in a single-user mode. In typical scenarios, this + is no longer necessary, and should be avoided whenever possible, since it involves taking + the system down. It is also riskier, since it disables transaction ID wraparound safeguards + that are designed to prevent data loss. The only reason to use single-user mode in this + scenario is if you wish to TRUNCATE or DROP unneeded + tables to avoid needing to VACUUM them. The three-million-transaction + safety margin exists to let the administrator do this. See the + reference page for details about using single-user mode. + + + Multixacts and Wraparound @@ -727,6 +777,38 @@ HINT: Stop the postmaster and vacuum that database in single-user mode. have the oldest multixact-age. Both of these kinds of aggressive scans will occur even if autovacuum is nominally disabled. + + + Similar to the XID case, if autovacuum fails to clear old MXIDs from a table, the + system will begin to emit warning messages when the database's oldest MXIDs reach forty + million transactions from the wraparound point. And, just as an the XID case, if these + warnings are ignored, the system will refuse to generate new MXIDs once there are fewer + than three million left until wraparound. + + + + Normal operation when MXIDs are exhausted can be restored in much the same way as + when XIDs are exhausted. Follow the same steps in the previous section, but with the + following differences: + + + + Running transactions and prepared transactions can be ignored if there + is no chance that they might appear in a multixact. + + + MXID information is not directly visible in system views such as + pg_stat_activity; however, looking for old XIDs is still a good + way of determining which transactions are causing MXID wraparound problems. + + + XID exhaustion will block all write transactions, but MXID exhaustion will + only block a subset of write transactions, specifically those that involve + row locks that require an MXID. + + + + From 68f8472ae929dca0eeb7a9f38dee889d6a1aec85 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 16 Oct 2023 14:06:11 -0400 Subject: [PATCH 071/109] Ensure we have a snapshot while dropping ON COMMIT DROP temp tables. Dropping a temp table could entail TOAST table access to clean out toasted catalog entries, such as large pg_constraint.conbin strings for complex CHECK constraints. If we did that via ON COMMIT DROP, we triggered the assertion in init_toast_snapshot(), because there was no provision for setting up a snapshot for the drop actions. Fix that. (I assume here that the adjacent truncation actions for ON COMMIT DELETE ROWS don't have a similar problem: it doesn't seem like nontransactional truncations would need to touch any toasted fields. If that proves wrong, we could refactor a bit to have the same snapshot acquisition cover that too.) The test case added here does not fail before v15, because that assertion was added in 277692220 which was not back-patched. However, the race condition the assertion warns of surely exists further back, so back-patch to all supported branches. Per report from Richard Guo. Discussion: https://postgr.es/m/CAMbWs4-x26=_QxxgdJyNbiCDzvtr2WV5ZDso_v-CukKEe6cBZw@mail.gmail.com --- src/backend/commands/tablecmds.c | 8 ++++++++ src/test/regress/expected/temp.out | 20 ++++++++++++++++++++ src/test/regress/sql/temp.sql | 16 ++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index af7bc2c1d5c..801391b6f9e 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -20654,6 +20654,12 @@ PreCommit_on_commit_actions(void) add_exact_object_address(&object, targetObjects); } + /* + * Object deletion might involve toast table access (to clean up + * toasted catalog entries), so ensure we have a valid snapshot. + */ + PushActiveSnapshot(GetTransactionSnapshot()); + /* * Since this is an automatic drop, rather than one directly initiated * by the user, we pass the PERFORM_DELETION_INTERNAL flag. @@ -20661,6 +20667,8 @@ PreCommit_on_commit_actions(void) performMultipleDeletions(targetObjects, DROP_CASCADE, PERFORM_DELETION_INTERNAL | PERFORM_DELETION_QUIETLY); + PopActiveSnapshot(); + #ifdef USE_ASSERT_CHECKING /* diff --git a/src/test/regress/expected/temp.out b/src/test/regress/expected/temp.out index 46186faa221..ec705636482 100644 --- a/src/test/regress/expected/temp.out +++ b/src/test/regress/expected/temp.out @@ -108,6 +108,26 @@ SELECT * FROM temptest; 1 (1 row) +COMMIT; +SELECT * FROM temptest; +ERROR: relation "temptest" does not exist +LINE 1: SELECT * FROM temptest; + ^ +-- Test it with a CHECK condition that produces a toasted pg_constraint entry +BEGIN; +do $$ +begin + execute format($cmd$ + CREATE TEMP TABLE temptest (col text CHECK (col < %L)) ON COMMIT DROP + $cmd$, + (SELECT string_agg(g.i::text || ':' || random()::text, '|') + FROM generate_series(1, 100) g(i))); +end$$; +SELECT * FROM temptest; + col +----- +(0 rows) + COMMIT; SELECT * FROM temptest; ERROR: relation "temptest" does not exist diff --git a/src/test/regress/sql/temp.sql b/src/test/regress/sql/temp.sql index 2eeeb4a65e2..75616b53e1a 100644 --- a/src/test/regress/sql/temp.sql +++ b/src/test/regress/sql/temp.sql @@ -101,6 +101,22 @@ COMMIT; SELECT * FROM temptest; +-- Test it with a CHECK condition that produces a toasted pg_constraint entry +BEGIN; +do $$ +begin + execute format($cmd$ + CREATE TEMP TABLE temptest (col text CHECK (col < %L)) ON COMMIT DROP + $cmd$, + (SELECT string_agg(g.i::text || ':' || random()::text, '|') + FROM generate_series(1, 100) g(i))); +end$$; + +SELECT * FROM temptest; +COMMIT; + +SELECT * FROM temptest; + -- ON COMMIT is only allowed for TEMP CREATE TABLE temptest(col int) ON COMMIT DELETE ROWS; From 52e789302b79ae082c4aad5beccc9497324e903c Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Tue, 17 Oct 2023 10:42:12 -0500 Subject: [PATCH 072/109] Avoid calling proc_exit() in processes forked by system(). The SIGTERM handler for the startup process immediately calls proc_exit() for the duration of the restore_command, i.e., a call to system(). This system() call forks a new process to execute the shell command, and this child process inherits the parent's signal handlers. If both the parent and child processes receive SIGTERM, both will attempt to call proc_exit(). This can end badly. For example, both processes will try to remove themselves from the PGPROC shared array. To fix this problem, this commit adds a check in StartupProcShutdownHandler() to see whether MyProcPid == getpid(). If they match, this is the parent process, and we can proc_exit() like before. If they do not match, this is a child process, and we just emit a message to STDERR (in a signal safe manner) and _exit(), thereby skipping any problematic exit callbacks. This commit also adds checks in proc_exit(), ProcKill(), and AuxiliaryProcKill() that verify they are not being called within such child processes. Suggested-by: Andres Freund Reviewed-by: Thomas Munro, Andres Freund Discussion: https://postgr.es/m/Y9nGDSgIm83FHcad%40paquier.xyz Discussion: https://postgr.es/m/20230223231503.GA743455%40nathanxps13 Backpatch-through: 11 --- src/backend/postmaster/startup.c | 17 ++++++++++++++++- src/backend/utils/error/elog.c | 28 ++++++++++++++++++++++++++++ src/include/utils/elog.h | 5 +++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/backend/postmaster/startup.c b/src/backend/postmaster/startup.c index c8752b5cc79..a9fbf6d52d8 100644 --- a/src/backend/postmaster/startup.c +++ b/src/backend/postmaster/startup.c @@ -19,6 +19,8 @@ */ #include "postgres.h" +#include + #include "access/xlog.h" #include "libpq/pqsignal.h" #include "miscadmin.h" @@ -103,7 +105,20 @@ StartupProcShutdownHandler(SIGNAL_ARGS) int save_errno = errno; if (in_restore_command) - proc_exit(1); + { + /* + * If we are in a child process (e.g., forked by system() in + * RestoreArchivedFile()), we don't want to call any exit callbacks. + * The parent will take care of that. + */ + if (MyProcPid == (int) getpid()) + proc_exit(1); + else + { + write_stderr_signal_safe("StartupProcShutdownHandler() called in child process\n"); + _exit(1); + } + } else shutdown_requested = true; WakeupRecovery(); diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 859ac9e56fa..2f78f5750c2 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -5056,6 +5056,34 @@ write_stderr(const char *fmt,...) va_end(ap); } +/* + * Write a message to STDERR using only async-signal-safe functions. This can + * be used to safely emit a message from a signal handler. + * + * TODO: It is likely possible to safely do a limited amount of string + * interpolation (e.g., %s and %d), but that is not presently supported. + */ +void +write_stderr_signal_safe(const char *str) +{ + int nwritten = 0; + int ntotal = strlen(str); + + while (nwritten < ntotal) + { + int rc; + + rc = write(STDERR_FILENO, str + nwritten, ntotal - nwritten); + + /* Just give up on error. There isn't much else we can do. */ + if (rc == -1) + return; + + nwritten += rc; + } +} + + /* * Adjust the level of a recovery-related message per trace_recovery_messages. * diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h index 0f61a3ae8b9..534d9f0bef4 100644 --- a/src/include/utils/elog.h +++ b/src/include/utils/elog.h @@ -607,5 +607,10 @@ extern bool gp_log_stack_trace_lines; /* session GUC, controls line info in st extern const char *SegvBusIllName(int signal); extern void StandardHandlerForSigillSigsegvSigbus_OnMainThread(char * processName, SIGNAL_ARGS); +/* + * Write a message to STDERR using only async-signal-safe functions. This can + * be used to safely emit a message from a signal handler. + */ +extern void write_stderr_signal_safe(const char *fmt); #endif /* ELOG_H */ From 2b05121d9a7c797a7d8c4c85ab945a49f288189e Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 17 Oct 2023 13:55:45 -0400 Subject: [PATCH 073/109] Back-patch test cases for timetz_zone/timetz_izone. Per code coverage reports, we had zero regression test coverage of these functions. That came back to bite us, as apparently that's allowed us to miss discovering misbehavior of this code with AIX's xlc compiler. Install relevant portions of the test cases added in 97957fdba, 2f0472030, 19fa97731. (Assuming the expected outcome that the xlc problem does appear in back branches, a code fix will follow.) Discussion: https://postgr.es/m/CA+hUKGK=DOC+hE-62FKfZy=Ybt5uLkrg3zCZD-jFykM-iPn8yw@mail.gmail.com --- src/test/regress/expected/timetz.out | 60 ++++++++++++++++++++++++++++ src/test/regress/sql/timetz.sql | 20 ++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/test/regress/expected/timetz.out b/src/test/regress/expected/timetz.out index 62084eb5f64..7f0e5e64ac8 100644 --- a/src/test/regress/expected/timetz.out +++ b/src/test/regress/expected/timetz.out @@ -275,3 +275,63 @@ SELECT date_part('epoch', TIME WITH TIME ZONE '2020-05-26 13:30:25.575401- 63025.575401 (1 row) +-- +-- Test timetz_zone, timetz_izone +-- +BEGIN; +SET LOCAL TimeZone TO 'UTC'; +CREATE VIEW timetz_local_view AS + SELECT f1 AS dat, + f1 AT TIME ZONE current_setting('TimeZone') AS dat_at_tz, + f1 AT TIME ZONE INTERVAL '00:00' AS dat_at_int + FROM TIMETZ_TBL + ORDER BY f1; +SELECT pg_get_viewdef('timetz_local_view', true); + pg_get_viewdef +---------------------------------------------------------------------------------- + SELECT timetz_tbl.f1 AS dat, + + (timetz_tbl.f1 AT TIME ZONE current_setting('TimeZone'::text)) AS dat_at_tz,+ + (timetz_tbl.f1 AT TIME ZONE '@ 0'::interval) AS dat_at_int + + FROM timetz_tbl + + ORDER BY timetz_tbl.f1; +(1 row) + +TABLE timetz_local_view; + dat | dat_at_tz | dat_at_int +----------------+----------------+---------------- + 00:01:00-07 | 07:01:00+00 | 07:01:00+00 + 01:00:00-07 | 08:00:00+00 | 08:00:00+00 + 02:03:00-07 | 09:03:00+00 | 09:03:00+00 + 08:08:00-04 | 12:08:00+00 | 12:08:00+00 + 07:07:00-08 | 15:07:00+00 | 15:07:00+00 + 11:59:00-07 | 18:59:00+00 | 18:59:00+00 + 12:00:00-07 | 19:00:00+00 | 19:00:00+00 + 12:01:00-07 | 19:01:00+00 | 19:01:00+00 + 15:36:39-04 | 19:36:39+00 | 19:36:39+00 + 15:36:39-05 | 20:36:39+00 | 20:36:39+00 + 23:59:00-07 | 06:59:00+00 | 06:59:00+00 + 23:59:59.99-07 | 06:59:59.99+00 | 06:59:59.99+00 +(12 rows) + +SELECT f1 AS dat, + f1 AT TIME ZONE 'UTC+10' AS dat_at_tz, + f1 AT TIME ZONE INTERVAL '-10:00' AS dat_at_int + FROM TIMETZ_TBL + ORDER BY f1; + dat | dat_at_tz | dat_at_int +----------------+----------------+---------------- + 00:01:00-07 | 21:01:00-10 | 21:01:00-10 + 01:00:00-07 | 22:00:00-10 | 22:00:00-10 + 02:03:00-07 | 23:03:00-10 | 23:03:00-10 + 08:08:00-04 | 02:08:00-10 | 02:08:00-10 + 07:07:00-08 | 05:07:00-10 | 05:07:00-10 + 11:59:00-07 | 08:59:00-10 | 08:59:00-10 + 12:00:00-07 | 09:00:00-10 | 09:00:00-10 + 12:01:00-07 | 09:01:00-10 | 09:01:00-10 + 15:36:39-04 | 09:36:39-10 | 09:36:39-10 + 15:36:39-05 | 10:36:39-10 | 10:36:39-10 + 23:59:00-07 | 20:59:00-10 | 20:59:00-10 + 23:59:59.99-07 | 20:59:59.99-10 | 20:59:59.99-10 +(12 rows) + +ROLLBACK; diff --git a/src/test/regress/sql/timetz.sql b/src/test/regress/sql/timetz.sql index 308e095eb42..eca8ceab12b 100644 --- a/src/test/regress/sql/timetz.sql +++ b/src/test/regress/sql/timetz.sql @@ -92,3 +92,23 @@ SELECT date_part('microsecond', TIME WITH TIME ZONE '2020-05-26 13:30:25.575401- SELECT date_part('millisecond', TIME WITH TIME ZONE '2020-05-26 13:30:25.575401-04'); SELECT date_part('second', TIME WITH TIME ZONE '2020-05-26 13:30:25.575401-04'); SELECT date_part('epoch', TIME WITH TIME ZONE '2020-05-26 13:30:25.575401-04'); + +-- +-- Test timetz_zone, timetz_izone +-- +BEGIN; +SET LOCAL TimeZone TO 'UTC'; +CREATE VIEW timetz_local_view AS + SELECT f1 AS dat, + f1 AT TIME ZONE current_setting('TimeZone') AS dat_at_tz, + f1 AT TIME ZONE INTERVAL '00:00' AS dat_at_int + FROM TIMETZ_TBL + ORDER BY f1; +SELECT pg_get_viewdef('timetz_local_view', true); +TABLE timetz_local_view; +SELECT f1 AS dat, + f1 AT TIME ZONE 'UTC+10' AS dat_at_tz, + f1 AT TIME ZONE INTERVAL '-10:00' AS dat_at_int + FROM TIMETZ_TBL + ORDER BY f1; +ROLLBACK; From 9362091cdb18cfa21a005eeaec988c6e776757b8 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Tue, 17 Oct 2023 16:11:03 -0500 Subject: [PATCH 074/109] windows: msvc: Define STDIN/OUT/ERR_FILENO. This commit (c290e79cf0) was originally back-patched to v15. Commit 97550c0711 added a new use of STDERR_FILENO, and it was back-patched all the way to v11, thus breaking MSVC builds for v11 through v14. Since STDERR_FILENO is now needed on older versions, let's back-patch c290e79cf0 down to v11, too. Here follows the original commit message describing the change: Because they are not available we've used _fileno(stdin) in some places, but that doesn't reliably work, because stdin might be closed. This is the prerequisite of the subsequent commit, fixing a failure introduced in 76e38b37a5. Author: Andres Freund Reported-By: Sandeep Thakkar Message-Id: 20220520164558.ozb7lm6unakqzezi@alap3.anarazel.de (on pgsql-packagers) Discussion: https://postgr.es/m/20231017164517.GA613565%40nathanxps13 Backpatch-through: 11 --- src/include/port/win32_msvc/unistd.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/include/port/win32_msvc/unistd.h b/src/include/port/win32_msvc/unistd.h index b63f4770a1c..b7795ba03c4 100644 --- a/src/include/port/win32_msvc/unistd.h +++ b/src/include/port/win32_msvc/unistd.h @@ -1 +1,9 @@ /* src/include/port/win32_msvc/unistd.h */ + +/* + * MSVC does not define these, nor does _fileno(stdin) etc reliably work + * (returns -1 if stdin/out/err are closed). + */ +#define STDIN_FILENO 0 +#define STDOUT_FILENO 1 +#define STDERR_FILENO 2 From 8452699e2c662c9757669e7368071608f7007e49 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Wed, 18 Oct 2023 22:09:05 +1300 Subject: [PATCH 075/109] jit: Support opaque pointers in LLVM 16. Remove use of LLVMGetElementType() and provide the type of all pointers to LLVMBuildXXX() functions when emitting IR, as required by modern LLVM versions[1]. * For LLVM <= 14, we'll still use the old LLVMBuildXXX() functions. * For LLVM == 15, we'll continue to do the same, explicitly opting out of opaque pointer mode. * For LLVM >= 16, we'll use the new LLVMBuildXXX2() functions that take the extra type argument. The difference is hidden behind some new IR emitting wrapper functions l_load(), l_gep(), l_call() etc. The change is mostly mechanical, except that at each site the correct type had to be provided. In some places we needed to do some extra work to get functions types, including some new wrappers for C++ APIs that are not yet exposed by in LLVM's C API, and some new "example" functions in llvmjit_types.c because it's no longer possible to start from the function pointer type and ask for the function type. Back-patch to 12, because it's a little tricker in 11 and we agreed not to put the latest LLVM support into the upcoming final release of 11. [1] https://llvm.org/docs/OpaquePointers.html Reviewed-by: Dmitry Dolgov <9erthalion6@gmail.com> Reviewed-by: Ronan Dunklau Reviewed-by: Andres Freund Discussion: https://postgr.es/m/CA%2BhUKGKNX_%3Df%2B1C4r06WETKTq0G4Z_7q4L4Fxn5WWpMycDj9Fw%40mail.gmail.com --- src/backend/jit/llvm/llvmjit.c | 57 ++-- src/backend/jit/llvm/llvmjit_deform.c | 119 ++++---- src/backend/jit/llvm/llvmjit_expr.c | 397 ++++++++++++++++---------- src/backend/jit/llvm/llvmjit_types.c | 39 ++- src/backend/jit/llvm/llvmjit_wrap.cpp | 12 + src/include/jit/llvmjit.h | 6 + src/include/jit/llvmjit_emit.h | 106 +++++-- 7 files changed, 475 insertions(+), 261 deletions(-) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index a649deab4b8..a58ac6d11db 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -85,8 +85,11 @@ LLVMTypeRef StructExprState; LLVMTypeRef StructAggState; LLVMTypeRef StructAggStatePerGroupData; LLVMTypeRef StructAggStatePerTransData; +LLVMTypeRef StructPlanState; LLVMValueRef AttributeTemplate; +LLVMValueRef ExecEvalSubroutineTemplate; +LLVMValueRef ExecEvalBoolSubroutineTemplate; LLVMModuleRef llvm_types_module = NULL; @@ -384,11 +387,7 @@ llvm_pg_var_type(const char *varname) if (!v_srcvar) elog(ERROR, "variable %s not in llvmjit_types.c", varname); - /* look at the contained type */ - typ = LLVMTypeOf(v_srcvar); - Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMPointerTypeKind); - typ = LLVMGetElementType(typ); - Assert(typ != NULL); + typ = LLVMGlobalGetValueType(v_srcvar); return typ; } @@ -400,12 +399,14 @@ llvm_pg_var_type(const char *varname) LLVMTypeRef llvm_pg_var_func_type(const char *varname) { - LLVMTypeRef typ = llvm_pg_var_type(varname); + LLVMValueRef v_srcvar; + LLVMTypeRef typ; + + v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname); + if (!v_srcvar) + elog(ERROR, "function %s not in llvmjit_types.c", varname); - /* look at the contained type */ - Assert(LLVMGetTypeKind(typ) == LLVMPointerTypeKind); - typ = LLVMGetElementType(typ); - Assert(typ != NULL && LLVMGetTypeKind(typ) == LLVMFunctionTypeKind); + typ = LLVMGetFunctionType(v_srcvar); return typ; } @@ -435,7 +436,7 @@ llvm_pg_func(LLVMModuleRef mod, const char *funcname) v_fn = LLVMAddFunction(mod, funcname, - LLVMGetElementType(LLVMTypeOf(v_srcfn))); + LLVMGetFunctionType(v_srcfn)); llvm_copy_attributes(v_srcfn, v_fn); return v_fn; @@ -531,7 +532,7 @@ llvm_function_reference(LLVMJitContext *context, fcinfo->flinfo->fn_oid); v_fn = LLVMGetNamedGlobal(mod, funcname); if (v_fn != 0) - return LLVMBuildLoad(builder, v_fn, ""); + return l_load(builder, TypePGFunction, v_fn, ""); v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction); @@ -541,7 +542,7 @@ llvm_function_reference(LLVMJitContext *context, LLVMSetLinkage(v_fn, LLVMPrivateLinkage); LLVMSetUnnamedAddr(v_fn, true); - return LLVMBuildLoad(builder, v_fn, ""); + return l_load(builder, TypePGFunction, v_fn, ""); } /* check if function already has been added */ @@ -549,7 +550,7 @@ llvm_function_reference(LLVMJitContext *context, if (v_fn != 0) return v_fn; - v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction)); + v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate)); return v_fn; } @@ -801,12 +802,15 @@ llvm_session_initialize(void) LLVMInitializeNativeAsmParser(); /* - * When targeting an LLVM version with opaque pointers enabled by - * default, turn them off for the context we build our code in. We don't - * need to do so for other contexts (e.g. llvm_ts_context). Once the IR is - * generated, it carries the necessary information. + * When targeting LLVM 15, turn off opaque pointers for the context we + * build our code in. We don't need to do so for other contexts (e.g. + * llvm_ts_context). Once the IR is generated, it carries the necessary + * information. + * + * For 16 and above, opaque pointers must be used, and we have special + * code for that. */ -#if LLVM_VERSION_MAJOR > 14 +#if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false); #endif @@ -968,15 +972,7 @@ load_return_type(LLVMModuleRef mod, const char *name) if (!value) elog(ERROR, "function %s is unknown", name); - /* get type of function pointer */ - typ = LLVMTypeOf(value); - Assert(typ != NULL); - /* dereference pointer */ - typ = LLVMGetElementType(typ); - Assert(typ != NULL); - /* and look at return type */ - typ = LLVMGetReturnType(typ); - Assert(typ != NULL); + typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */ return typ; } @@ -1031,12 +1027,17 @@ llvm_create_types(void) StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot"); StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot"); StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData"); + StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData"); StructTupleDescData = llvm_pg_var_type("StructTupleDescData"); StructAggState = llvm_pg_var_type("StructAggState"); StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData"); StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData"); + StructPlanState = llvm_pg_var_type("StructPlanState"); + StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData"); AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate"); + ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate"); + ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate"); } /* diff --git a/src/backend/jit/llvm/llvmjit_deform.c b/src/backend/jit/llvm/llvmjit_deform.c index 008cd617f6c..60a677ba439 100644 --- a/src/backend/jit/llvm/llvmjit_deform.c +++ b/src/backend/jit/llvm/llvmjit_deform.c @@ -171,13 +171,13 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_slot = LLVMGetParam(v_deform_fn, 0); v_tts_values = - l_load_struct_gep(b, v_slot, FIELDNO_TUPLETABLESLOT_VALUES, + l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES, "tts_values"); v_tts_nulls = - l_load_struct_gep(b, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL, + l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL, "tts_ISNULL"); - v_flagsp = LLVMBuildStructGEP(b, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, ""); - v_nvalidp = LLVMBuildStructGEP(b, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, ""); + v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, ""); + v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, ""); if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple) { @@ -188,9 +188,9 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_slot, l_ptr(StructHeapTupleTableSlot), "heapslot"); - v_slotoffp = LLVMBuildStructGEP(b, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, ""); + v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, ""); v_tupleheaderp = - l_load_struct_gep(b, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE, + l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE, "tupleheader"); } @@ -203,9 +203,15 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_slot, l_ptr(StructMinimalTupleTableSlot), "minimalslot"); - v_slotoffp = LLVMBuildStructGEP(b, v_minimalslot, FIELDNO_MINIMALTUPLETABLESLOT_OFF, ""); + v_slotoffp = l_struct_gep(b, + StructMinimalTupleTableSlot, + v_minimalslot, + FIELDNO_MINIMALTUPLETABLESLOT_OFF, ""); v_tupleheaderp = - l_load_struct_gep(b, v_minimalslot, FIELDNO_MINIMALTUPLETABLESLOT_TUPLE, + l_load_struct_gep(b, + StructMinimalTupleTableSlot, + v_minimalslot, + FIELDNO_MINIMALTUPLETABLESLOT_TUPLE, "tupleheader"); } else @@ -215,21 +221,29 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, } v_tuplep = - l_load_struct_gep(b, v_tupleheaderp, FIELDNO_HEAPTUPLEDATA_DATA, + l_load_struct_gep(b, + StructHeapTupleData, + v_tupleheaderp, + FIELDNO_HEAPTUPLEDATA_DATA, "tuple"); v_bits = LLVMBuildBitCast(b, - LLVMBuildStructGEP(b, v_tuplep, - FIELDNO_HEAPTUPLEHEADERDATA_BITS, - ""), + l_struct_gep(b, + StructHeapTupleHeaderData, + v_tuplep, + FIELDNO_HEAPTUPLEHEADERDATA_BITS, + ""), l_ptr(LLVMInt8Type()), "t_bits"); v_infomask1 = - l_load_struct_gep(b, v_tuplep, + l_load_struct_gep(b, + StructHeapTupleHeaderData, + v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK, "infomask1"); v_infomask2 = l_load_struct_gep(b, + StructHeapTupleHeaderData, v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2, "infomask2"); @@ -254,19 +268,21 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, */ v_hoff = LLVMBuildZExt(b, - l_load_struct_gep(b, v_tuplep, + l_load_struct_gep(b, + StructHeapTupleHeaderData, + v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_HOFF, ""), LLVMInt32Type(), "t_hoff"); - v_tupdata_base = - LLVMBuildGEP(b, - LLVMBuildBitCast(b, - v_tuplep, - l_ptr(LLVMInt8Type()), - ""), - &v_hoff, 1, - "v_tupdata_base"); + v_tupdata_base = l_gep(b, + LLVMInt8Type(), + LLVMBuildBitCast(b, + v_tuplep, + l_ptr(LLVMInt8Type()), + ""), + &v_hoff, 1, + "v_tupdata_base"); /* * Load tuple start offset from slot. Will be reset below in case there's @@ -275,7 +291,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, { LLVMValueRef v_off_start; - v_off_start = LLVMBuildLoad(b, v_slotoffp, "v_slot_off"); + v_off_start = l_load(b, LLVMInt32Type(), v_slotoffp, "v_slot_off"); v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, ""); LLVMBuildStore(b, v_off_start, v_offp); } @@ -315,6 +331,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, else { LLVMValueRef v_params[3]; + LLVMValueRef f; /* branch if not all columns available */ LLVMBuildCondBr(b, @@ -331,14 +348,16 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_params[0] = v_slot; v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32Type(), ""); v_params[2] = l_int32_const(natts); - LLVMBuildCall(b, llvm_pg_func(mod, "slot_getmissingattrs"), - v_params, lengthof(v_params), ""); + f = llvm_pg_func(mod, "slot_getmissingattrs"); + l_call(b, + LLVMGetFunctionType(f), f, + v_params, lengthof(v_params), ""); LLVMBuildBr(b, b_find_start); } LLVMPositionBuilderAtEnd(b, b_find_start); - v_nvalid = LLVMBuildLoad(b, v_nvalidp, ""); + v_nvalid = l_load(b, LLVMInt16Type(), v_nvalidp, ""); /* * Build switch to go from nvalid to the right startblock. Callers @@ -440,7 +459,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, v_nullbyteno = l_int32_const(attnum >> 3); v_nullbytemask = l_int8_const(1 << ((attnum) & 0x07)); - v_nullbyte = l_load_gep1(b, v_bits, v_nullbyteno, "attnullbyte"); + v_nullbyte = l_load_gep1(b, LLVMInt8Type(), v_bits, v_nullbyteno, "attnullbyte"); v_nullbit = LLVMBuildICmp(b, LLVMIntEQ, @@ -457,11 +476,11 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, /* store null-byte */ LLVMBuildStore(b, l_int8_const(1), - LLVMBuildGEP(b, v_tts_nulls, &l_attno, 1, "")); + l_gep(b, LLVMInt8Type(), v_tts_nulls, &l_attno, 1, "")); /* store zero datum */ LLVMBuildStore(b, l_sizet_const(0), - LLVMBuildGEP(b, v_tts_values, &l_attno, 1, "")); + l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, "")); LLVMBuildBr(b, b_next); attguaranteedalign = false; @@ -520,10 +539,10 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, /* don't know if short varlena or not */ attguaranteedalign = false; - v_off = LLVMBuildLoad(b, v_offp, ""); + v_off = l_load(b, TypeSizeT, v_offp, ""); v_possible_padbyte = - l_load_gep1(b, v_tupdata_base, v_off, "padbyte"); + l_load_gep1(b, LLVMInt8Type(), v_tupdata_base, v_off, "padbyte"); v_ispad = LLVMBuildICmp(b, LLVMIntEQ, v_possible_padbyte, l_int8_const(0), @@ -542,7 +561,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, /* translation of alignment code (cf TYPEALIGN()) */ { LLVMValueRef v_off_aligned; - LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, ""); + LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); /* ((ALIGNVAL) - 1) */ LLVMValueRef v_alignval = l_sizet_const(alignto - 1); @@ -631,18 +650,18 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, /* compute address to load data from */ { - LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, ""); + LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); v_attdatap = - LLVMBuildGEP(b, v_tupdata_base, &v_off, 1, ""); + l_gep(b, LLVMInt8Type(), v_tupdata_base, &v_off, 1, ""); } /* compute address to store value at */ - v_resultp = LLVMBuildGEP(b, v_tts_values, &l_attno, 1, ""); + v_resultp = l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, ""); /* store null-byte (false) */ LLVMBuildStore(b, l_int8_const(0), - LLVMBuildGEP(b, v_tts_nulls, &l_attno, 1, "")); + l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, "")); /* * Store datum. For byval: datums copy the value, extend to Datum's @@ -651,12 +670,12 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, if (att->attbyval) { LLVMValueRef v_tmp_loaddata; - LLVMTypeRef vartypep = - LLVMPointerType(LLVMIntType(att->attlen * 8), 0); + LLVMTypeRef vartype = LLVMIntType(att->attlen * 8); + LLVMTypeRef vartypep = LLVMPointerType(vartype, 0); v_tmp_loaddata = LLVMBuildPointerCast(b, v_attdatap, vartypep, ""); - v_tmp_loaddata = LLVMBuildLoad(b, v_tmp_loaddata, "attr_byval"); + v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval"); v_tmp_loaddata = LLVMBuildZExt(b, v_tmp_loaddata, TypeSizeT, ""); LLVMBuildStore(b, v_tmp_loaddata, v_resultp); @@ -681,18 +700,20 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, } else if (att->attlen == -1) { - v_incby = LLVMBuildCall(b, - llvm_pg_func(mod, "varsize_any"), - &v_attdatap, 1, - "varsize_any"); + v_incby = l_call(b, + llvm_pg_var_func_type("varsize_any"), + llvm_pg_func(mod, "varsize_any"), + &v_attdatap, 1, + "varsize_any"); l_callsite_ro(v_incby); l_callsite_alwaysinline(v_incby); } else if (att->attlen == -2) { - v_incby = LLVMBuildCall(b, - llvm_pg_func(mod, "strlen"), - &v_attdatap, 1, "strlen"); + v_incby = l_call(b, + llvm_pg_var_func_type("strlen"), + llvm_pg_func(mod, "strlen"), + &v_attdatap, 1, "strlen"); l_callsite_ro(v_incby); @@ -712,7 +733,7 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, } else { - LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, ""); + LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset"); LLVMBuildStore(b, v_off, v_offp); @@ -738,13 +759,13 @@ slot_compile_deform(LLVMJitContext *context, TupleDesc desc, LLVMPositionBuilderAtEnd(b, b_out); { - LLVMValueRef v_off = LLVMBuildLoad(b, v_offp, ""); + LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); LLVMValueRef v_flags; LLVMBuildStore(b, l_int16_const(natts), v_nvalidp); v_off = LLVMBuildTrunc(b, v_off, LLVMInt32Type(), ""); LLVMBuildStore(b, v_off, v_slotoffp); - v_flags = LLVMBuildLoad(b, v_flagsp, "tts_flags"); + v_flags = l_load(b, LLVMInt16Type(), v_flagsp, "tts_flags"); v_flags = LLVMBuildOr(b, v_flags, l_int16_const(TTS_FLAG_SLOW), ""); LLVMBuildStore(b, v_flags, v_flagsp); LLVMBuildRetVoid(b); diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c index 84c5f5980ab..b1c29039468 100644 --- a/src/backend/jit/llvm/llvmjit_expr.c +++ b/src/backend/jit/llvm/llvmjit_expr.c @@ -150,7 +150,7 @@ llvm_compile_expr(ExprState *state) /* create function */ eval_fn = LLVMAddFunction(mod, funcname, - llvm_pg_var_func_type("TypeExprStateEvalFunc")); + llvm_pg_var_func_type("ExecInterpExprStillValid")); LLVMSetLinkage(eval_fn, LLVMExternalLinkage); LLVMSetVisibility(eval_fn, LLVMDefaultVisibility); llvm_copy_attributes(AttributeTemplate, eval_fn); @@ -164,61 +164,95 @@ llvm_compile_expr(ExprState *state) LLVMPositionBuilderAtEnd(b, entry); - v_tmpvaluep = LLVMBuildStructGEP(b, v_state, - FIELDNO_EXPRSTATE_RESVALUE, - "v.state.resvalue"); - v_tmpisnullp = LLVMBuildStructGEP(b, v_state, - FIELDNO_EXPRSTATE_RESNULL, - "v.state.resnull"); - v_parent = l_load_struct_gep(b, v_state, + v_tmpvaluep = l_struct_gep(b, + StructExprState, + v_state, + FIELDNO_EXPRSTATE_RESVALUE, + "v.state.resvalue"); + v_tmpisnullp = l_struct_gep(b, + StructExprState, + v_state, + FIELDNO_EXPRSTATE_RESNULL, + "v.state.resnull"); + v_parent = l_load_struct_gep(b, + StructExprState, + v_state, FIELDNO_EXPRSTATE_PARENT, "v.state.parent"); /* build global slots */ - v_scanslot = l_load_struct_gep(b, v_econtext, + v_scanslot = l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_SCANTUPLE, "v_scanslot"); - v_innerslot = l_load_struct_gep(b, v_econtext, + v_innerslot = l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_INNERTUPLE, "v_innerslot"); - v_outerslot = l_load_struct_gep(b, v_econtext, + v_outerslot = l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_OUTERTUPLE, "v_outerslot"); - v_resultslot = l_load_struct_gep(b, v_state, + v_resultslot = l_load_struct_gep(b, + StructExprState, + v_state, FIELDNO_EXPRSTATE_RESULTSLOT, "v_resultslot"); /* build global values/isnull pointers */ - v_scanvalues = l_load_struct_gep(b, v_scanslot, + v_scanvalues = l_load_struct_gep(b, + StructTupleTableSlot, + v_scanslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_scanvalues"); - v_scannulls = l_load_struct_gep(b, v_scanslot, + v_scannulls = l_load_struct_gep(b, + StructTupleTableSlot, + v_scanslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_scannulls"); - v_innervalues = l_load_struct_gep(b, v_innerslot, + v_innervalues = l_load_struct_gep(b, + StructTupleTableSlot, + v_innerslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_innervalues"); - v_innernulls = l_load_struct_gep(b, v_innerslot, + v_innernulls = l_load_struct_gep(b, + StructTupleTableSlot, + v_innerslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_innernulls"); - v_outervalues = l_load_struct_gep(b, v_outerslot, + v_outervalues = l_load_struct_gep(b, + StructTupleTableSlot, + v_outerslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_outervalues"); - v_outernulls = l_load_struct_gep(b, v_outerslot, + v_outernulls = l_load_struct_gep(b, + StructTupleTableSlot, + v_outerslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_outernulls"); - v_resultvalues = l_load_struct_gep(b, v_resultslot, + v_resultvalues = l_load_struct_gep(b, + StructTupleTableSlot, + v_resultslot, FIELDNO_TUPLETABLESLOT_VALUES, "v_resultvalues"); - v_resultnulls = l_load_struct_gep(b, v_resultslot, + v_resultnulls = l_load_struct_gep(b, + StructTupleTableSlot, + v_resultslot, FIELDNO_TUPLETABLESLOT_ISNULL, "v_resultnulls"); /* aggvalues/aggnulls */ - v_aggvalues = l_load_struct_gep(b, v_econtext, + v_aggvalues = l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_AGGVALUES, "v.econtext.aggvalues"); - v_aggnulls = l_load_struct_gep(b, v_econtext, + v_aggnulls = l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_AGGNULLS, "v.econtext.aggnulls"); @@ -252,8 +286,8 @@ llvm_compile_expr(ExprState *state) LLVMValueRef v_tmpisnull; LLVMValueRef v_tmpvalue; - v_tmpvalue = LLVMBuildLoad(b, v_tmpvaluep, ""); - v_tmpisnull = LLVMBuildLoad(b, v_tmpisnullp, ""); + v_tmpvalue = l_load(b, TypeSizeT, v_tmpvaluep, ""); + v_tmpisnull = l_load(b, TypeStorageBool, v_tmpisnullp, ""); LLVMBuildStore(b, v_tmpisnull, v_isnullp); @@ -296,7 +330,9 @@ llvm_compile_expr(ExprState *state) * whether deforming is required. */ v_nvalid = - l_load_struct_gep(b, v_slot, + l_load_struct_gep(b, + StructTupleTableSlot, + v_slot, FIELDNO_TUPLETABLESLOT_NVALID, ""); LLVMBuildCondBr(b, @@ -327,8 +363,10 @@ llvm_compile_expr(ExprState *state) params[0] = v_slot; - LLVMBuildCall(b, l_jit_deform, - params, lengthof(params), ""); + l_call(b, + LLVMGetFunctionType(l_jit_deform), + l_jit_deform, + params, lengthof(params), ""); } else { @@ -337,9 +375,10 @@ llvm_compile_expr(ExprState *state) params[0] = v_slot; params[1] = l_int32_const(op->d.fetch.last_var); - LLVMBuildCall(b, - llvm_pg_func(mod, "slot_getsomeattrs_int"), - params, lengthof(params), ""); + l_call(b, + llvm_pg_var_func_type("slot_getsomeattrs_int"), + llvm_pg_func(mod, "slot_getsomeattrs_int"), + params, lengthof(params), ""); } LLVMBuildBr(b, opblocks[opno + 1]); @@ -373,8 +412,8 @@ llvm_compile_expr(ExprState *state) } v_attnum = l_int32_const(op->d.var.attnum); - value = l_load_gep1(b, v_values, v_attnum, ""); - isnull = l_load_gep1(b, v_nulls, v_attnum, ""); + value = l_load_gep1(b, TypeSizeT, v_values, v_attnum, ""); + isnull = l_load_gep1(b, TypeStorageBool, v_nulls, v_attnum, ""); LLVMBuildStore(b, value, v_resvaluep); LLVMBuildStore(b, isnull, v_resnullp); @@ -439,15 +478,19 @@ llvm_compile_expr(ExprState *state) /* load data */ v_attnum = l_int32_const(op->d.assign_var.attnum); - v_value = l_load_gep1(b, v_values, v_attnum, ""); - v_isnull = l_load_gep1(b, v_nulls, v_attnum, ""); + v_value = l_load_gep1(b, TypeSizeT, v_values, v_attnum, ""); + v_isnull = l_load_gep1(b, TypeStorageBool, v_nulls, v_attnum, ""); /* compute addresses of targets */ v_resultnum = l_int32_const(op->d.assign_var.resultnum); - v_rvaluep = LLVMBuildGEP(b, v_resultvalues, - &v_resultnum, 1, ""); - v_risnullp = LLVMBuildGEP(b, v_resultnulls, - &v_resultnum, 1, ""); + v_rvaluep = l_gep(b, + TypeSizeT, + v_resultvalues, + &v_resultnum, 1, ""); + v_risnullp = l_gep(b, + TypeStorageBool, + v_resultnulls, + &v_resultnum, 1, ""); /* and store */ LLVMBuildStore(b, v_value, v_rvaluep); @@ -468,15 +511,15 @@ llvm_compile_expr(ExprState *state) size_t resultnum = op->d.assign_tmp.resultnum; /* load data */ - v_value = LLVMBuildLoad(b, v_tmpvaluep, ""); - v_isnull = LLVMBuildLoad(b, v_tmpisnullp, ""); + v_value = l_load(b, TypeSizeT, v_tmpvaluep, ""); + v_isnull = l_load(b, TypeStorageBool, v_tmpisnullp, ""); /* compute addresses of targets */ v_resultnum = l_int32_const(resultnum); v_rvaluep = - LLVMBuildGEP(b, v_resultvalues, &v_resultnum, 1, ""); + l_gep(b, TypeSizeT, v_resultvalues, &v_resultnum, 1, ""); v_risnullp = - LLVMBuildGEP(b, v_resultnulls, &v_resultnum, 1, ""); + l_gep(b, TypeStorageBool, v_resultnulls, &v_resultnum, 1, ""); /* store nullness */ LLVMBuildStore(b, v_isnull, v_risnullp); @@ -500,9 +543,10 @@ llvm_compile_expr(ExprState *state) LLVMPositionBuilderAtEnd(b, b_notnull); v_params[0] = v_value; v_value = - LLVMBuildCall(b, - llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), - v_params, lengthof(v_params), ""); + l_call(b, + llvm_pg_var_func_type("MakeExpandedObjectReadOnlyInternal"), + llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), + v_params, lengthof(v_params), ""); /* * Falling out of the if () with builder in b_notnull, @@ -665,8 +709,8 @@ llvm_compile_expr(ExprState *state) if (opcode == EEOP_BOOL_AND_STEP_FIRST) LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp); - v_boolnull = LLVMBuildLoad(b, v_resnullp, ""); - v_boolvalue = LLVMBuildLoad(b, v_resvaluep, ""); + v_boolnull = l_load(b, TypeStorageBool, v_resnullp, ""); + v_boolvalue = l_load(b, TypeSizeT, v_resvaluep, ""); /* set resnull to boolnull */ LLVMBuildStore(b, v_boolnull, v_resnullp); @@ -707,7 +751,7 @@ llvm_compile_expr(ExprState *state) /* Build block that continues if bool is TRUE. */ LLVMPositionBuilderAtEnd(b, b_boolcont); - v_boolanynull = LLVMBuildLoad(b, v_boolanynullp, ""); + v_boolanynull = l_load(b, TypeStorageBool, v_boolanynullp, ""); /* set value to NULL if any previous values were NULL */ LLVMBuildCondBr(b, @@ -761,8 +805,8 @@ llvm_compile_expr(ExprState *state) if (opcode == EEOP_BOOL_OR_STEP_FIRST) LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp); - v_boolnull = LLVMBuildLoad(b, v_resnullp, ""); - v_boolvalue = LLVMBuildLoad(b, v_resvaluep, ""); + v_boolnull = l_load(b, TypeStorageBool, v_resnullp, ""); + v_boolvalue = l_load(b, TypeSizeT, v_resvaluep, ""); /* set resnull to boolnull */ LLVMBuildStore(b, v_boolnull, v_resnullp); @@ -802,7 +846,7 @@ llvm_compile_expr(ExprState *state) /* build block that continues if bool is FALSE */ LLVMPositionBuilderAtEnd(b, b_boolcont); - v_boolanynull = LLVMBuildLoad(b, v_boolanynullp, ""); + v_boolanynull = l_load(b, TypeStorageBool, v_boolanynullp, ""); /* set value to NULL if any previous values were NULL */ LLVMBuildCondBr(b, @@ -826,8 +870,8 @@ llvm_compile_expr(ExprState *state) LLVMValueRef v_boolnull; LLVMValueRef v_negbool; - v_boolnull = LLVMBuildLoad(b, v_resnullp, ""); - v_boolvalue = LLVMBuildLoad(b, v_resvaluep, ""); + v_boolnull = l_load(b, TypeStorageBool, v_resnullp, ""); + v_boolvalue = l_load(b, TypeSizeT, v_resvaluep, ""); v_negbool = LLVMBuildZExt(b, LLVMBuildICmp(b, LLVMIntEQ, @@ -854,8 +898,8 @@ llvm_compile_expr(ExprState *state) b_qualfail = l_bb_before_v(opblocks[opno + 1], "op.%d.qualfail", opno); - v_resvalue = LLVMBuildLoad(b, v_resvaluep, ""); - v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + v_resvalue = l_load(b, TypeSizeT, v_resvaluep, ""); + v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); v_nullorfalse = LLVMBuildOr(b, @@ -893,7 +937,7 @@ llvm_compile_expr(ExprState *state) /* Transfer control if current result is null */ - v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, @@ -909,7 +953,7 @@ llvm_compile_expr(ExprState *state) /* Transfer control if current result is non-null */ - v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, @@ -928,8 +972,8 @@ llvm_compile_expr(ExprState *state) /* Transfer control if current result is null or false */ - v_resvalue = LLVMBuildLoad(b, v_resvaluep, ""); - v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + v_resvalue = l_load(b, TypeSizeT, v_resvaluep, ""); + v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); v_nullorfalse = LLVMBuildOr(b, @@ -948,7 +992,7 @@ llvm_compile_expr(ExprState *state) case EEOP_NULLTEST_ISNULL: { - LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + LLVMValueRef v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMValueRef v_resvalue; v_resvalue = @@ -967,7 +1011,7 @@ llvm_compile_expr(ExprState *state) case EEOP_NULLTEST_ISNOTNULL: { - LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + LLVMValueRef v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMValueRef v_resvalue; v_resvalue = @@ -1003,7 +1047,7 @@ llvm_compile_expr(ExprState *state) { LLVMBasicBlockRef b_isnull, b_notnull; - LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + LLVMValueRef v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); b_isnull = l_bb_before_v(opblocks[opno + 1], "op.%d.isnull", opno); @@ -1047,7 +1091,7 @@ llvm_compile_expr(ExprState *state) else { LLVMValueRef v_value = - LLVMBuildLoad(b, v_resvaluep, ""); + l_load(b, TypeSizeT, v_resvaluep, ""); v_value = LLVMBuildZExt(b, LLVMBuildICmp(b, LLVMIntEQ, @@ -1075,20 +1119,19 @@ llvm_compile_expr(ExprState *state) case EEOP_PARAM_CALLBACK: { - LLVMTypeRef v_functype; LLVMValueRef v_func; LLVMValueRef v_params[3]; - v_functype = llvm_pg_var_func_type("TypeExecEvalSubroutine"); v_func = l_ptr_const(op->d.cparam.paramfunc, - LLVMPointerType(v_functype, 0)); + llvm_pg_var_type("TypeExecEvalSubroutine")); v_params[0] = v_state; v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep)); v_params[2] = v_econtext; - LLVMBuildCall(b, - v_func, - v_params, lengthof(v_params), ""); + l_call(b, + LLVMGetFunctionType(ExecEvalSubroutineTemplate), + v_func, + v_params, lengthof(v_params), ""); LLVMBuildBr(b, opblocks[opno + 1]); break; @@ -1097,21 +1140,20 @@ llvm_compile_expr(ExprState *state) case EEOP_SBSREF_SUBSCRIPTS: { int jumpdone = op->d.sbsref_subscript.jumpdone; - LLVMTypeRef v_functype; LLVMValueRef v_func; LLVMValueRef v_params[3]; LLVMValueRef v_ret; - v_functype = llvm_pg_var_func_type("TypeExecEvalBoolSubroutine"); v_func = l_ptr_const(op->d.sbsref_subscript.subscriptfunc, - LLVMPointerType(v_functype, 0)); + llvm_pg_var_type("TypeExecEvalBoolSubroutine")); v_params[0] = v_state; v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep)); v_params[2] = v_econtext; - v_ret = LLVMBuildCall(b, - v_func, - v_params, lengthof(v_params), ""); + v_ret = l_call(b, + LLVMGetFunctionType(ExecEvalBoolSubroutineTemplate), + v_func, + v_params, lengthof(v_params), ""); v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, ""); LLVMBuildCondBr(b, @@ -1126,20 +1168,19 @@ llvm_compile_expr(ExprState *state) case EEOP_SBSREF_ASSIGN: case EEOP_SBSREF_FETCH: { - LLVMTypeRef v_functype; LLVMValueRef v_func; LLVMValueRef v_params[3]; - v_functype = llvm_pg_var_func_type("TypeExecEvalSubroutine"); v_func = l_ptr_const(op->d.sbsref.subscriptfunc, - LLVMPointerType(v_functype, 0)); + llvm_pg_var_type("TypeExecEvalSubroutine")); v_params[0] = v_state; v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep)); v_params[2] = v_econtext; - LLVMBuildCall(b, - v_func, - v_params, lengthof(v_params), ""); + l_call(b, + LLVMGetFunctionType(ExecEvalSubroutineTemplate), + v_func, + v_params, lengthof(v_params), ""); LLVMBuildBr(b, opblocks[opno + 1]); break; @@ -1174,8 +1215,8 @@ llvm_compile_expr(ExprState *state) /* if casetest != NULL */ LLVMPositionBuilderAtEnd(b, b_avail); - v_casevalue = LLVMBuildLoad(b, v_casevaluep, ""); - v_casenull = LLVMBuildLoad(b, v_casenullp, ""); + v_casevalue = l_load(b, TypeSizeT, v_casevaluep, ""); + v_casenull = l_load(b, TypeStorageBool, v_casenullp, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); @@ -1183,10 +1224,14 @@ llvm_compile_expr(ExprState *state) /* if casetest == NULL */ LLVMPositionBuilderAtEnd(b, b_notavail); v_casevalue = - l_load_struct_gep(b, v_econtext, + l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_CASEDATUM, ""); v_casenull = - l_load_struct_gep(b, v_econtext, + l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_CASENULL, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); @@ -1211,7 +1256,7 @@ llvm_compile_expr(ExprState *state) v_nullp = l_ptr_const(op->d.make_readonly.isnull, l_ptr(TypeStorageBool)); - v_null = LLVMBuildLoad(b, v_nullp, ""); + v_null = l_load(b, TypeStorageBool, v_nullp, ""); /* store null isnull value in result */ LLVMBuildStore(b, v_null, v_resnullp); @@ -1228,13 +1273,14 @@ llvm_compile_expr(ExprState *state) v_valuep = l_ptr_const(op->d.make_readonly.value, l_ptr(TypeSizeT)); - v_value = LLVMBuildLoad(b, v_valuep, ""); + v_value = l_load(b, TypeSizeT, v_valuep, ""); v_params[0] = v_value; v_ret = - LLVMBuildCall(b, - llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), - v_params, lengthof(v_params), ""); + l_call(b, + llvm_pg_var_func_type("MakeExpandedObjectReadOnlyInternal"), + llvm_pg_func(mod, "MakeExpandedObjectReadOnlyInternal"), + v_params, lengthof(v_params), ""); LLVMBuildStore(b, v_ret, v_resvaluep); LLVMBuildBr(b, opblocks[opno + 1]); @@ -1280,12 +1326,14 @@ llvm_compile_expr(ExprState *state) v_fcinfo_in = l_ptr_const(fcinfo_in, l_ptr(StructFunctionCallInfoData)); v_fcinfo_in_isnullp = - LLVMBuildStructGEP(b, v_fcinfo_in, - FIELDNO_FUNCTIONCALLINFODATA_ISNULL, - "v_fcinfo_in_isnull"); + l_struct_gep(b, + StructFunctionCallInfoData, + v_fcinfo_in, + FIELDNO_FUNCTIONCALLINFODATA_ISNULL, + "v_fcinfo_in_isnull"); /* output functions are not called on nulls */ - v_resnull = LLVMBuildLoad(b, v_resnullp, ""); + v_resnull = l_load(b, TypeStorageBool, v_resnullp, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, v_resnull, l_sbool_const(1), ""), @@ -1297,7 +1345,7 @@ llvm_compile_expr(ExprState *state) LLVMBuildBr(b, b_input); LLVMPositionBuilderAtEnd(b, b_calloutput); - v_resvalue = LLVMBuildLoad(b, v_resvaluep, ""); + v_resvalue = l_load(b, TypeSizeT, v_resvaluep, ""); /* set arg[0] */ LLVMBuildStore(b, @@ -1307,8 +1355,10 @@ llvm_compile_expr(ExprState *state) l_sbool_const(0), l_funcnullp(b, v_fcinfo_out, 0)); /* and call output function (can never return NULL) */ - v_output = LLVMBuildCall(b, v_fn_out, &v_fcinfo_out, - 1, "funccall_coerce_out"); + v_output = l_call(b, + LLVMGetFunctionType(v_fn_out), + v_fn_out, &v_fcinfo_out, + 1, "funccall_coerce_out"); LLVMBuildBr(b, b_input); /* build block handling input function call */ @@ -1362,8 +1412,10 @@ llvm_compile_expr(ExprState *state) /* reset fcinfo_in->isnull */ LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_in_isnullp); /* and call function */ - v_retval = LLVMBuildCall(b, v_fn_in, &v_fcinfo_in, 1, - "funccall_iocoerce_in"); + v_retval = l_call(b, + LLVMGetFunctionType(v_fn_in), + v_fn_in, &v_fcinfo_in, 1, + "funccall_iocoerce_in"); LLVMBuildStore(b, v_retval, v_resvaluep); @@ -1696,7 +1748,7 @@ llvm_compile_expr(ExprState *state) */ v_cmpresult = LLVMBuildTrunc(b, - LLVMBuildLoad(b, v_resvaluep, ""), + l_load(b, TypeSizeT, v_resvaluep, ""), LLVMInt32Type(), ""); switch (rctype) @@ -1789,8 +1841,8 @@ llvm_compile_expr(ExprState *state) /* if casetest != NULL */ LLVMPositionBuilderAtEnd(b, b_avail); - v_casevalue = LLVMBuildLoad(b, v_casevaluep, ""); - v_casenull = LLVMBuildLoad(b, v_casenullp, ""); + v_casevalue = l_load(b, TypeSizeT, v_casevaluep, ""); + v_casenull = l_load(b, TypeStorageBool, v_casenullp, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); LLVMBuildStore(b, v_casenull, v_resnullp); LLVMBuildBr(b, opblocks[opno + 1]); @@ -1798,11 +1850,15 @@ llvm_compile_expr(ExprState *state) /* if casetest == NULL */ LLVMPositionBuilderAtEnd(b, b_notavail); v_casevalue = - l_load_struct_gep(b, v_econtext, + l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_DOMAINDATUM, ""); v_casenull = - l_load_struct_gep(b, v_econtext, + l_load_struct_gep(b, + StructExprContext, + v_econtext, FIELDNO_EXPRCONTEXT_DOMAINNULL, ""); LLVMBuildStore(b, v_casevalue, v_resvaluep); @@ -1869,8 +1925,8 @@ llvm_compile_expr(ExprState *state) v_aggno = l_int32_const(op->d.aggref.aggno); /* load agg value / null */ - value = l_load_gep1(b, v_aggvalues, v_aggno, "aggvalue"); - isnull = l_load_gep1(b, v_aggnulls, v_aggno, "aggnull"); + value = l_load_gep1(b, TypeSizeT, v_aggvalues, v_aggno, "aggvalue"); + isnull = l_load_gep1(b, TypeStorageBool, v_aggnulls, v_aggno, "aggnull"); /* and store result */ LLVMBuildStore(b, value, v_resvaluep); @@ -1982,12 +2038,12 @@ llvm_compile_expr(ExprState *state) */ v_wfuncnop = l_ptr_const(&wfunc->wfuncno, l_ptr(LLVMInt32Type())); - v_wfuncno = LLVMBuildLoad(b, v_wfuncnop, "v_wfuncno"); + v_wfuncno = l_load(b, LLVMInt32Type(), v_wfuncnop, "v_wfuncno"); /* load window func value / null */ - value = l_load_gep1(b, v_aggvalues, v_wfuncno, + value = l_load_gep1(b, TypeSizeT, v_aggvalues, v_wfuncno, "windowvalue"); - isnull = l_load_gep1(b, v_aggnulls, v_wfuncno, + isnull = l_load_gep1(b, TypeStorageBool, v_aggnulls, v_wfuncno, "windownull"); LLVMBuildStore(b, value, v_resvaluep); @@ -2101,14 +2157,14 @@ llvm_compile_expr(ExprState *state) b_argnotnull = b_checknulls[argno + 1]; if (opcode == EEOP_AGG_STRICT_INPUT_CHECK_NULLS) - v_argisnull = l_load_gep1(b, v_nullsp, v_argno, ""); + v_argisnull = l_load_gep1(b, TypeStorageBool, v_nullsp, v_argno, ""); else { LLVMValueRef v_argn; - v_argn = LLVMBuildGEP(b, v_argsp, &v_argno, 1, ""); + v_argn = l_gep(b, StructNullableDatum, v_argsp, &v_argno, 1, ""); v_argisnull = - l_load_struct_gep(b, v_argn, + l_load_struct_gep(b, StructNullableDatum, v_argn, FIELDNO_NULLABLE_DATUM_ISNULL, ""); } @@ -2142,13 +2198,16 @@ llvm_compile_expr(ExprState *state) v_aggstatep = LLVMBuildBitCast(b, v_parent, l_ptr(StructAggState), ""); - v_allpergroupsp = l_load_struct_gep(b, v_aggstatep, + v_allpergroupsp = l_load_struct_gep(b, + StructAggState, + v_aggstatep, FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); v_setoff = l_int32_const(op->d.agg_plain_pergroup_nullcheck.setoff); - v_pergroup_allaggs = l_load_gep1(b, v_allpergroupsp, v_setoff, ""); + v_pergroup_allaggs = l_load_gep1(b, l_ptr(StructAggStatePerGroupData), + v_allpergroupsp, v_setoff, ""); LLVMBuildCondBr(b, LLVMBuildICmp(b, LLVMIntEQ, @@ -2212,15 +2271,19 @@ llvm_compile_expr(ExprState *state) * [op->d.agg_init_trans_check.transno]; */ v_allpergroupsp = - l_load_struct_gep(b, v_aggstatep, + l_load_struct_gep(b, + StructAggState, + v_aggstatep, FIELDNO_AGGSTATE_ALL_PERGROUPS, "aggstate.all_pergroups"); v_setoff = l_int32_const(op->d.agg_trans.setoff); v_transno = l_int32_const(op->d.agg_trans.transno); v_pergroupp = - LLVMBuildGEP(b, - l_load_gep1(b, v_allpergroupsp, v_setoff, ""), - &v_transno, 1, ""); + l_gep(b, + StructAggStatePerGroupData, + l_load_gep1(b, l_ptr(StructAggStatePerGroupData), + v_allpergroupsp, v_setoff, ""), + &v_transno, 1, ""); if (opcode == EEOP_AGG_PLAIN_TRANS_INIT_STRICT_BYVAL || @@ -2231,7 +2294,9 @@ llvm_compile_expr(ExprState *state) LLVMBasicBlockRef b_no_init; v_notransvalue = - l_load_struct_gep(b, v_pergroupp, + l_load_struct_gep(b, + StructAggStatePerGroupData, + v_pergroupp, FIELDNO_AGGSTATEPERGROUPDATA_NOTRANSVALUE, "notransvalue"); @@ -2260,10 +2325,11 @@ llvm_compile_expr(ExprState *state) params[2] = v_pergroupp; params[3] = v_aggcontext; - LLVMBuildCall(b, - llvm_pg_func(mod, "ExecAggInitGroup"), - params, lengthof(params), - ""); + l_call(b, + llvm_pg_var_func_type("ExecAggInitGroup"), + llvm_pg_func(mod, "ExecAggInitGroup"), + params, lengthof(params), + ""); LLVMBuildBr(b, opblocks[opno + 1]); @@ -2283,7 +2349,9 @@ llvm_compile_expr(ExprState *state) b_strictpass = l_bb_before_v(opblocks[opno + 1], "op.%d.strictpass", opno); v_transnull = - l_load_struct_gep(b, v_pergroupp, + l_load_struct_gep(b, + StructAggStatePerGroupData, + v_pergroupp, FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL, "transnull"); @@ -2303,20 +2371,23 @@ llvm_compile_expr(ExprState *state) l_ptr(StructExprContext)); v_current_setp = - LLVMBuildStructGEP(b, - v_aggstatep, - FIELDNO_AGGSTATE_CURRENT_SET, - "aggstate.current_set"); + l_struct_gep(b, + StructAggState, + v_aggstatep, + FIELDNO_AGGSTATE_CURRENT_SET, + "aggstate.current_set"); v_curaggcontext = - LLVMBuildStructGEP(b, - v_aggstatep, - FIELDNO_AGGSTATE_CURAGGCONTEXT, - "aggstate.curaggcontext"); + l_struct_gep(b, + StructAggState, + v_aggstatep, + FIELDNO_AGGSTATE_CURAGGCONTEXT, + "aggstate.curaggcontext"); v_current_pertransp = - LLVMBuildStructGEP(b, - v_aggstatep, - FIELDNO_AGGSTATE_CURPERTRANS, - "aggstate.curpertrans"); + l_struct_gep(b, + StructAggState, + v_aggstatep, + FIELDNO_AGGSTATE_CURPERTRANS, + "aggstate.curpertrans"); /* set aggstate globals */ LLVMBuildStore(b, v_aggcontext, v_curaggcontext); @@ -2332,19 +2403,25 @@ llvm_compile_expr(ExprState *state) /* store transvalue in fcinfo->args[0] */ v_transvaluep = - LLVMBuildStructGEP(b, v_pergroupp, - FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUE, - "transvalue"); + l_struct_gep(b, + StructAggStatePerGroupData, + v_pergroupp, + FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUE, + "transvalue"); v_transnullp = - LLVMBuildStructGEP(b, v_pergroupp, - FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL, - "transnullp"); + l_struct_gep(b, + StructAggStatePerGroupData, + v_pergroupp, + FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL, + "transnullp"); LLVMBuildStore(b, - LLVMBuildLoad(b, v_transvaluep, - "transvalue"), + l_load(b, + TypeSizeT, + v_transvaluep, + "transvalue"), l_funcvaluep(b, v_fcinfo, 0)); LLVMBuildStore(b, - LLVMBuildLoad(b, v_transnullp, "transnull"), + l_load(b, TypeStorageBool, v_transnullp, "transnull"), l_funcnullp(b, v_fcinfo, 0)); /* and invoke transition function */ @@ -2377,8 +2454,8 @@ llvm_compile_expr(ExprState *state) b_nocall = l_bb_before_v(opblocks[opno + 1], "op.%d.transnocall", opno); - v_transvalue = LLVMBuildLoad(b, v_transvaluep, ""); - v_transnull = LLVMBuildLoad(b, v_transnullp, ""); + v_transvalue = l_load(b, TypeSizeT, v_transvaluep, ""); + v_transnull = l_load(b, TypeStorageBool, v_transnullp, ""); /* * DatumGetPointer(newVal) != @@ -2404,9 +2481,11 @@ llvm_compile_expr(ExprState *state) v_fn = llvm_pg_func(mod, "ExecAggTransReparent"); v_newval = - LLVMBuildCall(b, v_fn, - params, lengthof(params), - ""); + l_call(b, + LLVMGetFunctionType(v_fn), + v_fn, + params, lengthof(params), + ""); /* store trans value */ LLVMBuildStore(b, v_newval, v_transvaluep); @@ -2516,15 +2595,17 @@ BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, v_fn = llvm_function_reference(context, b, mod, fcinfo); v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData)); - v_fcinfo_isnullp = LLVMBuildStructGEP(b, v_fcinfo, - FIELDNO_FUNCTIONCALLINFODATA_ISNULL, - "v_fcinfo_isnull"); + v_fcinfo_isnullp = l_struct_gep(b, + StructFunctionCallInfoData, + v_fcinfo, + FIELDNO_FUNCTIONCALLINFODATA_ISNULL, + "v_fcinfo_isnull"); LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_isnullp); - v_retval = LLVMBuildCall(b, v_fn, &v_fcinfo, 1, "funccall"); + v_retval = l_call(b, LLVMGetFunctionType(AttributeTemplate), v_fn, &v_fcinfo, 1, "funccall"); if (v_fcinfo_isnull) - *v_fcinfo_isnull = LLVMBuildLoad(b, v_fcinfo_isnullp, ""); + *v_fcinfo_isnull = l_load(b, TypeStorageBool, v_fcinfo_isnullp, ""); /* * Add lifetime-end annotation, signaling that writes to memory don't have @@ -2536,11 +2617,11 @@ BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b, params[0] = l_int64_const(sizeof(NullableDatum) * fcinfo->nargs); params[1] = l_ptr_const(fcinfo->args, l_ptr(LLVMInt8Type())); - LLVMBuildCall(b, v_lifetime, params, lengthof(params), ""); + l_call(b, LLVMGetFunctionType(v_lifetime), v_lifetime, params, lengthof(params), ""); params[0] = l_int64_const(sizeof(fcinfo->isnull)); params[1] = l_ptr_const(&fcinfo->isnull, l_ptr(LLVMInt8Type())); - LLVMBuildCall(b, v_lifetime, params, lengthof(params), ""); + l_call(b, LLVMGetFunctionType(v_lifetime), v_lifetime, params, lengthof(params), ""); } return v_retval; @@ -2572,7 +2653,7 @@ build_EvalXFuncInt(LLVMBuilderRef b, LLVMModuleRef mod, const char *funcname, for (int i = 0; i < nargs; i++) params[argno++] = v_args[i]; - v_ret = LLVMBuildCall(b, v_fn, params, argno, ""); + v_ret = l_call(b, LLVMGetFunctionType(v_fn), v_fn, params, argno, ""); pfree(params); diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c index 2deb65c5b5c..58c5d1359c1 100644 --- a/src/backend/jit/llvm/llvmjit_types.c +++ b/src/backend/jit/llvm/llvmjit_types.c @@ -48,7 +48,7 @@ PGFunction TypePGFunction; size_t TypeSizeT; bool TypeStorageBool; -ExprStateEvalFunc TypeExprStateEvalFunc; + ExecEvalSubroutine TypeExecEvalSubroutine; ExecEvalBoolSubroutine TypeExecEvalBoolSubroutine; @@ -61,11 +61,14 @@ ExprEvalStep StructExprEvalStep; ExprState StructExprState; FunctionCallInfoBaseData StructFunctionCallInfoData; HeapTupleData StructHeapTupleData; +HeapTupleHeaderData StructHeapTupleHeaderData; MemoryContextData StructMemoryContextData; TupleTableSlot StructTupleTableSlot; HeapTupleTableSlot StructHeapTupleTableSlot; MinimalTupleTableSlot StructMinimalTupleTableSlot; TupleDescData StructTupleDescData; +PlanState StructPlanState; +MinimalTupleData StructMinimalTupleData; /* @@ -77,9 +80,42 @@ extern Datum AttributeTemplate(PG_FUNCTION_ARGS); Datum AttributeTemplate(PG_FUNCTION_ARGS) { + AssertVariableIsOfType(&AttributeTemplate, PGFunction); + PG_RETURN_NULL(); } +/* + * And some more "templates" to give us examples of function types + * corresponding to function pointer types. + */ + +extern void ExecEvalSubroutineTemplate(ExprState *state, + struct ExprEvalStep *op, + ExprContext *econtext); +void +ExecEvalSubroutineTemplate(ExprState *state, + struct ExprEvalStep *op, + ExprContext *econtext) +{ + AssertVariableIsOfType(&ExecEvalSubroutineTemplate, + ExecEvalSubroutine); +} + +extern bool ExecEvalBoolSubroutineTemplate(ExprState *state, + struct ExprEvalStep *op, + ExprContext *econtext); +bool +ExecEvalBoolSubroutineTemplate(ExprState *state, + struct ExprEvalStep *op, + ExprContext *econtext) +{ + AssertVariableIsOfType(&ExecEvalBoolSubroutineTemplate, + ExecEvalBoolSubroutine); + + return false; +} + /* * Clang represents stdbool.h style booleans that are returned by functions * differently (as i1) than stored ones (as i8). Therefore we do not just need @@ -136,4 +172,5 @@ void *referenced_functions[] = slot_getsomeattrs_int, strlen, varsize_any, + ExecInterpExprStillValid, }; diff --git a/src/backend/jit/llvm/llvmjit_wrap.cpp b/src/backend/jit/llvm/llvmjit_wrap.cpp index 692483d3b93..ffda94a9938 100644 --- a/src/backend/jit/llvm/llvmjit_wrap.cpp +++ b/src/backend/jit/llvm/llvmjit_wrap.cpp @@ -76,3 +76,15 @@ LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx) */ return LLVMGetAttributeCountAtIndex(F, Idx); } + +LLVMTypeRef +LLVMGetFunctionReturnType(LLVMValueRef r) +{ + return llvm::wrap(llvm::unwrap(r)->getReturnType()); +} + +LLVMTypeRef +LLVMGetFunctionType(LLVMValueRef r) +{ + return llvm::wrap(llvm::unwrap(r)->getFunctionType()); +} diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h index 3560715e329..ffc2307ee14 100644 --- a/src/include/jit/llvmjit.h +++ b/src/include/jit/llvmjit.h @@ -67,6 +67,8 @@ extern LLVMTypeRef TypeStorageBool; extern LLVMTypeRef StructNullableDatum; extern LLVMTypeRef StructTupleDescData; extern LLVMTypeRef StructHeapTupleData; +extern LLVMTypeRef StructHeapTupleHeaderData; +extern LLVMTypeRef StructMinimalTupleData; extern LLVMTypeRef StructTupleTableSlot; extern LLVMTypeRef StructHeapTupleTableSlot; extern LLVMTypeRef StructMinimalTupleTableSlot; @@ -80,6 +82,8 @@ extern LLVMTypeRef StructAggStatePerTransData; extern LLVMTypeRef StructAggStatePerGroupData; extern LLVMValueRef AttributeTemplate; +extern LLVMValueRef ExecEvalBoolSubroutineTemplate; +extern LLVMValueRef ExecEvalSubroutineTemplate; extern void llvm_enter_fatal_on_oom(void); @@ -133,6 +137,8 @@ extern char *LLVMGetHostCPUFeatures(void); #endif extern unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx); +extern LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r); +extern LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r); #ifdef __cplusplus } /* extern "C" */ diff --git a/src/include/jit/llvmjit_emit.h b/src/include/jit/llvmjit_emit.h index a9e8d65dfee..c7e31249baa 100644 --- a/src/include/jit/llvmjit_emit.h +++ b/src/include/jit/llvmjit_emit.h @@ -16,6 +16,7 @@ #ifdef USE_LLVM #include +#include #include "jit/llvmjit.h" @@ -103,26 +104,65 @@ l_pbool_const(bool i) return LLVMConstInt(TypeParamBool, (int) i, false); } +static inline LLVMValueRef +l_struct_gep(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, int32 idx, const char *name) +{ +#if LLVM_VERSION_MAJOR < 16 + return LLVMBuildStructGEP(b, v, idx, ""); +#else + return LLVMBuildStructGEP2(b, t, v, idx, ""); +#endif +} + +static inline LLVMValueRef +l_gep(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, LLVMValueRef *indices, int32 nindices, const char *name) +{ +#if LLVM_VERSION_MAJOR < 16 + return LLVMBuildGEP(b, v, indices, nindices, name); +#else + return LLVMBuildGEP2(b, t, v, indices, nindices, name); +#endif +} + +static inline LLVMValueRef +l_load(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, const char *name) +{ +#if LLVM_VERSION_MAJOR < 16 + return LLVMBuildLoad(b, v, name); +#else + return LLVMBuildLoad2(b, t, v, name); +#endif +} + +static inline LLVMValueRef +l_call(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef fn, LLVMValueRef *args, int32 nargs, const char *name) +{ +#if LLVM_VERSION_MAJOR < 16 + return LLVMBuildCall(b, fn, args, nargs, name); +#else + return LLVMBuildCall2(b, t, fn, args, nargs, name); +#endif +} + /* * Load a pointer member idx from a struct. */ static inline LLVMValueRef -l_load_struct_gep(LLVMBuilderRef b, LLVMValueRef v, int32 idx, const char *name) +l_load_struct_gep(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, int32 idx, const char *name) { - LLVMValueRef v_ptr = LLVMBuildStructGEP(b, v, idx, ""); - - return LLVMBuildLoad(b, v_ptr, name); + return l_load(b, + LLVMStructGetTypeAtIndex(t, idx), + l_struct_gep(b, t, v, idx, ""), + name); } /* * Load value of a pointer, after applying one index operation. */ static inline LLVMValueRef -l_load_gep1(LLVMBuilderRef b, LLVMValueRef v, LLVMValueRef idx, const char *name) +l_load_gep1(LLVMBuilderRef b, LLVMTypeRef t, LLVMValueRef v, LLVMValueRef idx, const char *name) { - LLVMValueRef v_ptr = LLVMBuildGEP(b, v, &idx, 1, ""); - - return LLVMBuildLoad(b, v_ptr, name); + return l_load(b, t, l_gep(b, t, v, &idx, 1, ""), name); } /* separate, because pg_attribute_printf(2, 3) can't appear in definition */ @@ -210,7 +250,7 @@ l_mcxt_switch(LLVMModuleRef mod, LLVMBuilderRef b, LLVMValueRef nc) if (!(cur = LLVMGetNamedGlobal(mod, cmc))) cur = LLVMAddGlobal(mod, l_ptr(StructMemoryContextData), cmc); - ret = LLVMBuildLoad(b, cur, cmc); + ret = l_load(b, l_ptr(StructMemoryContextData), cur, cmc); LLVMBuildStore(b, nc, cur); return ret; @@ -225,13 +265,21 @@ l_funcnullp(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno) LLVMValueRef v_args; LLVMValueRef v_argn; - v_args = LLVMBuildStructGEP(b, - v_fcinfo, - FIELDNO_FUNCTIONCALLINFODATA_ARGS, - ""); - v_argn = LLVMBuildStructGEP(b, v_args, argno, ""); - - return LLVMBuildStructGEP(b, v_argn, FIELDNO_NULLABLE_DATUM_ISNULL, ""); + v_args = l_struct_gep(b, + StructFunctionCallInfoData, + v_fcinfo, + FIELDNO_FUNCTIONCALLINFODATA_ARGS, + ""); + v_argn = l_struct_gep(b, + LLVMArrayType(StructNullableDatum, 0), + v_args, + argno, + ""); + return l_struct_gep(b, + StructNullableDatum, + v_argn, + FIELDNO_NULLABLE_DATUM_ISNULL, + ""); } /* @@ -243,13 +291,21 @@ l_funcvaluep(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno) LLVMValueRef v_args; LLVMValueRef v_argn; - v_args = LLVMBuildStructGEP(b, - v_fcinfo, - FIELDNO_FUNCTIONCALLINFODATA_ARGS, - ""); - v_argn = LLVMBuildStructGEP(b, v_args, argno, ""); - - return LLVMBuildStructGEP(b, v_argn, FIELDNO_NULLABLE_DATUM_DATUM, ""); + v_args = l_struct_gep(b, + StructFunctionCallInfoData, + v_fcinfo, + FIELDNO_FUNCTIONCALLINFODATA_ARGS, + ""); + v_argn = l_struct_gep(b, + LLVMArrayType(StructNullableDatum, 0), + v_args, + argno, + ""); + return l_struct_gep(b, + StructNullableDatum, + v_argn, + FIELDNO_NULLABLE_DATUM_DATUM, + ""); } /* @@ -258,7 +314,7 @@ l_funcvaluep(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno) static inline LLVMValueRef l_funcnull(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno) { - return LLVMBuildLoad(b, l_funcnullp(b, v_fcinfo, argno), ""); + return l_load(b, TypeStorageBool, l_funcnullp(b, v_fcinfo, argno), ""); } /* @@ -267,7 +323,7 @@ l_funcnull(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno) static inline LLVMValueRef l_funcvalue(LLVMBuilderRef b, LLVMValueRef v_fcinfo, size_t argno) { - return LLVMBuildLoad(b, l_funcvaluep(b, v_fcinfo, argno), ""); + return l_load(b, TypeSizeT, l_funcvaluep(b, v_fcinfo, argno), ""); } #endif /* USE_LLVM */ From 2525f53207d010ef5cc8b032f11cd85ab5ef5bed Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Thu, 19 Oct 2023 03:01:55 +1300 Subject: [PATCH 076/109] jit: Supply LLVMGlobalGetValueType() for LLVM < 8. Commit 37d5babb used this C API function while adding support for LLVM 16 and opaque pointers, but it's not available in LLVM 7 and older. Provide it in our own llvmjit_wrap.cpp. It just calls a C++ function that pre-dates LLVM 3.9, our minimum target. Back-patch to 12, like 37d5babb. Discussion: https://postgr.es/m/CA%2BhUKGKnLnJnWrkr%3D4mSGhE5FuTK55FY15uULR7%3Dzzc%3DwX4Nqw%40mail.gmail.com --- src/backend/jit/llvm/llvmjit_wrap.cpp | 8 ++++++++ src/include/jit/llvmjit.h | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/backend/jit/llvm/llvmjit_wrap.cpp b/src/backend/jit/llvm/llvmjit_wrap.cpp index ffda94a9938..8008b9abd06 100644 --- a/src/backend/jit/llvm/llvmjit_wrap.cpp +++ b/src/backend/jit/llvm/llvmjit_wrap.cpp @@ -88,3 +88,11 @@ LLVMGetFunctionType(LLVMValueRef r) { return llvm::wrap(llvm::unwrap(r)->getFunctionType()); } + +#if LLVM_VERSION_MAJOR < 8 +LLVMTypeRef +LLVMGlobalGetValueType(LLVMValueRef g) +{ + return llvm::wrap(llvm::unwrap(g)->getValueType()); +} +#endif diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h index ffc2307ee14..60cb5d51f5a 100644 --- a/src/include/jit/llvmjit.h +++ b/src/include/jit/llvmjit.h @@ -140,6 +140,10 @@ extern unsigned LLVMGetAttributeCountAtIndexPG(LLVMValueRef F, uint32 Idx); extern LLVMTypeRef LLVMGetFunctionReturnType(LLVMValueRef r); extern LLVMTypeRef LLVMGetFunctionType(LLVMValueRef r); +#if LLVM_MAJOR_VERSION < 8 +extern LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef g); +#endif + #ifdef __cplusplus } /* extern "C" */ #endif From 1264b2c8f27116cecd825be6a61774210a5d0adb Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Wed, 18 Oct 2023 22:15:54 +1300 Subject: [PATCH 077/109] jit: Changes for LLVM 17. Changes required by https://llvm.org/docs/NewPassManager.html. Back-patch to 12, leaving the final release of 11 unchanged, consistent with earlier decision not to back-patch LLVM 16 support either. Author: Dmitry Dolgov <9erthalion6@gmail.com> Reviewed-by: Andres Freund Reviewed-by: Thomas Munro Discussion: https://postgr.es/m/CA%2BhUKG%2BWXznXCyTgCADd%3DHWkP9Qksa6chd7L%3DGCnZo-MBgg9Lg%40mail.gmail.com --- src/backend/jit/llvm/llvmjit.c | 31 +++++++++++++++++++++++++++ src/backend/jit/llvm/llvmjit_wrap.cpp | 6 ++++++ 2 files changed, 37 insertions(+) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index a58ac6d11db..b888fad028d 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -18,6 +18,9 @@ #include #include #include +#if LLVM_VERSION_MAJOR > 16 +#include +#endif #if LLVM_VERSION_MAJOR > 11 #include #include @@ -27,12 +30,14 @@ #endif #include #include +#if LLVM_VERSION_MAJOR < 17 #include #include #include #if LLVM_VERSION_MAJOR > 6 #include #endif +#endif #include "jit/llvmjit.h" #include "jit/llvmjit_emit.h" @@ -561,6 +566,7 @@ llvm_function_reference(LLVMJitContext *context, static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module) { +#if LLVM_VERSION_MAJOR < 17 LLVMPassManagerBuilderRef llvm_pmb; LLVMPassManagerRef llvm_mpm; LLVMPassManagerRef llvm_fpm; @@ -624,6 +630,31 @@ llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module) LLVMDisposePassManager(llvm_mpm); LLVMPassManagerBuilderDispose(llvm_pmb); +#else + LLVMPassBuilderOptionsRef options; + LLVMErrorRef err; + const char *passes; + + if (context->base.flags & PGJIT_OPT3) + passes = "default"; + else + passes = "default,mem2reg"; + + options = LLVMCreatePassBuilderOptions(); + +#ifdef LLVM_PASS_DEBUG + LLVMPassBuilderOptionsSetDebugLogging(options, 1); +#endif + + LLVMPassBuilderOptionsSetInlinerThreshold(options, 512); + + err = LLVMRunPasses(module, passes, NULL, options); + + if (err) + elog(ERROR, "failed to JIT module: %s", llvm_error_message(err)); + + LLVMDisposePassBuilderOptions(options); +#endif } /* diff --git a/src/backend/jit/llvm/llvmjit_wrap.cpp b/src/backend/jit/llvm/llvmjit_wrap.cpp index 8008b9abd06..2962083c50f 100644 --- a/src/backend/jit/llvm/llvmjit_wrap.cpp +++ b/src/backend/jit/llvm/llvmjit_wrap.cpp @@ -23,8 +23,14 @@ extern "C" #include #include +#if LLVM_VERSION_MAJOR < 17 #include +#endif +#if LLVM_VERSION_MAJOR > 16 +#include +#else #include +#endif #include "jit/llvmjit.h" From c51dbb55f75f13d34906524a6887b49347012c3b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 18 Oct 2023 20:43:17 -0400 Subject: [PATCH 078/109] Improve pglz_decompress's defenses against corrupt compressed data. When processing a match tag, check to see if the claimed "off" is more than the distance back to the output buffer start. If it is, then the data is corrupt, and what's more we would fetch from outside the buffer boundaries and potentially incur a SIGSEGV. (Although the odds of that seem relatively low, given that "off" can't be more than 4K.) Back-patch to v13; before that, this function wasn't really trying to protect against bad data. Report and fix by Flavien Guedez. Discussion: https://postgr.es/m/01fc0593-e31e-463d-902c-dd43174acee2@oopacity.net --- src/common/pg_lzcompress.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index a30a2c2eb83..02e3b2a0087 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -735,11 +735,15 @@ pglz_decompress(const char *source, int32 slen, char *dest, /* * Check for corrupt data: if we fell off the end of the - * source, or if we obtained off = 0, we have problems. (We - * must check this, else we risk an infinite loop below in the - * face of corrupt data.) + * source, or if we obtained off = 0, or if off is more than + * the distance back to the buffer start, we have problems. + * (We must check for off = 0, else we risk an infinite loop + * below in the face of corrupt data. Likewise, the upper + * limit on off prevents accessing outside the buffer + * boundaries.) */ - if (unlikely(sp > srcend || off == 0)) + if (unlikely(sp > srcend || off == 0 || + off > (dp - (unsigned char *) dest))) return -1; /* From 991dcb5d84887f2a09242ec91b7b35b7f2cef861 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 20 Oct 2023 13:01:02 -0400 Subject: [PATCH 079/109] Doc: update CREATE OPERATOR's statement about => as an operator. This doco said that use of => as an operator "is deprecated". It's been fully disallowed since 865f14a2d back in 9.5, but evidently that commit missed updating this statement. Do so now. --- doc/src/sgml/ref/create_operator.sgml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/ref/create_operator.sgml b/doc/src/sgml/ref/create_operator.sgml index e27512ff391..d4d45ec357d 100644 --- a/doc/src/sgml/ref/create_operator.sgml +++ b/doc/src/sgml/ref/create_operator.sgml @@ -52,7 +52,8 @@ CREATE OPERATOR name ( There are a few restrictions on your choice of name: - -- and /* cannot appear anywhere in an operator name, + + -- and /* cannot appear anywhere in an operator name, since they will be taken as the start of a comment. @@ -72,8 +73,8 @@ CREATE OPERATOR name ( - The use of => as an operator name is deprecated. It may - be disallowed altogether in a future release. + The symbol => is reserved by the SQL grammar, + so it cannot be used as an operator name. From 811bf12f221e860cad5c420a25f99f89c7325ffc Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 20 Oct 2023 13:40:15 -0400 Subject: [PATCH 080/109] Dodge a compiler bug affecting timetz_zone/timetz_izone. This avoids a compiler bug occurring in AIX's xlc, even in pretty late-model revisions. Buildfarm testing has now confirmed that only 64-bit xlc is affected. Although we are contemplating dropping support for xlc in v17, it's still supported in the back branches, so we need this fix. Back-patch of code changes from HEAD commit 19fa97731. (The test cases were already back-patched, in 4a427b82c et al.) Discussion: https://postgr.es/m/CA+hUKGK=DOC+hE-62FKfZy=Ybt5uLkrg3zCZD-jFykM-iPn8yw@mail.gmail.com --- src/backend/utils/adt/date.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/utils/adt/date.c b/src/backend/utils/adt/date.c index 70c8791b611..861364c7ab1 100644 --- a/src/backend/utils/adt/date.c +++ b/src/backend/utils/adt/date.c @@ -3176,10 +3176,11 @@ timetz_zone(PG_FUNCTION_ARGS) result = (TimeTzADT *) palloc(sizeof(TimeTzADT)); result->time = t->time + (t->zone - tz) * USECS_PER_SEC; + /* C99 modulo has the wrong sign convention for negative input */ while (result->time < INT64CONST(0)) result->time += USECS_PER_DAY; - while (result->time >= USECS_PER_DAY) - result->time -= USECS_PER_DAY; + if (result->time >= USECS_PER_DAY) + result->time %= USECS_PER_DAY; result->zone = tz; @@ -3209,10 +3210,11 @@ timetz_izone(PG_FUNCTION_ARGS) result = (TimeTzADT *) palloc(sizeof(TimeTzADT)); result->time = time->time + (time->zone - tz) * USECS_PER_SEC; + /* C99 modulo has the wrong sign convention for negative input */ while (result->time < INT64CONST(0)) result->time += USECS_PER_DAY; - while (result->time >= USECS_PER_DAY) - result->time -= USECS_PER_DAY; + if (result->time >= USECS_PER_DAY) + result->time %= USECS_PER_DAY; result->zone = tz; From bbba8bc1caaadf40e6e7282048cc5d0b5f45199b Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sun, 22 Oct 2023 10:04:55 +1300 Subject: [PATCH 081/109] Fix min_dynamic_shared_memory on Windows. When min_dynamic_shared_memory is set above 0, we try to find space in a pre-allocated region of the main shared memory area instead of calling dsm_impl_XXX() routines to allocate more. The dsm_pin_segment() and dsm_unpin_segment() routines had a bug: they called dsm_impl_XXX() routines even for main region segments. Nobody noticed before now because those routines do nothing on Unix, but on Windows they'd fail while attempting to duplicate an invalid Windows HANDLE. Add the missing gating. Back-patch to 14, where commit 84b1c63a added this feature. Fixes pgsql-bugs bug #18165. Reported-by: Maxime Boyer Tested-by: Alexander Lakhin Discussion: https://postgr.es/m/18165-bf4f525cea6e51de%40postgresql.org --- src/backend/storage/ipc/dsm.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/storage/ipc/dsm.c b/src/backend/storage/ipc/dsm.c index 02e344a46c3..0c10a2711a0 100644 --- a/src/backend/storage/ipc/dsm.c +++ b/src/backend/storage/ipc/dsm.c @@ -922,7 +922,7 @@ dsm_unpin_mapping(dsm_segment *seg) void dsm_pin_segment(dsm_segment *seg) { - void *handle; + void *handle = NULL; /* * Bump reference count for this segment in shared memory. This will @@ -933,7 +933,8 @@ dsm_pin_segment(dsm_segment *seg) LWLockAcquire(DynamicSharedMemoryControlLock, LW_EXCLUSIVE); if (dsm_control->item[seg->control_slot].pinned) elog(ERROR, "cannot pin a segment that is already pinned"); - dsm_impl_pin_segment(seg->handle, seg->impl_private, &handle); + if (!is_main_region_dsm_handle(seg->handle)) + dsm_impl_pin_segment(seg->handle, seg->impl_private, &handle); dsm_control->item[seg->control_slot].pinned = true; dsm_control->item[seg->control_slot].refcnt++; dsm_control->item[seg->control_slot].impl_private_pm_handle = handle; @@ -990,8 +991,9 @@ dsm_unpin_segment(dsm_handle handle) * releasing the lock, because impl_private_pm_handle may get modified by * dsm_impl_unpin_segment. */ - dsm_impl_unpin_segment(handle, - &dsm_control->item[control_slot].impl_private_pm_handle); + if (!is_main_region_dsm_handle(handle)) + dsm_impl_unpin_segment(handle, + &dsm_control->item[control_slot].impl_private_pm_handle); /* Note that 1 means no references (0 means unused slot). */ if (--dsm_control->item[control_slot].refcnt == 1) From 57604e6ba345cafe07e66ed0d73981ec5052b6be Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sun, 22 Oct 2023 14:17:00 +1300 Subject: [PATCH 082/109] Log LLVM library version in configure output. When scanning build farm results, it's useful to be able to see which version is in use. For the Meson build system, this information was already displayed. Back-patch to all supported branches. Discussion: https://postgr.es/m/4022690.1697852728%40sss.pgh.pa.us --- config/llvm.m4 | 1 + configure | 2 ++ 2 files changed, 3 insertions(+) diff --git a/config/llvm.m4 b/config/llvm.m4 index 3a75cd8b4df..21d8cd4f90f 100644 --- a/config/llvm.m4 +++ b/config/llvm.m4 @@ -28,6 +28,7 @@ AC_DEFUN([PGAC_LLVM_SUPPORT], if echo $pgac_llvm_version | $AWK -F '.' '{ if ([$]1 >= 4 || ([$]1 == 3 && [$]2 >= 9)) exit 1; else exit 0;}';then AC_MSG_ERROR([$LLVM_CONFIG version is $pgac_llvm_version but at least 3.9 is required]) fi + AC_MSG_NOTICE([using llvm $pgac_llvm_version]) # need clang to create some bitcode files AC_ARG_VAR(CLANG, [path to clang compiler to generate bitcode]) diff --git a/configure b/configure index 7e446a1ba75..950021d36f3 100755 --- a/configure +++ b/configure @@ -5458,6 +5458,8 @@ fi if echo $pgac_llvm_version | $AWK -F '.' '{ if ($1 >= 4 || ($1 == 3 && $2 >= 9)) exit 1; else exit 0;}';then as_fn_error $? "$LLVM_CONFIG version is $pgac_llvm_version but at least 3.9 is required" "$LINENO" 5 fi + { $as_echo "$as_me:${as_lineno-$LINENO}: using llvm $pgac_llvm_version" >&5 +$as_echo "$as_me: using llvm $pgac_llvm_version" >&6;} # need clang to create some bitcode files From 9aedd8613b596850f523bb100fd597a22d0dd29c Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Tue, 24 Oct 2023 09:27:21 -0700 Subject: [PATCH 083/109] Doc: indexUnchanged is strictly a hint. Clearly spell out the limitations of aminsert()'s indexUnchanged hinting mechanism in the index AM documentation. Oversight in commit 9dc718bd, which added the "logically unchanged index" hint (which is used to trigger bottom-up index deletion). Author: Peter Geoghegan Reported-By: Tom Lane Reviewed-By: Tom Lane Discussion: https://postgr.es/m/CAH2-WzmU_BQ=-H9L+bxTSMQBqHMjp1DSwGypvL0gKs+dTOfkKg@mail.gmail.com Backpatch: 14-, where indexUnchanged hinting was introduced. --- doc/src/sgml/indexam.sgml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 4f83970c851..1a787179a34 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -319,9 +319,13 @@ aminsert (Relation indexRelation, modify any columns covered by the index, but nevertheless requires a new version in the index. The index AM may use this hint to decide to apply bottom-up index deletion in parts of the index where many - versions of the same logical row accumulate. Note that updating a - non-key column does not affect the value of - indexUnchanged. + versions of the same logical row accumulate. Note that updating a non-key + column or a column that only appears in a partial index predicate does not + affect the value of indexUnchanged. The core code + determines each tuple's indexUnchanged value using a low + overhead approach that allows both false positives and false negatives. + Index AMs must not treat indexUnchanged as an + authoritative source of information about tuple visibility or versioning. From 1ec72fea04beda5f1b1599c3c659052c5e0b4101 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Tue, 24 Oct 2023 14:48:28 -0400 Subject: [PATCH 084/109] Fix problems when a plain-inheritance parent table is excluded. When an UPDATE/DELETE/MERGE's target table is an old-style inheritance tree, it's possible for the parent to get excluded from the plan while some children are not. (I believe this is only possible if we can prove that a CHECK ... NO INHERIT constraint on the parent contradicts the query WHERE clause, so it's a very unusual case.) In such a case, ExecInitModifyTable mistakenly concluded that the first surviving child is the target table, leading to at least two bugs: 1. The wrong table's statement-level triggers would get fired. 2. In v16 and up, it was possible to fail with "invalid perminfoindex 0 in RTE with relid nnnn" due to the child RTE not having permissions data included in the query plan. This was hard to reproduce reliably because it did not occur unless the update triggered some non-HOT index updates. In v14 and up, this is easy to fix by defining ModifyTable.rootRelation to be the parent RTE in plain inheritance as well as partitioned cases. While the wrong-triggers bug also appears in older branches, the relevant code in both the planner and executor is quite a bit different, so it would take a good deal of effort to develop and test a suitable patch. Given the lack of field complaints about the trigger issue, I'll desist for now. (Patching v11 for this seems unwise anyway, given that it will have no more releases after next month.) Per bug #18147 from Hans Buschmann. Amit Langote and Tom Lane Discussion: https://postgr.es/m/18147-6fc796538913ee88@postgresql.org --- src/backend/executor/nodeModifyTable.c | 9 +++++---- src/backend/optimizer/plan/planner.c | 14 ++++--------- src/backend/optimizer/util/pathnode.c | 2 +- src/include/nodes/pathnodes.h | 2 +- src/include/nodes/plannodes.h | 13 +++++++------ src/test/regress/expected/inherit.out | 27 ++++++++++++++++++++++++++ src/test/regress/sql/inherit.sql | 19 ++++++++++++++++++ 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 4e718bd7b31..8c11a1e826e 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -3172,10 +3172,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) * must be converted, and * - the root partitioned table used for tuple routing. * - * If it's a partitioned table, the root partition doesn't appear - * elsewhere in the plan and its RT index is given explicitly in - * node->rootRelation. Otherwise (i.e. table inheritance) the target - * relation is the first relation in the node->resultRelations list. + * If it's a partitioned or inherited table, the root partition or + * appendrel RTE doesn't appear elsewhere in the plan and its RT index is + * given explicitly in node->rootRelation. Otherwise, the target relation + * is the sole relation in the node->resultRelations list. *---------- */ if (node->rootRelation > 0) @@ -3186,6 +3186,7 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) } else { + Assert(list_length(node->resultRelations) == 1); mtstate->rootResultRelInfo = mtstate->resultRelInfo; ExecInitResultRelation(estate, mtstate->resultRelInfo, linitial_int(node->resultRelations)); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 7b55eded430..cdb2ed911e1 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -2463,6 +2463,9 @@ grouping_planner(PlannerInfo *root, double tuple_fraction) parse->resultRelation); int resultRelation = -1; + /* Pass the root result rel forward to the executor. */ + rootRelation = parse->resultRelation; + /* Add only leaf children to ModifyTable. */ while ((resultRelation = bms_next_member(root->leaf_result_relids, resultRelation)) >= 0) @@ -2547,6 +2550,7 @@ grouping_planner(PlannerInfo *root, double tuple_fraction) else { /* Single-relation INSERT/UPDATE/DELETE. */ + rootRelation = 0; /* there's no separate root rel */ resultRelations = list_make1_int(parse->resultRelation); if (parse->commandType == CMD_UPDATE) updateColnosLists = list_make1(root->update_colnos); @@ -2556,16 +2560,6 @@ grouping_planner(PlannerInfo *root, double tuple_fraction) returningLists = list_make1(parse->returningList); } - /* - * If target is a partition root table, we need to mark the - * ModifyTable node appropriately for that. - */ - if (rt_fetch(parse->resultRelation, parse->rtable)->relkind == - RELKIND_PARTITIONED_TABLE) - rootRelation = parse->resultRelation; - else - rootRelation = 0; - /* * If there was a FOR [KEY] UPDATE/SHARE clause, the LockRows node * will have dealt with fetching non-locked marked rows, else we diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 4648a2f2719..04db89e92f9 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -5815,7 +5815,7 @@ create_lockrows_path(PlannerInfo *root, RelOptInfo *rel, * 'operation' is the operation type * 'canSetTag' is true if we set the command tag/es_processed * 'nominalRelation' is the parent RT index for use of EXPLAIN - * 'rootRelation' is the partitioned table root RT index, or 0 if none + * 'rootRelation' is the partitioned/inherited table root RTI, or 0 if none * 'partColsUpdated' is true if any partitioning columns are being updated, * either from the target relation or a descendent partitioned table. * 'resultRelations' is an integer list of actual RT indexes of target rel(s) diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 9281ea90729..d08a5841b18 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -2369,7 +2369,7 @@ typedef struct ModifyTablePath CmdType operation; /* INSERT, UPDATE, or DELETE */ bool canSetTag; /* do we set the command tag/es_processed? */ Index nominalRelation; /* Parent RT index for use of EXPLAIN */ - Index rootRelation; /* Root RT index, if target is partitioned */ + Index rootRelation; /* Root RT index, if partitioned/inherited */ bool partColsUpdated; /* some part key in hierarchy updated? */ bool splitUpdate; /* if distribution key is updated */ List *resultRelations; /* integer list of RT indexes */ diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index dd214cb9996..5392015a738 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -389,11 +389,12 @@ typedef struct ProjectSet * Apply rows produced by outer plan to result table(s), * by inserting, updating, or deleting. * - * If the originally named target table is a partitioned table, both - * nominalRelation and rootRelation contain the RT index of the partition - * root, which is not otherwise mentioned in the plan. Otherwise rootRelation - * is zero. However, nominalRelation will always be set, as it's the rel that - * EXPLAIN should claim is the INSERT/UPDATE/DELETE target. + * If the originally named target table is a partitioned table or inheritance + * tree, both nominalRelation and rootRelation contain the RT index of the + * partition root or appendrel RTE, which is not otherwise mentioned in the + * plan. Otherwise rootRelation is zero. However, nominalRelation will + * always be set, as it's the rel that EXPLAIN should claim is the + * INSERT/UPDATE/DELETE target. * * Note that rowMarks and epqParam are presumed to be valid for all the * table(s); they can't contain any info that varies across tables. @@ -405,7 +406,7 @@ typedef struct ModifyTable CmdType operation; /* INSERT, UPDATE, or DELETE */ bool canSetTag; /* do we set the command tag/es_processed? */ Index nominalRelation; /* Parent RT index for use of EXPLAIN */ - Index rootRelation; /* Root RT index, if target is partitioned */ + Index rootRelation; /* Root RT index, if partitioned/inherited */ bool partColsUpdated; /* some part key in hierarchy updated? */ bool splitUpdate; /* if it's split update */ List *resultRelations; /* integer list of RT indexes */ diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index 7d20cc2bdc4..e74710dd51d 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -539,6 +539,33 @@ CREATE TEMP TABLE z (b TEXT, PRIMARY KEY(aa, b)) inherits (a); INSERT INTO z VALUES (NULL, 'text'); -- should fail ERROR: null value in column "aa" of relation "z" violates not-null constraint DETAIL: Failing row contains (null, text). +-- Check inherited UPDATE with first child excluded +create table some_tab (f1 int, f2 int, f3 int, check (f1 < 10) no inherit); +create table some_tab_child () inherits(some_tab); +insert into some_tab_child select i, i+1, 0 from generate_series(1,1000) i; +create index on some_tab_child(f1, f2); +-- while at it, also check that statement-level triggers fire +create function some_tab_stmt_trig_func() returns trigger as +$$begin raise notice 'updating some_tab'; return NULL; end;$$ +language plpgsql; +create trigger some_tab_stmt_trig + before update on some_tab execute function some_tab_stmt_trig_func(); +explain (costs off) +update some_tab set f3 = 11 where f1 = 12 and f2 = 13; + QUERY PLAN +------------------------------------------------------------------------------------ + Update on some_tab + Update on some_tab_child some_tab_1 + -> Result + -> Index Scan using some_tab_child_f1_f2_idx on some_tab_child some_tab_1 + Index Cond: ((f1 = 12) AND (f2 = 13)) +(5 rows) + +update some_tab set f3 = 11 where f1 = 12 and f2 = 13; +NOTICE: updating some_tab +drop table some_tab cascade; +NOTICE: drop cascades to table some_tab_child +drop function some_tab_stmt_trig_func(); -- Check inherited UPDATE with all children excluded create table some_tab (a int, b int) distributed randomly; create table some_tab_child () inherits (some_tab); diff --git a/src/test/regress/sql/inherit.sql b/src/test/regress/sql/inherit.sql index e5f9b980776..c6acd74cb84 100644 --- a/src/test/regress/sql/inherit.sql +++ b/src/test/regress/sql/inherit.sql @@ -97,6 +97,25 @@ SELECT relname, d.* FROM ONLY d, pg_class where d.tableoid = pg_class.oid; CREATE TEMP TABLE z (b TEXT, PRIMARY KEY(aa, b)) inherits (a); INSERT INTO z VALUES (NULL, 'text'); -- should fail +-- Check inherited UPDATE with first child excluded +create table some_tab (f1 int, f2 int, f3 int, check (f1 < 10) no inherit); +create table some_tab_child () inherits(some_tab); +insert into some_tab_child select i, i+1, 0 from generate_series(1,1000) i; +create index on some_tab_child(f1, f2); +-- while at it, also check that statement-level triggers fire +create function some_tab_stmt_trig_func() returns trigger as +$$begin raise notice 'updating some_tab'; return NULL; end;$$ +language plpgsql; +create trigger some_tab_stmt_trig + before update on some_tab execute function some_tab_stmt_trig_func(); + +explain (costs off) +update some_tab set f3 = 11 where f1 = 12 and f2 = 13; +update some_tab set f3 = 11 where f1 = 12 and f2 = 13; + +drop table some_tab cascade; +drop function some_tab_stmt_trig_func(); + -- Check inherited UPDATE with all children excluded create table some_tab (a int, b int) distributed randomly; create table some_tab_child () inherits (some_tab); From 9de790c388410a0f27e5215cefd3e7cc6086e42c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 25 Oct 2023 09:41:13 +0900 Subject: [PATCH 085/109] doc: Fix some typos and grammar Author: Ekaterina Kiryanova, Elena Indrupskaya, Oleg Sibiryakov, Maxim Yablokov Discussion: https://postgr.es/m/7aad518b-3e6d-47f3-9184-b1d69cb412e7@postgrespro.ru Backpatch-through: 11 --- doc/src/sgml/ref/create_foreign_table.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/create_foreign_table.sgml b/doc/src/sgml/ref/create_foreign_table.sgml index 7ad22ea0783..d1717fade90 100644 --- a/doc/src/sgml/ref/create_foreign_table.sgml +++ b/doc/src/sgml/ref/create_foreign_table.sgml @@ -423,7 +423,7 @@ int totalNumberOfSegments = getgpsegmentCount(); an UPDATE that changes the partition key value can cause a row to be moved from a local partition to a foreign-table partition, provided the foreign data wrapper supports tuple routing. - However it is not currently possible to move a row from a + However, it is not currently possible to move a row from a foreign-table partition to another partition. An UPDATE that would require doing that will fail due to the partitioning constraint, assuming that that is properly From c5fdcb7b003c9f40d9f6c1b50bfccaa5ae291f6a Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 25 Oct 2023 17:34:47 -0400 Subject: [PATCH 086/109] Doc: remove misleading info about ecpg's CONNECT/DISCONNECT DEFAULT. As far as I can see, ecpg has no notion of a "default" open connection. You can do "CONNECT TO DEFAULT" but that just specifies letting libpq use all its default connection parameters --- the resulting connection is not special subsequently. In particular, SET CONNECTION = DEFAULT and DISCONNECT DEFAULT simply act on a connection named DEFAULT, if you've made one; they do not have special lookup rules. But the documentation of these commands makes it look like they do. Simplest fix, I think, is just to remove the paras suggesting that DEFAULT is special here. Also, SET CONNECTION *does* have one special lookup rule, which is that it recognizes CURRENT as an alias for the currently selected connection. SET CONNECTION = CURRENT is a no-op, so it's pretty useless, but nonetheless it does something different from selecting a connection by name; so we'd better document it. Per report from Sylvain Frandaz. Back-patch to all supported versions. Discussion: https://postgr.es/m/169824721149.1769274.1553568436817652238@wrigleys.postgresql.org --- doc/src/sgml/ecpg.sgml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 9df09df4d77..46187a563be 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -412,12 +412,6 @@ EXEC SQL DISCONNECT connection; - - - DEFAULT - - - CURRENT @@ -7093,7 +7087,6 @@ EXEC SQL DEALLOCATE DESCRIPTOR mydesc; DISCONNECT connection_name DISCONNECT [ CURRENT ] -DISCONNECT DEFAULT DISCONNECT ALL @@ -7134,15 +7127,6 @@ DISCONNECT ALL - - DEFAULT - - - Close the default connection. - - - - ALL @@ -7161,13 +7145,11 @@ DISCONNECT ALL int main(void) { - EXEC SQL CONNECT TO testdb AS DEFAULT USER testuser; EXEC SQL CONNECT TO testdb AS con1 USER testuser; EXEC SQL CONNECT TO testdb AS con2 USER testuser; EXEC SQL CONNECT TO testdb AS con3 USER testuser; EXEC SQL DISCONNECT CURRENT; /* close con3 */ - EXEC SQL DISCONNECT DEFAULT; /* close DEFAULT */ EXEC SQL DISCONNECT ALL; /* close con2 and con1 */ return 0; @@ -7736,10 +7718,10 @@ SET CONNECTION [ TO | = ] connection_name - DEFAULT + CURRENT - Set the connection to the default connection. + Set the connection to the current connection (thus, nothing happens). From b22833922738954833f92c09995b06e8e4152435 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Fri, 27 Oct 2023 17:56:27 +0200 Subject: [PATCH 087/109] Fix overflow when calculating timestamp distance in BRIN When calculating distances for timestamp values for BRIN minmax-multi indexes, we need to be careful about overflows for extreme values. If the value overflows into a negative value, the index may be inefficient. The new regression test checks this for the timestamp type by adding a table with enough values to force range compaction/merging. The values are close to min/max, which means a risk of overflow. Fixed by converting the int64 values to double first, before calculating the distance. This prevents the overflow. We may lose some precision, of course, but that's good enough. In the worst case we build a slightly less efficient index, but for large distances this won't matter. This only affects minmax-multi indexes on timestamp columns, with ranges containing values sufficiently distant to cause an overflow. That seems like a fairly rare case in practice. Backpatch to 14, where minmax-multi indexes were introduced. Reported-by: Ashutosh Bapat Reviewed-by: Ashutosh Bapat, Dean Rasheed Backpatch-through: 14 Discussion: https://postgr.es/m/eef0ea8c-4aaa-8d0d-027f-58b1f35dd170@enterprisedb.com --- src/backend/access/brin/brin_minmax_multi.c | 2 +- src/test/regress/expected/brin_multi.out | 15 +++++++++++++++ src/test/regress/sql/brin_multi.sql | 21 +++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c index b4e50937609..270fe7a4baf 100644 --- a/src/backend/access/brin/brin_minmax_multi.c +++ b/src/backend/access/brin/brin_minmax_multi.c @@ -2138,7 +2138,7 @@ brin_minmax_multi_distance_timestamp(PG_FUNCTION_ARGS) if (TIMESTAMP_NOT_FINITE(dt1) || TIMESTAMP_NOT_FINITE(dt2)) PG_RETURN_FLOAT8(0); - delta = dt2 - dt1; + delta = (float8) dt2 - (float8) dt1; Assert(delta >= 0); diff --git a/src/test/regress/expected/brin_multi.out b/src/test/regress/expected/brin_multi.out index 677fb45f1fd..e24935298bc 100644 --- a/src/test/regress/expected/brin_multi.out +++ b/src/test/regress/expected/brin_multi.out @@ -477,3 +477,18 @@ EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE b = 1; Optimizer: Postgres query optimizer (4 rows) +-- test overflows during CREATE INDEX with extreme timestamp values +CREATE TABLE brin_timestamp_test(a TIMESTAMPTZ); +SET datestyle TO iso; +-- values close to timetamp minimum +INSERT INTO brin_timestamp_test +SELECT '4713-01-01 00:00:01 BC'::timestamptz + (i || ' seconds')::interval + FROM generate_series(1,30) s(i); +-- values close to timetamp maximum +INSERT INTO brin_timestamp_test +SELECT '294276-12-01 00:00:01'::timestamptz + (i || ' seconds')::interval + FROM generate_series(1,30) s(i); +CREATE INDEX ON brin_timestamp_test USING brin (a timestamptz_minmax_multi_ops) WITH (pages_per_range=1); +DROP TABLE brin_timestamp_test; +RESET enable_seqscan; +RESET datestyle; diff --git a/src/test/regress/sql/brin_multi.sql b/src/test/regress/sql/brin_multi.sql index 327a4215bf9..06ed60b3dd0 100644 --- a/src/test/regress/sql/brin_multi.sql +++ b/src/test/regress/sql/brin_multi.sql @@ -424,3 +424,24 @@ VACUUM ANALYZE brin_test_multi; EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE a = 1; -- Ensure brin index is not used when values are not correlated EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE b = 1; + +-- test overflows during CREATE INDEX with extreme timestamp values +CREATE TABLE brin_timestamp_test(a TIMESTAMPTZ); + +SET datestyle TO iso; + +-- values close to timetamp minimum +INSERT INTO brin_timestamp_test +SELECT '4713-01-01 00:00:01 BC'::timestamptz + (i || ' seconds')::interval + FROM generate_series(1,30) s(i); + +-- values close to timetamp maximum +INSERT INTO brin_timestamp_test +SELECT '294276-12-01 00:00:01'::timestamptz + (i || ' seconds')::interval + FROM generate_series(1,30) s(i); + +CREATE INDEX ON brin_timestamp_test USING brin (a timestamptz_minmax_multi_ops) WITH (pages_per_range=1); +DROP TABLE brin_timestamp_test; + +RESET enable_seqscan; +RESET datestyle; From 272981f4f1583a9264c4980c5335176a5caf82c0 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Fri, 27 Oct 2023 17:57:11 +0200 Subject: [PATCH 088/109] Fix calculation in brin_minmax_multi_distance_date When calculating the distance between date values, make sure to subtract them in the right order, i.e. (larger - smaller). The distance is used to determine which values to merge, and is expected to be a positive value. The code unfortunately did the subtraction in the opposite order, i.e. (smaller - larger), thus producing negative values and merging values the most distant values first. The resulting index is correct (i.e. produces correct results), but may be significantly less efficient. This affects all minmax-multi indexes on date columns. Backpatch to 14, where minmax-multi indexes were introduced. Reported-by: Ashutosh Bapat Reviewed-by: Ashutosh Bapat, Dean Rasheed Backpatch-through: 14 Discussion: https://postgr.es/m/eef0ea8c-4aaa-8d0d-027f-58b1f35dd170@enterprisedb.com --- src/backend/access/brin/brin_minmax_multi.c | 7 ++++++- src/test/regress/expected/brin_multi.out | 20 ++++++++++++++++++++ src/test/regress/sql/brin_multi.sql | 18 ++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c index 270fe7a4baf..407b7d8992a 100644 --- a/src/backend/access/brin/brin_minmax_multi.c +++ b/src/backend/access/brin/brin_minmax_multi.c @@ -2075,13 +2075,18 @@ brin_minmax_multi_distance_uuid(PG_FUNCTION_ARGS) Datum brin_minmax_multi_distance_date(PG_FUNCTION_ARGS) { + float8 delta = 0; DateADT dateVal1 = PG_GETARG_DATEADT(0); DateADT dateVal2 = PG_GETARG_DATEADT(1); if (DATE_NOT_FINITE(dateVal1) || DATE_NOT_FINITE(dateVal2)) PG_RETURN_FLOAT8(0); - PG_RETURN_FLOAT8(dateVal1 - dateVal2); + delta = (float8) dateVal2 - (float8) dateVal1; + + Assert(delta >= 0); + + PG_RETURN_FLOAT8(delta); } /* diff --git a/src/test/regress/expected/brin_multi.out b/src/test/regress/expected/brin_multi.out index e24935298bc..6f305c651a5 100644 --- a/src/test/regress/expected/brin_multi.out +++ b/src/test/regress/expected/brin_multi.out @@ -490,5 +490,25 @@ SELECT '294276-12-01 00:00:01'::timestamptz + (i || ' seconds')::interval FROM generate_series(1,30) s(i); CREATE INDEX ON brin_timestamp_test USING brin (a timestamptz_minmax_multi_ops) WITH (pages_per_range=1); DROP TABLE brin_timestamp_test; +-- test overflows during CREATE INDEX with extreme date values +CREATE TABLE brin_date_test(a DATE); +-- insert values close to date minimum +INSERT INTO brin_date_test SELECT '4713-01-01 BC'::date + i FROM generate_series(1, 30) s(i); +-- insert values close to date minimum +INSERT INTO brin_date_test SELECT '5874897-12-01'::date + i FROM generate_series(1, 30) s(i); +CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +-- make sure the ranges were built correctly and 2023-01-01 eliminates all +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; + QUERY PLAN +------------------------------------------------------------------------- + Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) + Recheck Cond: (a = '2023-01-01'::date) + -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '2023-01-01'::date) +(4 rows) + +DROP TABLE brin_date_test; RESET enable_seqscan; RESET datestyle; diff --git a/src/test/regress/sql/brin_multi.sql b/src/test/regress/sql/brin_multi.sql index 06ed60b3dd0..f764ac931a7 100644 --- a/src/test/regress/sql/brin_multi.sql +++ b/src/test/regress/sql/brin_multi.sql @@ -443,5 +443,23 @@ SELECT '294276-12-01 00:00:01'::timestamptz + (i || ' seconds')::interval CREATE INDEX ON brin_timestamp_test USING brin (a timestamptz_minmax_multi_ops) WITH (pages_per_range=1); DROP TABLE brin_timestamp_test; +-- test overflows during CREATE INDEX with extreme date values +CREATE TABLE brin_date_test(a DATE); + +-- insert values close to date minimum +INSERT INTO brin_date_test SELECT '4713-01-01 BC'::date + i FROM generate_series(1, 30) s(i); + +-- insert values close to date minimum +INSERT INTO brin_date_test SELECT '5874897-12-01'::date + i FROM generate_series(1, 30) s(i); + +CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_per_range=1); + +SET enable_seqscan = off; + +-- make sure the ranges were built correctly and 2023-01-01 eliminates all +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; + +DROP TABLE brin_date_test; RESET enable_seqscan; RESET datestyle; From ccdf008446a1b3d2ae4916be9bfe4ef29dc3aec5 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Fri, 27 Oct 2023 17:57:28 +0200 Subject: [PATCH 089/109] Fix minmax-multi on infinite date/timestamp values Make sure that infinite values in date/timestamp columns are treated as if in infinite distance. Infinite values should not be merged with other values, leaving them as outliers. The code however returned distance 0 in this case, so that infinite values were merged first. While this does not break the index (i.e. it still produces correct query results), it may make it much less efficient. We don't need explicit handling of infinite date/timestamp values when calculating distances, because those values are represented as extreme but regular values (e.g. INT64_MIN/MAX for the timestamp type). We don't need an exact distance, just a value that is much larger than distanced between regular values. With the added cast to double values, we can simply subtract the values. The regression test queries a value in the "gap" and checks the range was properly eliminated by the BRIN index. This only affects minmax-multi indexes on timestamp/date columns with infinite values, which is not very common in practice. The affected indexes may need to be rebuilt. Backpatch to 14, where minmax-multi indexes were introduced. Reported-by: Ashutosh Bapat Reviewed-by: Ashutosh Bapat, Dean Rasheed Backpatch-through: 14 Discussion: https://postgr.es/m/eef0ea8c-4aaa-8d0d-027f-58b1f35dd170@enterprisedb.com --- src/backend/access/brin/brin_minmax_multi.c | 6 --- src/test/regress/expected/brin_multi.out | 57 +++++++++++++++++++++ src/test/regress/sql/brin_multi.sql | 39 ++++++++++++++ 3 files changed, 96 insertions(+), 6 deletions(-) diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c index 407b7d8992a..9f7d6448a7c 100644 --- a/src/backend/access/brin/brin_minmax_multi.c +++ b/src/backend/access/brin/brin_minmax_multi.c @@ -2079,9 +2079,6 @@ brin_minmax_multi_distance_date(PG_FUNCTION_ARGS) DateADT dateVal1 = PG_GETARG_DATEADT(0); DateADT dateVal2 = PG_GETARG_DATEADT(1); - if (DATE_NOT_FINITE(dateVal1) || DATE_NOT_FINITE(dateVal2)) - PG_RETURN_FLOAT8(0); - delta = (float8) dateVal2 - (float8) dateVal1; Assert(delta >= 0); @@ -2140,9 +2137,6 @@ brin_minmax_multi_distance_timestamp(PG_FUNCTION_ARGS) Timestamp dt1 = PG_GETARG_TIMESTAMP(0); Timestamp dt2 = PG_GETARG_TIMESTAMP(1); - if (TIMESTAMP_NOT_FINITE(dt1) || TIMESTAMP_NOT_FINITE(dt2)) - PG_RETURN_FLOAT8(0); - delta = (float8) dt2 - (float8) dt1; Assert(delta >= 0); diff --git a/src/test/regress/expected/brin_multi.out b/src/test/regress/expected/brin_multi.out index 6f305c651a5..5c3b46235e4 100644 --- a/src/test/regress/expected/brin_multi.out +++ b/src/test/regress/expected/brin_multi.out @@ -509,6 +509,63 @@ SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; Index Cond: (a = '2023-01-01'::date) (4 rows) +DROP TABLE brin_date_test; +RESET enable_seqscan; +-- test handling of infinite timestamp values +CREATE TABLE brin_timestamp_test(a TIMESTAMP); +INSERT INTO brin_timestamp_test VALUES ('-infinity'), ('infinity'); +INSERT INTO brin_timestamp_test +SELECT i FROM generate_series('2000-01-01'::timestamp, '2000-02-09'::timestamp, '1 day'::interval) s(i); +CREATE INDEX ON brin_timestamp_test USING brin (a timestamp_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_timestamp_test WHERE a = '2023-01-01'::timestamp; + QUERY PLAN +------------------------------------------------------------------------------ + Bitmap Heap Scan on brin_timestamp_test (actual rows=0 loops=1) + Recheck Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) + -> Bitmap Index Scan on brin_timestamp_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) +(4 rows) + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_timestamp_test WHERE a = '1900-01-01'::timestamp; + QUERY PLAN +------------------------------------------------------------------------------ + Bitmap Heap Scan on brin_timestamp_test (actual rows=0 loops=1) + Recheck Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) + -> Bitmap Index Scan on brin_timestamp_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) +(4 rows) + +DROP TABLE brin_timestamp_test; +RESET enable_seqscan; +-- test handling of infinite date values +CREATE TABLE brin_date_test(a DATE); +INSERT INTO brin_date_test VALUES ('-infinity'), ('infinity'); +INSERT INTO brin_date_test SELECT '2000-01-01'::date + i FROM generate_series(1, 40) s(i); +CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; + QUERY PLAN +------------------------------------------------------------------------- + Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) + Recheck Cond: (a = '2023-01-01'::date) + -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '2023-01-01'::date) +(4 rows) + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; + QUERY PLAN +------------------------------------------------------------------------- + Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) + Recheck Cond: (a = '1900-01-01'::date) + -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '1900-01-01'::date) +(4 rows) + DROP TABLE brin_date_test; RESET enable_seqscan; RESET datestyle; diff --git a/src/test/regress/sql/brin_multi.sql b/src/test/regress/sql/brin_multi.sql index f764ac931a7..3028d6f2238 100644 --- a/src/test/regress/sql/brin_multi.sql +++ b/src/test/regress/sql/brin_multi.sql @@ -460,6 +460,45 @@ SET enable_seqscan = off; EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; +DROP TABLE brin_date_test; +RESET enable_seqscan; + +-- test handling of infinite timestamp values +CREATE TABLE brin_timestamp_test(a TIMESTAMP); + +INSERT INTO brin_timestamp_test VALUES ('-infinity'), ('infinity'); +INSERT INTO brin_timestamp_test +SELECT i FROM generate_series('2000-01-01'::timestamp, '2000-02-09'::timestamp, '1 day'::interval) s(i); + +CREATE INDEX ON brin_timestamp_test USING brin (a timestamp_minmax_multi_ops) WITH (pages_per_range=1); + +SET enable_seqscan = off; + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_timestamp_test WHERE a = '2023-01-01'::timestamp; + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_timestamp_test WHERE a = '1900-01-01'::timestamp; + +DROP TABLE brin_timestamp_test; +RESET enable_seqscan; + +-- test handling of infinite date values +CREATE TABLE brin_date_test(a DATE); + +INSERT INTO brin_date_test VALUES ('-infinity'), ('infinity'); +INSERT INTO brin_date_test SELECT '2000-01-01'::date + i FROM generate_series(1, 40) s(i); + +CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_per_range=1); + +SET enable_seqscan = off; + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; + DROP TABLE brin_date_test; RESET enable_seqscan; RESET datestyle; From 8157b187b9fa554d0a3e6dc6d9ba2751a85acc64 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Fri, 27 Oct 2023 17:57:44 +0200 Subject: [PATCH 090/109] Fix minmax-multi distance for extreme interval values When calculating distance for interval values, the code mostly mimicked interval_mi, i.e. it built a new interval value for the difference. That however does not work for sufficiently distant interval values, when the difference overflows the interval range. Instead, we can calculate the distance directly, without constructing the intermediate (and unnecessary) interval value. Backpatch to 14, where minmax-multi indexes were introduced. Reported-by: Dean Rasheed Reviewed-by: Ashutosh Bapat, Dean Rasheed Backpatch-through: 14 Discussion: https://postgr.es/m/eef0ea8c-4aaa-8d0d-027f-58b1f35dd170@enterprisedb.com --- src/backend/access/brin/brin_minmax_multi.c | 33 +++------------------ src/test/regress/expected/brin_multi.out | 29 ++++++++++++++++++ src/test/regress/sql/brin_multi.sql | 21 +++++++++++++ 3 files changed, 54 insertions(+), 29 deletions(-) diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c index 9f7d6448a7c..1e1159d6382 100644 --- a/src/backend/access/brin/brin_minmax_multi.c +++ b/src/backend/access/brin/brin_minmax_multi.c @@ -2154,45 +2154,20 @@ brin_minmax_multi_distance_interval(PG_FUNCTION_ARGS) Interval *ia = PG_GETARG_INTERVAL_P(0); Interval *ib = PG_GETARG_INTERVAL_P(1); - Interval *result; int64 dayfraction; int64 days; - result = (Interval *) palloc(sizeof(Interval)); - - result->month = ib->month - ia->month; - /* overflow check copied from int4mi */ - if (!SAMESIGN(ib->month, ia->month) && - !SAMESIGN(result->month, ib->month)) - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("interval out of range"))); - - result->day = ib->day - ia->day; - if (!SAMESIGN(ib->day, ia->day) && - !SAMESIGN(result->day, ib->day)) - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("interval out of range"))); - - result->time = ib->time - ia->time; - if (!SAMESIGN(ib->time, ia->time) && - !SAMESIGN(result->time, ib->time)) - ereport(ERROR, - (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), - errmsg("interval out of range"))); - /* * Delta is (fractional) number of days between the intervals. Assume * months have 30 days for consistency with interval_cmp_internal. We * don't need to be exact, in the worst case we'll build a bit less * efficient ranges. But we should not contradict interval_cmp. */ - dayfraction = result->time % USECS_PER_DAY; - days = result->time / USECS_PER_DAY; - days += result->month * INT64CONST(30); - days += result->day; + dayfraction = (ib->time % USECS_PER_DAY) - (ia->time % USECS_PER_DAY); + days = (ib->time / USECS_PER_DAY) - (ia->time / USECS_PER_DAY); + days += (int64) ib->day - (int64) ia->day; + days += ((int64) ib->month - (int64) ia->month) * INT64CONST(30); /* convert to double precision */ delta = (double) days + dayfraction / (double) USECS_PER_DAY; diff --git a/src/test/regress/expected/brin_multi.out b/src/test/regress/expected/brin_multi.out index 5c3b46235e4..987f010273b 100644 --- a/src/test/regress/expected/brin_multi.out +++ b/src/test/regress/expected/brin_multi.out @@ -569,3 +569,32 @@ SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; DROP TABLE brin_date_test; RESET enable_seqscan; RESET datestyle; +-- test handling of overflow for interval values +CREATE TABLE brin_interval_test(a INTERVAL); +INSERT INTO brin_interval_test SELECT (i || ' years')::interval FROM generate_series(-178000000, -177999980) s(i); +INSERT INTO brin_interval_test SELECT (i || ' years')::interval FROM generate_series( 177999980, 178000000) s(i); +CREATE INDEX ON brin_interval_test USING brin (a interval_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_interval_test WHERE a = '-30 years'::interval; + QUERY PLAN +----------------------------------------------------------------------------- + Bitmap Heap Scan on brin_interval_test (actual rows=0 loops=1) + Recheck Cond: (a = '@ 30 years ago'::interval) + -> Bitmap Index Scan on brin_interval_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '@ 30 years ago'::interval) +(4 rows) + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_interval_test WHERE a = '30 years'::interval; + QUERY PLAN +----------------------------------------------------------------------------- + Bitmap Heap Scan on brin_interval_test (actual rows=0 loops=1) + Recheck Cond: (a = '@ 30 years'::interval) + -> Bitmap Index Scan on brin_interval_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '@ 30 years'::interval) +(4 rows) + +DROP TABLE brin_interval_test; +RESET enable_seqscan; +RESET datestyle; diff --git a/src/test/regress/sql/brin_multi.sql b/src/test/regress/sql/brin_multi.sql index 3028d6f2238..8051ec997e4 100644 --- a/src/test/regress/sql/brin_multi.sql +++ b/src/test/regress/sql/brin_multi.sql @@ -502,3 +502,24 @@ SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; DROP TABLE brin_date_test; RESET enable_seqscan; RESET datestyle; + +-- test handling of overflow for interval values +CREATE TABLE brin_interval_test(a INTERVAL); + +INSERT INTO brin_interval_test SELECT (i || ' years')::interval FROM generate_series(-178000000, -177999980) s(i); + +INSERT INTO brin_interval_test SELECT (i || ' years')::interval FROM generate_series( 177999980, 178000000) s(i); + +CREATE INDEX ON brin_interval_test USING brin (a interval_minmax_multi_ops) WITH (pages_per_range=1); + +SET enable_seqscan = off; + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_interval_test WHERE a = '-30 years'::interval; + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_interval_test WHERE a = '30 years'::interval; + +DROP TABLE brin_interval_test; +RESET enable_seqscan; +RESET datestyle; From 3fbe319a51dbec90ffcbf9b46db0a66b01321b39 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 28 Oct 2023 11:54:40 -0400 Subject: [PATCH 091/109] Remove PHOT from our default timezone abbreviations list. Debian recently decided to split out a bunch of "obsolete" timezone names into a new tzdata-legacy package, which isn't installed by default. One of these zone names is Pacific/Enderbury, and that breaks our regression tests (on --with-system-tzdata builds) because our default timezone abbreviations list defines PHOT as Pacific/Enderbury. Pacific/Enderbury got renamed to Pacific/Kanton in tzdata 2021b, so that in distros that still have this entry it's just a symlink to Pacific/Kanton anyway. So one answer would be to redefine PHOT as Pacific/Kanton. However, then things would fail if the installed tzdata predates 2021b, which is recent enough that that seems like a real problem. Instead, let's just remove PHOT from the default list. That seems likely to affect nobody in the real world, because (a) it was an abbreviation that the tzdb crew made up in the first place, with no evidence of real-world usage, and (b) the total human population of the Phoenix Islands is less than two dozen persons, per Wikipedia. If anyone does use this zone abbreviation they can easily put it back via a custom abbreviations file. We'll keep PHOT in the Pacific.txt reference file, but change it to Pacific/Kanton there, as that definition seems more likely to be useful to future readers of that file. Per report from Victor Wagner. Back-patch to all supported branches. Discussion: https://postgr.es/m/20231027152049.4b5c8044@wagner.wagner.home --- src/timezone/tznames/Default | 1 - src/timezone/tznames/Pacific.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/timezone/tznames/Default b/src/timezone/tznames/Default index b76d170b968..5e692423b5a 100644 --- a/src/timezone/tznames/Default +++ b/src/timezone/tznames/Default @@ -616,7 +616,6 @@ NZST 43200 # New Zealand Standard Time # (Antarctica/McMurdo) # (Pacific/Auckland) PGT 36000 # Papua New Guinea Time (obsolete) -PHOT Pacific/Enderbury # Phoenix Islands Time (Kiribati) (obsolete) PONT 39600 # Ponape Time (Micronesia) (obsolete) PWT 32400 # Palau Time (obsolete) TAHT -36000 # Tahiti Time (obsolete) diff --git a/src/timezone/tznames/Pacific.txt b/src/timezone/tznames/Pacific.txt index c30008cb049..556a370af58 100644 --- a/src/timezone/tznames/Pacific.txt +++ b/src/timezone/tznames/Pacific.txt @@ -50,7 +50,7 @@ NZST 43200 # New Zealand Standard Time # (Antarctica/McMurdo) # (Pacific/Auckland) PGT 36000 # Papua New Guinea Time (obsolete) -PHOT Pacific/Enderbury # Phoenix Islands Time (Kiribati) (obsolete) +PHOT Pacific/Kanton # Phoenix Islands Time (Kiribati) (obsolete) PONT 39600 # Ponape Time (Micronesia) (obsolete) # CONFLICT! PST is not unique # Other timezones: From fe3962fa1f9743f0fe4dc8384fc3bbe56923fd61 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 28 Oct 2023 14:04:43 -0400 Subject: [PATCH 092/109] Fix intra-query memory leak when a SRF returns zero rows. When looping around after finding that the set-returning function returned zero rows for the current input tuple, ExecProjectSet neglected to reset either of the two memory contexts it's responsible for cleaning out. Typically this wouldn't cause much problem, because once the SRF does return at least one row, the contexts would get reset on the next call. However, if the SRF returns no rows for many input tuples in succession, quite a lot of memory could be transiently consumed. To fix, make sure we reset both contexts while looping around. Per bug #18172 from Sergei Kornilov. Back-patch to all supported branches. Discussion: https://postgr.es/m/18172-9b8c5fc1d676ded3@postgresql.org --- src/backend/executor/nodeProjectSet.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/backend/executor/nodeProjectSet.c b/src/backend/executor/nodeProjectSet.c index 07be814d7b5..d441a57cc0e 100644 --- a/src/backend/executor/nodeProjectSet.c +++ b/src/backend/executor/nodeProjectSet.c @@ -72,20 +72,22 @@ ExecProjectSet(PlanState *pstate) return resultSlot; } - /* - * Reset argument context to free any expression evaluation storage - * allocated in the previous tuple cycle. Note this can't happen until - * we're done projecting out tuples from a scan tuple, as ValuePerCall - * functions are allowed to reference the arguments for each returned - * tuple. - */ - MemoryContextReset(node->argcontext); - /* * Get another input tuple and project SRFs from it. */ for (;;) { + /* + * Reset argument context to free any expression evaluation storage + * allocated in the previous tuple cycle. Note this can't happen + * until we're done projecting out tuples from a scan tuple, as + * ValuePerCall functions are allowed to reference the arguments for + * each returned tuple. However, if we loop around after finding that + * no rows are produced from a scan tuple, we should reset, to avoid + * leaking memory when many successive scan tuples produce no rows. + */ + MemoryContextReset(node->argcontext); + /* * Retrieve tuples from the outer plan until there are no more. */ @@ -111,6 +113,12 @@ ExecProjectSet(PlanState *pstate) */ if (resultSlot) return resultSlot; + + /* + * When we do loop back, we'd better reset the econtext again, just in + * case the SRF leaked some memory there. + */ + ResetExprContext(econtext); } return NULL; From 745f2db1e2818553039b85b7b218e1a1b3471af4 Mon Sep 17 00:00:00 2001 From: Dean Rasheed Date: Sun, 29 Oct 2023 11:14:34 +0000 Subject: [PATCH 093/109] btree_gin: Fix calculation of leftmost interval value. Formerly, the value computed by leftmostvalue_interval() was a long way short of the minimum possible interval value. As a result, an index scan on a GIN index on an interval column with < or <= operators would miss large negative interval values. Fix by setting all fields of the leftmost interval to their minimum values, ensuring that the result is less than any other possible interval. Since this only affects index searches, no index rebuild is necessary. Back-patch to all supported branches. Dean Rasheed, reviewed by Heikki Linnakangas. Discussion: https://postgr.es/m/CAEZATCV80%2BgOfF8ehNUUfaKBZgZMDfCfL-g1HhWGb6kC3rpDfw%40mail.gmail.com --- contrib/btree_gin/btree_gin.c | 6 +++--- contrib/btree_gin/expected/interval.out | 16 +++++++++++----- contrib/btree_gin/sql/interval.sql | 4 +++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/contrib/btree_gin/btree_gin.c b/contrib/btree_gin/btree_gin.c index c50d68ce186..b09bb8df964 100644 --- a/contrib/btree_gin/btree_gin.c +++ b/contrib/btree_gin/btree_gin.c @@ -306,9 +306,9 @@ leftmostvalue_interval(void) { Interval *v = palloc(sizeof(Interval)); - v->time = DT_NOBEGIN; - v->day = 0; - v->month = 0; + v->time = PG_INT64_MIN; + v->day = PG_INT32_MIN; + v->month = PG_INT32_MIN; return IntervalPGetDatum(v); } diff --git a/contrib/btree_gin/expected/interval.out b/contrib/btree_gin/expected/interval.out index 1f6ef54070e..8bb9806650d 100644 --- a/contrib/btree_gin/expected/interval.out +++ b/contrib/btree_gin/expected/interval.out @@ -3,30 +3,34 @@ CREATE TABLE test_interval ( i interval ); INSERT INTO test_interval VALUES + ( '-178000000 years' ), ( '03:55:08' ), ( '04:55:08' ), ( '05:55:08' ), ( '08:55:08' ), ( '09:55:08' ), - ( '10:55:08' ) + ( '10:55:08' ), + ( '178000000 years' ) ; CREATE INDEX idx_interval ON test_interval USING gin (i); SELECT * FROM test_interval WHERE i<'08:55:08'::interval ORDER BY i; i -------------------------- + @ 178000000 years ago @ 3 hours 55 mins 8 secs @ 4 hours 55 mins 8 secs @ 5 hours 55 mins 8 secs -(3 rows) +(4 rows) SELECT * FROM test_interval WHERE i<='08:55:08'::interval ORDER BY i; i -------------------------- + @ 178000000 years ago @ 3 hours 55 mins 8 secs @ 4 hours 55 mins 8 secs @ 5 hours 55 mins 8 secs @ 8 hours 55 mins 8 secs -(4 rows) +(5 rows) SELECT * FROM test_interval WHERE i='08:55:08'::interval ORDER BY i; i @@ -40,12 +44,14 @@ SELECT * FROM test_interval WHERE i>='08:55:08'::interval ORDER BY i; @ 8 hours 55 mins 8 secs @ 9 hours 55 mins 8 secs @ 10 hours 55 mins 8 secs -(3 rows) + @ 178000000 years +(4 rows) SELECT * FROM test_interval WHERE i>'08:55:08'::interval ORDER BY i; i --------------------------- @ 9 hours 55 mins 8 secs @ 10 hours 55 mins 8 secs -(2 rows) + @ 178000000 years +(3 rows) diff --git a/contrib/btree_gin/sql/interval.sql b/contrib/btree_gin/sql/interval.sql index e3851587833..7a2f3ac0d85 100644 --- a/contrib/btree_gin/sql/interval.sql +++ b/contrib/btree_gin/sql/interval.sql @@ -5,12 +5,14 @@ CREATE TABLE test_interval ( ); INSERT INTO test_interval VALUES + ( '-178000000 years' ), ( '03:55:08' ), ( '04:55:08' ), ( '05:55:08' ), ( '08:55:08' ), ( '09:55:08' ), - ( '10:55:08' ) + ( '10:55:08' ), + ( '178000000 years' ) ; CREATE INDEX idx_interval ON test_interval USING gin (i); From 176640109d8f21e8771067a99faeba9e171509f0 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Mon, 30 Oct 2023 14:46:05 -0700 Subject: [PATCH 094/109] amcheck: Distinguish interrupted page deletion from corruption. This prevents false-positive reports about "the first child of leftmost target page is not leftmost of its level", "block %u is not leftmost" and "left link/right link pair". They appeared if amcheck ran before VACUUM cleaned things, after a cluster exited recovery between the first-stage and second-stage WAL records of a deletion. Back-patch to v11 (all supported versions). Reviewed by Peter Geoghegan. Discussion: https://postgr.es/m/20231005025232.c7.nmisch@google.com --- contrib/amcheck/t/005_pitr.pl | 89 +++++++++++++++++++++++++++++++++ contrib/amcheck/verify_nbtree.c | 83 ++++++++++++++++++++++++++++-- 2 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 contrib/amcheck/t/005_pitr.pl diff --git a/contrib/amcheck/t/005_pitr.pl b/contrib/amcheck/t/005_pitr.pl new file mode 100644 index 00000000000..07187a799be --- /dev/null +++ b/contrib/amcheck/t/005_pitr.pl @@ -0,0 +1,89 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +# Test integrity of intermediate states by PITR to those states +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# origin node: generate WAL records of interest. +my $origin = PostgreSQL::Test::Cluster->new('origin'); +$origin->init(has_archiving => 1, allows_streaming => 1); +$origin->append_conf('postgresql.conf', 'autovacuum = off'); +$origin->start; +$origin->backup('my_backup'); +# Create a table with each of 6 PK values spanning 1/4 of a block. Delete the +# first four, so one index leaf is eligible for deletion. Make a replication +# slot just so pg_waldump will always have access to later WAL. +my $setup = <safe_psql('postgres', $setup); +my $before_vacuum_walfile = + $origin->safe_psql('postgres', "SELECT pg_walfile_name(pg_current_wal_lsn())"); +# VACUUM to delete the aforementioned leaf page. Force an XLogFlush() by +# dropping a permanent table. That way, the XLogReader infrastructure can +# always see VACUUM's records, even under synchronous_commit=off. Finally, +# find the LSN of that VACUUM's last UNLINK_PAGE record. +my $vacuum = <safe_psql('postgres', $vacuum); +$origin->stop; +my $unlink_lsn = do { + local %ENV = $origin->_get_env(); + my $stdout; + run_log(['pg_waldump', '-p', $origin->data_dir . '/pg_wal', + $before_vacuum_walfile, $after_unlink_walfile], + '>', \$stdout); + $stdout =~ m|^rmgr: Btree .*, lsn: ([/0-9A-F]+), .*, desc: UNLINK_PAGE left|m; + $1; +}; +die "did not find UNLINK_PAGE record" unless $unlink_lsn; + +# replica node: amcheck at notable points in the WAL stream +my $replica = PostgreSQL::Test::Cluster->new('replica'); +$replica->init_from_backup($origin, 'my_backup', has_restoring => 1); +$replica->append_conf('postgresql.conf', + "recovery_target_lsn = '$unlink_lsn'"); +$replica->append_conf('postgresql.conf', 'recovery_target_inclusive = off'); +$replica->append_conf('postgresql.conf', 'recovery_target_action = promote'); +$replica->start; +$replica->poll_query_until('postgres', "SELECT pg_is_in_recovery() = 'f';") + or die "Timed out while waiting for PITR promotion"; +# recovery done; run amcheck +my $debug = "SET client_min_messages = 'debug1'"; +my ($rc, $stderr); +$rc = $replica->psql( + 'postgres', + "$debug; SELECT bt_index_parent_check('not_leftmost_pk', true)", + stderr => \$stderr); +print STDERR $stderr, "\n"; +is($rc, 0, "bt_index_parent_check passes"); +like( + $stderr, + qr/interrupted page deletion detected/, + "bt_index_parent_check: interrupted page deletion detected"); +$rc = $replica->psql( + 'postgres', + "$debug; SELECT bt_index_check('not_leftmost_pk', true)", + stderr => \$stderr); +print STDERR $stderr, "\n"; +is($rc, 0, "bt_index_check passes"); + +done_testing(); diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index 6c3b3c27ccc..a1b581871e6 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -146,6 +146,9 @@ static void bt_check_every_level(Relation rel, Relation heaprel, bool rootdescend); static BtreeLevel bt_check_level_from_leftmost(BtreeCheckState *state, BtreeLevel level); +static bool bt_leftmost_ignoring_half_dead(BtreeCheckState *state, + BlockNumber start, + BTPageOpaque start_opaque); static void bt_recheck_sibling_links(BtreeCheckState *state, BlockNumber btpo_prev_from_target, BlockNumber leftcurrent); @@ -775,7 +778,7 @@ bt_check_level_from_leftmost(BtreeCheckState *state, BtreeLevel level) */ if (state->readonly) { - if (!P_LEFTMOST(opaque)) + if (!bt_leftmost_ignoring_half_dead(state, current, opaque)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("block %u is not leftmost in index \"%s\"", @@ -829,8 +832,16 @@ bt_check_level_from_leftmost(BtreeCheckState *state, BtreeLevel level) */ } - /* Sibling links should be in mutual agreement */ - if (opaque->btpo_prev != leftcurrent) + /* + * Sibling links should be in mutual agreement. There arises + * leftcurrent == P_NONE && btpo_prev != P_NONE when the left sibling + * of the parent's low-key downlink is half-dead. (A half-dead page + * has no downlink from its parent.) Under heavyweight locking, the + * last bt_leftmost_ignoring_half_dead() validated this btpo_prev. + * Without heavyweight locking, validation of the P_NONE case remains + * unimplemented. + */ + if (opaque->btpo_prev != leftcurrent && leftcurrent != P_NONE) bt_recheck_sibling_links(state, opaque->btpo_prev, leftcurrent); /* Check level */ @@ -911,6 +922,66 @@ bt_check_level_from_leftmost(BtreeCheckState *state, BtreeLevel level) return nextleveldown; } +/* + * Like P_LEFTMOST(start_opaque), but accept an arbitrarily-long chain of + * half-dead, sibling-linked pages to the left. If a half-dead page appears + * under state->readonly, the database exited recovery between the first-stage + * and second-stage WAL records of a deletion. + */ +static bool +bt_leftmost_ignoring_half_dead(BtreeCheckState *state, + BlockNumber start, + BTPageOpaque start_opaque) +{ + BlockNumber reached = start_opaque->btpo_prev, + reached_from = start; + bool all_half_dead = true; + + /* + * To handle the !readonly case, we'd need to accept BTP_DELETED pages and + * potentially observe nbtree/README "Page deletion and backwards scans". + */ + Assert(state->readonly); + + while (reached != P_NONE && all_half_dead) + { + Page page = palloc_btree_page(state, reached); + BTPageOpaque reached_opaque = (BTPageOpaque) PageGetSpecialPointer(page); + + CHECK_FOR_INTERRUPTS(); + + /* + * Try to detect btpo_prev circular links. _bt_unlink_halfdead_page() + * writes that side-links will continue to point to the siblings. + * Check btpo_next for that property. + */ + all_half_dead = P_ISHALFDEAD(reached_opaque) && + reached != start && + reached != reached_from && + reached_opaque->btpo_next == reached_from; + if (all_half_dead) + { + XLogRecPtr pagelsn = PageGetLSN(page); + + /* pagelsn should point to an XLOG_BTREE_MARK_PAGE_HALFDEAD */ + ereport(DEBUG1, + (errcode(ERRCODE_NO_DATA), + errmsg_internal("harmless interrupted page deletion detected in index \"%s\"", + RelationGetRelationName(state->rel)), + errdetail_internal("Block=%u right block=%u page lsn=%X/%X.", + reached, reached_from, + LSN_FORMAT_ARGS(pagelsn)))); + + reached_from = reached; + reached = reached_opaque->btpo_prev; + } + + pfree(page); + } + + return all_half_dead; +} + /* * Raise an error when target page's left link does not point back to the * previous target page, called leftcurrent here. The leftcurrent page's @@ -951,6 +1022,9 @@ bt_recheck_sibling_links(BtreeCheckState *state, BlockNumber btpo_prev_from_target, BlockNumber leftcurrent) { + /* taking BTPageOpaque from metapage would give irrelevant findings */ + Assert(leftcurrent != P_NONE); + if (!state->readonly) { Buffer lbuf; @@ -1934,7 +2008,8 @@ bt_child_highkey_check(BtreeCheckState *state, opaque = (BTPageOpaque) PageGetSpecialPointer(page); /* The first page we visit at the level should be leftmost */ - if (first && !BlockNumberIsValid(state->prevrightlink) && !P_LEFTMOST(opaque)) + if (first && !BlockNumberIsValid(state->prevrightlink) && + !bt_leftmost_ignoring_half_dead(state, blkno, opaque)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("the first child of leftmost target page is not leftmost of its level in index \"%s\"", From 5018fb3ae184a018cdd5b1e228587c7e999d5d03 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Mon, 30 Oct 2023 14:46:05 -0700 Subject: [PATCH 095/109] Diagnose !indisvalid in more SQL functions. pgstatindex failed with ERRCODE_DATA_CORRUPTED, of the "can't-happen" class XX. The other functions succeeded on an empty index; they might have malfunctioned if the failed index build left torn I/O or other complex state. Report an ERROR in statistics functions pgstatindex, pgstatginindex, pgstathashindex, and pgstattuple. Report DEBUG1 and skip all index I/O in maintenance functions brin_desummarize_range, brin_summarize_new_values, brin_summarize_range, and gin_clean_pending_list. Back-patch to v11 (all supported versions). Discussion: https://postgr.es/m/20231001195309.a3@google.com --- contrib/pgstattuple/pgstatindex.c | 26 ++++++++++++++++++++++++++ contrib/pgstattuple/pgstattuple.c | 7 +++++++ src/backend/access/brin/brin.c | 27 +++++++++++++++++++++------ src/backend/access/gin/ginfast.c | 23 ++++++++++++++++++++--- 4 files changed, 74 insertions(+), 9 deletions(-) diff --git a/contrib/pgstattuple/pgstatindex.c b/contrib/pgstattuple/pgstatindex.c index f7763b2e2fc..fdfaef925cd 100644 --- a/contrib/pgstattuple/pgstatindex.c +++ b/contrib/pgstattuple/pgstatindex.c @@ -237,6 +237,18 @@ pgstatindex_impl(Relation rel, FunctionCallInfo fcinfo) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot access temporary tables of other sessions"))); + /* + * A !indisready index could lead to ERRCODE_DATA_CORRUPTED later, so exit + * early. We're capable of assessing an indisready&&!indisvalid index, + * but the results could be confusing. For example, the index's size + * could be too low for a valid index of the table. + */ + if (!rel->rd_index->indisvalid) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(rel)))); + /* * Read metapage */ @@ -542,6 +554,13 @@ pgstatginindex_internal(Oid relid, FunctionCallInfo fcinfo) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot access temporary indexes of other sessions"))); + /* see pgstatindex_impl */ + if (!rel->rd_index->indisvalid) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(rel)))); + /* * Read metapage */ @@ -619,6 +638,13 @@ pgstathashindex(PG_FUNCTION_ARGS) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot access temporary indexes of other sessions"))); + /* see pgstatindex_impl */ + if (!rel->rd_index->indisvalid) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(rel)))); + /* Get the information we need from the metapage. */ memset(&stats, 0, sizeof(stats)); metabuf = _hash_getbuf(rel, HASH_METAPAGE, HASH_READ, LH_META_PAGE); diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index ae0f09fd05e..cafe43fec3c 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -264,6 +264,13 @@ pgstat_relation(Relation rel, FunctionCallInfo fcinfo) case RELKIND_DIRECTORY_TABLE: return pgstat_heap(rel, fcinfo); case RELKIND_INDEX: + /* see pgstatindex_impl */ + if (!rel->rd_index->indisvalid) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(rel)))); + switch (rel->rd_rel->relam) { case BTREE_AM_OID: diff --git a/src/backend/access/brin/brin.c b/src/backend/access/brin/brin.c index fc565a55cf3..2a9d6c22810 100644 --- a/src/backend/access/brin/brin.c +++ b/src/backend/access/brin/brin.c @@ -1241,8 +1241,14 @@ brin_summarize_range_internal(PG_FUNCTION_ARGS) errmsg("could not open parent table of index \"%s\"", RelationGetRelationName(indexRel)))); - /* OK, do it */ - brinsummarize(indexRel, heapRel, heapBlk, true, &numSummarized, NULL); + /* see gin_clean_pending_list() */ + if (indexRel->rd_index->indisvalid) + brinsummarize(indexRel, heapRel, heapBlk, true, &numSummarized, NULL); + else + ereport(DEBUG1, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(indexRel)))); /* Roll back any GUC changes executed by index functions */ AtEOXact_GUC(false, save_nestlevel); @@ -1327,12 +1333,21 @@ brin_desummarize_range(PG_FUNCTION_ARGS) errmsg("could not open parent table of index \"%s\"", RelationGetRelationName(indexRel)))); - /* the revmap does the hard work */ - do + /* see gin_clean_pending_list() */ + if (indexRel->rd_index->indisvalid) { - done = brinRevmapDesummarizeRange(indexRel, heapBlk); + /* the revmap does the hard work */ + do + { + done = brinRevmapDesummarizeRange(indexRel, heapBlk); + } + while (!done); } - while (!done); + else + ereport(DEBUG1, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(indexRel)))); relation_close(indexRel, ShareUpdateExclusiveLock); relation_close(heapRel, ShareUpdateExclusiveLock); diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c index 8679698b1f2..3f84e90b260 100644 --- a/src/backend/access/gin/ginfast.c +++ b/src/backend/access/gin/ginfast.c @@ -1035,7 +1035,6 @@ gin_clean_pending_list(PG_FUNCTION_ARGS) Oid indexoid = PG_GETARG_OID(0); Relation indexRel = index_open(indexoid, RowExclusiveLock); IndexBulkDeleteResult stats; - GinState ginstate; if (RecoveryInProgress()) ereport(ERROR, @@ -1067,8 +1066,26 @@ gin_clean_pending_list(PG_FUNCTION_ARGS) RelationGetRelationName(indexRel)); memset(&stats, 0, sizeof(stats)); - initGinState(&ginstate, indexRel); - ginInsertCleanup(&ginstate, true, true, true, &stats); + + /* + * Can't assume anything about the content of an !indisready index. Make + * those a no-op, not an error, so users can just run this function on all + * indexes of the access method. Since an indisready&&!indisvalid index + * is merely awaiting missed aminsert calls, we're capable of processing + * it. Decline to do so, out of an abundance of caution. + */ + if (indexRel->rd_index->indisvalid) + { + GinState ginstate; + + initGinState(&ginstate, indexRel); + ginInsertCleanup(&ginstate, true, true, true, &stats); + } + else + ereport(DEBUG1, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("index \"%s\" is not valid", + RelationGetRelationName(indexRel)))); index_close(indexRel, RowExclusiveLock); From 04aa112c98506043b7b9cd6d0cb9e5dd5cdaa121 Mon Sep 17 00:00:00 2001 From: David Rowley Date: Tue, 31 Oct 2023 16:43:28 +1300 Subject: [PATCH 096/109] Adjust the order of the prechecks in pgrowlocks() 4b8266415 added a precheck to pgrowlocks() to ensure the given object's pg_class.relam is HEAP_TABLE_AM_OID, however, that check was put before another check which was checking if the given object was a partitioned table. Since the pg_class.relam is always InvalidOid for partitioned tables, if pgrowlocks() was called passing a partitioned table, then the "only heap AM is supported" error would be raised instead of the intended error about the given object being a partitioned table. Here we simply move the pg_class.relam check to after the check that verifies that we are in fact working with a normal (non-partitioned) table. Reported-by: jian he Discussion: https://postgr.es/m/CACJufxFaSp_WguFCf0X98951zFVX+dXFnF1mxAb-G3g1HiHOow@mail.gmail.com Backpatch-through: 12, where 4b8266415 was introduced. --- contrib/pgrowlocks/pgrowlocks.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/pgrowlocks/pgrowlocks.c b/contrib/pgrowlocks/pgrowlocks.c index fd6d21d0bd1..d4d6a8c7299 100644 --- a/contrib/pgrowlocks/pgrowlocks.c +++ b/contrib/pgrowlocks/pgrowlocks.c @@ -107,10 +107,6 @@ pgrowlocks(PG_FUNCTION_ARGS) relrv = makeRangeVarFromNameList(textToQualifiedNameList(relname)); rel = relation_openrv(relrv, AccessShareLock); - if (rel->rd_rel->relam != HEAP_TABLE_AM_OID) - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only heap AM is supported"))); - if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), @@ -122,6 +118,10 @@ pgrowlocks(PG_FUNCTION_ARGS) (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a table", RelationGetRelationName(rel)))); + else if (rel->rd_rel->relam != HEAP_TABLE_AM_OID) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only heap AM is supported"))); /* * check permissions: must have SELECT on table or be in From f4df4ea972a2a85d56e70ebc00e5d0ab3d7c17c7 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 31 Oct 2023 09:10:35 -0400 Subject: [PATCH 097/109] doc: 1-byte varlena headers can be used for user PLAIN storage This also updates some C comments. Reported-by: suchithjn22@gmail.com Discussion: https://postgr.es/m/167336599095.2667301.15497893107226841625@wrigleys.postgresql.org Author: Laurenz Albe (doc patch) Backpatch-through: 11 --- doc/src/sgml/storage.sgml | 4 +--- src/backend/access/common/heaptuple.c | 11 ++++++++++- src/backend/utils/adt/rangetypes.c | 3 ++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 41891c96b7d..d4155e5f47e 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -461,9 +461,7 @@ for storing TOAST-able columns on disk: PLAIN prevents either compression or - out-of-line storage; furthermore it disables use of single-byte headers - for varlena types. - This is the only possible strategy for + out-of-line storage. This is the only possible strategy for columns of non-TOAST-able data types. diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index c4ad0d6c3d6..8933bdd0397 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -73,7 +73,16 @@ #include "catalog/pg_type.h" #include "cdb/cdbvars.h" /* GpIdentity.segindex */ -/* Does att's datatype allow packing into the 1-byte-header varlena format? */ +/* + * Does att's datatype allow packing into the 1-byte-header varlena format? + * While functions that use TupleDescAttr() and assign attstorage = + * TYPSTORAGE_PLAIN cannot use packed varlena headers, functions that call + * TupleDescInitEntry() use typeForm->typstorage (TYPSTORAGE_EXTENDED) and + * can use packed varlena headers, e.g.: + * CREATE TABLE test(a VARCHAR(10000) STORAGE PLAIN); + * INSERT INTO test VALUES (repeat('A',10)); + * This can be verified with pageinspect. + */ #define ATT_IS_PACKABLE(att) \ ((att)->attlen == -1 && (att)->attstorage != TYPSTORAGE_PLAIN) /* Use this if it's already known varlena */ diff --git a/src/backend/utils/adt/rangetypes.c b/src/backend/utils/adt/rangetypes.c index 815175a654e..e3ff545fd3f 100644 --- a/src/backend/utils/adt/rangetypes.c +++ b/src/backend/utils/adt/rangetypes.c @@ -2513,7 +2513,8 @@ range_contains_elem_internal(TypeCacheEntry *typcache, const RangeType *r, Datum * values into a range object. They are modeled after heaptuple.c's * heap_compute_data_size() and heap_fill_tuple(), but we need not handle * null values here. TYPE_IS_PACKABLE must test the same conditions as - * heaptuple.c's ATT_IS_PACKABLE macro. + * heaptuple.c's ATT_IS_PACKABLE macro. See the comments thare for more + * details. */ /* Does datatype allow packing into the 1-byte-header varlena format? */ From 05cbb58c6d5ed1fd6ee9be6d791dadf2adef0fc6 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 31 Oct 2023 09:23:09 -0400 Subject: [PATCH 098/109] doc: add function argument and query parameter limits Also reorder entries and add commas. Reported-by: David G. Johnston Discussion: https://postgr.es/m/CAKFQuwYeNPxeocV3_0+Zx=_Xwvg+sNyEMdzyG5s2E2e0hZLQhg@mail.gmail.com Author: David G. Johnston (partial) Backpatch-through: 12 --- doc/src/sgml/limits.sgml | 42 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/doc/src/sgml/limits.sgml b/doc/src/sgml/limits.sgml index d5b2b627ddf..c549447013f 100644 --- a/doc/src/sgml/limits.sgml +++ b/doc/src/sgml/limits.sgml @@ -57,14 +57,14 @@ columns per table - 1600 + 1,600 further limited by tuple size fitting on a single page; see note below columns in a result set - 1664 + 1,664 @@ -74,12 +74,6 @@ - - identifier length - 63 bytes - can be increased by recompiling PostgreSQL - - indexes per table unlimited @@ -92,11 +86,29 @@ can be increased by recompiling PostgreSQL - - partition keys - 32 - can be increased by recompiling PostgreSQL - + + partition keys + 32 + can be increased by recompiling PostgreSQL + + + + identifier length + 63 bytes + can be increased by recompiling PostgreSQL + + + + function arguments + 100 + can be increased by recompiling PostgreSQL + + + + query parameters + 65,535 + +
@@ -104,9 +116,9 @@ The maximum number of columns for a table is further reduced as the tuple being stored must fit in a single 8192-byte heap page. For example, - excluding the tuple header, a tuple made up of 1600 int columns + excluding the tuple header, a tuple made up of 1,600 int columns would consume 6400 bytes and could be stored in a heap page, but a tuple of - 1600 bigint columns would consume 12800 bytes and would + 1,600 bigint columns would consume 12800 bytes and would therefore not fit inside a heap page. Variable-length fields of types such as text, varchar, and char From cabfa0a95a22917259ce7aa826a1d1669c907e11 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Tue, 31 Oct 2023 10:21:32 -0400 Subject: [PATCH 099/109] doc: improve ALTER SYSTEM description of value list quoting Reported-by: splarv@ya.ru Discussion: https://postgr.es/m/167105927893.1897.13227723035830709578@wrigleys.postgresql.org Backpatch-through: 11 --- doc/src/sgml/ref/alter_system.sgml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/alter_system.sgml b/doc/src/sgml/ref/alter_system.sgml index 5e41f7f6444..ab8d1502a7c 100644 --- a/doc/src/sgml/ref/alter_system.sgml +++ b/doc/src/sgml/ref/alter_system.sgml @@ -21,7 +21,7 @@ PostgreSQL documentation -ALTER SYSTEM SET configuration_parameter { TO | = } { value | 'value' | DEFAULT } +ALTER SYSTEM SET configuration_parameter { TO | = } { value [, ...] | DEFAULT } ALTER SYSTEM RESET configuration_parameter ALTER SYSTEM RESET ALL @@ -82,9 +82,17 @@ ALTER SYSTEM RESET ALL New value of the parameter. Values can be specified as string constants, identifiers, numbers, or comma-separated lists of these, as appropriate for the particular parameter. + Values that are neither numbers nor valid identifiers must be quoted. DEFAULT can be written to specify removing the parameter and its value from postgresql.auto.conf. + + + For some list-accepting parameters, quoted values will produce + double-quoted output to preserve whitespace and commas; for others, + double-quotes must be used inside single-quoted strings to get + this effect. + From d954cf7dc8fafb43a74aaf27a8b64ff8d89228c7 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 2 Nov 2023 07:33:02 +0900 Subject: [PATCH 100/109] doc: Replace reference to ERRCODE_RAISE_EXCEPTION by "raise_exception" This part of the documentation refers to exceptions as handled by PL/pgSQL, and using the internal error code is confusing. Per thinko in 66bde49d96a9. Reported-by: Euler Taveira, Bruce Momjian Discussion: https://postgr.es/m/ZUEUnLevXyW7DlCs@momjian.us Backpatch-through: 11 --- doc/src/sgml/plpgsql.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/plpgsql.sgml b/doc/src/sgml/plpgsql.sgml index 8017ba392a4..aecf8ffd846 100644 --- a/doc/src/sgml/plpgsql.sgml +++ b/doc/src/sgml/plpgsql.sgml @@ -3903,7 +3903,7 @@ RAISE unique_violation USING MESSAGE = 'Duplicate user ID: ' || user_id; If no condition name nor SQLSTATE is specified in a RAISE EXCEPTION command, the default is to use - ERRCODE_RAISE_EXCEPTION (P0001). + raise_exception (P0001). If no message text is specified, the default is to use the condition name or SQLSTATE as message text. From fcff79dcf3cd030ece86d86db44b10be422ccaac Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 3 Nov 2023 09:51:53 -0400 Subject: [PATCH 101/109] doc: ALTER DEFAULT PRIVILEGES does not affect inherited roles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported-by: Jordi Gutiérrez Hermoso Discussion: https://postgr.es/m/72652d72e1816bfc3c05d40f9e0e0373d07823c8.camel@octave.org Co-authored-by: Laurenz Albe Backpatch-through: 11 --- doc/src/sgml/ref/alter_default_privileges.sgml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/ref/alter_default_privileges.sgml b/doc/src/sgml/ref/alter_default_privileges.sgml index f1d54f5aa35..8a6006188d3 100644 --- a/doc/src/sgml/ref/alter_default_privileges.sgml +++ b/doc/src/sgml/ref/alter_default_privileges.sgml @@ -137,7 +137,11 @@ REVOKE [ GRANT OPTION FOR ] The name of an existing role of which the current role is a member. - If FOR ROLE is omitted, the current role is assumed. + Default access privileges are not inherited, so member roles + must use SET ROLE to access these privileges, + or ALTER DEFAULT PRIVILEGES must be run for + each member role. If FOR ROLE is omitted, + the current role is assumed. From c3e5f11cf0c48c9be7403dc160546f429d1a5d5d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 3 Nov 2023 11:48:23 -0400 Subject: [PATCH 102/109] Doc: update CREATE RULE ref page's hoary discussion of views. This text left one with the impression that an ON SELECT rule could be attached to a plain table, which has not been true since commit 264c06820 (meaning the text was already misleading when written, evidently by me in 96bd67f61). However, it didn't get really bad until b23cd185f removed the convert-a-table-to-a-view logic, which had made it possible for scripts that thought they were attaching ON SELECTs to tables to still work. Rewrite into a form that makes it clear that an ON SELECT rule is better regarded as an implementation detail of a view. Pre-v16, point out that adding ON SELECT to a table actually converts it to a view. Per bug #18178 from Joshua Uyehara. Back-patch to all supported branches. Discussion: https://postgr.es/m/18178-05534d7064044d2d@postgresql.org --- doc/src/sgml/ref/create_rule.sgml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/src/sgml/ref/create_rule.sgml b/doc/src/sgml/ref/create_rule.sgml index dbf4c937841..76029abf515 100644 --- a/doc/src/sgml/ref/create_rule.sgml +++ b/doc/src/sgml/ref/create_rule.sgml @@ -59,15 +59,17 @@ CREATE [ OR REPLACE ] RULE name AS - Presently, ON SELECT rules must be unconditional - INSTEAD rules and must have actions that consist - of a single SELECT command. Thus, an - ON SELECT rule effectively turns the table into - a view, whose visible contents are the rows returned by the rule's - SELECT command rather than whatever had been - stored in the table (if anything). It is considered better style - to write a CREATE VIEW command than to create a - real table and define an ON SELECT rule for it. + Presently, ON SELECT rules can only be attached + to views. (Attaching one to a table converts the table into a view.) + Such a rule must be named "_RETURN", + must be an unconditional INSTEAD rule, and must have + an action that consists of a single SELECT command. + This command defines the visible contents of the view. (The view + itself is basically a dummy table with no storage.) It's best to + regard such a rule as an implementation detail. While a view can be + redefined via CREATE OR REPLACE RULE "_RETURN" AS + ..., it's better style to use CREATE OR REPLACE + VIEW. From 6e3e44d820dae6f0474cd6adff61a634bdeee52d Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 3 Nov 2023 11:50:25 -0400 Subject: [PATCH 103/109] pg_upgrade: Add missing newline to message This was the backport of 2e3dc8c148, but in older releases the newline must be in the message. --- src/bin/pg_upgrade/check.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 80a8f251126..289a0d485e3 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -1583,7 +1583,7 @@ check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, if (check_for_data_type_usage(cluster, typename, output_path)) { - pg_log(PG_REPORT, "fatal"); + pg_log(PG_REPORT, "fatal\n"); pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" "The \"%s\" type has been removed in PostgreSQL version %s,\n" "so this cluster cannot currently be upgraded. You can drop the\n" From 6581de03f467b49e560acc0425013b76676f5123 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 3 Nov 2023 13:39:50 -0400 Subject: [PATCH 104/109] doc: CREATE DATABASE doesn't copy db-level perms. from template Reported-by: david@kapitaltrading.com Discussion: https://postgr.es/m/166007719137.995877.13951579839074751714@wrigleys.postgresql.org Backpatch-through: 11 --- doc/src/sgml/manage-ag.sgml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/src/sgml/manage-ag.sgml b/doc/src/sgml/manage-ag.sgml index 74055a47065..116e5e88506 100644 --- a/doc/src/sgml/manage-ag.sgml +++ b/doc/src/sgml/manage-ag.sgml @@ -207,6 +207,12 @@ createdb -O rolename dbname + + However, CREATE DATABASE does not copy database-level + GRANT permissions attached to the source database. + The new database has default database-level permissions. + + There is a second standard system database named template0.template0 This From aadc71ad3a4419ff9c750004c8d95fa1c68cb66a Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Fri, 3 Nov 2023 13:57:59 -0400 Subject: [PATCH 105/109] doc: \copy can get data values \. and end-of-input confused Reported-by: Svante Richter Discussion: https://postgr.es/m/fcd57e4-8f23-4c3e-a5db-2571d09208e2@beta.fastmail.com Backpatch-through: 11 --- doc/src/sgml/ref/psql-ref.sgml | 4 ++++ src/bin/psql/copy.c | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml index f7c1ccad02e..bc921fcb840 100644 --- a/doc/src/sgml/ref/psql-ref.sgml +++ b/doc/src/sgml/ref/psql-ref.sgml @@ -1100,6 +1100,10 @@ testdb=> destination, because all data must pass through the client/server connection. For large amounts of data the SQL command might be preferable. + Also, because of this pass-through method, \copy + ... from in CSV mode will erroneously + treat a \. data value alone on a line as an + end-of-input marker. diff --git a/src/bin/psql/copy.c b/src/bin/psql/copy.c index 1e83ad06ee2..9528bf7f583 100644 --- a/src/bin/psql/copy.c +++ b/src/bin/psql/copy.c @@ -712,6 +712,7 @@ handleCopyIn(PGconn *conn, FILE *copystream, bool isbinary, PGresult **res) * This code erroneously assumes '\.' on a line alone * inside a quoted CSV string terminates the \copy. * https://www.postgresql.org/message-id/E1TdNVQ-0001ju-GO@wrigleys.postgresql.org + * https://www.postgresql.org/message-id/bfcd57e4-8f23-4c3e-a5db-2571d09208e2@beta.fastmail.com */ if (strcmp(buf, "\\.\n") == 0 || strcmp(buf, "\\.\r\n") == 0) From 72998fb61b73bc536180c06b73a2f3e58ae1ea2b Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 25 Aug 2026 06:22:34 +0000 Subject: [PATCH 106/109] Backport fixups --- src/backend/optimizer/path/joinpath.c | 3 +++ src/bin/pg_upgrade/check.c | 34 --------------------------- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index 5defbdd3613..82678b8a522 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -367,6 +367,7 @@ add_paths_to_join_relation(PlannerInfo *root, hash_inner_and_outer(root, joinrel, outerrel, innerrel, jointype, &extra); +#ifdef NOT_USED /* * createplan.c does not currently support handling of pseudoconstant * clauses assigned to joins pushed down by extensions; check if the @@ -379,6 +380,8 @@ add_paths_to_join_relation(PlannerInfo *root, consider_join_pushdown = !has_pseudoconstant_clauses(root, restrictlist); +#endif + /* * 5. If inner and outer relations are foreign tables (or joins) belonging * to the same server and assigned to the same user to check access diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 289a0d485e3..ae364b76ef3 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -1561,40 +1561,6 @@ check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, check_ok(); } -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal\n"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s\n\n", datatype, datatype, version, output_path); - } - else - check_ok(); -} - /* * check_for_jsonb_9_4_usage() From a1bb62c8fdd54ee2e9e03bfd57291bf1fe10d05f Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Tue, 25 Aug 2026 23:23:35 +0500 Subject: [PATCH 107/109] Update regression expected files to match new backport --- .../expected/brin_multi_optimizer_1.out | 141 ++++++++- .../regress/expected/inherit_optimizer.out | 27 ++ src/test/regress/expected/opr_sanity.out | 8 +- .../expected/partition_prune_optimizer.out | 268 ++++++++++++++++-- src/test/regress/expected/rowtypes.out | 29 +- 5 files changed, 425 insertions(+), 48 deletions(-) diff --git a/src/test/regress/expected/brin_multi_optimizer_1.out b/src/test/regress/expected/brin_multi_optimizer_1.out index d56000ba8fb..eb0eb462a21 100644 --- a/src/test/regress/expected/brin_multi_optimizer_1.out +++ b/src/test/regress/expected/brin_multi_optimizer_1.out @@ -616,18 +616,135 @@ EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE a = 1; Recheck Cond: (a = 1) -> Bitmap Index Scan on brin_test_multi_a_idx Index Cond: (a = 1) - Optimizer: Pivotal Optimizer (GPORCA) +Optimizer: Pivotal Optimizer (GPORCA) (6 rows) --- Ensure brin index is not used when values are not correlated -EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE b = 1; - QUERY PLAN --------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) - -> Bitmap Heap Scan on brin_test_multi - Recheck Cond: (b = 1) - -> Bitmap Index Scan on brin_test_multi_b_idx - Index Cond: (b = 1) - Optimizer: Pivotal Optimizer (GPORCA) -(6 rows) +-- test overflows during CREATE INDEX with extreme timestamp values +CREATE TABLE brin_timestamp_test(a TIMESTAMPTZ); +SET datestyle TO iso; +-- values close to timetamp minimum +INSERT INTO brin_timestamp_test +SELECT '4713-01-01 00:00:01 BC'::timestamptz + (i || ' seconds')::interval + FROM generate_series(1,30) s(i); +-- values close to timetamp maximum +INSERT INTO brin_timestamp_test +SELECT '294276-12-01 00:00:01'::timestamptz + (i || ' seconds')::interval + FROM generate_series(1,30) s(i); +CREATE INDEX ON brin_timestamp_test USING brin (a timestamptz_minmax_multi_ops) WITH (pages_per_range=1); +DROP TABLE brin_timestamp_test; +-- test overflows during CREATE INDEX with extreme date values +CREATE TABLE brin_date_test(a DATE); +-- insert values close to date minimum +INSERT INTO brin_date_test SELECT '4713-01-01 BC'::date + i FROM generate_series(1, 30) s(i); +-- insert values close to date minimum +INSERT INTO brin_date_test SELECT '5874897-12-01'::date + i FROM generate_series(1, 30) s(i); +CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +-- make sure the ranges were built correctly and 2023-01-01 eliminates all +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) + Recheck Cond: (a = '2023-01-01'::date) + -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '2023-01-01'::date) +GP_IGNORE:(6 rows) + +DROP TABLE brin_date_test; +RESET enable_seqscan; +-- test handling of infinite timestamp values +CREATE TABLE brin_timestamp_test(a TIMESTAMP); +INSERT INTO brin_timestamp_test VALUES ('-infinity'), ('infinity'); +INSERT INTO brin_timestamp_test +SELECT i FROM generate_series('2000-01-01'::timestamp, '2000-02-09'::timestamp, '1 day'::interval) s(i); +CREATE INDEX ON brin_timestamp_test USING brin (a timestamp_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_timestamp_test WHERE a = '2023-01-01'::timestamp; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_timestamp_test (actual rows=0 loops=1) + Recheck Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) + -> Bitmap Index Scan on brin_timestamp_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) +GP_IGNORE:(6 rows) + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_timestamp_test WHERE a = '1900-01-01'::timestamp; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_timestamp_test (actual rows=0 loops=1) + Recheck Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) + -> Bitmap Index Scan on brin_timestamp_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) +GP_IGNORE:(6 rows) + +DROP TABLE brin_timestamp_test; +RESET enable_seqscan; +-- test handling of infinite date values +CREATE TABLE brin_date_test(a DATE); +INSERT INTO brin_date_test VALUES ('-infinity'), ('infinity'); +INSERT INTO brin_date_test SELECT '2000-01-01'::date + i FROM generate_series(1, 40) s(i); +CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) + Recheck Cond: (a = '2023-01-01'::date) + -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '2023-01-01'::date) +GP_IGNORE:(6 rows) + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) + Recheck Cond: (a = '1900-01-01'::date) + -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '1900-01-01'::date) +GP_IGNORE:(6 rows) + +DROP TABLE brin_date_test; +RESET enable_seqscan; +RESET datestyle; +-- test handling of overflow for interval values +CREATE TABLE brin_interval_test(a INTERVAL); +INSERT INTO brin_interval_test SELECT (i || ' years')::interval FROM generate_series(-178000000, -177999980) s(i); +INSERT INTO brin_interval_test SELECT (i || ' years')::interval FROM generate_series( 177999980, 178000000) s(i); +CREATE INDEX ON brin_interval_test USING brin (a interval_minmax_multi_ops) WITH (pages_per_range=1); +SET enable_seqscan = off; +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_interval_test WHERE a = '-30 years'::interval; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_interval_test (actual rows=0 loops=1) + Recheck Cond: (a = '@ 30 years ago'::interval) + -> Bitmap Index Scan on brin_interval_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '@ 30 years ago'::interval) +GP_IGNORE:(6 rows) + +EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) +SELECT * FROM brin_interval_test WHERE a = '30 years'::interval; +QUERY PLAN +___________ + Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) + -> Bitmap Heap Scan on brin_interval_test (actual rows=0 loops=1) + Recheck Cond: (a = '@ 30 years'::interval) + -> Bitmap Index Scan on brin_interval_test_a_idx (actual rows=0 loops=1) + Index Cond: (a = '@ 30 years'::interval) +GP_IGNORE:(6 rows) + +DROP TABLE brin_interval_test; +RESET enable_seqscan; +RESET datestyle; diff --git a/src/test/regress/expected/inherit_optimizer.out b/src/test/regress/expected/inherit_optimizer.out index ef4a44eac33..24c8be33eda 100644 --- a/src/test/regress/expected/inherit_optimizer.out +++ b/src/test/regress/expected/inherit_optimizer.out @@ -539,6 +539,33 @@ CREATE TEMP TABLE z (b TEXT, PRIMARY KEY(aa, b)) inherits (a); INSERT INTO z VALUES (NULL, 'text'); -- should fail ERROR: null value in column "aa" of relation "z" violates not-null constraint DETAIL: Failing row contains (null, text). +-- Check inherited UPDATE with first child excluded +create table some_tab (f1 int, f2 int, f3 int, check (f1 < 10) no inherit); +create table some_tab_child () inherits(some_tab); +insert into some_tab_child select i, i+1, 0 from generate_series(1,1000) i; +create index on some_tab_child(f1, f2); +-- while at it, also check that statement-level triggers fire +create function some_tab_stmt_trig_func() returns trigger as +$$begin raise notice 'updating some_tab'; return NULL; end;$$ +language plpgsql; +create trigger some_tab_stmt_trig + before update on some_tab execute function some_tab_stmt_trig_func(); +ERROR: Triggers for statements are not yet supported +explain (costs off) +update some_tab set f3 = 11 where f1 = 12 and f2 = 13; +QUERY PLAN +___________ + Update on some_tab + Update on some_tab_child some_tab_1 + -> Result + -> Index Scan using some_tab_child_f1_f2_idx on some_tab_child some_tab_1 + Index Cond: ((f1 = 12) AND (f2 = 13)) +GP_IGNORE:(6 rows) + +update some_tab set f3 = 11 where f1 = 12 and f2 = 13; +drop table some_tab cascade; +NOTICE: drop cascades to table some_tab_child +drop function some_tab_stmt_trig_func(); -- Check inherited UPDATE with all children excluded create table some_tab (a int, b int) distributed randomly; create table some_tab_child () inherits (some_tab); diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 4ab11e74d6e..be4221fac9f 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -1531,10 +1531,10 @@ WHERE a.aggfnoid = p.oid AND ); aggfnoid | proname | oid | proname ----------+--------------------+------+--------------------- - 9189 | gp_percentile_cont | 9188 | gp_percentile_final - 9190 | gp_percentile_cont | 9188 | gp_percentile_final - 9191 | gp_percentile_cont | 9188 | gp_percentile_final 9192 | gp_percentile_cont | 9188 | gp_percentile_final + 9191 | gp_percentile_cont | 9188 | gp_percentile_final + 9190 | gp_percentile_cont | 9188 | gp_percentile_final + 9189 | gp_percentile_cont | 9188 | gp_percentile_final (4 rows) -- If transfn is strict then either initval should be non-NULL, or @@ -2240,7 +2240,7 @@ ORDER BY 1, 2, 3; | record_ops | record_ops | record | tsquery_ops | tsquery_ops | tsquery | tsvector_ops | tsvector_ops | tsvector -(16 rows) +(17 rows) -- **************** pg_index **************** -- Look for illegal values in pg_index fields. diff --git a/src/test/regress/expected/partition_prune_optimizer.out b/src/test/regress/expected/partition_prune_optimizer.out index 72e87807c8a..ad5baa972c8 100644 --- a/src/test/regress/expected/partition_prune_optimizer.out +++ b/src/test/regress/expected/partition_prune_optimizer.out @@ -1959,11 +1959,10 @@ explain (costs off) select * from hp where a = 1 and b = 'abcde' and (c = 2 or c = 3); QUERY PLAN -------------------------- - Result - One-Time Filter: false -(2 rows) + Result + One-Time Filter: false +GP_IGNORE:(3 rows) -drop table hp; -- -- Test runtime partition pruning -- @@ -2097,6 +2096,28 @@ explain (analyze, costs off, summary off, timing off) execute ab_q3 (2, 2); Optimizer: Postgres query optimizer (12 rows) +-- +-- Test runtime pruning with hash partitioned tables +-- +-- recreate partitions dropped above +create table hp1 partition of hp for values with (modulus 4, remainder 1); +create table hp2 partition of hp for values with (modulus 4, remainder 2); +create table hp3 partition of hp for values with (modulus 4, remainder 3); +-- Ensure we correctly prune unneeded partitions when there is an IS NULL qual +prepare hp_q1 (text) as +select * from hp where a is null and b = $1; +explain (costs off) execute hp_q1('xxx'); +QUERY PLAN +___________ + Gather Motion XXX + -> Append + Subplans Removed: 3 + -> Seq Scan on hp2 hp_1 + Filter: ((a IS NULL) AND (b = $1)) +GP_IGNORE:(6 rows) + +deallocate hp_q1; +drop table hp; -- Test a backwards Append scan create table list_part (a int) partition by list (a); create table list_part1 partition of list_part for values in (1); @@ -4204,25 +4225,236 @@ explain (costs off) select * from rp_prefix_test3 where a >= 1 and b >= 1 and b Gather Motion 3:1 (slice1; segments: 3) -> Seq Scan on rp_prefix_test3_p2 rp_prefix_test3 Filter: ((a >= 1) AND (b >= 1) AND (d >= 0) AND (c >= 1) AND (b = 2) AND (c = 2)) - Optimizer: Postgres query optimizer -(4 rows) - -create table hp_prefix_test (a int, b int, c int, d int) partition by hash (a part_test_int4_ops, b part_test_int4_ops, c part_test_int4_ops, d part_test_int4_ops); -create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 2, remainder 0); -create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 2, remainder 1); --- Test that get_steps_using_prefix() handles non-NULL step_nullkeys -explain (costs off) select * from hp_prefix_test where a = 1 and b is null and c = 1 and d = 1; - QUERY PLAN -------------------------------------------------------------------- - Gather Motion 1:1 (slice1; segments: 1) - -> Seq Scan on hp_prefix_test_p1 hp_prefix_test - Filter: ((b IS NULL) AND (a = 1) AND (c = 1) AND (d = 1)) - Optimizer: Postgres query optimizer + Optimizer: Postgres query optimizer (4 rows) drop table rp_prefix_test1; drop table rp_prefix_test2; drop table rp_prefix_test3; +-- +-- Test that get_steps_using_prefix() handles IS NULL clauses correctly +-- +create table hp_prefix_test (a int, b int, c int, d int) + partition by hash (a part_test_int4_ops, b part_test_int4_ops, c part_test_int4_ops, d part_test_int4_ops); +-- create 8 partitions +select 'create table hp_prefix_test_p' || x::text || ' partition of hp_prefix_test for values with (modulus 8, remainder ' || x::text || ');' +from generate_Series(0,7) x; + ?column? +------------------------------------------------------------------------------------------------------ + create table hp_prefix_test_p0 partition of hp_prefix_test for values with (modulus 8, remainder 0); + create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 8, remainder 1); + create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 8, remainder 2); + create table hp_prefix_test_p3 partition of hp_prefix_test for values with (modulus 8, remainder 3); + create table hp_prefix_test_p4 partition of hp_prefix_test for values with (modulus 8, remainder 4); + create table hp_prefix_test_p5 partition of hp_prefix_test for values with (modulus 8, remainder 5); + create table hp_prefix_test_p6 partition of hp_prefix_test for values with (modulus 8, remainder 6); + create table hp_prefix_test_p7 partition of hp_prefix_test for values with (modulus 8, remainder 7); +(8 rows) + +\gexec +create table hp_prefix_test_p0 partition of hp_prefix_test for values with (modulus 8, remainder 0); +create table hp_prefix_test_p1 partition of hp_prefix_test for values with (modulus 8, remainder 1); +create table hp_prefix_test_p2 partition of hp_prefix_test for values with (modulus 8, remainder 2); +create table hp_prefix_test_p3 partition of hp_prefix_test for values with (modulus 8, remainder 3); +create table hp_prefix_test_p4 partition of hp_prefix_test for values with (modulus 8, remainder 4); +create table hp_prefix_test_p5 partition of hp_prefix_test for values with (modulus 8, remainder 5); +create table hp_prefix_test_p6 partition of hp_prefix_test for values with (modulus 8, remainder 6); +create table hp_prefix_test_p7 partition of hp_prefix_test for values with (modulus 8, remainder 7); +-- insert 16 rows, one row for each test to perform. +insert into hp_prefix_test +select + case a when 0 then null else 1 end, + case b when 0 then null else 2 end, + case c when 0 then null else 3 end, + case d when 0 then null else 4 end +from + generate_series(0,1) a, + generate_series(0,1) b, + generate_Series(0,1) c, + generate_Series(0,1) d; +-- Ensure partition pruning works correctly for each combination of IS NULL +-- and equality quals. This may seem a little excessive, but there have been +-- a number of bugs in this area over the years. We make use of row only +-- output to reduce the size of the expected results. +\t on +select + 'explain (costs off) select tableoid::regclass,* from hp_prefix_test where ' || + string_agg(c.colname || case when g.s & (1 << c.colpos) = 0 then ' is null' else ' = ' || (colpos+1)::text end, ' and ' order by c.colpos) +from (values('a',0),('b',1),('c',2),('d',3)) c(colname, colpos), generate_Series(0,15) g(s) +group by g.s +order by g.s; + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + +\gexec +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p0 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d IS NULL)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p1 hp_prefix_test + Filter: ((b IS NULL) AND (c IS NULL) AND (d IS NULL) AND (a = 1)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p2 hp_prefix_test + Filter: ((a IS NULL) AND (c IS NULL) AND (d IS NULL) AND (b = 2)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((c IS NULL) AND (d IS NULL) AND (a = 1) AND (b = 2)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p3 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (d IS NULL) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p7 hp_prefix_test + Filter: ((b IS NULL) AND (d IS NULL) AND (a = 1) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (d IS NULL) AND (b = 2) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((d IS NULL) AND (a = 1) AND (b = 2) AND (c = 3)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((b IS NULL) AND (c IS NULL) AND (a = 1) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((a IS NULL) AND (c IS NULL) AND (b = 2) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((c IS NULL) AND (a = 1) AND (b = 2) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c = 3) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((b IS NULL) AND (a = 1) AND (c = 3) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((a IS NULL) AND (b = 2) AND (c = 3) AND (d = 4)) + +explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a = 1) AND (b = 2) AND (c = 3) AND (d = 4)) + +-- And ensure we get exactly 1 row from each. Again, all 16 possible combinations. +select + 'select tableoid::regclass,* from hp_prefix_test where ' || + string_agg(c.colname || case when g.s & (1 << c.colpos) = 0 then ' is null' else ' = ' || (colpos+1)::text end, ' and ' order by c.colpos) +from (values('a',0),('b',1),('c',2),('d',3)) c(colname, colpos), generate_Series(0,15) g(s) +group by g.s +order by g.s; + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + +\gexec +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null + hp_prefix_test_p0 | | | | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null + hp_prefix_test_p1 | 1 | | | + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null + hp_prefix_test_p2 | | 2 | | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null + hp_prefix_test_p4 | 1 | 2 | | + +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null + hp_prefix_test_p3 | | | 3 | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null + hp_prefix_test_p7 | 1 | | 3 | + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null + hp_prefix_test_p4 | | 2 | 3 | + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null + hp_prefix_test_p5 | 1 | 2 | 3 | + +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 + hp_prefix_test_p4 | | | | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 + hp_prefix_test_p6 | 1 | | | 4 + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 + hp_prefix_test_p5 | | 2 | | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 + hp_prefix_test_p6 | 1 | 2 | | 4 + +select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 + hp_prefix_test_p4 | | | 3 | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 + hp_prefix_test_p5 | 1 | | 3 | 4 + +select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 + hp_prefix_test_p6 | | 2 | 3 | 4 + +select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 + hp_prefix_test_p4 | 1 | 2 | 3 | 4 + +\t off drop table hp_prefix_test; -- -- Check that gen_partprune_steps() detects self-contradiction from clauses diff --git a/src/test/regress/expected/rowtypes.out b/src/test/regress/expected/rowtypes.out index 2c85583cce0..dd93e7c8ca7 100644 --- a/src/test/regress/expected/rowtypes.out +++ b/src/test/regress/expected/rowtypes.out @@ -1243,22 +1243,23 @@ with cte(c) as materialized (select row(1, 2)), select * from cte2 as t where (select * from (select c as c1) s where (select (c1).f1 > 0)) is not null; - QUERY PLAN --------------------------------------------- - CTE Scan on cte - Output: cte.c - Filter: ((SubPlan 3) IS NOT NULL) - CTE cte - -> Result - Output: '(1,2)'::record - SubPlan 3 + QUERY PLAN +------------------------------------------ + Subquery Scan on t + Output: t.c + Filter: ((SubPlan 2) IS NOT NULL) + -> Shared Scan (share slice:id 0:0) + Output: share0_ref1."row" + -> Result + Output: ROW(1, 2) + SubPlan 2 -> Result - Output: cte.c - One-Time Filter: $2 - InitPlan 2 (returns $2) + Output: t.c + One-Time Filter: $1 + InitPlan 1 (returns $1) -> Result - Output: ((cte.c).f1 > 0) -(13 rows) + Output: ((t.c).f1 > 0) +GP_IGNORE:(16 rows) with cte(c) as materialized (select row(1, 2)), cte2(c) as (select * from cte) From a32e9c210db10d813700809b91da57104acc93e8 Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Wed, 26 Aug 2026 10:24:31 +0500 Subject: [PATCH 108/109] Update non-optimizer regression expected files for PG/CBDB plan differences --- src/test/regress/expected/brin_multi.out | 14 +-- src/test/regress/expected/inherit.out | 4 +- src/test/regress/expected/partition_prune.out | 93 +++++++++++-------- 3 files changed, 64 insertions(+), 47 deletions(-) diff --git a/src/test/regress/expected/brin_multi.out b/src/test/regress/expected/brin_multi.out index 987f010273b..8d57eb7708b 100644 --- a/src/test/regress/expected/brin_multi.out +++ b/src/test/regress/expected/brin_multi.out @@ -507,7 +507,7 @@ SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; Recheck Cond: (a = '2023-01-01'::date) -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '2023-01-01'::date) -(4 rows) +GP_IGNORE:(6 rows) DROP TABLE brin_date_test; RESET enable_seqscan; @@ -526,7 +526,7 @@ SELECT * FROM brin_timestamp_test WHERE a = '2023-01-01'::timestamp; Recheck Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) -> Bitmap Index Scan on brin_timestamp_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) -(4 rows) +GP_IGNORE:(6 rows) EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_timestamp_test WHERE a = '1900-01-01'::timestamp; @@ -536,7 +536,7 @@ SELECT * FROM brin_timestamp_test WHERE a = '1900-01-01'::timestamp; Recheck Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) -> Bitmap Index Scan on brin_timestamp_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) -(4 rows) +GP_IGNORE:(6 rows) DROP TABLE brin_timestamp_test; RESET enable_seqscan; @@ -554,7 +554,7 @@ SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; Recheck Cond: (a = '2023-01-01'::date) -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '2023-01-01'::date) -(4 rows) +GP_IGNORE:(6 rows) EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; @@ -564,7 +564,7 @@ SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; Recheck Cond: (a = '1900-01-01'::date) -> Bitmap Index Scan on brin_date_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '1900-01-01'::date) -(4 rows) +GP_IGNORE:(6 rows) DROP TABLE brin_date_test; RESET enable_seqscan; @@ -583,7 +583,7 @@ SELECT * FROM brin_interval_test WHERE a = '-30 years'::interval; Recheck Cond: (a = '@ 30 years ago'::interval) -> Bitmap Index Scan on brin_interval_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '@ 30 years ago'::interval) -(4 rows) +GP_IGNORE:(6 rows) EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_interval_test WHERE a = '30 years'::interval; @@ -593,7 +593,7 @@ SELECT * FROM brin_interval_test WHERE a = '30 years'::interval; Recheck Cond: (a = '@ 30 years'::interval) -> Bitmap Index Scan on brin_interval_test_a_idx (actual rows=0 loops=1) Index Cond: (a = '@ 30 years'::interval) -(4 rows) +GP_IGNORE:(6 rows) DROP TABLE brin_interval_test; RESET enable_seqscan; diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index e74710dd51d..d90b7c444c3 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -550,6 +550,7 @@ $$begin raise notice 'updating some_tab'; return NULL; end;$$ language plpgsql; create trigger some_tab_stmt_trig before update on some_tab execute function some_tab_stmt_trig_func(); +ERROR: Triggers for statements are not yet supported explain (costs off) update some_tab set f3 = 11 where f1 = 12 and f2 = 13; QUERY PLAN @@ -559,10 +560,9 @@ update some_tab set f3 = 11 where f1 = 12 and f2 = 13; -> Result -> Index Scan using some_tab_child_f1_f2_idx on some_tab_child some_tab_1 Index Cond: ((f1 = 12) AND (f2 = 13)) -(5 rows) +GP_IGNORE:(6 rows) update some_tab set f3 = 11 where f1 = 12 and f2 = 13; -NOTICE: updating some_tab drop table some_tab cascade; NOTICE: drop cascades to table some_tab_child drop function some_tab_stmt_trig_func(); diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index b3e82a7b229..b5785c4f3e2 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -2178,13 +2178,14 @@ create table hp3 partition of hp for values with (modulus 4, remainder 3); prepare hp_q1 (text) as select * from hp where a is null and b = $1; explain (costs off) execute hp_q1('xxx'); - QUERY PLAN + QUERY PLAN -------------------------------------------- - Append - Subplans Removed: 3 - -> Seq Scan on hp2 hp_1 - Filter: ((a IS NULL) AND (b = $1)) -(4 rows) + Gather Motion XXX + -> Append + Subplans Removed: 3 + -> Seq Scan on hp2 hp_1 + Filter: ((a IS NULL) AND (b = $1)) +GP_IGNORE:(6 rows) deallocate hp_q1; drop table hp; @@ -4453,68 +4454,84 @@ order by g.s; \gexec explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d is null - Seq Scan on hp_prefix_test_p0 hp_prefix_test - Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d IS NULL)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p0 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d IS NULL)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d is null - Seq Scan on hp_prefix_test_p1 hp_prefix_test - Filter: ((b IS NULL) AND (c IS NULL) AND (d IS NULL) AND (a = 1)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p1 hp_prefix_test + Filter: ((b IS NULL) AND (c IS NULL) AND (d IS NULL) AND (a = 1)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d is null - Seq Scan on hp_prefix_test_p2 hp_prefix_test - Filter: ((a IS NULL) AND (c IS NULL) AND (d IS NULL) AND (b = 2)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p2 hp_prefix_test + Filter: ((a IS NULL) AND (c IS NULL) AND (d IS NULL) AND (b = 2)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d is null - Seq Scan on hp_prefix_test_p4 hp_prefix_test - Filter: ((c IS NULL) AND (d IS NULL) AND (a = 1) AND (b = 2)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((c IS NULL) AND (d IS NULL) AND (a = 1) AND (b = 2)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d is null - Seq Scan on hp_prefix_test_p3 hp_prefix_test - Filter: ((a IS NULL) AND (b IS NULL) AND (d IS NULL) AND (c = 3)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p3 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (d IS NULL) AND (c = 3)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d is null - Seq Scan on hp_prefix_test_p7 hp_prefix_test - Filter: ((b IS NULL) AND (d IS NULL) AND (a = 1) AND (c = 3)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p7 hp_prefix_test + Filter: ((b IS NULL) AND (d IS NULL) AND (a = 1) AND (c = 3)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d is null - Seq Scan on hp_prefix_test_p4 hp_prefix_test - Filter: ((a IS NULL) AND (d IS NULL) AND (b = 2) AND (c = 3)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (d IS NULL) AND (b = 2) AND (c = 3)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d is null - Seq Scan on hp_prefix_test_p5 hp_prefix_test - Filter: ((d IS NULL) AND (a = 1) AND (b = 2) AND (c = 3)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((d IS NULL) AND (a = 1) AND (b = 2) AND (c = 3)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c is null and d = 4 - Seq Scan on hp_prefix_test_p4 hp_prefix_test - Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c IS NULL) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c is null and d = 4 - Seq Scan on hp_prefix_test_p6 hp_prefix_test - Filter: ((b IS NULL) AND (c IS NULL) AND (a = 1) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((b IS NULL) AND (c IS NULL) AND (a = 1) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c is null and d = 4 - Seq Scan on hp_prefix_test_p5 hp_prefix_test - Filter: ((a IS NULL) AND (c IS NULL) AND (b = 2) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((a IS NULL) AND (c IS NULL) AND (b = 2) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c is null and d = 4 - Seq Scan on hp_prefix_test_p6 hp_prefix_test - Filter: ((c IS NULL) AND (a = 1) AND (b = 2) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((c IS NULL) AND (a = 1) AND (b = 2) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b is null and c = 3 and d = 4 - Seq Scan on hp_prefix_test_p4 hp_prefix_test - Filter: ((a IS NULL) AND (b IS NULL) AND (c = 3) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a IS NULL) AND (b IS NULL) AND (c = 3) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b is null and c = 3 and d = 4 - Seq Scan on hp_prefix_test_p5 hp_prefix_test - Filter: ((b IS NULL) AND (a = 1) AND (c = 3) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p5 hp_prefix_test + Filter: ((b IS NULL) AND (a = 1) AND (c = 3) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a is null and b = 2 and c = 3 and d = 4 - Seq Scan on hp_prefix_test_p6 hp_prefix_test - Filter: ((a IS NULL) AND (b = 2) AND (c = 3) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p6 hp_prefix_test + Filter: ((a IS NULL) AND (b = 2) AND (c = 3) AND (d = 4)) explain (costs off) select tableoid::regclass,* from hp_prefix_test where a = 1 and b = 2 and c = 3 and d = 4 - Seq Scan on hp_prefix_test_p4 hp_prefix_test - Filter: ((a = 1) AND (b = 2) AND (c = 3) AND (d = 4)) + Gather Motion XXX + -> Seq Scan on hp_prefix_test_p4 hp_prefix_test + Filter: ((a = 1) AND (b = 2) AND (c = 3) AND (d = 4)) -- And ensure we get exactly 1 row from each. Again, all 16 possible combinations. select From 7526a4fd77ee565a4907213461f779e787c2bfdd Mon Sep 17 00:00:00 2001 From: Kirill Reshke Date: Wed, 26 Aug 2026 16:54:01 +0500 Subject: [PATCH 109/109] Fix remaining regression expected diffs for optimizer variants --- .../regress/expected/bfv_cte_optimizer.out | 2 +- .../expected/brin_multi_optimizer_1.out | 43 +++++++++++-------- .../regress/expected/inherit_optimizer.out | 4 +- .../expected/partition_prune_optimizer.out | 11 +++-- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/test/regress/expected/bfv_cte_optimizer.out b/src/test/regress/expected/bfv_cte_optimizer.out index ec2ee57605f..62c35b26f79 100644 --- a/src/test/regress/expected/bfv_cte_optimizer.out +++ b/src/test/regress/expected/bfv_cte_optimizer.out @@ -512,8 +512,8 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support -- end_matchsubs -- Filter out irrelevant LOG messages from segments other than seg2. \! cat /tmp/bfv_cte.out | grep -P '^(?!LOG)|^(LOG.*seg2)' | grep -vP 'LOG.*fault|decreased xslice state refcount' -LOG: SISC (shareid=0, slice=2): initialized xslice state (seg2 slice2 127.0.1.1:7004 pid=1048240) LOG: SISC READER (shareid=0, slice=2): wrote notify_done (seg2 slice2 127.0.1.1:7004 pid=1048240) +LOG: SISC (shareid=0, slice=2): initialized xslice state (seg2 slice2 127.0.1.1:7004 pid=1048240) LOG: SISC READER (shareid=0, slice=4): wrote notify_done (seg2 slice4 127.0.1.1:7004 pid=1048252) LOG: SISC WRITER (shareid=0, slice=1): No tuplestore yet, creating tuplestore (seg2 slice1 127.0.1.1:7004 pid=1048234) LOG: SISC WRITER (shareid=0, slice=1): wrote notify_ready (seg2 slice1 127.0.1.1:7004 pid=1048234) diff --git a/src/test/regress/expected/brin_multi_optimizer_1.out b/src/test/regress/expected/brin_multi_optimizer_1.out index eb0eb462a21..2fc40356c49 100644 --- a/src/test/regress/expected/brin_multi_optimizer_1.out +++ b/src/test/regress/expected/brin_multi_optimizer_1.out @@ -616,8 +616,18 @@ EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE a = 1; Recheck Cond: (a = 1) -> Bitmap Index Scan on brin_test_multi_a_idx Index Cond: (a = 1) -Optimizer: Pivotal Optimizer (GPORCA) -(6 rows) +GP_IGNORE:(6 rows) + +-- Ensure brin index is not used when values are not correlated +EXPLAIN (COSTS OFF) SELECT * FROM brin_test_multi WHERE b = 1; + QUERY PLAN +-------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Bitmap Heap Scan on brin_test_multi + Recheck Cond: (b = 1) + -> Bitmap Index Scan on brin_test_multi_b_idx + Index Cond: (b = 1) +GP_IGNORE:(6 rows) -- test overflows during CREATE INDEX with extreme timestamp values CREATE TABLE brin_timestamp_test(a TIMESTAMPTZ); @@ -643,8 +653,8 @@ SET enable_seqscan = off; -- make sure the ranges were built correctly and 2023-01-01 eliminates all EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; -QUERY PLAN -___________ + QUERY PLAN +------------------------------------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) Recheck Cond: (a = '2023-01-01'::date) @@ -663,8 +673,8 @@ CREATE INDEX ON brin_timestamp_test USING brin (a timestamp_minmax_multi_ops) WI SET enable_seqscan = off; EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_timestamp_test WHERE a = '2023-01-01'::timestamp; -QUERY PLAN -___________ + QUERY PLAN +------------------------------------------------------------------------------------ Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_timestamp_test (actual rows=0 loops=1) Recheck Cond: (a = '2023-01-01 00:00:00'::timestamp without time zone) @@ -674,8 +684,8 @@ GP_IGNORE:(6 rows) EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_timestamp_test WHERE a = '1900-01-01'::timestamp; -QUERY PLAN -___________ + QUERY PLAN +------------------------------------------------------------------------------------ Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_timestamp_test (actual rows=0 loops=1) Recheck Cond: (a = '1900-01-01 00:00:00'::timestamp without time zone) @@ -693,8 +703,8 @@ CREATE INDEX ON brin_date_test USING brin (a date_minmax_multi_ops) WITH (pages_ SET enable_seqscan = off; EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_date_test WHERE a = '2023-01-01'::date; -QUERY PLAN -___________ + QUERY PLAN +------------------------------------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) Recheck Cond: (a = '2023-01-01'::date) @@ -704,8 +714,8 @@ GP_IGNORE:(6 rows) EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_date_test WHERE a = '1900-01-01'::date; -QUERY PLAN -___________ + QUERY PLAN +------------------------------------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_date_test (actual rows=0 loops=1) Recheck Cond: (a = '1900-01-01'::date) @@ -724,8 +734,8 @@ CREATE INDEX ON brin_interval_test USING brin (a interval_minmax_multi_ops) WITH SET enable_seqscan = off; EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_interval_test WHERE a = '-30 years'::interval; -QUERY PLAN -___________ + QUERY PLAN +----------------------------------------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_interval_test (actual rows=0 loops=1) Recheck Cond: (a = '@ 30 years ago'::interval) @@ -735,8 +745,8 @@ GP_IGNORE:(6 rows) EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF) SELECT * FROM brin_interval_test WHERE a = '30 years'::interval; -QUERY PLAN -___________ + QUERY PLAN +----------------------------------------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) (actual rows=0 loops=1) -> Bitmap Heap Scan on brin_interval_test (actual rows=0 loops=1) Recheck Cond: (a = '@ 30 years'::interval) @@ -747,4 +757,3 @@ GP_IGNORE:(6 rows) DROP TABLE brin_interval_test; RESET enable_seqscan; RESET datestyle; - diff --git a/src/test/regress/expected/inherit_optimizer.out b/src/test/regress/expected/inherit_optimizer.out index 24c8be33eda..4939fb69e41 100644 --- a/src/test/regress/expected/inherit_optimizer.out +++ b/src/test/regress/expected/inherit_optimizer.out @@ -553,8 +553,8 @@ create trigger some_tab_stmt_trig ERROR: Triggers for statements are not yet supported explain (costs off) update some_tab set f3 = 11 where f1 = 12 and f2 = 13; -QUERY PLAN -___________ + QUERY PLAN +------------------------------------------------------------------------------------ Update on some_tab Update on some_tab_child some_tab_1 -> Result diff --git a/src/test/regress/expected/partition_prune_optimizer.out b/src/test/regress/expected/partition_prune_optimizer.out index ad5baa972c8..3ec4285920c 100644 --- a/src/test/regress/expected/partition_prune_optimizer.out +++ b/src/test/regress/expected/partition_prune_optimizer.out @@ -1959,8 +1959,8 @@ explain (costs off) select * from hp where a = 1 and b = 'abcde' and (c = 2 or c = 3); QUERY PLAN -------------------------- - Result - One-Time Filter: false + Result + One-Time Filter: false GP_IGNORE:(3 rows) -- @@ -2107,8 +2107,8 @@ create table hp3 partition of hp for values with (modulus 4, remainder 3); prepare hp_q1 (text) as select * from hp where a is null and b = $1; explain (costs off) execute hp_q1('xxx'); -QUERY PLAN -___________ + QUERY PLAN +-------------------------------------------------- Gather Motion XXX -> Append Subplans Removed: 3 @@ -4224,8 +4224,7 @@ explain (costs off) select * from rp_prefix_test3 where a >= 1 and b >= 1 and b ------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) -> Seq Scan on rp_prefix_test3_p2 rp_prefix_test3 - Filter: ((a >= 1) AND (b >= 1) AND (d >= 0) AND (c >= 1) AND (b = 2) AND (c = 2)) - Optimizer: Postgres query optimizer + Filter: ((a >= 1) AND (b >= 1) AND (d >= 0) AND (c >= 1) AND (b = 2) AND (c = 2)) (4 rows) drop table rp_prefix_test1;