diff --git a/.gitmodules b/.gitmodules index 784e93fbb62c..6acf8b86ad42 100644 --- a/.gitmodules +++ b/.gitmodules @@ -36,7 +36,7 @@ url = https://github.com/ClickHouse/jemalloc [submodule "contrib/google-protobuf"] path = contrib/google-protobuf - url = https://github.com/ClickHouse/google-protobuf.git + url = https://github.com/protocolbuffers/protobuf.git [submodule "contrib/boost"] path = contrib/boost url = https://github.com/ClickHouse/boost @@ -401,4 +401,4 @@ url = https://github.com/ClickHouse/silk.git [submodule "contrib/xsimd"] path = contrib/xsimd - url = https://github.com/xtensor-stack/xsimd + url = https://github.com/ClickHouse/xsimd diff --git a/base/poco/XML/CMakeLists.txt b/base/poco/XML/CMakeLists.txt index dda7ab845a3d..0bcbb22dc492 100644 --- a/base/poco/XML/CMakeLists.txt +++ b/base/poco/XML/CMakeLists.txt @@ -13,6 +13,7 @@ target_compile_options (_poco_xml_expat -Wno-extra-semi-stmt -Wno-implicit-fallthrough -Wno-reserved-identifier + -Wno-unreachable-code-fallthrough -Wno-unused-macros -Wno-implicit-int-conversion ) @@ -47,6 +48,7 @@ target_compile_options (_poco_xml -Wno-tautological-unsigned-zero-compare -Wno-unreachable-code -Wno-unreachable-code-break + -Wno-unreachable-code-fallthrough -Wno-unused-macros -Wno-unused-parameter -Wno-zero-as-null-pointer-constant diff --git a/base/poco/XML/include/Poco/XML/expat.h b/base/poco/XML/include/Poco/XML/expat.h index 1d8ccf6aac7e..7b3c0dd40580 100644 --- a/base/poco/XML/include/Poco/XML/expat.h +++ b/base/poco/XML/include/Poco/XML/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 1 +# define XML_MICRO_VERSION 2 # ifdef __cplusplus } diff --git a/base/poco/XML/src/fallthrough.h b/base/poco/XML/src/fallthrough.h new file mode 100644 index 000000000000..cae4c6274da3 --- /dev/null +++ b/base/poco/XML/src/fallthrough.h @@ -0,0 +1,49 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Nick Begg + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#ifndef FALLTHROUGH_H +# define FALLTHROUGH_H 1 + +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. + +# define EXPAT_FALLTHROUGH \ + do { \ + } while (0) + +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# undef EXPAT_FALLTHROUGH +# define EXPAT_FALLTHROUGH __attribute__((fallthrough)) +# endif +# endif + +#endif // FALLTHROUGH_H diff --git a/base/poco/XML/src/siphash.h b/base/poco/XML/src/siphash.h index 26f4b36507f4..906a37613e9e 100644 --- a/base/poco/XML/src/siphash.h +++ b/base/poco/XML/src/siphash.h @@ -101,6 +101,8 @@ #include /* size_t */ #include /* uint64_t uint32_t uint8_t */ +#include "fallthrough.h" + /* * Workaround to not require a C++11 compiler for using ULL suffix * if this code is included and compiled as C++; related GCC warning is: @@ -240,25 +242,25 @@ static uint64_t sip24_final(struct siphash * H) { case 7: b |= (uint64_t)H->buf[6] << 48; - /* fall through */ + EXPAT_FALLTHROUGH; case 6: b |= (uint64_t)H->buf[5] << 40; - /* fall through */ + EXPAT_FALLTHROUGH; case 5: b |= (uint64_t)H->buf[4] << 32; - /* fall through */ + EXPAT_FALLTHROUGH; case 4: b |= (uint64_t)H->buf[3] << 24; - /* fall through */ + EXPAT_FALLTHROUGH; case 3: b |= (uint64_t)H->buf[2] << 16; - /* fall through */ + EXPAT_FALLTHROUGH; case 2: b |= (uint64_t)H->buf[1] << 8; - /* fall through */ + EXPAT_FALLTHROUGH; case 1: b |= (uint64_t)H->buf[0] << 0; - /* fall through */ + EXPAT_FALLTHROUGH; case 0: break; } diff --git a/base/poco/XML/src/xcsinc.c b/base/poco/XML/src/xcsinc.c new file mode 100644 index 000000000000..3597c2480bc9 --- /dev/null +++ b/base/poco/XML/src/xcsinc.c @@ -0,0 +1,48 @@ +/* This file is included from other .c files! + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +static size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index baa6c750e53e..f019aef201c9 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -1,4 +1,4 @@ -/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) +/* 5de44e6750c6cc78818f06ed552f522a1241df0299395250e1792cb339389daf (2.8.2+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -47,6 +47,9 @@ Copyright (c) 2026 Rosen Penev Copyright (c) 2026 Francesco Bertolaccini Copyright (c) 2026 Christian Ng + Copyright (c) 2026 Nick Begg + Copyright (c) 2026 Kartik Kenchi + Copyright (c) 2026 Haris Hussain Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -90,7 +93,7 @@ #include #include /* memset(), memcpy() */ #include -#include /* INT_MAX, UINT_MAX */ +#include /* INT_MAX, LLONG_MAX, LONG_MAX, UINT_MAX */ #include /* fprintf */ #include /* getenv */ #include /* SIZE_MAX, uintptr_t */ @@ -119,6 +122,7 @@ #include "ascii.h" #include "Poco/XML/expat.h" #include "siphash.h" +#include "xcsinc.c" #if defined(HAVE_ARC4RANDOM) # include "random_arc4random.h" @@ -162,7 +166,7 @@ * Windows >=Vista (rand_s): _WIN32. \ \ If you insist on not using any of these, bypass this error by defining \ - XML_POOR_ENTROPY; you have been warned. \ + XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ to the build system, please open a bug. Thank you! @@ -213,6 +217,12 @@ typedef char ICHAR; #endif +#ifdef XML_LARGE_SIZE +# define XML_INDEX_MAX LLONG_MAX +#else +# define XML_INDEX_MAX LONG_MAX +#endif + /* Round up n to be a multiple of sz, where sz is a power of 2. */ #define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) @@ -276,8 +286,8 @@ typedef struct binding { struct binding *prevPrefixBinding; const struct attribute_id *attId; XML_Char *uri; - int uriLen; - int uriAlloc; + size_t uriLen; + size_t uriAlloc; } BINDING; typedef struct prefix { @@ -289,9 +299,9 @@ typedef struct { const XML_Char *str; const XML_Char *localPart; const XML_Char *prefix; - int strLen; - int uriLen; - int prefixLen; + size_t strLen; + size_t uriLen; + size_t prefixLen; } TAG_NAME; /* TAG represents an open element. @@ -390,8 +400,8 @@ typedef struct { const XML_Char *name; PREFIX *prefix; const ATTRIBUTE_ID *idAtt; - int nDefaultAtts; - int allocDefaultAtts; + size_t nDefaultAtts; + size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; @@ -423,6 +433,7 @@ typedef struct { unsigned scaffCount; int scaffLevel; int *scaffIndex; + size_t scaffIndexSize; } DTD; enum EntityType { @@ -594,6 +605,7 @@ static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); +static bool FASTCALL poolGrowUntil(STRING_POOL *pool, size_t needed); static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s); static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, @@ -648,16 +660,40 @@ static XML_Parser getRootParserOf(XML_Parser parser, static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); +static bool poolAppendChar(STRING_POOL *pool, XML_Char c); + +static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) #define poolDiscard(pool) ((pool)->ptr = (pool)->start) #define poolFinish(pool) ((pool)->start = (pool)->ptr) -#define poolAppendChar(pool, c) \ - (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \ - ? 0 \ - : ((*((pool)->ptr)++ = c), 1)) + +bool +poolAppendChar(STRING_POOL *pool, XML_Char c) { + if (pool->ptr == pool->end && ! poolGrow(pool)) + return false; + + *(pool->ptr)++ = c; + return true; +} + +bool +poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) { + // Detect and prevent integer overflow + if (len > SIZE_MAX / sizeof(XML_Char)) + return false; + + if (! poolGrowUntil(pool, len)) + return false; + + memcpy(pool->ptr, s, len * sizeof(XML_Char)); + pool->ptr += len; + + return true; +} #if ! defined(XML_TESTING) const @@ -736,11 +772,9 @@ struct XML_ParserStruct { const char *m_eventEndPtr; const char *m_positionPtr; OPEN_INTERNAL_ENTITY *m_openInternalEntities; - OPEN_INTERNAL_ENTITY *m_freeInternalEntities; OPEN_INTERNAL_ENTITY *m_openAttributeEntities; - OPEN_INTERNAL_ENTITY *m_freeAttributeEntities; OPEN_INTERNAL_ENTITY *m_openValueEntities; - OPEN_INTERNAL_ENTITY *m_freeValueEntities; + OPEN_INTERNAL_ENTITY *m_freeEntities; XML_Bool m_defaultExpandInternalEntities; int m_tagLevel; ENTITY *m_declEntity; @@ -760,7 +794,7 @@ struct XML_ParserStruct { TAG *m_freeTagList; BINDING *m_inheritedBindings; BINDING *m_freeBindingList; - int m_attsSize; + size_t m_attsSize; int m_nSpecifiedAtts; int m_idAttIndex; ATTRIBUTE *m_atts; @@ -774,7 +808,7 @@ struct XML_ParserStruct { STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; char *m_groupConnector; - unsigned int m_groupSize; + size_t m_groupSize; XML_Char m_namespaceSeparator; XML_Parser m_parentParser; XML_ParsingStatus m_parsingStatus; @@ -791,6 +825,7 @@ struct XML_ParserStruct { ENTITY_STATS m_entity_stats; #endif XML_Bool m_reenter; + unsigned m_handlerCallDepth; }; #if XML_GE == 1 @@ -1151,6 +1186,23 @@ generate_hash_secret_salt(void) { #endif /* defined(EXPAT_POCO) */ } +static void +beforeHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth < UINT_MAX); + parser->m_handlerCallDepth++; +} + +static void +afterHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth > 0); + parser->m_handlerCallDepth--; +} + +static bool +isCalledFromInsideHandler(XML_Parser parser) { + return parser->m_handlerCallDepth > 0; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1387,9 +1439,7 @@ parserCreate(const XML_Char *encodingName, parser->m_freeBindingList = NULL; parser->m_freeTagList = NULL; - parser->m_freeInternalEntities = NULL; - parser->m_freeAttributeEntities = NULL; - parser->m_freeValueEntities = NULL; + parser->m_freeEntities = NULL; parser->m_groupSize = 0; parser->m_groupConnector = NULL; @@ -1505,6 +1555,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_parsingStatus.parsing = XML_INITIALIZED; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; + parser->m_handlerCallDepth = 0; #ifdef XML_DTD parser->m_isParamEntity = XML_FALSE; parser->m_useForeignDTD = XML_FALSE; @@ -1538,12 +1589,22 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { } } +/* Moves a list of entities onto the start of another list. */ +static void +moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) { + for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = head; + head = head->next; + openEntity->next = *dst; + *dst = openEntity; + } +} + XML_Bool XMLCALL XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; - OPEN_INTERNAL_ENTITY *openEntityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_FALSE; if (parser->m_parentParser) @@ -1558,32 +1619,14 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { tag->bindings = NULL; parser->m_freeTagList = tag; } - /* move m_openInternalEntities to m_freeInternalEntities */ - openEntityList = parser->m_openInternalEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } - /* move m_openAttributeEntities to m_freeAttributeEntities (i.e. same task but - * for attributes) */ - openEntityList = parser->m_openAttributeEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; - } - /* move m_openValueEntities to m_freeValueEntities (i.e. same task but - * for value entities) */ - openEntityList = parser->m_openValueEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; - } + /* move m_openInternalEntities to m_freeEntities */ + moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities); + /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for + * attributes) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities); + /* move m_openValueEntities to m_freeEntities (i.e. same task but for value + * entities) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) @@ -1735,11 +1778,6 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, newDtd = oldDtd; #endif /* XML_DTD */ - /* Note that the magical uses of the pre-processor to make field - access look more like C++ require that `parser' be overwritten - here. This makes this function more painful to follow than it - would be otherwise. - */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); @@ -1829,8 +1867,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser) { void XMLCALL XML_ParserFree(XML_Parser parser) { TAG *tagList; - OPEN_INTERNAL_ENTITY *entityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; @@ -1848,48 +1885,35 @@ XML_ParserFree(XML_Parser parser) { destroyBindings(p->bindings, parser); FREE(parser, p); } - /* free m_openInternalEntities and m_freeInternalEntities */ - entityList = parser->m_openInternalEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeInternalEntities == NULL) - break; - entityList = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = NULL; - } - openEntity = entityList; + /* free m_openInternalEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openAttributeEntities and m_freeAttributeEntities */ - entityList = parser->m_openAttributeEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeAttributeEntities == NULL) - break; - entityList = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = NULL; - } - openEntity = entityList; + /* free m_openAttributeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openValueEntities and m_freeValueEntities */ - entityList = parser->m_openValueEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeValueEntities == NULL) - break; - entityList = parser->m_freeValueEntities; - parser->m_freeValueEntities = NULL; - } - openEntity = entityList; + /* free m_openValueEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } + /* free m_freeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + parser->m_freeEntities = NULL; destroyBindings(parser->m_freeBindingList, parser); destroyBindings(parser->m_inheritedBindings, parser); poolDestroy(&parser->m_tempPool); @@ -2286,6 +2310,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } + if (isCalledFromInsideHandler(parser)) + return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2298,7 +2324,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } @@ -2309,7 +2335,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2340,7 +2366,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return XML_STATUS_OK; } - /* fall through */ + EXPAT_FALLTHROUGH; default: result = XML_STATUS_OK; } @@ -2395,7 +2421,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { const char *start; enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (len < 0) { @@ -2421,11 +2447,19 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } + // Detect and avoid integer overflow + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } + start = parser->m_bufferPtr; parser->m_positionPtr = start; parser->m_bufferEnd += len; @@ -2451,6 +2485,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; /* should not happen */ } } @@ -2461,9 +2496,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { return result; } +/* Modifies `parser`’s buffer to be backed by `newBuf`. */ +static void +setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) { + parser->m_bufferLim = newBuf + newBufSize; + if (parser->m_bufferPtr) { + const int parsing + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf + parsing + keep; + parser->m_bufferPtr = newBuf + keep; + } else { + /* This must be a brand new buffer with no data in it yet */ + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf; + parser->m_bufferPtr = newBuf; + } +} + void *XMLCALL XML_GetBuffer(XML_Parser parser, int len) { - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return NULL; if (len < 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2484,9 +2542,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_lastBufferRequestSize = len; if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) || parser->m_buffer == NULL) { -#if XML_CONTEXT_BYTES > 0 - int keep; -#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2496,7 +2551,9 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } #if XML_CONTEXT_BYTES > 0 - keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + const int parsed + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + int keep = parsed; if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; /* Detect and prevent integer overflow */ @@ -2504,16 +2561,16 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - neededSize += keep; +#else + int keep = 0; #endif /* XML_CONTEXT_BYTES > 0 */ + neededSize += keep; if (parser->m_buffer && parser->m_bufferPtr && neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { #if XML_CONTEXT_BYTES > 0 - if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { - int offset - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - - keep; + if (keep < parsed) { + int offset = parsed - keep; /* The buffer pointers cannot be NULL here; we have at least some bytes * in the buffer */ memmove(parser->m_buffer, &parser->m_buffer[offset], @@ -2530,7 +2587,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr = parser->m_buffer; #endif /* XML_CONTEXT_BYTES > 0 */ } else { - char *newBuf; int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) @@ -2545,49 +2601,12 @@ XML_GetBuffer(XML_Parser parser, int len) { } // NOTE: We are avoiding MALLOC(..) here to leave limiting // the input size to the application using Expat. - newBuf = (char*)parser->m_mem.malloc_fcn(bufferSize); + char *const newBuf = (char*)parser->m_mem.malloc_fcn(bufferSize); if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - parser->m_bufferLim = newBuf + bufferSize; -#if XML_CONTEXT_BYTES > 0 - if (parser->m_bufferPtr) { - memcpy(newBuf, &parser->m_bufferPtr[-keep], - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep); - // NOTE: We are avoiding FREE(..) here because parser->m_buffer - // is not being allocated with MALLOC(..) but with plain - // .malloc_fcn(..). - parser->m_mem.free_fcn(parser->m_buffer); - parser->m_buffer = newBuf; - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep; - parser->m_bufferPtr = parser->m_buffer + keep; - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - parser->m_bufferPtr = parser->m_buffer = newBuf; - } -#else - if (parser->m_bufferPtr) { - memcpy(newBuf, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - // NOTE: We are avoiding FREE(..) here because parser->m_buffer - // is not being allocated with MALLOC(..) but with plain - // .malloc_fcn(..). - parser->m_mem.free_fcn(parser->m_buffer); - parser->m_bufferEnd - = newBuf - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - } - parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* XML_CONTEXT_BYTES > 0 */ + setParserBuffer(parser, newBuf, bufferSize, keep); } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2640,7 +2659,7 @@ enum XML_Status XMLCALL XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (parser->m_parsingStatus.parsing != XML_SUSPENDED) { parser->m_errorCode = XML_ERROR_NOT_SUSPENDED; @@ -2666,6 +2685,7 @@ XML_ResumeParser(XML_Parser parser) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; } } @@ -2727,7 +2747,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* XML_CONTEXT_BYTES > 0 */ - return (const char *)0; + return NULL; } XML_Size XMLCALL @@ -3086,7 +3106,7 @@ storeRawNames(XML_Parser parser) { */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); /* Detect and prevent integer overflow. */ - if (rawNameLen > (size_t)INT_MAX - nameLen) + if (rawNameLen > SIZE_MAX - nameLen) return XML_FALSE; bufSize = nameLen + rawNameLen; if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { @@ -3213,7 +3233,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default: start = next; } @@ -3292,7 +3312,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventEndPP = end; if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3345,9 +3367,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); #endif /* XML_GE == 1 */ - if (parser->m_characterDataHandler) + if (parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3367,9 +3391,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, else if (! entity->is_internal) return XML_ERROR_ENTITY_DECLARED_IN_PE; } else if (! entity) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3380,10 +3406,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (entity->textPtr) { enum XML_Error result; if (! parser->m_defaultExpandInternalEntities) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3397,9 +3425,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, entity->open = XML_FALSE; if (! context) return XML_ERROR_NO_MEMORY; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, context, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, context, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; poolDiscard(&parser->m_tempPool); } else if (parser->m_defaultHandler) @@ -3407,7 +3438,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; } case XML_TOK_START_TAG_NO_ATTS: - /* fall through */ case XML_TOK_START_TAG_WITH_ATTS: { TAG *tag; enum XML_Error result; @@ -3439,11 +3469,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *fromPtr = tag->rawName; toPtr = tag->buf.str; for (;;) { - int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - tag->buf.str); + const size_t convLen = (size_t)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; @@ -3468,16 +3497,17 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; - if (parser->m_startElementHandler) + if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, tag->name.str, (const XML_Char **)parser->m_atts); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); poolClear(&parser->m_tempPool); break; } case XML_TOK_EMPTY_ELEMENT_NO_ATTS: - /* fall through */ case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: { const char *rawName = s + enc->minBytesPerChar; enum XML_Error result; @@ -3497,14 +3527,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } poolFinish(&parser->m_tempPool); if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (parser->m_endElementHandler) { if (parser->m_startElementHandler) *eventPP = *eventEndPP; + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, name.str); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (noElmHandlers && parser->m_defaultHandler) @@ -3562,14 +3596,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } *uri = XML_T('\0'); } + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, tag->name.str); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); while (tag->bindings) { BINDING *b = tag->bindings; - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } tag->bindings = tag->bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; parser->m_freeBindingList = b; @@ -3592,8 +3631,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return XML_ERROR_BAD_CHAR_REF; if (parser->m_characterDataHandler) { XML_Char buf[XML_ENCODE_MAX]; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf)); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3602,32 +3643,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; case XML_TOK_CDATA_SECT_OPEN: { enum XML_Error result; - if (parser->m_startCdataSectionHandler) + if (parser->m_startCdataSectionHandler) { + beforeHandler(parser); parser->m_startCdataSectionHandler(parser->m_handlerArg); - /* BEGIN disabled code */ - /* Suppose you doing a transformation on a document that involves - changing only the character data. You set up a defaultHandler - and a characterDataHandler. The defaultHandler simply copies - characters through. The characterDataHandler does the - transformation and writes the characters out escaping them as - necessary. This case will fail to work if we leave out the - following two lines (because & and < inside CDATA sections will - be incorrectly escaped). - - However, now we have a start/endCdataSectionHandler, so it seems - easier to let the user deal with this. - */ - else if ((0) && parser->m_characterDataHandler) + afterHandler(parser); + /* BEGIN disabled code */ + /* Suppose you doing a transformation on a document that involves + changing only the character data. You set up a defaultHandler + and a characterDataHandler. The defaultHandler simply copies + characters through. The characterDataHandler does the + transformation and writes the characters out escaping them as + necessary. This case will fail to work if we leave out the + following two lines (because & and < inside CDATA sections will + be incorrectly escaped). + + However, now we have a start/endCdataSectionHandler, so it seems + easier to let the user deal with this. + */ + } else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); @@ -3647,13 +3694,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (MUST_CONVERT(enc, s)) { ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf; XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - } else + afterHandler(parser); + } else { + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3678,16 +3730,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3725,7 +3782,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -3745,8 +3802,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) { /* m_startNamespaceDeclHandler will have been called for this * binding in addBindings(), so call the end handler now. */ - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } bindings = bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; @@ -3770,20 +3830,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - ELEMENT_TYPE *elementType; - int nDefaultAtts; - const XML_Char **appAtts; /* the attribute list for the application */ int attIndex = 0; - int prefixLen; - int i; - int n; XML_Char *uri; int nPrefixes = 0; BINDING *binding; const XML_Char *localPart; /* lookup the element type name */ - elementType + ELEMENT_TYPE *elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0); if (! elementType) { const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str); @@ -3798,75 +3852,71 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } - nDefaultAtts = elementType->nDefaultAtts; + const size_t nDefaultAtts = elementType->nDefaultAtts; + + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); + size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, + parser->m_atts); /* Detect and prevent integer overflow */ - if (n > INT_MAX - nDefaultAtts) { + if (n > SIZE_MAX - nDefaultAtts) { return XML_ERROR_NO_MEMORY; } if (n + nDefaultAtts > parser->m_attsSize) { - int oldAttsSize = parser->m_attsSize; - ATTRIBUTE *temp; -#ifdef XML_ATTR_INFO - XML_AttrInfo *temp2; -#endif + size_t oldAttsSize = parser->m_attsSize; /* Detect and prevent integer overflow */ - if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE) - || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { + if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE) + || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { return XML_ERROR_NO_MEMORY; } parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -#endif - temp = (ATTRIBUTE*)REALLOC(parser, parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + ATTRIBUTE *const temp = (ATTRIBUTE*)REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; #ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -# if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -# endif - temp2 = (XML_AttrInfo*)REALLOC(parser, parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); + XML_AttrInfo *const temp2 = REALLOC( + parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); if (temp2 == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_attInfo = temp2; #endif - if (n > oldAttsSize) - XmlGetAttributes(enc, attStr, n, parser->m_atts); + if (n > oldAttsSize) { + /* Detect and prevent integer overflow. */ + if (n > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; + XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); + } } - appAtts = (const XML_Char **)parser->m_atts; - for (i = 0; i < n; i++) { + /* the attribute list for the application */ + const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; + for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; #ifdef XML_ATTR_INFO XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; @@ -3899,13 +3949,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, (attId->name)[-1] = 1; appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - enum XML_Error result; XML_Bool isCdata = XML_TRUE; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - int j; - for (j = 0; j < nDefaultAtts; j++) { + for (size_t j = 0; j < nDefaultAtts; j++) { if (attId == elementType->defaultAtts[j].id) { isCdata = elementType->defaultAtts[j].isCdata; break; @@ -3914,7 +3962,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* normalize the attribute value */ - result = storeAttributeValue( + const enum XML_Error result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) @@ -3952,7 +4000,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ parser->m_nSpecifiedAtts = attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (i = 0; i < attIndex; i += 2) + for (int i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { parser->m_idAttIndex = i; break; @@ -3961,7 +4009,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, parser->m_idAttIndex = -1; /* do attribute defaulting */ - for (i = 0; i < nDefaultAtts; i++) { + for (size_t i = 0; i < nDefaultAtts; i++) { const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i; if (! (da->id->name)[-1] && da->value) { if (da->id->prefix) { @@ -3987,7 +4035,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - i = 0; + int i = 0; if (nPrefixes) { unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; @@ -4002,7 +4050,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ - NS_ATT *temp; /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4030,7 +4077,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } #endif - temp = (NS_ATT*)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); + NS_ATT *const temp + = (NS_ATT*)REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -4051,9 +4099,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, for (; i < attIndex; i += 2) { const XML_Char *s = appAtts[i]; if (s[-1] == 2) { /* prefixed */ - ATTRIBUTE_ID *id; - const BINDING *b; - unsigned long uriHash; struct siphash sip_state; struct sipkey sip_key; @@ -4061,7 +4106,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_init(&sip_state, &sip_key); ((XML_Char *)s)[-1] = 0; /* clear flag */ - id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); + ATTRIBUTE_ID *const id + = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (! id || ! id->prefix) { /* This code is walking through the appAtts array, dealing * with (in this case) a prefixed attribute name. To be in @@ -4069,7 +4115,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * has to have passed through the hash table lookup once * already. That implies that an entry for it already * exists, so the lookup above will return a pointer to - * already allocated memory. There is no opportunaity for + * already allocated memory. There is no opportunity for * the allocator to fail, so the condition above cannot be * fulfilled. * @@ -4079,15 +4125,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, */ return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ } - b = id->prefix->binding; + const BINDING *const b = id->prefix->binding; if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < (unsigned int)b->uriLen; j++) { - const XML_Char c = b->uri[j]; - if (! poolAppendChar(&parser->m_tempPool, c)) - return XML_ERROR_NO_MEMORY; - } + if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen)) + return XML_ERROR_NO_MEMORY; sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); @@ -4096,12 +4139,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); - do { /* copies null terminator */ - if (! poolAppendChar(&parser->m_tempPool, *s)) + { + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) return XML_ERROR_NO_MEMORY; - } while (*s++); + } - uriHash = (unsigned long)sip24_final(&sip_state); + const unsigned long uriHash = (unsigned long)sip24_final(&sip_state); { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). @@ -4129,10 +4173,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns_triplets) { /* append namespace separator and prefix */ parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator; s = b->prefix->name; - do { - if (! poolAppendChar(&parser->m_tempPool, *s)) - return XML_ERROR_NO_MEMORY; - } while (*s++); + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) + return XML_ERROR_NO_MEMORY; } /* store expanded name in attribute list */ @@ -4175,48 +4218,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, localPart = tagNamePtr->str; } else return XML_ERROR_NONE; - prefixLen = 0; - if (parser->m_ns_triplets && binding->prefix->name) { - while (binding->prefix->name[prefixLen++]) - ; /* prefixLen includes null terminator */ - } + size_t prefixLen = 0; + if (parser->m_ns_triplets && binding->prefix->name) + prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1; tagNamePtr->localPart = localPart; tagNamePtr->uriLen = binding->uriLen; tagNamePtr->prefix = binding->prefix->name; tagNamePtr->prefixLen = prefixLen; - for (i = 0; localPart[i++];) - ; /* i includes null terminator */ + + const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ - if (binding->uriLen > INT_MAX - prefixLen - || i > INT_MAX - (binding->uriLen + prefixLen)) { + if (binding->uriLen > SIZE_MAX - prefixLen + || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) { return XML_ERROR_NO_MEMORY; } - n = i + binding->uriLen + prefixLen; - if (n > binding->uriAlloc) { - TAG *p; - + const size_t totalLen = localPartLen + binding->uriLen + prefixLen; + if (totalLen > binding->uriAlloc) { /* Detect and prevent integer overflow */ - if (n > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { + if (totalLen > SIZE_MAX - EXPAND_SPARE + || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - uri = (XML_Char*)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = (XML_Char*)MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; - binding->uriAlloc = n + EXPAND_SPARE; + binding->uriAlloc = totalLen + EXPAND_SPARE; memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char)); - for (p = parser->m_tagStack; p; p = p->parent) + for (TAG *p = parser->m_tagStack; p; p = p->parent) if (p->name.str == binding->uri) p->name.str = uri; FREE(parser, binding->uri); @@ -4224,10 +4255,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* if m_namespaceSeparator != '\0' then uri includes it already */ uri = binding->uri + binding->uriLen; - memcpy(uri, localPart, i * sizeof(XML_Char)); + /* Detect and prevent integer overflow */ + if (localPartLen > SIZE_MAX / sizeof(XML_Char)) { + return XML_ERROR_NO_MEMORY; + } + memcpy(uri, localPart, localPartLen * sizeof(XML_Char)); /* we always have a namespace separator between localPart and prefix */ if (prefixLen) { - uri += i - 1; + uri += localPartLen - 1; *uri = parser->m_namespaceSeparator; /* replace null terminator */ memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char)); } @@ -4362,7 +4397,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; - static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1; // "http://www.w3.org/2000/xmlns/" static const XML_Char xmlnsNamespace[] = {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, @@ -4370,15 +4405,14 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'}; - static const int xmlnsLen - = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; XML_Bool mustBeXML = XML_FALSE; XML_Bool isXML = XML_TRUE; XML_Bool isXMLNS = XML_TRUE; BINDING *b; - int len; + size_t len; /* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */ if (*uri == XML_T('\0') && prefix->name) @@ -4397,6 +4431,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } for (len = 0; uri[len]; len++) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len])) isXML = XML_FALSE; @@ -4437,25 +4475,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, if (isXMLNS) return XML_ERROR_RESERVED_NAMESPACE_URI; - if (parser->m_namespaceSeparator) + if (parser->m_namespaceSeparator) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } len++; + } if (parser->m_freeBindingList) { b = parser->m_freeBindingList; if (len > b->uriAlloc) { /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif XML_Char *temp = (XML_Char*)REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); @@ -4471,18 +4505,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, return XML_ERROR_NO_MEMORY; /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { - return XML_ERROR_NO_MEMORY; - } -#endif b->uri = (XML_Char*)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { @@ -4506,9 +4532,12 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, b->nextTagBinding = *bindingsPtr; *bindingsPtr = b; /* if attId == NULL then we are not starting a namespace scope */ - if (attId && parser->m_startNamespaceDeclHandler) + if (attId && parser->m_startNamespaceDeclHandler) { + beforeHandler(parser); parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name, prefix->binding ? uri : 0); + afterHandler(parser); + } return XML_ERROR_NONE; } @@ -4570,15 +4599,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: - if (parser->m_endCdataSectionHandler) + if (parser->m_endCdataSectionHandler) { + beforeHandler(parser); parser->m_endCdataSectionHandler(parser->m_handlerArg); + afterHandler(parser); + } /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if ((0) && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); *startPtr = next; *nextPtr = next; @@ -4589,7 +4623,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; @@ -4602,16 +4638,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = next; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4656,7 +4697,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -4850,8 +4891,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, if (! storedversion) return XML_ERROR_NO_MEMORY; } + beforeHandler(parser); parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, parser->m_encoding, s, next); if (parser->m_protocolEncodingName == NULL) { @@ -4901,8 +4944,11 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { info.convert = NULL; info.data = NULL; info.release = NULL; - if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, - encodingName, &info)) { + beforeHandler(parser); + const int status = parser->m_unknownEncodingHandler( + parser->m_unknownEncodingHandlerData, encodingName, &info); + afterHandler(parser); + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = (void*)MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { @@ -5279,9 +5325,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_DOCTYPE_INTERNAL_SUBSET: if (parser->m_startDoctypeDeclHandler) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 1); + afterHandler(parser); parser->m_doctypeName = NULL; poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; @@ -5320,7 +5368,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; goto alreadyChecked; } - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_PUBLIC_ID: if (! XmlIsPublicId(enc, s, next, eventPP)) return XML_ERROR_PUBLICID; @@ -5348,9 +5396,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_doctypeName) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 0); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5377,14 +5427,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_useForeignDTD) entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5397,7 +5455,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif /* XML_DTD */ if (parser->m_endDoctypeDeclHandler) { + beforeHandler(parser); parser->m_endDoctypeDeclHandler(parser->m_handlerArg); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5417,14 +5477,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5517,10 +5585,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5555,10 +5625,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5573,16 +5645,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + return XML_ERROR_NO_MEMORY; + } parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool)); poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } else @@ -5600,10 +5678,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5634,9 +5714,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #ifdef XML_DTD && ! parser->m_paramEntityParsing #endif /* XML_DTD */ - && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } #ifndef XML_DTD break; #else /* XML_DTD */ @@ -5648,7 +5732,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declEntity->publicId = NULL; } #endif /* XML_DTD */ - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_SYSTEM_ID: if (dtd->keepProcessing && parser->m_declEntity) { parser->m_declEntity->systemId @@ -5679,10 +5763,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5695,17 +5781,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&dtd->pool); if (parser->m_unparsedEntityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_unparsedEntityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } else if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5812,9 +5902,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! systemId) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, systemId, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5822,9 +5914,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_NOTATION_NO_SYSTEM_ID: if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, 0, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5858,41 +5952,18 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_GROUP_OPEN: if (parser->m_prologState.level >= parser->m_groupSize) { if (parser->m_groupSize) { - { - /* Detect and prevent integer overflow */ - if (parser->m_groupSize > (unsigned int)(-1) / 2u) { - return XML_ERROR_NO_MEMORY; - } - - char *const new_connector = (char*)REALLOC( - parser, parser->m_groupConnector, parser->m_groupSize *= 2); - if (new_connector == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - parser->m_groupConnector = new_connector; + /* Detect and prevent integer overflow */ + if (parser->m_groupSize > SIZE_MAX / 2) { + return XML_ERROR_NO_MEMORY; } - if (dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } -#endif - - int *const new_scaff_index = (int*)REALLOC( - parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - dtd->scaffIndex = new_scaff_index; + char *const new_connector = (char*)REALLOC(parser, parser->m_groupConnector, + parser->m_groupSize *= 2); + if (new_connector == NULL) { + parser->m_groupSize /= 2; + return XML_ERROR_NO_MEMORY; } + parser->m_groupConnector = new_connector; } else { parser->m_groupConnector = (char*)MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { @@ -5907,6 +5978,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (myindex < 0) return XML_ERROR_NO_MEMORY; assert(dtd->scaffIndex != NULL); + if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) { + /* Detect and prevent integer overflow */ + if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) { + return XML_ERROR_NO_MEMORY; + } + assert(dtd->scaffIndexSize > 0); + const size_t new_size = dtd->scaffIndexSize * 2; + int *const new_scaff_index + = (int*)REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int)); + if (new_scaff_index == NULL) { + return XML_ERROR_NO_MEMORY; + } + dtd->scaffIndex = new_scaff_index; + dtd->scaffIndexSize = new_size; + } dtd->scaffIndex[dtd->scaffLevel] = myindex; dtd->scaffLevel++; dtd->scaffold[myindex].type = XML_CTYPE_SEQ; @@ -5987,7 +6073,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* cannot report skipped entities in declarations */ if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -6008,9 +6096,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6028,9 +6119,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } #endif /* XML_DTD */ - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } break; /* Element declaration stuff */ @@ -6065,8 +6160,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY : XML_CTYPE_EMPTY); *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, content); + afterHandler(parser); handleDefault = XML_FALSE; } dtd->in_eldecl = XML_FALSE; @@ -6110,9 +6207,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; name = el->name; dtd->scaffold[myindex].name = name; - nameLen = 0; - while (name[nameLen++]) - ; + nameLen = xcslen(name) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ if (nameLen > UINT_MAX - dtd->contentStringLen) { @@ -6148,8 +6243,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! model) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, model); + afterHandler(parser); } dtd->in_eldecl = XML_FALSE; dtd->contentStringLen = 0; @@ -6211,7 +6308,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default: s = next; tok = XmlPrologTok(enc, s, end, &next); @@ -6291,7 +6388,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; parser->m_eventPtr = s = next; } @@ -6301,20 +6398,18 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, enum EntityType type) { - OPEN_INTERNAL_ENTITY *openEntity, **openEntityList, **freeEntityList; + OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; + OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; switch (type) { case ENTITY_INTERNAL: parser->m_processor = internalEntityProcessor; openEntityList = &parser->m_openInternalEntities; - freeEntityList = &parser->m_freeInternalEntities; break; case ENTITY_ATTRIBUTE: openEntityList = &parser->m_openAttributeEntities; - freeEntityList = &parser->m_freeAttributeEntities; break; case ENTITY_VALUE: openEntityList = &parser->m_openValueEntities; - freeEntityList = &parser->m_freeValueEntities; break; /* default case serves merely as a safety net in case of a * wrong entityType. Therefore we exclude the following lines @@ -6431,8 +6526,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; if (parser->m_openInternalEntities == NULL) { parser->m_processor = entity->is_param ? prologProcessor : contentProcessor; @@ -6508,8 +6603,8 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6562,7 +6657,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_INVALID_TOKEN; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, ptr); if (n < 0) { if (enc == parser->m_encoding) @@ -6582,10 +6676,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (! poolAppendChar(pool, buf[i])) - return XML_ERROR_NO_MEMORY; - } + + if (! poolAppendChars(pool, buf, n)) + return XML_ERROR_NO_MEMORY; } break; case XML_TOK_DATA_CHARS: if (! poolAppend(pool, enc, ptr, next)) @@ -6593,7 +6686,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; case XML_TOK_TRAILING_CR: next = ptr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_ATTRIBUTE_VALUE_S: case XML_TOK_DATA_NEWLINE: if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) @@ -6647,8 +6740,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } else if (! entity) { /* Cannot report skipped entity here - see comments on parser->m_skippedEntityHandler. - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ /* Cannot call the default handler because this would be out of sync with the call to the startElementHandler. @@ -6781,8 +6877,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, /* not a well-formedness error - see XML 1.0: WFC Entity Declared */ /* cannot report skipped entity here - see comments on parser->m_skippedEntityHandler - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ dtd->keepProcessing = dtd->standalone; goto endEntityValue; @@ -6798,9 +6897,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6836,17 +6938,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, break; case XML_TOK_TRAILING_CR: next = entityTextPtr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_DATA_NEWLINE: - if (pool->end == pool->ptr && ! poolGrow(pool)) { + if (! poolAppendChar(pool, 0xA)) { result = XML_ERROR_NO_MEMORY; goto endEntityValue; } - *(pool->ptr)++ = 0xA; break; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, entityTextPtr); if (n < 0) { if (enc == parser->m_encoding) @@ -6864,12 +6964,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (pool->end == pool->ptr && ! poolGrow(pool)) { - result = XML_ERROR_NO_MEMORY; - goto endEntityValue; - } - *(pool->ptr)++ = buf[i]; + if (! poolAppendChars(pool, buf, n)) { + result = XML_ERROR_NO_MEMORY; + goto endEntityValue; } } break; case XML_TOK_PARTIAL: @@ -6966,8 +7063,8 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6997,6 +7094,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { return XML_ERROR_NO_MEMORY; } + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(pool) > (size_t)INT_MAX) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } entity->textPtr = poolStart(pool); entity->textLen = (int)(poolLength(pool)); poolFinish(pool); @@ -7049,7 +7151,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_processingInstructionHandler(parser->m_handlerArg, target, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -7069,7 +7173,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_commentHandler(parser->m_handlerArg, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -7110,15 +7216,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); *eventPP = s; } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); - } else + } else { + beforeHandler(parser); parser->m_defaultHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } static int @@ -7129,48 +7240,34 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttsNames), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } if (type->nDefaultAtts == type->allocDefaultAtts) { - if (type->allocDefaultAtts == 0) { - type->allocDefaultAtts = 8; - type->defaultAtts - = (DEFAULT_ATTRIBUTE*)MALLOC(parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); - if (! type->defaultAtts) { - type->allocDefaultAtts = 0; - return 0; - } - } else { - DEFAULT_ATTRIBUTE *temp; - - /* Detect and prevent integer overflow */ - if (type->allocDefaultAtts > INT_MAX / 2) { - return 0; - } - - int count = type->allocDefaultAtts * 2; + /* Detect and prevent integer overflow */ + if (type->allocDefaultAtts > SIZE_MAX / 2) { + return 0; + } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { - return 0; - } -#endif + size_t count = type->allocDefaultAtts * 2; + if (count == 0) { + count = 8; + } - temp = (DEFAULT_ATTRIBUTE*)REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); - if (temp == NULL) - return 0; - type->allocDefaultAtts = count; - type->defaultAtts = temp; + /* Detect and prevent integer overflow. */ + if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + return 0; } + + DEFAULT_ATTRIBUTE *const temp = (DEFAULT_ATTRIBUTE*)REALLOC( + parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE))); + if (temp == NULL) + return 0; + type->allocDefaultAtts = count; + type->defaultAtts = temp; } att = type->defaultAtts + type->nDefaultAtts; att->id = attId; @@ -7179,8 +7276,8 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound = (NAMED *)lookup( - parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + NAMED *const nameAddedOrFound + = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); if (! nameAddedOrFound) return 0; @@ -7253,13 +7350,14 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, } else { int i; for (i = 0; name[i]; i++) { + /* Detect and prevent signed integer overflow */ + if (i == INT_MAX) { + return NULL; + } /* attributes without prefix are *not* in the default namespace */ if (name[i] == XML_T(ASCII_COLON)) { - int j; - for (j = 0; j < i; j++) { - if (! poolAppendChar(&dtd->pool, name[j])) - return NULL; - } + if (! poolAppendChars(&dtd->pool, name, i)) + return NULL; if (! poolAppendChar(&dtd->pool, XML_T('\0'))) return NULL; id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, @@ -7287,46 +7385,39 @@ getContext(XML_Parser parser) { XML_Bool needSep = XML_FALSE; if (dtd->defaultPrefix.binding) { - int i; - int len; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = dtd->defaultPrefix.binding->uriLen; + size_t len = dtd->defaultPrefix.binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) { - if (! poolAppendChar(&parser->m_tempPool, - dtd->defaultPrefix.binding->uri[i])) { - /* Because of memory caching, I don't believe this line can be - * executed. - * - * This is part of a loop copying the default prefix binding - * URI into the parser's temporary string pool. Previously, - * that URI was copied into the same string pool, with a - * terminating NUL character, as part of setContext(). When - * the pool was cleared, that leaves a block definitely big - * enough to hold the URI on the free block list of the pool. - * The URI copy in getContext() therefore cannot run out of - * memory. - * - * If the pool is used between the setContext() and - * getContext() calls, the worst it can do is leave a bigger - * block on the front of the free list. Given that this is - * all somewhat inobvious and program logic can be changed, we - * don't delete the line but we do exclude it from the test - * coverage statistics. - */ - return NULL; /* LCOV_EXCL_LINE */ - } + if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri, + len)) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ } needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->prefixes)); for (;;) { - int i; - int len; - const XML_Char *s; PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (! prefix) break; @@ -7341,23 +7432,21 @@ getContext(XML_Parser parser) { } if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = prefix->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->name, + xcslen(prefix->name))) + return NULL; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = prefix->binding->uriLen; + size_t len = prefix->binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i])) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) + return NULL; needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->generalEntities)); for (;;) { - const XML_Char *s; ENTITY *e = (ENTITY *)hashTableIterNext(&iter); if (! e) break; @@ -7365,9 +7454,8 @@ getContext(XML_Parser parser) { continue; if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = e->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return 0; + if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) + return NULL; needSep = XML_TRUE; } @@ -7489,6 +7577,7 @@ dtdCreate(XML_Parser parser) { p->in_eldecl = XML_FALSE; p->scaffIndex = NULL; + p->scaffIndexSize = 0; p->scaffold = NULL; p->scaffLevel = 0; p->scaffSize = 0; @@ -7510,8 +7599,7 @@ dtdReset(DTD *p, XML_Parser parser) { if (! e) break; hashTableDestroy(&(e->defaultAttsNames)); - if (e->allocDefaultAtts != 0) - FREE(parser, e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD @@ -7530,6 +7618,7 @@ dtdReset(DTD *p, XML_Parser parser) { FREE(parser, p->scaffIndex); p->scaffIndex = NULL; + p->scaffIndexSize = 0; FREE(parser, p->scaffold); p->scaffold = NULL; @@ -7552,8 +7641,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { if (! e) break; hashTableDestroy(&(e->defaultAttsNames)); - if (e->allocDefaultAtts != 0) - FREE(parser, e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -7632,7 +7720,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableIterInit(&iter, &(oldDtd->elementTypes)); for (;;) { - int i; ELEMENT_TYPE *newE; const XML_Char *name; const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter); @@ -7650,15 +7737,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableInit(&(newE->defaultAttsNames), parser); if (oldE->nDefaultAtts) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } -#endif newE->defaultAtts = (DEFAULT_ATTRIBUTE*)MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { @@ -7672,7 +7754,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (oldE->prefix) newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); - for (i = 0; i < newE->nDefaultAtts; i++) { + for (size_t i = 0; i < newE->nDefaultAtts; i++) { const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( oldParser, &(newDtd->attributeIds), attributeName, 0); @@ -7685,8 +7767,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = (NAMED *)lookup( - parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), + attributeName, sizeof(NAMED)); if (! nameAddedOrFound) { return 0; } @@ -7716,6 +7798,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newDtd->scaffSize = oldDtd->scaffSize; newDtd->scaffLevel = oldDtd->scaffLevel; newDtd->scaffIndex = oldDtd->scaffIndex; + newDtd->scaffIndexSize = oldDtd->scaffIndexSize; return 1; } /* End dtdCopy */ @@ -7787,18 +7870,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, static XML_Bool FASTCALL keyeq(KEY s1, KEY s2) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +# else for (; *s1 == *s2; s1++, s2++) if (*s1 == 0) return XML_TRUE; return XML_FALSE; +# endif +#else + return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +#endif } static size_t keylen(KEY s) { - size_t len = 0; - for (; *s; s++, len++) - ; - return len; + return xcslen(s); } static void @@ -8016,10 +8104,8 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s) { - do { - if (! poolAppendChar(pool, *s)) - return NULL; - } while (*s++); + if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -8058,10 +8144,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { */ return NULL; /* LCOV_EXCL_LINE */ } - for (; n > 0; --n, s++) { - if (! poolAppendChar(pool, *s)) - return NULL; - } + if (n > 0 && ! poolAppendChars(pool, s, n)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -8069,11 +8153,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, const XML_Char *s) { - while (*s) { - if (! poolAppendChar(pool, *s)) - return NULL; - s++; - } + if (! poolAppendChars(pool, s, xcslen(s))) + return NULL; return pool->start; } @@ -8082,9 +8163,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end) { if (! poolAppend(pool, enc, ptr, end)) return NULL; - if (pool->ptr == pool->end && ! poolGrow(pool)) + if (! poolAppendChar(pool, 0)) return NULL; - *(pool->ptr)++ = 0; return pool->start; } @@ -8219,6 +8299,19 @@ poolGrow(STRING_POOL *pool) { return XML_TRUE; } +static bool FASTCALL +poolGrowUntil(STRING_POOL *pool, size_t needed) { + for (;;) { + const size_t available = pool->end - pool->ptr; + if (available >= needed) { + return true; + } + if (! poolGrow(pool)) { + return false; + } + } +} + static int FASTCALL nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ @@ -8226,18 +8319,14 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX + /* Detect and prevent integer overflow. */ if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } -#endif dtd->scaffIndex = (int*)MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; + dtd->scaffIndexSize = parser->m_groupSize; dtd->scaffIndex[0] = 0; } @@ -8398,12 +8487,21 @@ build_model(XML_Parser parser) { const XML_Char *src; dest->name = str; src = dtd->scaffold[src_node].name; - for (;;) { - *str++ = *src; - if (! *src) - break; - src++; + + const size_t nameLen = xcslen(src) + /* null terminator*/ 1; + + // Detect and prevent integer overflow + if (nameLen > SIZE_MAX / sizeof(XML_Char)) { + // NOTE: We are avoiding FREE(..) here because the model + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(ret); + return NULL; } + + memcpy(str, src, nameLen * sizeof(XML_Char)); + str += nameLen; + dest->numchildren = 0; dest->children = NULL; } else { @@ -8450,22 +8548,24 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, static XML_Char * copyString(const XML_Char *s, XML_Parser parser) { - size_t charsRequired = 0; - XML_Char *result; - /* First determine how long the string is */ - while (s[charsRequired] != 0) { - charsRequired++; - } - /* Include the terminator */ - charsRequired++; + const size_t charsRequired = xcslen(s) + /*null terminator*/ 1; + + /* Detect and prevent integer overflow */ + if (charsRequired > SIZE_MAX / sizeof(XML_Char)) + return NULL; + + const size_t bytesRequired = charsRequired * sizeof(XML_Char); /* Now allocate space for the copy */ - result = (XML_Char*)MALLOC(parser, charsRequired * sizeof(XML_Char)); + XML_Char *const result = (XML_Char*)MALLOC(parser, bytesRequired); + if (result == NULL) return NULL; + /* Copy the original into place */ - memcpy(result, s, charsRequired * sizeof(XML_Char)); + memcpy(result, s, bytesRequired); + return result; } @@ -8632,15 +8732,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *const entityName = entity->name; # endif + const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2; + const int maxLimitedDepth = 10; // somewhat arbitrary + const int candidateIndentDepth + = (int)rootParser->m_entity_stats.currentDepth - 1; + const bool limitingNeeded + = limitingWanted && (candidateIndentDepth > maxLimitedDepth); + const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : ""; + const int indentDepth + = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) + : candidateIndentDepth; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, - rootParser->m_entity_stats.maximumDepthSeen, - ((int)rootParser->m_entity_stats.currentDepth - 1) * 2, "", - entity->is_param ? "%" : "&", entityName, action, entity->textLen, - sourceLine); + rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", + ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, + entity->textLen, sourceLine); } static void diff --git a/base/poco/XML/src/xmltok.c b/base/poco/XML/src/xmltok.c index 19626ea7f195..fe8a780b625d 100644 --- a/base/poco/XML/src/xmltok.c +++ b/base/poco/XML/src/xmltok.c @@ -24,24 +24,26 @@ Copyright (c) 2022 Martin Ettl Copyright (c) 2022 Sean McBride Copyright (c) 2023 Hanno Böck + Copyright (c) 2025 Alfonso Gregory + Copyright (c) 2026 Nick Begg Licensed under the MIT license: - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit - persons to whom the Software is furnished to do so, subject to the + persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ @@ -58,6 +60,7 @@ #include "Poco/XML/expat_external.h" #include "internal.h" +#include "fallthrough.h" #include "xmltok.h" #include "nametab.h" @@ -641,7 +644,7 @@ unicode_byte_type(char hi, char lo) { *(*toP)++ = lo; \ break; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case 0x1: \ case 0x2: \ case 0x3: \ @@ -1557,7 +1560,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, case 0xEF: /* possibly first byte of UTF-8 BOM */ if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE) break; - /* fall through */ + EXPAT_FALLTHROUGH; case 0x00: case 0x3C: return XML_TOK_PARTIAL; diff --git a/base/poco/XML/src/xmltok_impl.c b/base/poco/XML/src/xmltok_impl.c index a598fa58e2f1..d41b8e9e1a0c 100644 --- a/base/poco/XML/src/xmltok_impl.c +++ b/base/poco/XML/src/xmltok_impl.c @@ -17,6 +17,7 @@ Copyright (c) 2019 David Loffredo Copyright (c) 2020 Boris Kolpackov Copyright (c) 2022 Martin Ettl + Copyright (c) 2026 Nick Begg Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -83,7 +84,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ case BT_DIGIT: \ @@ -112,7 +113,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ ptr += MINBPC(enc); \ @@ -209,7 +210,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_CR: case BT_LF: @@ -323,7 +324,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr + MINBPC(enc); return tok; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -615,7 +616,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_EQUALS: { int open; # ifdef XML_NS @@ -898,7 +899,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_AMP: case BT_LT: case BT_NONXML: @@ -1059,7 +1060,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* indicate that this might be part of a CR/LF pair */ return -XML_TOK_PROLOG_S; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_LF: for (;;) { @@ -1074,7 +1075,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* don't split CR/LF pair */ if (ptr + MINBPC(enc) != end) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_PROLOG_S; @@ -1189,7 +1190,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, tok = XML_TOK_NMTOKEN; break; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -1484,7 +1485,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, case BT_NMSTRT: if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f)) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: switch (BYTE_TO_ASCII(enc, ptr)) { case 0x24: /* $ */ diff --git a/ci/jobs/integration_test_job.py b/ci/jobs/integration_test_job.py index 9578dfb40e7d..9285fa08402a 100644 --- a/ci/jobs/integration_test_job.py +++ b/ci/jobs/integration_test_job.py @@ -235,6 +235,7 @@ def matches_substring(substring, log, is_regex): "docker_compose_iceberg_hms_catalog.yml", "docker_compose_iceberg_lakekeeper_catalog.yml", "docker_compose_iceberg_nessie_catalog.yml", + "docker_compose_iceberg_seaweedfs_catalog.yml", ], "hms_catalog": ["docker_compose_iceberg_hms_catalog.yml"], "glue_catalog": ["docker_compose_glue_catalog.yml"], diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 787132b884c7..3af76bb330f7 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -1,14 +1,14 @@ -# This variables autochanged by tests/ci/version_helper.py: +# This variables autochanged by ci/jobs/scripts/create_release.py: # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54512) +SET(VERSION_REVISION 54514) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 6) -SET(VERSION_PATCH 2) -SET(VERSION_GITHASH 6615864161fd5332edfcb2864658bd9ccc65197b) -SET(VERSION_DESCRIBE v26.6.2.20001.altinityantalya) -SET(VERSION_STRING 26.6.2.20001.altinityantalya) +SET(VERSION_PATCH 4) +SET(VERSION_GITHASH c40659002db83ab4ba5d3c7152a5ef8236ee2fab) +SET(VERSION_DESCRIBE v26.6.4.20001.altinityantalya) +SET(VERSION_STRING 26.6.4.20001.altinityantalya) # end of autochange SET(VERSION_TWEAK 20001) diff --git a/contrib/AMQP-CPP b/contrib/AMQP-CPP index 99b52d1a7d74..4fccd7f84318 160000 --- a/contrib/AMQP-CPP +++ b/contrib/AMQP-CPP @@ -1 +1 @@ -Subproject commit 99b52d1a7d74a23b84cdb90ee214cd46539febc3 +Subproject commit 4fccd7f84318fe3a022c63a5b621a831c99e3a32 diff --git a/contrib/arrow b/contrib/arrow index 6a0df9e8d6d0..949bccfd6c66 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 6a0df9e8d6d008bc54e311675f6fd1242cea0c3c +Subproject commit 949bccfd6c664b15dfad5b3e50e5428021852973 diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 2c56dd43f267..2f373b84c2be 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -39,12 +39,12 @@ endif() # We require a C++20 compliant compiler set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(ARROW_VERSION "23.0.1") +set(ARROW_VERSION "25.0.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}") -set(ARROW_VERSION_MAJOR "23") +set(ARROW_VERSION_MAJOR "25") set(ARROW_VERSION_MINOR "0") -set(ARROW_VERSION_PATCH "1") +set(ARROW_VERSION_PATCH "0") if(ARROW_VERSION_MAJOR STREQUAL "0") # Arrow 0.x.y => SO version is "x", full SO version is "x.y.0" @@ -82,7 +82,10 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc COMMAND ${PROTOBUF_EXECUTABLE} -I ${PROTO_DIR} --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" - "${PROTO_DIR}/orc_proto.proto") + "${PROTO_DIR}/orc_proto.proto" + # Without these, the generated sources are not regenerated when the schema or + # `protoc` itself changes, and a stale `orc_proto.pb.h` is compiled instead. + DEPENDS "${PROTO_DIR}/orc_proto.proto" ${PROTOBUF_EXECUTABLE}) # === flatbuffers set(FLATBUFFERS_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/flatbuffers") @@ -145,6 +148,8 @@ set(ORC_SRCS "${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc" "${ORC_SOURCE_SRC_DIR}/ColumnWriter.hh" "${ORC_SOURCE_SRC_DIR}/Common.cc" + "${ORC_SOURCE_SRC_DIR}/Dictionary.cc" + "${ORC_SOURCE_SRC_DIR}/DictionaryLoader.cc" "${ORC_SOURCE_SRC_DIR}/Compression.cc" "${ORC_SOURCE_SRC_DIR}/Compression.hh" "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.cc" @@ -153,6 +158,7 @@ set(ORC_SRCS "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.hh" "${ORC_SOURCE_SRC_DIR}/Dispatch.hh" "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" + "${ORC_SOURCE_SRC_DIR}/Geospatial.cc" "${ORC_SOURCE_SRC_DIR}/Int128.cc" "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc" "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.hh" @@ -340,6 +346,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/extension_type.cc" "${LIBRARY_DIR}/extension/bool8.cc" "${LIBRARY_DIR}/extension/json.cc" + "${LIBRARY_DIR}/extension/parquet_variant.cc" "${LIBRARY_DIR}/extension/uuid.cc" "${LIBRARY_DIR}/integration/c_data_integration_internal.cc" "${LIBRARY_DIR}/io/buffered.cc" @@ -389,7 +396,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/bitmap_ops.cc" "${LIBRARY_DIR}/util/bpacking.cc" "${LIBRARY_DIR}/util/bpacking_scalar.cc" - "${LIBRARY_DIR}/util/bpacking_simd_default.cc" + "${LIBRARY_DIR}/util/bpacking_simd_128.cc" "${LIBRARY_DIR}/util/byte_size.cc" "${LIBRARY_DIR}/util/byte_stream_split_internal.cc" "${LIBRARY_DIR}/util/cancel.cc" @@ -428,6 +435,7 @@ set(ARROW_SRCS "${LIBRARY_DIR}/util/trie.cc" "${LIBRARY_DIR}/util/union_util.cc" "${LIBRARY_DIR}/util/unreachable.cc" + "${LIBRARY_DIR}/util/ulp_distance.cc" "${LIBRARY_DIR}/util/uri.cc" "${LIBRARY_DIR}/util/utf8.cc" "${LIBRARY_DIR}/util/value_parsing.cc" @@ -501,7 +509,7 @@ elseif (ARCH_AMD64 AND X86_ARCH_LEVEL VERSION_GREATER_EQUAL 3) ARROW_HAVE_RUNTIME_SSE4_2 ARROW_HAVE_RUNTIME_AVX2 ARROW_HAVE_RUNTIME_BMI2) # bpacking's unpack() has no compile-time AVX2 path, only a runtime-dispatched one, so its source is needed. SET(ARROW_SRCS ${ARROW_SRCS} - "${LIBRARY_DIR}/util/bpacking_simd_avx2.cc" + "${LIBRARY_DIR}/util/bpacking_simd_256.cc" "${LIBRARY_DIR}/util/byte_stream_split_internal_avx2.cc") if (X86_ARCH_LEVEL VERSION_GREATER_EQUAL 4) list (APPEND ARROW_SIMD_DEFS ARROW_HAVE_AVX512 ARROW_HAVE_RUNTIME_AVX512) @@ -558,7 +566,6 @@ set(PARQUET_SRCS "${LIBRARY_DIR}/arrow/reader_internal.cc" "${LIBRARY_DIR}/arrow/schema.cc" "${LIBRARY_DIR}/arrow/schema_internal.cc" - "${LIBRARY_DIR}/arrow/variant_internal.cc" "${LIBRARY_DIR}/arrow/writer.cc" "${LIBRARY_DIR}/benchmark_util.cc" "${LIBRARY_DIR}/bloom_filter.cc" @@ -611,6 +618,14 @@ set(PARQUET_SRCS "${GEN_LIBRARY_DIR}/parquet_types.cpp" ) #list(TRANSFORM PARQUET_SRCS PREPEND "${LIBRARY_DIR}/") # cmake 3.12 + +# The Bloom filter dispatch table references `FindHashBlockAvx2` whenever AVX2 is available, so the +# kernel has to be built. Unlike `level_comparison_avx2.cc` this one is x86-only (it instantiates +# `xsimd::batch`), so it must not be added on other architectures. +if (ARCH_AMD64 AND X86_ARCH_LEVEL VERSION_GREATER_EQUAL 3) + list (APPEND PARQUET_SRCS "${LIBRARY_DIR}/bloom_filter_avx2.cc") +endif () + add_library(_parquet ${PARQUET_SRCS}) add_library(ch_contrib::parquet ALIAS _parquet) target_include_directories(_parquet SYSTEM BEFORE diff --git a/contrib/curl b/contrib/curl index a05f34973e6c..68720b483728 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit a05f34973e6c4bb629d018f7cb51487be1c904d8 +Subproject commit 68720b4837284335b2d63cb358f8f6ce65f5bc55 diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 87646ef92114..2396e9c1e854 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -17,11 +17,14 @@ set (SRCS "${LIBRARY_DIR}/lib/cf-haproxy.c" "${LIBRARY_DIR}/lib/cf-https-connect.c" "${LIBRARY_DIR}/lib/cf-ip-happy.c" + "${LIBRARY_DIR}/lib/cf-recvbuf.c" + "${LIBRARY_DIR}/lib/cf-setup.c" "${LIBRARY_DIR}/lib/cf-socket.c" "${LIBRARY_DIR}/lib/cfilters.c" "${LIBRARY_DIR}/lib/conncache.c" "${LIBRARY_DIR}/lib/connect.c" "${LIBRARY_DIR}/lib/content_encoding.c" + "${LIBRARY_DIR}/lib/creds.c" "${LIBRARY_DIR}/lib/cshutdn.c" "${LIBRARY_DIR}/lib/curl_addrinfo.c" "${LIBRARY_DIR}/lib/curl_endian.c" @@ -62,8 +65,9 @@ set (SRCS "${LIBRARY_DIR}/lib/multi.c" "${LIBRARY_DIR}/lib/multi_ev.c" "${LIBRARY_DIR}/lib/multi_ntfy.c" - "${LIBRARY_DIR}/lib/noproxy.c" "${LIBRARY_DIR}/lib/parsedate.c" + "${LIBRARY_DIR}/lib/peer.c" + "${LIBRARY_DIR}/lib/proxy.c" "${LIBRARY_DIR}/lib/progress.c" "${LIBRARY_DIR}/lib/protocol.c" "${LIBRARY_DIR}/lib/rand.c" @@ -98,6 +102,7 @@ set (SRCS "${LIBRARY_DIR}/lib/vtls/keylog.c" "${LIBRARY_DIR}/lib/vtls/openssl.c" "${LIBRARY_DIR}/lib/vtls/vtls.c" + "${LIBRARY_DIR}/lib/vtls/vtls_config.c" "${LIBRARY_DIR}/lib/vtls/vtls_scache.c" "${LIBRARY_DIR}/lib/curlx/base64.c" "${LIBRARY_DIR}/lib/curlx/basename.c" diff --git a/contrib/google-protobuf b/contrib/google-protobuf index 74211c0dfc27..35cd01f9fe9a 160000 --- a/contrib/google-protobuf +++ b/contrib/google-protobuf @@ -1 +1 @@ -Subproject commit 74211c0dfc2777318ab53c2cd2c317a2ef9012de +Subproject commit 35cd01f9fe9afbeea38cc7b979a3b6bfcde82c03 diff --git a/contrib/google-protobuf-cmake/CMakeLists.txt b/contrib/google-protobuf-cmake/CMakeLists.txt index 004df6b9cc56..fa353c59b5f3 100644 --- a/contrib/google-protobuf-cmake/CMakeLists.txt +++ b/contrib/google-protobuf-cmake/CMakeLists.txt @@ -115,7 +115,9 @@ target_link_libraries(_libprotobuf-lite if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(_libprotobuf-lite log) endif() -target_include_directories(_libprotobuf-lite SYSTEM PUBLIC ${protobuf_source_dir}/src) +target_include_directories(_libprotobuf-lite SYSTEM PUBLIC + ${protobuf_source_dir}/src + ${protobuf_source_dir}/third_party/utf8_range) add_library(protobuf::libprotobuf-lite ALIAS _libprotobuf-lite) @@ -186,6 +188,7 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.cc ${protobuf_source_dir}/src/google/protobuf/service.cc + ${protobuf_source_dir}/src/google/protobuf/symbol_checker.cc ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc ${protobuf_source_dir}/src/google/protobuf/text_format.cc ${protobuf_source_dir}/src/google/protobuf/unknown_field_set.cc @@ -211,7 +214,9 @@ target_link_libraries(_libprotobuf if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(_libprotobuf log) endif() -target_include_directories(_libprotobuf SYSTEM PUBLIC ${protobuf_source_dir}/src) +target_include_directories(_libprotobuf SYSTEM PUBLIC + ${protobuf_source_dir}/src + ${protobuf_source_dir}/third_party/utf8_range) add_library(protobuf::libprotobuf ALIAS _libprotobuf) @@ -318,6 +323,7 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/python/helpers.cc ${protobuf_source_dir}/src/google/protobuf/compiler/python/pyi_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/retention.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/rbs_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/ruby/ruby_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/accessor_case.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/accessors/accessors.cc @@ -333,6 +339,7 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/rust/context.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/crate_mapping.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/enum.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/rust/extension.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/message.cc ${protobuf_source_dir}/src/google/protobuf/compiler/rust/naming.cc @@ -370,6 +377,7 @@ set(libprotoc_files ${protobuf_source_dir}/upb/mini_descriptor/link.c ${protobuf_source_dir}/upb/mini_table/compat.c ${protobuf_source_dir}/upb/mini_table/extension_registry.c + ${protobuf_source_dir}/upb/mini_table/generated_registry.c ${protobuf_source_dir}/upb/mini_table/internal/message.c ${protobuf_source_dir}/upb/mini_table/message.c ${protobuf_source_dir}/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c @@ -392,10 +400,12 @@ set(libprotoc_files ${protobuf_source_dir}/upb/reflection/service_def.c ${protobuf_source_dir}/upb/wire/decode.c ${protobuf_source_dir}/upb/wire/encode.c + ${protobuf_source_dir}/upb/wire/eps_copy_input_stream.c + ${protobuf_source_dir}/upb/wire/internal/decoder.c + ${protobuf_source_dir}/upb/wire/reader.c ${protobuf_source_dir}/upb_generator/common.cc ${protobuf_source_dir}/upb_generator/common/names.cc ${protobuf_source_dir}/upb_generator/file_layout.cc - ${protobuf_source_dir}/upb_generator/minitable/fasttable.cc ${protobuf_source_dir}/upb_generator/minitable/generator.cc ${protobuf_source_dir}/upb_generator/minitable/names.cc ${protobuf_source_dir}/upb_generator/minitable/names_internal.cc diff --git a/contrib/grpc b/contrib/grpc index 64256b574aa1..a703d1f9448f 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 64256b574aa1c3ee21dfcb86c6c7780d31451604 +Subproject commit a703d1f9448ff77deca3beb36dfc7d337a209d3e diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake index 98eb4b83257f..419802aeb70b 100644 --- a/contrib/grpc-cmake/grpc.cmake +++ b/contrib/grpc-cmake/grpc.cmake @@ -97,7 +97,6 @@ add_library(gpr ${_gRPC_SOURCE_DIR}/src/core/config/config_vars_non_generated.cc ${_gRPC_SOURCE_DIR}/src/core/config/load_config.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/thread_local.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc ${_gRPC_SOURCE_DIR}/src/core/util/alloc.cc ${_gRPC_SOURCE_DIR}/src/core/util/log.cc ${_gRPC_SOURCE_DIR}/src/core/util/string.cc @@ -124,6 +123,15 @@ add_library(gpr ${_gRPC_SOURCE_DIR}/src/core/util/posix/env.cc ${_gRPC_SOURCE_DIR}/src/core/util/posix/stat.cc ${_gRPC_SOURCE_DIR}/src/core/util/posix/thd.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/cpu.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/env.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/stat.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/string.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/string_util.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/sync.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/thd.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/time.cc + ${_gRPC_SOURCE_DIR}/src/core/util/windows/tmpfile.cc ) target_compile_features(gpr PUBLIC cxx_std_17) @@ -183,9 +191,15 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/call/server_call.cc ${_gRPC_SOURCE_DIR}/src/core/call/status_util.cc ${_gRPC_SOURCE_DIR}/src/core/channelz/channel_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/property_list.cc ${_gRPC_SOURCE_DIR}/src/core/channelz/channelz.cc ${_gRPC_SOURCE_DIR}/src/core/channelz/channelz_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/text_encode.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/v2tov1/convert.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/v2tov1/legacy_api.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/v2tov1/property_list.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/backup_poller.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/buffered_call.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/client_channel.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/client_channel_factory.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/client_channel_filter.cc @@ -205,7 +219,10 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_pool_interface.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_client.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_limiter.cc ${_gRPC_SOURCE_DIR}/src/core/config/core_configuration.cc + ${_gRPC_SOURCE_DIR}/src/core/config/experiment_env_var.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/call/call_creds_registry_init.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/call_creds_util.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/composite/composite_call_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/external/aws_external_account_credentials.cc @@ -219,8 +236,11 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt/json_token.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt/jwt_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt/jwt_verifier.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt_token_file/jwt_token_file_call_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt_util.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/oauth2/oauth2_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/plugin/plugin_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/call/regional_access_boundary_fetcher.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/token_fetcher/token_fetcher_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/alts/alts_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/alts/alts_security_connector.cc @@ -248,12 +268,14 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_certificate_distributor.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_certificate_match.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_certificate_provider.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_certificate_selector.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_certificate_verifier.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_credentials_options.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/grpc_tls_crl_provider.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/load_system_roots_fallback.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/load_system_roots_supported.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/load_system_roots_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/spiffe_utils.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/ssl_utils.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/tls_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/transport/tls/tls_security_connector.cc @@ -265,9 +287,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/idle_filter_state.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_filter.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/gcp_authentication/gcp_authentication_filter.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/filters/gcp_authentication/gcp_authentication_service_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc @@ -277,7 +297,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/rbac/rbac_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/rbac/rbac_service_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/stateful_session/stateful_session_filter.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/chttp2_plugin.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc @@ -296,23 +315,55 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_security.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/goaway.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_client_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_server_transport.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_stats_collector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_transport.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/keepalive.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_promise.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/transport_common.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_cycle.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_size_policy.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/legacy_inproc_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/cel/expr/checked.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/cel/expr/syntax.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/config/common/mutation_rules/v3/mutation_rules.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/config/core/v3/cel.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/call_credentials/access_token/v3/access_token_credentials.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/channel_credentials/tls/v3/tls_credentials.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/channel_credentials/xds/v3/xds_credentials.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/service.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/cel/expr/checked.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/cel/expr/syntax.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/config/common/mutation_rules/v3/mutation_rules.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/config/core/v3/cel.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/common/matching/v3/extension_matcher.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/matcher/action/v3/skip_action.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/composite/v3/composite.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/property_list.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/promise.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/filters/http/composite/v3/composite.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/filters/common/matcher/action/v3/skip_action.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/common/matching/v3/extension_matcher.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.c @@ -603,6 +654,11 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/promise.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/property_list.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/service.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c @@ -636,16 +692,40 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/filter/auth/client_auth_filter.cc ${_gRPC_SOURCE_DIR}/src/core/filter/auth/server_auth_filter.cc - ${_gRPC_SOURCE_DIR}/src/core/filter/blackboard.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/composite/composite_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/fused_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_client_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/pipelined_secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cfsocket_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/endpoint_channel_arg_wrapper.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/file_descriptor_collection.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_interface_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_interface_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_write_event_sink.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/mpsc.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/wait_set.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/stream_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/telemetry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_tracker/resource_tracker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/promise_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/net/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/server/xds_server_config_fetcher_legacy.cc + ${_gRPC_SOURCE_DIR}/src/core/telemetry/context_list_entry.cc + ${_gRPC_SOURCE_DIR}/src/core/telemetry/instrument.cc + ${_gRPC_SOURCE_DIR}/src/core/transport/message_size_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/transport/session_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_telemetry_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/util/grpc_check.cc + ${_gRPC_SOURCE_DIR}/src/core/util/postmortem_emit.cc + ${_gRPC_SOURCE_DIR}/src/core/util/wait_for_single_owner.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/blackboard.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/endpoint_info/endpoint_info_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/handshaker_registry.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_handshaker.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_proxy_mapper.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_client_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_proxy_mapper.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/xds_http_proxy_mapper.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/proxy_mapper_registry.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/legacy_secure_endpoint.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/pipelined_secure_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/secure_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/security_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/parse_address.cc @@ -670,7 +750,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine_factory.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/event_engine.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc @@ -977,7 +1056,9 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_endpoint_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_health_status.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_composite_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_fault_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_filter_registry.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_gcp_authn_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_rbac_filter.cc @@ -985,6 +1066,11 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_lb_policy_registry.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_listener.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_listener_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_matcher.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_matcher_action.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_matcher_context.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_matcher_input.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_matcher_parse.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_metadata.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_metadata_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_route_config.cc @@ -1060,9 +1146,12 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/call/server_call.cc ${_gRPC_SOURCE_DIR}/src/core/call/status_util.cc ${_gRPC_SOURCE_DIR}/src/core/channelz/channel_trace.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/property_list.cc ${_gRPC_SOURCE_DIR}/src/core/channelz/channelz.cc ${_gRPC_SOURCE_DIR}/src/core/channelz/channelz_registry.cc + ${_gRPC_SOURCE_DIR}/src/core/channelz/text_encode.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/backup_poller.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/buffered_call.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/client_channel.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/client_channel_factory.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/client_channel_filter.cc @@ -1082,7 +1171,9 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_pool_interface.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_client.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_limiter.cc ${_gRPC_SOURCE_DIR}/src/core/config/core_configuration.cc + ${_gRPC_SOURCE_DIR}/src/core/config/experiment_env_var.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/call_creds_util.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/composite/composite_call_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/json_util.cc @@ -1111,7 +1202,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/idle_filter_state.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_filter.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc ${_gRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc @@ -1135,19 +1225,29 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_security.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/goaway.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_client_transport.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_server_transport.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings_manager.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_stats_collector.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_transport.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/keepalive.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_promise.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/transport_common.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_cycle.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/write_size_policy.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc @@ -1162,6 +1262,10 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/promise.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/property_list.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/service.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c @@ -1171,17 +1275,46 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/validate/validate.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/promise.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/property_list.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/service.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/filter/auth/client_auth_filter.cc ${_gRPC_SOURCE_DIR}/src/core/filter/auth/server_auth_filter.cc - ${_gRPC_SOURCE_DIR}/src/core/filter/blackboard.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/fused_filters.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_client_handshaker.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/pipelined_secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/cf_engine/cfsocket_listener.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/endpoint_channel_arg_wrapper.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/file_descriptor_collection.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_interface_posix.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_interface_windows.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/posix_write_event_sink.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/mpsc.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/promise/wait_set.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/stream_quota.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/telemetry.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/resource_tracker/resource_tracker.cc + ${_gRPC_SOURCE_DIR}/src/core/lib/transport/promise_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/net/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/telemetry/context_list_entry.cc + ${_gRPC_SOURCE_DIR}/src/core/telemetry/instrument.cc + ${_gRPC_SOURCE_DIR}/src/core/transport/message_size_service_config.cc + ${_gRPC_SOURCE_DIR}/src/core/transport/session_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/util/grpc_check.cc + ${_gRPC_SOURCE_DIR}/src/core/util/postmortem_emit.cc + ${_gRPC_SOURCE_DIR}/src/core/util/wait_for_single_owner.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/blackboard.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/endpoint_info/endpoint_info_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/handshaker_registry.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_handshaker.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_proxy_mapper.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_client_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_proxy_mapper.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/proxy_mapper_registry.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/legacy_secure_endpoint.cc - ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/secure_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/pipelined_secure_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/secure_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/security_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/lib/address_utils/parse_address.cc @@ -1206,7 +1339,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/default_event_engine_factory.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/event_engine.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/forkable.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc @@ -1506,6 +1638,7 @@ add_library(upb ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/alloc.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mem/arena.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/message.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/generated_registry.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/extension_registry.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/internal/message.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/message.c @@ -1541,6 +1674,9 @@ add_library(upb_mini_descriptor_lib ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/base92.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/internal/encode.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_descriptor/link.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/decode_fast/select.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/eps_copy_input_stream.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/reader.c ) target_compile_features(upb_mini_descriptor_lib PUBLIC cxx_std_17) @@ -1616,7 +1752,7 @@ add_library(upb_wire_lib ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/decode.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/encode.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/eps_copy_input_stream.c - ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/internal/decode_fast.c + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/internal/decoder.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/wire/reader.c ) @@ -1647,6 +1783,7 @@ target_link_libraries(upb_wire_lib ) add_library(upb_mini_table_lib + ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/generated_registry.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/extension_registry.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/internal/message.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/mini_table/message.c @@ -1794,6 +1931,8 @@ target_link_libraries(utf8_range_lib ) add_library(grpc++ + ${_gRPC_SOURCE_DIR}/src/core/client_channel/virtual_channel.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/call_context_registry.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/call_credentials.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_credentials.cc @@ -1869,7 +2008,11 @@ target_link_libraries(grpc++ ) add_library(grpc++_unsecure + ${_gRPC_SOURCE_DIR}/src/core/client_channel/virtual_channel.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/call_context_registry.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/call_credentials.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/channel_credentials.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/client_callback.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/client_context.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc @@ -1877,6 +2020,7 @@ add_library(grpc++_unsecure ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${_gRPC_SOURCE_DIR}/src/cpp/client/global_callback_hook.cc ${_gRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc ${_gRPC_SOURCE_DIR}/src/cpp/common/alarm.cc ${_gRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc @@ -1899,6 +2043,7 @@ add_library(grpc++_unsecure ${_gRPC_SOURCE_DIR}/src/cpp/server/server_callback.cc ${_gRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc ${_gRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${_gRPC_SOURCE_DIR}/src/cpp/server/server_credentials.cc ${_gRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc ${_gRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc ${_gRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc diff --git a/contrib/jwt-cpp b/contrib/jwt-cpp index a6927cb81408..b0ea29a58fc8 160000 --- a/contrib/jwt-cpp +++ b/contrib/jwt-cpp @@ -1 +1 @@ -Subproject commit a6927cb8140858c34e05d1a954626b9849fbcdfc +Subproject commit b0ea29a58fc852a67d4e896d266880c2c63b0c4c diff --git a/contrib/libcotp b/contrib/libcotp index 7725397cbd9c..3a7fa1a78071 160000 --- a/contrib/libcotp +++ b/contrib/libcotp @@ -1 +1 @@ -Subproject commit 7725397cbd9c268fd913dfa91f78f90673bf85b2 +Subproject commit 3a7fa1a780716534e800ba51f80fe929af77adf7 diff --git a/contrib/libcotp-cmake/CMakeLists.txt b/contrib/libcotp-cmake/CMakeLists.txt index 6c5550ed11b5..3b31cb680a4d 100644 --- a/contrib/libcotp-cmake/CMakeLists.txt +++ b/contrib/libcotp-cmake/CMakeLists.txt @@ -10,10 +10,20 @@ endif() set (LIBCOTP_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libcotp") set (LIBCOTP_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libcotp") +# Mirrors the upstream source list for the OpenSSL HMAC/hash backend, without +# the optional validation helpers (`COTP_ENABLE_VALIDATION`), which we do not use. set(SRCS + "${LIBCOTP_SOURCE_DIR}/src/ctx.c" "${LIBCOTP_SOURCE_DIR}/src/otp.c" + "${LIBCOTP_SOURCE_DIR}/src/strerror.c" + "${LIBCOTP_SOURCE_DIR}/src/yaotp.c" "${LIBCOTP_SOURCE_DIR}/src/utils/base32.c" + "${LIBCOTP_SOURCE_DIR}/src/utils/otpauth_uri.c" + "${LIBCOTP_SOURCE_DIR}/src/utils/pct.c" + "${LIBCOTP_SOURCE_DIR}/src/utils/secure_zero.c" + "${LIBCOTP_SOURCE_DIR}/src/utils/whash_openssl.c" "${LIBCOTP_SOURCE_DIR}/src/utils/whmac_openssl.c" + "${LIBCOTP_SOURCE_DIR}/src/utils/yaotp_uri.c" ) add_library (_libcotp ${SRCS}) diff --git a/contrib/libssh b/contrib/libssh index 47305a2f7257..50313883f3a0 160000 --- a/contrib/libssh +++ b/contrib/libssh @@ -1 +1 @@ -Subproject commit 47305a2f7257b56ca407260a72af85db058d551f +Subproject commit 50313883f3a077458cde4ea95bf46bfeb0771b34 diff --git a/contrib/libssh-cmake/CMakeLists.txt b/contrib/libssh-cmake/CMakeLists.txt index 0e1fae5880af..e6d58878f840 100644 --- a/contrib/libssh-cmake/CMakeLists.txt +++ b/contrib/libssh-cmake/CMakeLists.txt @@ -7,8 +7,8 @@ endif() # CMake variables needed by libssh_version.h.cmake, update them when you update libssh set(libssh_VERSION_MAJOR 0) -set(libssh_VERSION_MINOR 9) -set(libssh_VERSION_PATCH 8) +set(libssh_VERSION_MINOR 12) +set(libssh_VERSION_PATCH 0) set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh") set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh") @@ -37,6 +37,7 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/external/poly1305.c ${LIB_SOURCE_DIR}/src/external/sntrup761.c ${LIB_SOURCE_DIR}/src/getpass.c + ${LIB_SOURCE_DIR}/src/hybrid_mlkem.c ${LIB_SOURCE_DIR}/src/init.c ${LIB_SOURCE_DIR}/src/kdf.c ${LIB_SOURCE_DIR}/src/kex.c @@ -47,6 +48,7 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/match.c ${LIB_SOURCE_DIR}/src/messages.c ${LIB_SOURCE_DIR}/src/misc.c + ${LIB_SOURCE_DIR}/src/mlkem.c ${LIB_SOURCE_DIR}/src/options.c ${LIB_SOURCE_DIR}/src/packet.c ${LIB_SOURCE_DIR}/src/packet_cb.c @@ -79,6 +81,7 @@ set(libssh_SRCS ${LIB_SOURCE_DIR}/src/gzip.c ${LIB_SOURCE_DIR}/src/libcrypto.c ${LIB_SOURCE_DIR}/src/md_crypto.c + ${LIB_SOURCE_DIR}/src/mlkem_crypto.c ${LIB_SOURCE_DIR}/src/pki_crypto.c ${LIB_SOURCE_DIR}/src/pki_context.c ${LIB_SOURCE_DIR}/src/sntrup761.c diff --git a/contrib/libssh-cmake/darwin/config.h b/contrib/libssh-cmake/darwin/config.h index 12378a64ceaa..f748858a9055 100644 --- a/contrib/libssh-cmake/darwin/config.h +++ b/contrib/libssh-cmake/darwin/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/freebsd/config.h b/contrib/libssh-cmake/freebsd/config.h index 8a70acb473c0..857bdef90e6f 100644 --- a/contrib/libssh-cmake/freebsd/config.h +++ b/contrib/libssh-cmake/freebsd/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/aarch64-musl/config.h b/contrib/libssh-cmake/linux/aarch64-musl/config.h index 9cc21c1df4ad..30620274c4fb 100644 --- a/contrib/libssh-cmake/linux/aarch64-musl/config.h +++ b/contrib/libssh-cmake/linux/aarch64-musl/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/aarch64/config.h b/contrib/libssh-cmake/linux/aarch64/config.h index 7e21b1b4f683..bfab9f7f49c4 100644 --- a/contrib/libssh-cmake/linux/aarch64/config.h +++ b/contrib/libssh-cmake/linux/aarch64/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/loongarch64/config.h b/contrib/libssh-cmake/linux/loongarch64/config.h index 3e19e6ab945d..1856298aa39d 100644 --- a/contrib/libssh-cmake/linux/loongarch64/config.h +++ b/contrib/libssh-cmake/linux/loongarch64/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/ppc64le/config.h b/contrib/libssh-cmake/linux/ppc64le/config.h index 701ee00416bb..ae317dff7d1f 100644 --- a/contrib/libssh-cmake/linux/ppc64le/config.h +++ b/contrib/libssh-cmake/linux/ppc64le/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/riscv64/config.h b/contrib/libssh-cmake/linux/riscv64/config.h index ca0868072bdc..5e2ae515edfd 100644 --- a/contrib/libssh-cmake/linux/riscv64/config.h +++ b/contrib/libssh-cmake/linux/riscv64/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/s390x/config.h b/contrib/libssh-cmake/linux/s390x/config.h index 6c284c9d6d4f..ebc39273e5b3 100644 --- a/contrib/libssh-cmake/linux/s390x/config.h +++ b/contrib/libssh-cmake/linux/s390x/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/x86-64-musl/config.h b/contrib/libssh-cmake/linux/x86-64-musl/config.h index 9b325957c358..7a50d2df123f 100644 --- a/contrib/libssh-cmake/linux/x86-64-musl/config.h +++ b/contrib/libssh-cmake/linux/x86-64-musl/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ diff --git a/contrib/libssh-cmake/linux/x86-64/config.h b/contrib/libssh-cmake/linux/x86-64/config.h index f6316af195cd..181ece488eb5 100644 --- a/contrib/libssh-cmake/linux/x86-64/config.h +++ b/contrib/libssh-cmake/linux/x86-64/config.h @@ -14,6 +14,18 @@ /* Global client configuration file path */ #define GLOBAL_CLIENT_CONFIG "/etc/ssh/ssh_config" +/* Global configuration directory (libssh >= 0.12 uses it unconditionally) */ +#define GLOBAL_CONF_DIR "/etc/ssh" + +/* Define to 1 if we have support for ML-KEM in libgcrypt */ +/* #undef HAVE_GCRYPT_MLKEM */ + +/* Define to 1 if we have support for ML-KEM in OpenSSL */ +#define HAVE_OPENSSL_MLKEM 1 + +/* Define to 1 if we have support for ML-KEM1024 in either backend */ +#define HAVE_MLKEM1024 1 + /************************** HEADER FILES *************************/ /* Define to 1 if you have the header file. */ @@ -188,8 +200,6 @@ /* Define to 1 if we have support for blowfish */ /* #undef HAVE_BLOWFISH */ -/* Define to 1 if we have support for ML-KEM */ -/* #undef HAVE_MLKEM */ /*************************** LIBRARIES ***************************/ diff --git a/contrib/orc b/contrib/orc index 49e965750a94..8ca5eeed390f 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit 49e965750a94627d0ce0a3f3c781423b982cbc1d +Subproject commit 8ca5eeed390f57fe6d1d4f7f8d97da19de6c08cb diff --git a/contrib/postgres-cmake/CMakeLists.txt b/contrib/postgres-cmake/CMakeLists.txt index c843d857ba26..b804ab7e7b99 100644 --- a/contrib/postgres-cmake/CMakeLists.txt +++ b/contrib/postgres-cmake/CMakeLists.txt @@ -66,6 +66,12 @@ if(NOT OS_DARWIN) ) endif() +if(OS_DARWIN OR ARCH_PPC64LE) + set(SRCS ${SRCS} + "${POSTGRES_SOURCE_DIR}/src/port/explicit_bzero.c" + ) +endif() + add_library(_libpq ${SRCS}) add_definitions(-DFRONTEND) diff --git a/contrib/xsimd b/contrib/xsimd index 01bd143c82a2..f795779ccfad 160000 --- a/contrib/xsimd +++ b/contrib/xsimd @@ -1 +1 @@ -Subproject commit 01bd143c82a215973a643d709121617b7fc57dae +Subproject commit f795779ccfad12832ea47bfc02d02a65fd7f3576 diff --git a/docs/en/engines/table-engines/mergetree-family/textindexes.md b/docs/en/engines/table-engines/mergetree-family/textindexes.md index 7fa938dd316e..e4d22db14d36 100644 --- a/docs/en/engines/table-engines/mergetree-family/textindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/textindexes.md @@ -1365,6 +1365,12 @@ The text index currently has the following limitations: index materialization can happen directly (`ALTER TABLE MATERIALIZE INDEX `) or indirectly in part merges. - It is not possible to materialize text indexes on parts with more than 4.294.967.296 (= 2^32 = ca. 4.2 billion) rows. Without a materialized text index, queries fall back to slow brute-force search within the part. As a worst case estimation, assume a part contains a single column of type String and MergeTree setting `max_bytes_to_merge_at_max_space_in_pool` (default: 150 GB) was not changed. In this case, the situation happens if the column contains less than 29.5 characters per row on average. In practice, tables also contain other columns and the threshold is multiples times smaller than that (depending on the number, type and size of the other columns). +## Upgrade Notes {#upgrade-notes} + +The on-disk format version of text indexes is controlled by the table-level setting [`text_index_serialization_version`](/operations/settings/merge-tree-settings#text_index_serialization_version) (default: `v1_with_codec`). +The setting is a preference rather than a hard constraint: if the configured version cannot represent an index, a newer version that can represent it is chosen automatically, so writing a text index never fails because of this setting. +During a rolling upgrade, pin the format with the [`compatibility`](../../../operations/settings/settings#compatibility) setting on the already upgraded servers: when it is set to a version older than the one that introduced the corresponding format, `text_index_serialization_version` reverts to an older value automatically and newer servers keep writing the format that older servers can still read. + ## Text Indexes vs Bloom-Filter-Based Indexes {#text-index-vs-bloom-filter-indexes} String predicates can be sped up using text indexes and bloom-filter-based based indexes (index type `bloom_filter`, `ngrambf_v1`, `tokenbf_v1`, `sparse_grams`), yet both are fundamentally different in their design and intended use cases: diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index 86b8413662b6..8dfe35eaab00 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -135,37 +135,6 @@ background: var(--background-color-1); } - .inputs.unconnected { - height: 100vh; - } - .unconnected #params { - display: flex; - flex-flow: column nowrap; - justify-content: center; - align-items: center; - } - .unconnected #connection-params { - width: 50%; - - display: flex; - flex-flow: column nowrap; - } - .unconnected #url { - width: 100%; - } - .unconnected #button-options { - display: grid; - grid-auto-flow: column; - grid-auto-columns: 1fr; - gap: 0.3rem; - } - .unconnected #user { - margin-right: 0; - width: auto; - } - .unconnected #password { - width: auto; - } #user { margin-right: 0.25rem; width: 50%; @@ -173,9 +142,6 @@ #password { width: 49.5%; } - .unconnected input { - margin-bottom: 5px; - } #username-password { width: 100%; @@ -183,23 +149,11 @@ display: flex; flex-flow: row nowrap; } - .unconnected #username-password { - width: 100%; - - gap: 0.3rem; - - display: grid; - grid-template-columns: 1fr 1fr; - } .inputs #chart-params { display: block; } - .inputs.unconnected #chart-params { - display: none; - } - #connection-params { margin-bottom: 0.5rem; display: grid; @@ -251,19 +205,6 @@ filter: brightness(125%); } - #run { - background: var(--button-background-color); - color: var(--button-text-color); - font-weight: bold; - user-select: none; - cursor: pointer; - margin-bottom: 1rem; - } - - #run:hover { - filter: contrast(125%); - } - #add, #add-metrics, #reload, #edit, #search { padding: 0.25rem 0.5rem; text-align: center; @@ -346,10 +287,6 @@ color: var(--chart-button-hover-color); } - .disabled { - opacity: 0.5; - } - .query-editor { display: none; grid-template-columns: auto fit-content(10%); @@ -490,7 +427,7 @@ -
+
@@ -502,11 +439,11 @@
🌚🌞 - - - + + + - +
@@ -754,13 +691,6 @@ for (let [name, value] of Object.entries(params)) { insertParam(name, value); } - - let run = document.createElement('input'); - run.id = 'run'; - run.type = 'submit'; - run.value = 'Ok'; - - document.getElementById('chart-params').appendChild(run); } function updateParams() { @@ -829,7 +759,11 @@ refreshCustomized(true); saveState(); const idx = getCurrentIndex(); - draw(idx, chart, getParamsForURL(), q.query); + draw(idx, chart, getParamsForURL(), q.query).catch((e) => { + if (e.name != 'AbortError') { + showError(e.message); + } + }); } query_editor_confirm.addEventListener('click', editConfirm); @@ -994,7 +928,7 @@ chart.addEventListener('mouseenter', e => { edit_buttons.style.display = 'block'; }); chart.addEventListener('mouseleave', e => { edit_buttons.style.display = 'none'; }); - charts.appendChild(chart); + charts.insertBefore(chart, charts.children[i] || null); return {chart: chart, textarea: query_editor_textarea}; } @@ -1262,14 +1196,28 @@ } queries.unshift(...new_charts); hideMetricsEditor(); - regenerate(); + + /// Insert and draw only the new charts; the existing ones are not reloaded. + new_charts.forEach(q => findParamsInQuery(q.query, params)); + buildParams(); + plots.unshift(...new_charts.map(() => null)); + for (let i = 0; i < new_charts.length; i++) { + insertChart(i); + } + resize(); refreshCustomized(true); saveState(); - drawAll(); - const chart_divs = charts.querySelectorAll('.chart'); - if (chart_divs[0]) { - chart_divs[0].scrollIntoView(); + + const url_params = getParamsForURL(); + const chartsArray = charts.getElementsByClassName('chart'); + for (let i = 0; i < new_charts.length; i++) { + draw(i, chartsArray[i], url_params, queries[i].query).catch((e) => { + if (e.name != 'AbortError') { + showError(e.message); + } + }); } + chartsArray[0].scrollIntoView(); } document.getElementById('add-metrics').addEventListener('click', e => { @@ -1394,6 +1342,14 @@ } +/// Aborted and recreated on every reload, so stale in-flight queries never paint over a newer run. +let query_controller = new AbortController(); + +function cancelQueries() { + query_controller.abort(); + query_controller = new AbortController(); +} + async function doFetch(query, url_params = '') { host = document.getElementById('url').value || host; user = document.getElementById('user').value; @@ -1415,7 +1371,7 @@ let response, reply, error; try { - response = await fetch(url + url_params, { method: "POST", body: query, headers: { 'Authorization': 'never' } }); + response = await fetch(url + url_params, { method: "POST", body: query, headers: { 'Authorization': 'never' }, signal: query_controller.signal }); reply = await response.text(); if (response.ok) { reply = JSON.parse(reply); @@ -1426,6 +1382,9 @@ error = reply; } } catch (e) { + if (e.name == 'AbortError') { + throw e; + } console.log(e); error = e.toString(); } @@ -1460,7 +1419,11 @@ plots[idx] = null; } + const signal = query_controller.signal; let {reply, error} = await doFetch(query, url_params); + if (signal.aborted) { + return false; + } if (!error) { if (reply.rows == 0) { error = "Query returned empty result."; @@ -1641,70 +1604,37 @@ return true; } +/// The error is shown as a banner above the charts; the charts always stay visible. function showError(message) { - const charts = document.getElementById('charts'); - charts.style.height = '0px'; - charts.style.opacity = '0'; - document.getElementById('add').style.display = 'none'; - document.getElementById('add-metrics').style.display = 'none'; - document.getElementById('edit').style.display = 'none'; - const error = document.getElementById('global-error'); error.textContent = message; error.style.display = 'flex'; } function hideError() { - const charts = document.getElementById('charts'); - charts.style.height = 'auto'; - charts.style.opacity = '1'; - const error = document.getElementById('global-error'); error.textContent = ''; error.style.display = 'none'; } -let firstLoad = true; -let is_drawing = false; // Prevent race condition leading to duplicate/dangling charts. async function drawAll() { - if (is_drawing) return; - is_drawing = true; + /// Supersede any in-flight run: its queries are aborted, so it cannot paint stale charts. + cancelQueries(); + const signal = query_controller.signal; - try { - hideError(); + hideError(); - let params = getParamsForURL(); - const chartsArray = document.getElementsByClassName('chart'); + let params = getParamsForURL(); + const chartsArray = document.getElementsByClassName('chart'); - let had_global_error = false; - const results = await Promise.all([...Array(queries.length)].map(async (_, i) => { - return draw(i, chartsArray[i], params, queries[i].query).catch((e) => { + await Promise.all([...Array(queries.length)].map(async (_, i) => { + return draw(i, chartsArray[i], params, queries[i].query).catch((e) => { + if (e.name != 'AbortError' && !signal.aborted) { showError(e.message); - had_global_error = true; - return false; - }); - })); - - if (firstLoad) { - firstLoad = false; - } else { - enableButtons(); - } - - if (!had_global_error && results.length > 0) { - /// At least one chart was processed without a global error - /// (auth/connection). Show the connected UI. Individual charts - /// may still have per-chart query errors displayed in their divs. - const element = document.querySelector('.inputs'); - element.classList.remove('unconnected'); - document.getElementById('add').style.display = 'inline-block'; - document.getElementById('add-metrics').style.display = 'inline-block'; - document.getElementById('edit').style.display = 'inline-block'; - document.getElementById('search-span').style.display = ''; - } - } finally { - is_drawing = false; - } + } + return false; + }); + })); } function resize() { @@ -1718,46 +1648,17 @@ new ResizeObserver(resize).observe(document.body); -function disableButtons() { - const reloadButton = document.getElementById('reload'); - reloadButton.value = 'Reloading…'; - reloadButton.disabled = true; - reloadButton.classList.add('disabled'); - - const runButton = document.getElementById('run'); - if (runButton) { - runButton.value = 'Reloading…'; - runButton.disabled = true; - runButton.classList.add('disabled'); - } - - const searchButton = document.getElementById('search'); - searchButton.value = '…'; - searchButton.disabled = true; - searchButton.classList.add('disabled'); -} - -function enableButtons() { - const reloadButton = document.getElementById('reload'); - reloadButton.value = 'Reload'; - reloadButton.disabled = false; - reloadButton.classList.remove('disabled'); - - const runButton = document.getElementById('run'); - if (runButton) { - runButton.value = 'Ok'; - runButton.disabled = false; - runButton.classList.remove('disabled'); - } - - const searchButton = document.getElementById('search'); - searchButton.value = '🔎'; - searchButton.disabled = false; - searchButton.classList.remove('disabled'); +/// The buttons stay clickable while loading: pressing Reload again cancels the in-flight queries and restarts. +function setButtonsLoading(loading) { + document.getElementById('reload').value = loading ? 'Reloading…' : 'Reload'; + document.getElementById('search').value = loading ? '…' : '🔎'; } +let reload_epoch = 0; async function reloadAll(do_search) { - disableButtons(); + const epoch = ++reload_epoch; + cancelQueries(); + setButtonsLoading(true); try { updateParams(); if (do_search) { @@ -1771,9 +1672,13 @@ } await drawAll(); } catch (e) { - showError(e.message); + if (e.name != 'AbortError') { + showError(e.message); + } + } + if (epoch == reload_epoch) { + setButtonsLoading(false); } - enableButtons(); } document.getElementById('params').onsubmit = function(event) { @@ -1920,7 +1825,9 @@ } await populateSearchOptions(); } catch (e) { - showError(e.message); + if (e.name != 'AbortError') { + showError(e.message); + } } } diff --git a/src/Access/Common/OneTimePassword.cpp b/src/Access/Common/OneTimePassword.cpp index a0807acdf93e..bd9970ee2082 100644 --- a/src/Access/Common/OneTimePassword.cpp +++ b/src/Access/Common/OneTimePassword.cpp @@ -12,12 +12,9 @@ #include -constexpr int TOTP_SHA512 = SHA512; -constexpr int TOTP_SHA256 = SHA256; -constexpr int TOTP_SHA1 = SHA1; -#undef SHA512 -#undef SHA256 -#undef SHA1 +constexpr int TOTP_SHA512 = COTP_SHA512; +constexpr int TOTP_SHA256 = COTP_SHA256; +constexpr int TOTP_SHA1 = COTP_SHA1; #endif diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index c14df8d8a043..6e827acc5a0b 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -222,6 +222,31 @@ void SettingsConstraints::check(const Settings & current_settings, SettingsChang checkOrClamp(current_settings, changes, THROW_ON_VIOLATION, source); } +void SettingsConstraints::checkResetToDefault(const Settings & current_settings, const std::vector & names, SettingSource source) const +{ + /// A reset of a built-in setting is equivalent to assigning its declared default. The regular + /// check also deliberately permits a reset that does not change the value. + const Settings defaults; + for (const auto & name : names) + { + if (Settings::hasBuiltin(name)) + { + check(current_settings, SettingChange{name, defaults.get(name)}, source); + continue; + } + + /// Custom settings have no declared default: resetting one removes it. There cannot be a + /// value constraint for such a setting, but an existing value must still pass the readonly + /// and source checks. Do not check an absent custom setting, preserving its no-op behavior. + Field current_value; + if (current_settings.tryGet(name, current_value)) + { + SettingChange change{name, current_value}; + getChecker(current_settings, Settings::resolveName(name)).check(change, current_value, THROW_ON_VIOLATION, source); + } + } +} + void SettingsConstraints::check(const MergeTreeSettings & current_settings, const SettingChange & change) const { checkImpl(current_settings, const_cast(change), THROW_ON_VIOLATION); diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 8ae578df925d..7d8e1eec4529 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -84,6 +84,9 @@ class SettingsConstraints void check(const Settings & current_settings, const SettingsProfileElements & profile_elements, SettingSource source) const; void check(const Settings & current_settings, const AlterSettingsProfileElements & profile_elements, SettingSource source) const; + /// Checks whether resetting the specified settings to their defaults violates these constraints. + void checkResetToDefault(const Settings & current_settings, const std::vector & names, SettingSource source) const; + /// Checks whether `change` violates these constraints and throws an exception if so. (setting short name is expected inside `changes`) void check(const MergeTreeSettings & current_settings, const SettingChange & change) const; void check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const; diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 321cdae9452b..545332019d68 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -21,6 +21,19 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +namespace +{ + /// ParserSettingsProfileElement accepts only scalar literals, so a Map is emitted as a quoted + /// string holding the setting's canonical text. Custom settings are excluded: castValueUtil + /// returns their value unchanged, so a string would stay a String instead of becoming a Map. + std::optional settingValueToASTField(const String & setting_name, const std::optional & value) + { + if (!value || value->getType() != Field::Types::Map || !Settings::hasBuiltin(setting_name)) + return value; + return Field(Settings::valueToStringUtil(setting_name, *value)); + } +} + SettingsProfileElement::SettingsProfileElement(const ASTSettingsProfileElement & ast) { @@ -92,9 +105,9 @@ boost::intrusive_ptr SettingsProfileElement::toAST() ast->parent_profile = ::DB::toString(*parent_profile); ast->setting_name = setting_name; - ast->value = value; - ast->min_value = min_value; - ast->max_value = max_value; + ast->value = settingValueToASTField(setting_name, value); + ast->min_value = settingValueToASTField(setting_name, min_value); + ast->max_value = settingValueToASTField(setting_name, max_value); ast->disallowed_values = disallowed_values; ast->writability = writability; @@ -114,9 +127,9 @@ boost::intrusive_ptr SettingsProfileElement::toASTWit } ast->setting_name = setting_name; - ast->value = value; - ast->min_value = min_value; - ast->max_value = max_value; + ast->value = settingValueToASTField(setting_name, value); + ast->min_value = settingValueToASTField(setting_name, min_value); + ast->max_value = settingValueToASTField(setting_name, max_value); ast->disallowed_values = disallowed_values; ast->writability = writability; diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp index 31899c679caf..1675874586d1 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.cpp +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.cpp @@ -16,6 +16,7 @@ struct Settings; namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int INCORRECT_DATA; extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; @@ -388,14 +389,32 @@ void LinearModelData::returnWeights(IColumn & to) const val_to.push_back(bias); } -void LinearModelData::read(ReadBuffer & buf) +void LinearModelData::read(ReadBuffer & buf, UInt64 expected_param_num) { readBinary(bias, buf); readBinary(weights, buf); readBinary(iter_num, buf); readBinary(gradient_batch, buf); readBinary(batch_size, buf); - weights_updater->read(buf); + + if (weights.size() != expected_param_num) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed state of a machine learning aggregate function: it has {} weights, " + "while the type declares {} features", + weights.size(), expected_param_num); + + /// The gradient holds one value per weight plus one for the bias. The weights updaters rely on + /// that, so a state where the two disagree would make them read past the end of the gradient. + if (gradient_batch.size() != weights.size() + 1) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed state of a machine learning aggregate function: it has {} weights and a gradient of {} values", + weights.size(), gradient_batch.size()); + + /// The updaters keep their own vectors of the gradient size and index them by the weight + /// number as well, so they check the deserialized vectors against the same size. + weights_updater->read(buf, weights.size() + 1); } void LinearModelData::write(WriteBuffer & buf) const @@ -447,6 +466,22 @@ void LinearModelData::add(const IColumn ** columns, size_t row_num) } } +namespace +{ + /// The updater vectors hold one value per weight plus one for the bias, like the gradient. + /// An empty vector is also valid: versions before 23.2 serialized the vectors empty until + /// the first update, and the updaters treat an empty vector as "no accumulated data". + void checkUpdaterVectorSize(size_t size, UInt64 expected_size) + { + if (size != 0 && size != expected_size) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed state of a machine learning aggregate function: the weights updater holds " + "a vector of {} values, while {} are expected", + size, expected_size); + } +} + /// Weights updaters void Adam::write(WriteBuffer & buf) const @@ -455,10 +490,20 @@ void Adam::write(WriteBuffer & buf) const writeBinary(average_squared_gradient, buf); } -void Adam::read(ReadBuffer & buf) +void Adam::read(ReadBuffer & buf, UInt64 expected_size) { readBinary(average_gradient, buf); readBinary(average_squared_gradient, buf); + checkUpdaterVectorSize(average_gradient.size(), expected_size); + checkUpdaterVectorSize(average_squared_gradient.size(), expected_size); + + /// The two vectors are read and written together, so they must agree with each other as well. + if (average_gradient.size() != average_squared_gradient.size()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Malformed state of a machine learning aggregate function: the weights updater holds " + "an average gradient of {} values and an average squared gradient of {} values", + average_gradient.size(), average_squared_gradient.size()); } void Adam::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) @@ -524,9 +569,10 @@ void Adam::addToBatch( gradient_computer.compute(batch_gradient, weights, bias, l2_reg_coef, target, columns, row_num); } -void Nesterov::read(ReadBuffer & buf) +void Nesterov::read(ReadBuffer & buf, UInt64 expected_size) { readBinary(accumulated_gradient, buf); + checkUpdaterVectorSize(accumulated_gradient.size(), expected_size); } void Nesterov::write(WriteBuffer & buf) const @@ -537,6 +583,10 @@ void Nesterov::write(WriteBuffer & buf) const void Nesterov::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) { const auto & nesterov_rhs = static_cast(rhs); + + if (nesterov_rhs.accumulated_gradient.empty()) + return; + accumulated_gradient.resize(nesterov_rhs.accumulated_gradient.size(), Float64{0.0}); for (size_t i = 0; i < accumulated_gradient.size(); ++i) @@ -586,9 +636,10 @@ void Nesterov::addToBatch( gradient_computer.compute(batch_gradient, shifted_weights, shifted_bias, l2_reg_coef, target, columns, row_num); } -void Momentum::read(ReadBuffer & buf) +void Momentum::read(ReadBuffer & buf, UInt64 expected_size) { readBinary(accumulated_gradient, buf); + checkUpdaterVectorSize(accumulated_gradient.size(), expected_size); } void Momentum::write(WriteBuffer & buf) const @@ -599,6 +650,11 @@ void Momentum::write(WriteBuffer & buf) const void Momentum::merge(const IWeightsUpdater & rhs, Float64 frac, Float64 rhs_frac) { const auto & momentum_rhs = static_cast(rhs); + + if (momentum_rhs.accumulated_gradient.empty()) + return; + + accumulated_gradient.resize(momentum_rhs.accumulated_gradient.size(), Float64{0.0}); for (size_t i = 0; i < accumulated_gradient.size(); ++i) { accumulated_gradient[i] = accumulated_gradient[i] * frac + momentum_rhs.accumulated_gradient[i] * rhs_frac; @@ -657,6 +713,9 @@ void LogisticRegression::predict( Float64 bias, ContextPtr /*context*/) const { + if (weights.size() + 1 != arguments.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "In predict function number of arguments differs from the size of weights vector"); + size_t rows_num = arguments.front().column->size(); if (offset > rows_num || offset + limit > rows_num) @@ -727,7 +786,7 @@ void LinearRegression::predict( { if (weights.size() + 1 != arguments.size()) { - throw Exception(ErrorCodes::LOGICAL_ERROR, "In predict function number of arguments differs from the size of weights vector"); + throw Exception(ErrorCodes::INCORRECT_DATA, "In predict function number of arguments differs from the size of weights vector"); } size_t rows_num = arguments.front().column->size(); diff --git a/src/AggregateFunctions/AggregateFunctionMLMethod.h b/src/AggregateFunctions/AggregateFunctionMLMethod.h index 3dfd025d70fc..348161e0896e 100644 --- a/src/AggregateFunctions/AggregateFunctionMLMethod.h +++ b/src/AggregateFunctions/AggregateFunctionMLMethod.h @@ -135,8 +135,12 @@ class IWeightsUpdater /// Used for serialization when necessary virtual void write(WriteBuffer &) const {} - /// Used for serialization when necessary - virtual void read(ReadBuffer &) {} + /// Used for serialization when necessary. The state comes from the data, so the updaters that + /// store vectors must check them against `expected_size` (the size of the gradient, that is, + /// the number of weights plus one for the bias): the updaters index these vectors by the + /// weight number during `merge` and `addToBatch`. An empty vector is also valid: versions + /// before 23.2 serialized the vectors empty until the first update. + virtual void read(ReadBuffer &, UInt64 /* expected_size */) {} }; @@ -172,7 +176,7 @@ class Momentum : public IWeightsUpdater void write(WriteBuffer & buf) const override; - void read(ReadBuffer & buf) override; + void read(ReadBuffer & buf, UInt64 expected_size) override; private: Float64 alpha{0.1}; @@ -209,7 +213,7 @@ class Nesterov : public IWeightsUpdater void write(WriteBuffer & buf) const override; - void read(ReadBuffer & buf) override; + void read(ReadBuffer & buf, UInt64 expected_size) override; private: const Float64 alpha = 0.9; @@ -251,7 +255,7 @@ class Adam : public IWeightsUpdater void write(WriteBuffer & buf) const override; - void read(ReadBuffer & buf) override; + void read(ReadBuffer & buf, UInt64 expected_size) override; private: /// beta1 and beta2 hyperparameters have such recommended values @@ -287,7 +291,10 @@ class LinearModelData void write(WriteBuffer & buf) const; - void read(ReadBuffer & buf); + /// `expected_param_num` is the number of features declared by the type: the state comes from + /// the data and must agree with it, because everything downstream indexes the weights by the + /// feature number. + void read(ReadBuffer & buf, UInt64 expected_param_num); void predict( ColumnVector::Container & container, @@ -386,7 +393,10 @@ class AggregateFunctionMLMethod final : public IAggregateFunctionDataHelper /* version */) const override { this->data(place).write(buf); } - void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override { this->data(place).read(buf); } + void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional /* version */, Arena *) const override + { + this->data(place).read(buf, param_num); + } void predictValues( ConstAggregateDataPtr __restrict place, diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionResample.h b/src/AggregateFunctions/Combinators/AggregateFunctionResample.h index 69e488c3f3e0..51c56158ecf8 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionResample.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionResample.h @@ -24,6 +24,12 @@ class AggregateFunctionResample final : public IAggregateFunctionHelper max_state_size) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "Overflow in internal computations in function {}. The state is too large", getName()); + return result; } size_t alignOfData() const override diff --git a/src/AggregateFunctions/KeyHolderHelpers.h b/src/AggregateFunctions/KeyHolderHelpers.h index 2f9291b5dbc3..8e3cb8de62c8 100644 --- a/src/AggregateFunctions/KeyHolderHelpers.h +++ b/src/AggregateFunctions/KeyHolderHelpers.h @@ -34,7 +34,19 @@ template static void deserializeAndInsert(std::string_view str, IColumn & data_to) { if constexpr (is_plain_column) + { + /// `insertData` of the fixed-size columns ignores the length and always reads the width of + /// the value, so a shorter element of a crafted state would read past the end of the buffer. + if (data_to.valuesHaveFixedSize()) + { + const size_t expected_size = data_to.sizeOfValueIfFixed(); + if (str.size() != expected_size) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Element of an aggregation state is {} bytes, while {} bytes are expected", str.size(), expected_size); + } + data_to.insertData(str.data(), str.size()); + } else { ReadBufferFromString in(str); diff --git a/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.cpp b/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.cpp index 89b5c5fa873f..7a26dac8303b 100644 --- a/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.cpp +++ b/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -45,8 +47,19 @@ void OptimizeTrivialGroupByLimitPass::run(QueryTreeNodePtr & query_tree_node, Co auto * query = query_tree_node->as(); if (!query || !query->hasGroupBy() || !query->hasLimit() || query->hasHaving() || query->hasOrderBy() || query->hasWindow() - || query->hasLimitBy() || query->isGroupByWithTotals() || query->isGroupByWithRollup() || query->isGroupByWithCube() - || query->isGroupByWithGroupingSets() || hasAggregateFunctionNodes(query->getProjectionNode())) + || query->hasQualify() || query->hasLimitBy() || query->isDistinct() || query->isGroupByWithTotals() + || query->isGroupByWithRollup() || query->isGroupByWithCube() || query->isGroupByWithGroupingSets() + || hasAggregateFunctionNodes(query->getProjectionNode())) + return; + + /// Window functions and `arrayJoin` in the projection consume the aggregated rows after + /// GROUP BY, so the produced groups are not simply cut by LIMIT and keeping only the first + /// `LIMIT + OFFSET` groups changes the result: + /// - a window function is evaluated over all groups (`count() OVER ()` counts them); + /// - `arrayJoin` can expand or drop rows, so `LIMIT + OFFSET` groups may produce fewer + /// rows than the LIMIT while more groups exist. + /// `DISTINCT` and `QUALIFY` (checked above) collapse and filter the groups in the same way. + if (hasWindowFunctionNodes(query->getProjectionNode()) || hasFunctionNode(query->getProjectionNode(), "arrayJoin")) return; /// `group_by_overflow_mode` controls what happens when `max_rows_to_group_by` is exceeded. diff --git a/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.h b/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.h index d55e495d5eb5..f584b4e3a599 100644 --- a/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.h +++ b/src/Analyzer/Passes/OptimizeTrivialGroupByLimitPass.h @@ -5,11 +5,12 @@ namespace DB { -/// When a query has GROUP BY and LIMIT without HAVING, ORDER BY, WINDOW, LIMIT BY clauses, -/// GROUP BY modifiers or aggregate functions in the projection, we can optimize it by setting -/// max_rows_to_group_by to LIMIT + OFFSET with group_by_overflow_mode = 'any'. The optimization -/// is suppressed when the user has explicitly set a non-ANY group_by_overflow_mode or a tighter -/// max_rows_to_group_by, to preserve their explicit contract. +/// When a query has GROUP BY and LIMIT without HAVING, ORDER BY, WINDOW, QUALIFY, LIMIT BY, +/// DISTINCT clauses, GROUP BY modifiers, or aggregate functions, window functions and arrayJoin +/// in the projection, we can optimize it by setting max_rows_to_group_by to LIMIT + OFFSET with +/// group_by_overflow_mode = 'any'. The optimization is suppressed when the user has explicitly +/// set a non-ANY group_by_overflow_mode or a tighter max_rows_to_group_by, to preserve their +/// explicit contract. class OptimizeTrivialGroupByLimitPass final : public IQueryTreePass { public: diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp index 39d88491553d..6717f98c0b7e 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp @@ -11,6 +11,8 @@ #include +#include + namespace DB { @@ -66,6 +68,12 @@ class UniqInjectiveFunctionsEliminationVisitor : public InDepthQueryTreeVisitorW if (!arg_function->isInjective({})) return false; + /// The `Null` combinator makes `uniq*` skip rows where a Nullable argument is NULL: `uniq(tuple(x))` + /// counts the (NULL) row while `uniq(x)` skips it. + if (isNullableOrLowCardinalityNullable(arg->getResultType()) + != isNullableOrLowCardinalityNullable(arg_arguments_nodes[0]->getResultType())) + return false; + arg = arg_arguments_nodes[0]; return replaced_argument = true; }; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 75f318f6426a..be1b82a5a6ba 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -304,9 +305,21 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression( set_query.changes.removeSetting("offset"); } + /// A nested `SETTINGS` clause (a subquery, a CTE, a view's inner query) used to be applied to + /// the per-node context unchecked, letting a user override `readonly`, `CONST` and `MIN`/`MAX` + /// constraints - e.g. `additional_table_filters`, which the Planner reads from this context. + /// Clamp it instead of throwing, as done for other settings crossing execution contexts + /// (`getSQLSecurityOverriddenContext`, secondary queries, DDL replay): violating changes are + /// dropped, out-of-bounds values are clamped, so a view whose inner clause violates the + /// reader's constraints keeps working. A top-level clause still throws + /// (`applySettingsFromQuery`). Clamp a copy: the `QueryNode` keeps the clause as written, so + /// the tree's AST and hash are unchanged, and every node executing the subquery clamps it + /// against its own constraints. `SETTINGS name = DEFAULT` stays ignored here - see #115415. if (!set_query.changes.empty()) { - updated_context->applySettingsChanges(set_query.changes); + auto checked_changes = set_query.changes; + updated_context->clampToSettingsConstraints(checked_changes, SettingSource::QUERY); + updated_context->applySettingsChanges(checked_changes); settings_changes = set_query.changes; } } diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index e2cf9848905e..2a78d2ea207e 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include @@ -80,6 +81,7 @@ namespace Setting extern const SettingsBool analyzer_compatibility_allow_non_aggregate_in_having; extern const SettingsBool enable_streaming_queries; extern const SettingsBool analyzer_compatibility_join_using_top_level_identifier; + extern const SettingsBool analyzer_compatibility_multiple_joins_qualify_column_names; extern const SettingsBool analyzer_inline_views; extern const SettingsBool asterisk_include_alias_columns; extern const SettingsBool asterisk_include_materialized_columns; @@ -104,6 +106,7 @@ namespace Setting extern const SettingsString implicit_table_at_top_level; extern const SettingsBool parallel_replicas_for_cluster_engines; extern const SettingsBool enable_identifier_resolve_cache; + extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; } @@ -1680,6 +1683,46 @@ void QueryAnalyzer::qualifyColumnNodesWithProjectionNames(const QueryTreeNodes & size_t additional_column_qualification_parts_size = additional_column_qualification_parts.size(); const auto & table_expression_data = scope.getTableExpressionDataOrThrow(table_expression_node); + /** Compatibility mode that mimics the old analyzer's multiple-joins rewrite + * (`JoinToSubqueryTransformVisitor` with `multiple_joins_try_to_keep_original_names = false`): + * when there are two or more JOINs, every matcher-expanded column is unconditionally + * qualified with a single-part prefix (`.`) regardless of whether the + * bare name would be ambiguous. The qualifier is the table expression alias, a temporary + * table name, the bare table name (without database), or a CTE name; if none is available + * (e.g. an unaliased joined subquery) the column keeps its unqualified name. + */ + bool force_qualification = scope.joins_count >= 2 + && scope.context->getSettingsRef()[Setting::analyzer_compatibility_multiple_joins_qualify_column_names]; + + if (force_qualification) + { + std::string forced_qualifier; + if (table_expression_node->hasAlias()) + forced_qualifier = table_expression_node->getAlias(); + else if (auto * table_node = table_expression_node->as()) + { + if (!table_node->getTemporaryTableName().empty()) + forced_qualifier = table_node->getTemporaryTableName(); + else + forced_qualifier = table_node->getStorageID().getTableName(); + } + else if (auto * query_node = table_expression_node->as(); query_node && query_node->isCTE()) + forced_qualifier = query_node->getCTEName(); + else if (auto * union_node = table_expression_node->as(); union_node && union_node->isCTE()) + forced_qualifier = union_node->getCTEName(); + + for (const auto & column_node : column_nodes) + { + const auto & column_name = column_node->as().getColumnName(); + if (forced_qualifier.empty()) + node_to_projection_name.emplace(column_node, column_name); + else + node_to_projection_name.emplace(column_node, forced_qualifier + '.' + column_name); + } + + return; + } + /** For each matched column node iterate over additional column qualifications and apply them if column needs to be qualified. * To check if column needs to be qualified we check if column name can bind to any other table expression in scope or to scope aliases. */ @@ -2129,6 +2172,11 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( auto identifiers = matcher_node_typed.getColumnsIdentifiers(); result.reserve(identifiers.size()); + /// Old-analyzer parity: under two or more JOINs a list-form `COLUMNS` item keeps the written + /// identifier. Recorded on a clone, so the shared resolved node keeps its own name. + bool keep_written_names = nearest_query_scope->joins_count >= 2 + && scope.context->getSettingsRef()[Setting::analyzer_compatibility_multiple_joins_qualify_column_names]; + for (const auto & identifier : identifiers) { auto resolve_result = tryResolveIdentifier(IdentifierLookup{identifier, IdentifierLookupContext::EXPRESSION}, scope); @@ -2145,7 +2193,15 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher( identifier.getFullName(), resolve_result.resolved_identifier->getNodeTypeName(), scope.scope_node->formatASTForErrorMessage()); - result.emplace_back(resolve_result.resolved_identifier, resolved_column->getColumnName()); + + auto column_node = resolve_result.resolved_identifier; + if (keep_written_names) + { + column_node = column_node->clone(); + node_to_projection_name.emplace(column_node, identifier.getFullName()); + } + + result.emplace_back(column_node, resolved_column->getColumnName()); } return result; } @@ -2474,10 +2530,19 @@ ProjectionNames QueryAnalyzer::resolveMatcher(QueryTreeNodePtr & matcher_node, I auto it = scope.nullable_group_by_keys.find(node); if (it != scope.nullable_group_by_keys.end()) { + /// Look up the projection name before the clone replaces the node: the map is keyed + /// by node identity, so afterwards the original key is unreachable. + auto projection_name_it = node_to_projection_name.find(node); + /// See resolveExpressionNode: for a constant keep the matched node's own source /// expression instead of the stored key, which may be a different colliding constant. node = (node->getNodeType() == QueryTreeNodeType::CONSTANT ? node : it->second)->clone(); node->convertToNullable(); + + /// Keep the projection name computed for the original node, e.g. the qualified + /// `t.x` a matcher assigned to disambiguate columns of joined table expressions. + if (projection_name_it != node_to_projection_name.end()) + node_to_projection_name.emplace(node, projection_name_it->second); } } } @@ -3094,7 +3159,8 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( bool allow_lambda_expression, bool allow_table_expression, bool ignore_alias, - bool allow_niladic_functions) + bool allow_niladic_functions, + bool is_top_level_projection) { checkStackSize(); @@ -3161,6 +3227,23 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( result_projection_names.push_back(projection_name_it->second); } + /// Old-analyzer parity: under multiple JOINs a top-level unaliased public identifier + /// resolved into a plain column keeps its projection name exactly as written (`a.x` -> `a.x`). + if (is_top_level_projection + && resolved_identifier_node + && node_alias.empty() + && resolve_identifier_expression_result.isResolvedFromJoinTree() + && resolved_identifier_node->as()) + { + const auto * nearest_query_scope = scope.getNearestQueryScope(); + if (nearest_query_scope && nearest_query_scope->joins_count >= 2 + && scope.context->getSettingsRef()[Setting::analyzer_compatibility_multiple_joins_qualify_column_names]) + { + result_projection_names.clear(); + result_projection_names.push_back(unresolved_identifier.getFullName()); + } + } + if (!resolved_identifier_node && allow_lambda_expression) resolved_identifier_node = tryResolveIdentifier({unresolved_identifier, IdentifierLookupContext::FUNCTION}, scope, { .allow_to_resolve_niladic_functions = allow_niladic_functions }).resolved_identifier; @@ -3576,7 +3659,8 @@ ProjectionNames QueryAnalyzer::resolveExpressionNodeList( IdentifierResolveScope & scope, bool allow_lambda_expression, bool allow_table_expression, - bool allow_niladic_functions + bool allow_niladic_functions, + bool is_top_level_projection ) { auto & node_list_typed = node_list->as(); @@ -3590,7 +3674,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNodeList( for (auto & node : node_list_typed.getNodes()) { auto node_to_resolve = node; - auto expression_node_projection_names = resolveExpressionNode(node_to_resolve, scope, allow_lambda_expression, allow_table_expression, false /*ignore_alias*/, allow_niladic_functions); + auto expression_node_projection_names = resolveExpressionNode(node_to_resolve, scope, allow_lambda_expression, allow_table_expression, false /*ignore_alias*/, allow_niladic_functions, is_top_level_projection); size_t expected_projection_names_size = 1; if (auto * expression_list = node_to_resolve->as()) { @@ -4019,7 +4103,7 @@ void QueryAnalyzer::resolveWindowNodeList(QueryTreeNodePtr & window_node_list, I NamesAndTypes QueryAnalyzer::resolveProjectionExpressionNodeList(QueryTreeNodePtr & projection_node_list, IdentifierResolveScope & scope) { - ProjectionNames projection_names = resolveExpressionNodeList(projection_node_list, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + ProjectionNames projection_names = resolveExpressionNodeList(projection_node_list, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/, true /*allow_niladic_functions*/, true /*is_top_level_projection*/); auto projection_nodes = projection_node_list->as().getNodes(); size_t projection_nodes_size = projection_nodes.size(); @@ -5270,8 +5354,115 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS auto & join_using_list = join_node_typed.getJoinExpression()->as(); std::unordered_set join_using_identifiers; + /// SELECT-list alias map, computed lazily once per resolveJoin and reused (projection is not mutated here). + std::optional select_list_aliases; + + /// Set below when the identifier matched a nested SELECT-list alias (not a top-level projection alias); reset per identifier. + bool nested_alias_matched = false; + + /// Find a SELECT-list node aliased as the USING identifier: top-level projection aliases first (pick-first, kept for compatibility), then nested-subexpression aliases. + auto find_aliased_node_in_projection = [&select_list_aliases, &nested_alias_matched](const QueryNode * query_node_, + const String & identifier_full_name_) -> QueryTreeNodePtr + { + for (const auto & projection_node : query_node_->getProjection().getNodes()) + { + if (projection_node->hasAlias() && identifier_full_name_ == projection_node->getAlias()) + return projection_node; + } + + /// QueryExpressionsAliasVisitor applies SELECT-list scoping and stores clones of aliased nodes; it needs a mutable node, so clone first. + if (!select_list_aliases) + { + auto projection_list_clone = query_node_->getProjectionNode()->clone(); + select_list_aliases.emplace(); + QueryExpressionsAliasVisitor visitor(*select_list_aliases); + visitor.visit(projection_list_clone); + } + + /// A lambda alias must not become a USING column. + auto it = select_list_aliases->alias_name_to_expression_node.find(identifier_full_name_); + if (it == select_list_aliases->alias_name_to_expression_node.end()) + return nullptr; + + /// Do not pick an arbitrary expression among duplicated aliases. + for (const auto & duplicated_node : select_list_aliases->nodes_with_duplicated_aliases) + { + if (duplicated_node->hasAlias() && duplicated_node->getAlias() == identifier_full_name_) + return nullptr; + } + + nested_alias_matched = true; + return it->second; + }; + + /** While resolving JOIN USING identifier, try to resolve identifier from parent subquery projection. + * Example: SELECT a + 1 AS b FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b + * In this case `b` is not in the left table expression, but it is in the parent subquery projection. + */ + auto try_resolve_identifier_from_query_projection = [this, &find_aliased_node_in_projection]( + const String & identifier_full_name_, + const QueryTreeNodePtr & left_table_expression, + const IdentifierResolveScope & scope_) -> QueryTreeNodePtr + { + const QueryNode * query_node = scope_.scope_node ? scope_.scope_node->as() : nullptr; + if (!query_node) + return nullptr; + + auto matched_node = find_aliased_node_in_projection(query_node, identifier_full_name_); + if (!matched_node) + return nullptr; + + auto left_subquery = std::make_shared(query_node->getMutableContext()); + left_subquery->getProjection().getNodes().push_back(matched_node->clone()); + auto subquery_join_tree = left_table_expression; + if (subquery_join_tree->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) + subquery_join_tree = subquery_join_tree->as().getTableExpression(); + left_subquery->getJoinTree() = subquery_join_tree; + + IdentifierResolveScope & left_subquery_scope = createIdentifierResolveScope(left_subquery, nullptr /*parent_scope*/); + /// We are using alias column mechanism for USING column from projection. + /// It will be calculated right after reading, so column will be not nullable there. + left_subquery_scope.join_use_nulls = false; + + resolveQuery(left_subquery, left_subquery_scope); + + const auto & resolved_nodes = left_subquery->getProjection().getNodes(); + if (resolved_nodes.size() == 1) + { + /// Added column should not conflict with existing column names + NameSet existing_columns; + if (!getColumnsFromTableExpression(left_table_expression, existing_columns)) + return nullptr; + + NameAndTypePair column_name_type(identifier_full_name_, resolved_nodes.front()->getResultType()); + while (existing_columns.contains(column_name_type.name)) + column_name_type.name = "_" + column_name_type.name; + + auto [expression_source, is_single_source] = getExpressionSource(resolved_nodes.front()); + /// Do not support `SELECT t1.a + t2.a AS id ... USING id` + if (!is_single_source) + return nullptr; + + /// When expression has no table source (e.g. a constant like `concat('_1', 2, 2) AS id`), + /// and left_table_expression is a JOIN or CROSS_JOIN node, we must not assign the JOIN + /// as the column source. That would create a ColumnNode with a JOIN/CROSS_JOIN source + /// and non-ListNode expression, which CollectSourceColumnsVisitor doesn't expect. + if (!expression_source + && (left_table_expression->getNodeType() == QueryTreeNodeType::JOIN + || left_table_expression->getNodeType() == QueryTreeNodeType::CROSS_JOIN)) + return nullptr; + + /// Create ColumnNode with expression from parent projection + return std::make_shared(std::move(column_name_type), resolved_nodes.front(), + expression_source ? expression_source : left_table_expression); + } + return nullptr; + }; + for (auto & join_using_node : join_using_list.getNodes()) { + nested_alias_matched = false; + auto * identifier_node = join_using_node->as(); if (!identifier_node) throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -5291,72 +5482,6 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS const auto & settings = scope.context->getSettingsRef(); - /** While resolving JOIN USING identifier, try to resolve identifier from parent subquery projection. - * Example: SELECT a + 1 AS b FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) USING b - * In this case `b` is not in the left table expression, but it is in the parent subquery projection. - */ - auto try_resolve_identifier_from_query_projection = [this](const String & identifier_full_name_, - const QueryTreeNodePtr & left_table_expression, - const IdentifierResolveScope & scope_) -> QueryTreeNodePtr - { - const QueryNode * query_node = scope_.scope_node ? scope_.scope_node->as() : nullptr; - if (!query_node) - return nullptr; - - const auto & projection_list = query_node->getProjection(); - for (const auto & projection_node : projection_list.getNodes()) - { - if (projection_node->hasAlias() && identifier_full_name_ == projection_node->getAlias()) - { - auto left_subquery = std::make_shared(query_node->getMutableContext()); - left_subquery->getProjection().getNodes().push_back(projection_node->clone()); - auto subquery_join_tree = left_table_expression; - if (subquery_join_tree->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) - subquery_join_tree = subquery_join_tree->as().getTableExpression(); - left_subquery->getJoinTree() = subquery_join_tree; - - IdentifierResolveScope & left_subquery_scope = createIdentifierResolveScope(left_subquery, nullptr /*parent_scope*/); - /// We are using alias column mechanism for USING column from projection. - /// It will be calculated right after reading, so column will be not nullable there. - left_subquery_scope.join_use_nulls = false; - - resolveQuery(left_subquery, left_subquery_scope); - - const auto & resolved_nodes = left_subquery->getProjection().getNodes(); - if (resolved_nodes.size() == 1) - { - /// Added column should not conflict with existing column names - NameSet existing_columns; - if (!getColumnsFromTableExpression(left_table_expression, existing_columns)) - return nullptr; - - NameAndTypePair column_name_type(identifier_full_name_, resolved_nodes.front()->getResultType()); - while (existing_columns.contains(column_name_type.name)) - column_name_type.name = "_" + column_name_type.name; - - auto [expression_source, is_single_source] = getExpressionSource(resolved_nodes.front()); - /// Do not support `SELECT t1.a + t2.a AS id ... USING id` - if (!is_single_source) - return nullptr; - - /// When expression has no table source (e.g. a constant like `concat('_1', 2, 2) AS id`), - /// and left_table_expression is a JOIN or CROSS_JOIN node, we must not assign the JOIN - /// as the column source. That would create a ColumnNode with a JOIN/CROSS_JOIN source - /// and non-ListNode expression, which CollectSourceColumnsVisitor doesn't expect. - if (!expression_source - && (left_table_expression->getNodeType() == QueryTreeNodeType::JOIN - || left_table_expression->getNodeType() == QueryTreeNodeType::CROSS_JOIN)) - return nullptr; - - /// Create ColumnNode with expression from parent projection - return std::make_shared(std::move(column_name_type), resolved_nodes.front(), - expression_source ? expression_source : left_table_expression); - } - } - } - return nullptr; - }; - QueryTreeNodes result_table_expressions; QueryTreeNodePtr result_left_table_expression = nullptr; @@ -5368,6 +5493,48 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS if (settings[Setting::analyzer_compatibility_join_using_top_level_identifier]) result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpression(), scope); + /// A nested-alias USING key cannot ship to a remote server (rendered SQL keeps only top-level projection aliases), so disable parallel replicas for such a query. + if (result_left_table_expression && nested_alias_matched) + { + /// Independently-planned subqueries (`IN`/`FROM`/`JOIN`-right-side) are planned from their own context copies, + /// so disable on every `QueryNode`/`UnionNode` on the scope chain that contains this JOIN, not just the root. + bool disabled_any = false; + for (const IdentifierResolveScope * chain_scope = &scope; chain_scope; chain_scope = chain_scope->parent_scope) + { + ContextMutablePtr chain_context; + if (!chain_scope->scope_node) + continue; + if (auto * chain_query_node = chain_scope->scope_node->as()) + chain_context = chain_query_node->getMutableContext(); + else if (auto * chain_union_node = chain_scope->scope_node->as()) + chain_context = chain_union_node->getMutableContext(); + else + continue; /// expression/lambda scope - skip, keep walking + + /// Skip secondary (replica-side) queries to not corrupt task-based reading. The `canUseTaskBasedParallelReplicas` + /// gate (mirroring the planner's FINAL precedent) acts only when parallel replicas would actually run, leaving + /// custom-key modes to ship full SQL and fail loudly. Checked per node: subquery `SETTINGS` produce distinct contexts. + if (chain_context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY + || !chain_context->canUseTaskBasedParallelReplicas()) + continue; + + if (chain_context->getSettingsRef()[Setting::allow_experimental_parallel_reading_from_replicas] >= 2) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "JOIN USING identifier '{}' is resolved from an alias nested in the SELECT list, " + "which is not supported with parallel replicas", + identifier_full_name); + + chain_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); + disabled_any = true; + } + + if (disabled_any) + LOG_DEBUG(getLogger("QueryAnalyzer"), + "JOIN USING identifier '{}' is resolved from an alias nested in the SELECT list; " + "parallel replicas are disabled because the query sent to a remote server would not contain the alias", + identifier_full_name); + } + if (!result_left_table_expression) { IdentifierLookup identifier_lookup{identifier_node->getIdentifier(), IdentifierLookupContext::EXPRESSION}; @@ -5380,17 +5547,11 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS const QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; if (!settings[Setting::analyzer_compatibility_join_using_top_level_identifier] && query_node) { - for (const auto & projection_node : query_node->getProjection().getNodes()) - { - if (projection_node->hasAlias() && identifier_full_name == projection_node->getAlias()) - { - extra_message = fmt::format( - ", but alias '{}' is present in SELECT list." - " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause", - projection_node->formatASTForErrorMessage()); - break; - } - } + if (auto matched_node = find_aliased_node_in_projection(query_node, identifier_full_name)) + extra_message = fmt::format( + ", but alias '{}' is present in SELECT list." + " You may try to SET analyzer_compatibility_join_using_top_level_identifier = 1, to allow to use it in USING clause", + matched_node->formatASTForErrorMessage()); } throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, diff --git a/src/Analyzer/Resolve/QueryAnalyzer.h b/src/Analyzer/Resolve/QueryAnalyzer.h index 2de6c35feaba..d9877b0e1edd 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.h +++ b/src/Analyzer/Resolve/QueryAnalyzer.h @@ -253,9 +253,10 @@ class QueryAnalyzer bool allow_lambda_expression, bool allow_table_expression, bool ignore_alias = false, - bool allow_niladic_functions = true); + bool allow_niladic_functions = true, + bool is_top_level_projection = false); - ProjectionNames resolveExpressionNodeList(QueryTreeNodePtr & node_list, IdentifierResolveScope & scope, bool allow_lambda_expression, bool allow_table_expression, bool allow_niladic_functions = true); + ProjectionNames resolveExpressionNodeList(QueryTreeNodePtr & node_list, IdentifierResolveScope & scope, bool allow_lambda_expression, bool allow_table_expression, bool allow_niladic_functions = true, bool is_top_level_projection = false); ProjectionNames resolveSortNodeList(QueryTreeNodePtr & sort_node_list, IdentifierResolveScope & scope); diff --git a/src/Backups/BackupCoordinationFileInfos.cpp b/src/Backups/BackupCoordinationFileInfos.cpp index 25c36a260517..d11007a503b2 100644 --- a/src/Backups/BackupCoordinationFileInfos.cpp +++ b/src/Backups/BackupCoordinationFileInfos.cpp @@ -24,12 +24,17 @@ void BackupCoordinationFileInfos::addFileInfos(BackupFileInfos && file_infos_, c file_infos.emplace(host_id_, std::move(file_infos_)); } -BackupFileInfos BackupCoordinationFileInfos::getFileInfos(const String & host_id_) const +const BackupFileInfos & BackupCoordinationFileInfos::getFileInfos(const String & host_id_) const { prepare(); auto it = file_infos.find(host_id_); if (it == file_infos.end()) - return {}; + { + static const BackupFileInfos empty; + return empty; + } + /// Safe to return by reference: after prepare() the per-host vectors are never mutated (addFileInfos() throws + /// once prepared), and file_infos_for_all_hosts already holds raw pointers into them. return it->second; } diff --git a/src/Backups/BackupCoordinationFileInfos.h b/src/Backups/BackupCoordinationFileInfos.h index 99862794f283..5f7dcabb4901 100644 --- a/src/Backups/BackupCoordinationFileInfos.h +++ b/src/Backups/BackupCoordinationFileInfos.h @@ -38,7 +38,8 @@ class BackupCoordinationFileInfos void addFileInfos(BackupFileInfos && file_infos, const String & host_id); /// Returns file infos for the specified host after preparation. - BackupFileInfos getFileInfos(const String & host_id) const; + /// Returned by reference; the referenced storage is immutable after prepare() (see addFileInfos()). + const BackupFileInfos & getFileInfos(const String & host_id) const; /// Iterates the file infos of all hosts in place, without copying them into a vector. void forEachFileInfoForAllHosts(const std::function & callback) const; diff --git a/src/Backups/BackupCoordinationLocal.cpp b/src/Backups/BackupCoordinationLocal.cpp index 87a08a42fd64..95e7d1d700ef 100644 --- a/src/Backups/BackupCoordinationLocal.cpp +++ b/src/Backups/BackupCoordinationLocal.cpp @@ -110,7 +110,7 @@ void BackupCoordinationLocal::addFileInfos(BackupFileInfos && file_infos_) file_infos.addFileInfos(std::move(file_infos_), ""); } -BackupFileInfos BackupCoordinationLocal::getFileInfos() const +const BackupFileInfos & BackupCoordinationLocal::getFileInfos() const { std::lock_guard lock{file_infos_mutex}; return file_infos.getFileInfos(""); diff --git a/src/Backups/BackupCoordinationLocal.h b/src/Backups/BackupCoordinationLocal.h index 3688e8420119..9169a5ac3bde 100644 --- a/src/Backups/BackupCoordinationLocal.h +++ b/src/Backups/BackupCoordinationLocal.h @@ -62,7 +62,7 @@ class BackupCoordinationLocal : public IBackupCoordination String getKeeperMapDataPath(const String & table_zookeeper_root_path) const override; void addFileInfos(BackupFileInfos && file_infos) override; - BackupFileInfos getFileInfos() const override; + const BackupFileInfos & getFileInfos() const override; void forEachFileInfoForAllHosts(const std::function & callback) const override; bool startWritingFile(size_t data_file_index) override; diff --git a/src/Backups/BackupCoordinationOnCluster.cpp b/src/Backups/BackupCoordinationOnCluster.cpp index b0622fee3d20..2cb4a7904573 100644 --- a/src/Backups/BackupCoordinationOnCluster.cpp +++ b/src/Backups/BackupCoordinationOnCluster.cpp @@ -768,7 +768,7 @@ void BackupCoordinationOnCluster::addFileInfos(BackupFileInfos && file_infos_) serializeToMultipleZooKeeperNodes(zookeeper_path + "/file_infos/" + current_host, file_infos_str, "addFileInfos"); } -BackupFileInfos BackupCoordinationOnCluster::getFileInfos() const +const BackupFileInfos & BackupCoordinationOnCluster::getFileInfos() const { auto component_guard = Coordination::setCurrentComponent("BackupCoordinationOnCluster::getFileInfos"); std::lock_guard lock{file_infos_mutex}; diff --git a/src/Backups/BackupCoordinationOnCluster.h b/src/Backups/BackupCoordinationOnCluster.h index 299a69e2196e..365e6b531ef2 100644 --- a/src/Backups/BackupCoordinationOnCluster.h +++ b/src/Backups/BackupCoordinationOnCluster.h @@ -78,7 +78,7 @@ class BackupCoordinationOnCluster : public IBackupCoordination String getKeeperMapDataPath(const String & table_zookeeper_root_path) const override; void addFileInfos(BackupFileInfos && file_infos) override; - BackupFileInfos getFileInfos() const override; + const BackupFileInfos & getFileInfos() const override; void forEachFileInfoForAllHosts(const std::function & callback) const override; bool startWritingFile(size_t data_file_index) override; diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index 78f36357330f..3ac7df692668 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -1059,18 +1059,71 @@ String BackupImpl::getObjectKey(const String & file_name) const } size_t BackupImpl::copyFileToDisk(const String & file_name, - DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const + DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, bool sync) const { -#if CLICKHOUSE_CLOUD String object_key = getObjectKey(file_name); if (!object_key.empty()) + { + /// The optimized object-key copy exposes no buffer to fsync, so the sync case needs a buffered path. + if (sync) + return copyObjectKeyEntryToDiskSynced(object_key, destination_disk, destination_path, write_mode); +#if CLICKHOUSE_CLOUD return copyFileToDiskByObjectKey(object_key, destination_disk, destination_path, write_mode); #endif - return copyFileToDisk(getFileSizeAndChecksum(file_name), destination_disk, destination_path, write_mode); + } + return copyFileToDisk(getFileSizeAndChecksum(file_name), destination_disk, destination_path, write_mode, sync); +} + +size_t BackupImpl::copyObjectKeyEntryToDiskSynced( + const String & object_key, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const +{ + if (open_mode == OpenMode::WRITE) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup file should not be opened for writing. Something is wrong internally"); + + BackupFileInfo info; + { + std::lock_guard lock{mutex}; + auto it = lightweight_snapshot_file_infos.find(object_key); + if (it == lightweight_snapshot_file_infos.end()) + throw Exception( + ErrorCodes::BACKUP_ENTRY_NOT_FOUND, + "Backup {}: Entry with object key {} not found in the backup", + backup_name_for_logging, object_key); + info = it->second; + } + + if (info.encrypted_by_disk && !destination_disk->getDataSourceDescription().is_encrypted) + { + throw Exception( + ErrorCodes::CANNOT_RESTORE_TO_NONENCRYPTED_DISK, + "File {} is encrypted in the backup, it can be restored only to an encrypted disk", + info.data_file_name); + } + + auto read_buffer = readFileByObjectKey(info); + size_t buf_size = std::min(info.size ? info.size : DBMS_DEFAULT_BUFFER_SIZE, reader->getWriteBufferSize()); + std::unique_ptr write_buffer; + /// readFileByObjectKey returns the bytes as stored (still encrypted for encrypted-by-disk entries), + /// so write them through writeEncryptedFile to avoid re-encrypting, mirroring the generic copy path. + if (info.encrypted_by_disk) + write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, reader->getWriteSettings()); + else + write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, reader->getWriteSettings()); + copyData(*read_buffer, *write_buffer, info.size); + write_buffer->finalize(); + /// fdatasync the contents so a restored part survives power loss (see copyFileToDisk above). + write_buffer->sync(); + + { + std::lock_guard lock{mutex}; + ++num_read_files; + num_read_bytes += info.size; + } + return info.size; } size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, - DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const + DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, bool sync) const { if (open_mode == OpenMode::WRITE) throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup file should not be opened for writing. Something is wrong internally"); @@ -1080,8 +1133,18 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, /// Entry's data is empty. if (write_mode == WriteMode::Rewrite) { - /// Just create an empty file. - destination_disk->createFile(destination_path); + if (sync) + { + /// createFile() leaves the empty contents unsynced; a live buffer lets us fsync it. + auto write_buffer = destination_disk->writeFile(destination_path, DBMS_DEFAULT_BUFFER_SIZE, write_mode, reader->getWriteSettings()); + write_buffer->finalize(); + write_buffer->sync(); + } + else + { + /// Just create an empty file. + destination_disk->createFile(destination_path); + } } std::lock_guard lock{mutex}; ++num_read_files; @@ -1113,16 +1176,25 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, bool file_copied = false; - if (info.size && !info.base_size && !use_archive) + /// When `sync` is requested we must copy through a live destination buffer so we can fsync its + /// contents below. The optimized delegate paths (reader->copyFileToDisk / base backup) may use + /// fs::copy or an object-storage copy and expose no buffer, so skip them and take the buffered + /// branch, which is already correct for every source (this backup, base backup, archive). + if (!sync && info.size && !info.base_size && !use_archive) { - /// Data comes completely from this backup. + /// Data comes completely from this backup. The reader copies without exposing a write + /// buffer we could fsync, so this fast path is used only when `sync` isn't requested. reader->copyFileToDisk(info.data_file_name, info.size, info.encrypted_by_disk, destination_disk, destination_path, write_mode); file_copied = true; } else if (info.size && (info.size == info.base_size)) { - /// Data comes completely from the base backup (nothing comes from this backup). - getBaseBackup()->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode); + /// Data comes completely from the base backup (nothing comes from this backup). The base + /// backup is itself a BackupImpl that honours `sync` and can read its own encrypted-by-disk + /// entries, so forward the copy (and the `sync` request) there. Going through the generic + /// branch below instead would read the base via the public readFile(), which always requests + /// unencrypted data and would fail on an encrypted entry (CANNOT_RESTORE_TO_NONENCRYPTED_DISK). + getBaseBackup()->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode, sync); file_copied = true; } @@ -1137,7 +1209,7 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, { /// Use the generic way to copy data. `readFile()` will update `num_read_files`. auto read_buffer = readFileImpl(info.file_name, size_and_checksum, /* read_encrypted= */ info.encrypted_by_disk); - std::unique_ptr write_buffer; + std::unique_ptr write_buffer; size_t buf_size = std::min(info.size, reader->getWriteBufferSize()); if (info.encrypted_by_disk) write_buffer = destination_disk->writeEncryptedFile(destination_path, buf_size, write_mode, reader->getWriteSettings()); @@ -1145,6 +1217,10 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, write_buffer = destination_disk->writeFile(destination_path, buf_size, write_mode, reader->getWriteSettings()); copyData(*read_buffer, *write_buffer, info.size); write_buffer->finalize(); + /// fdatasync the contents so a restored part survives power loss, matching the durability + /// an inserted part gets from fsync_after_insert (the caller passes `sync` accordingly). + if (sync) + write_buffer->sync(); } return info.size; diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index 21e2d3e78273..77e89232b60e 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -82,8 +82,8 @@ class BackupImpl : public IBackup SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const override; std::unique_ptr readFile(const String & file_name) const override; std::unique_ptr readFile(const String & file_name, const SizeAndChecksum & size_and_checksum) const override; - size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; - size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const override; + size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, bool sync) const override; + size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, bool sync) const override; void writeFile(const BackupFileInfo & info, BackupEntryPtr entry) override; bool supportsWritingInMultipleThreads() const override { return !use_archive; } void finalizeWriting() override; @@ -109,6 +109,11 @@ class BackupImpl : public IBackup String getObjectKey(const String & file_name) const; std::unique_ptr readFileByObjectKey(const BackupFileInfo & info) const; + /// Copies a lightweight-snapshot (object-key) entry to the destination through a live write buffer + /// and fsyncs it (the optimized object-key copy exposes no buffer to fsync). Reached only in the + /// cloud build, where object keys are present; defined unconditionally so it is type-checked everywhere. + size_t copyObjectKeyEntryToDiskSynced(const String & object_key, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const; + /// Returns the base backup or null if there is no base backup. std::shared_ptr getBaseBackupUnlocked() const TSA_REQUIRES(mutex); diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp index 50d028b7920a..a95bc1b85895 100644 --- a/src/Backups/BackupsWorker.cpp +++ b/src/Backups/BackupsWorker.cpp @@ -196,6 +196,9 @@ enum class BackupsWorker::ThreadPoolId : uint8_t /// Making a list of files to copy or copying those files. BACKUP, + /// Dedicated pool for creating lightweight snapshots, so it is not starved by a concurrent heavy BACKUP sharing the BACKUP pool. + CREATE_SNAPSHOT, + /// Creating of tables and databases during RESTORE and filling them with data. RESTORE, @@ -203,6 +206,9 @@ enum class BackupsWorker::ThreadPoolId : uint8_t ASYNC_BACKGROUND_BACKUP, ASYNC_BACKGROUND_RESTORE, + /// Dedicated async-starter pool for lightweight snapshot creation, so a snapshot's top-level operation is not starved by a concurrent heavy BACKUP occupying ASYNC_BACKGROUND_BACKUP. + ASYNC_BACKGROUND_CREATE_SNAPSHOT, + /// We need background threads for coordination workers (see BackgroundCoordinationStageSync). ON_CLUSTER_COORDINATION_BACKUP, ON_CLUSTER_COORDINATION_RESTORE, @@ -250,7 +256,9 @@ class BackupsWorker::ThreadPools switch (thread_pool_id) { case ThreadPoolId::BACKUP: + case ThreadPoolId::CREATE_SNAPSHOT: case ThreadPoolId::ASYNC_BACKGROUND_BACKUP: + case ThreadPoolId::ASYNC_BACKGROUND_CREATE_SNAPSHOT: case ThreadPoolId::ON_CLUSTER_COORDINATION_BACKUP: case ThreadPoolId::ASYNC_BACKGROUND_INTERNAL_BACKUP: case ThreadPoolId::ON_CLUSTER_COORDINATION_INTERNAL_BACKUP: @@ -260,7 +268,7 @@ class BackupsWorker::ThreadPools metric_scheduled_threads = CurrentMetrics::BackupsThreadsScheduled; max_threads = num_backup_threads; /// We don't use thread pool queues for thread pools with a lot of tasks otherwise that queue could be memory-wasting. - use_queue = (thread_pool_id != ThreadPoolId::BACKUP); + use_queue = (thread_pool_id != ThreadPoolId::BACKUP && thread_pool_id != ThreadPoolId::CREATE_SNAPSHOT); break; } @@ -298,10 +306,12 @@ class BackupsWorker::ThreadPools /// and everything else is after those ones. ThreadPoolId::ASYNC_BACKGROUND_BACKUP, ThreadPoolId::ASYNC_BACKGROUND_RESTORE, + ThreadPoolId::ASYNC_BACKGROUND_CREATE_SNAPSHOT, ThreadPoolId::ASYNC_BACKGROUND_INTERNAL_BACKUP, ThreadPoolId::ASYNC_BACKGROUND_INTERNAL_RESTORE, /// Others: ThreadPoolId::BACKUP, + ThreadPoolId::CREATE_SNAPSHOT, ThreadPoolId::RESTORE, ThreadPoolId::ON_CLUSTER_COORDINATION_BACKUP, ThreadPoolId::ON_CLUSTER_COORDINATION_INTERNAL_BACKUP, @@ -575,8 +585,20 @@ std::pair BackupsWorker::startMakingBackup(cons try { - auto thread_pool_id = starter->is_internal_backup ? ThreadPoolId::ASYNC_BACKGROUND_INTERNAL_BACKUP: ThreadPoolId::ASYNC_BACKGROUND_BACKUP; - ThreadName thread_name = starter->is_internal_backup ? ThreadName::BACKUP_ASYNC_INTERNAL : ThreadName::BACKUP_ASYNC; + ThreadPoolId thread_pool_id = ThreadPoolId::ASYNC_BACKGROUND_BACKUP; + ThreadName thread_name = ThreadName::BACKUP_ASYNC; + if (starter->backup_settings.experimental_lightweight_snapshot) + { + /// Snapshot creation needs its own async-starter capacity: otherwise a concurrent heavy BACKUP can occupy all + /// ASYNC_BACKGROUND_BACKUP threads and the snapshot's doBackup would never start (and so never reach CREATE_SNAPSHOT). + thread_pool_id = ThreadPoolId::ASYNC_BACKGROUND_CREATE_SNAPSHOT; + thread_name = ThreadName::SNAPSHOT_ASYNC; + } + else if (starter->is_internal_backup) + { + thread_pool_id = ThreadPoolId::ASYNC_BACKGROUND_INTERNAL_BACKUP; + thread_name = ThreadName::BACKUP_ASYNC_INTERNAL; + } auto schedule = threadPoolCallbackRunnerUnsafe(thread_pools->getThreadPool(thread_pool_id), thread_name); schedule([starter] @@ -660,6 +682,10 @@ void BackupsWorker::doBackup( bool is_internal_backup = backup_settings.internal; + /// Snapshot creation uses a dedicated thread pool so it is not starved by a concurrent heavy BACKUP occupying the shared BACKUP pool. + const ThreadPoolId backup_thread_pool_id + = backup_settings.experimental_lightweight_snapshot ? ThreadPoolId::CREATE_SNAPSHOT : ThreadPoolId::BACKUP; + maybeSleepForTesting(); /// Write the backup. @@ -692,7 +718,7 @@ void BackupsWorker::doBackup( backup_coordination, read_settings, context, - getThreadPool(ThreadPoolId::BACKUP)); + getThreadPool(backup_thread_pool_id)); backup_entries = backup_entries_collector.run(); } @@ -700,8 +726,8 @@ void BackupsWorker::doBackup( chassert(backup); chassert(backup_coordination); chassert(context); - buildFileInfosForBackupEntries(backup, backup_entries, read_settings, backup_coordination, context->getProcessListElement()); - writeBackupEntries(backup, std::move(backup_entries), backup_id, backup_coordination, is_internal_backup, context->getProcessListElement()); + buildFileInfosForBackupEntries(backup, backup_entries, read_settings, backup_coordination, backup_thread_pool_id, context->getProcessListElement()); + writeBackupEntries(backup, std::move(backup_entries), backup_id, backup_coordination, is_internal_backup, backup_thread_pool_id, context->getProcessListElement()); /// We have written our backup entries (there is no need to sync it with other hosts because it's the last stage). backup_coordination->setStage(Stage::COMPLETED, "", /* sync = */ false); @@ -739,10 +765,10 @@ void BackupsWorker::doBackup( } -void BackupsWorker::buildFileInfosForBackupEntries(const BackupPtr & backup, const BackupEntries & backup_entries, const ReadSettings & read_settings, std::shared_ptr backup_coordination, QueryStatusPtr process_list_element) +void BackupsWorker::buildFileInfosForBackupEntries(const BackupPtr & backup, const BackupEntries & backup_entries, const ReadSettings & read_settings, std::shared_ptr backup_coordination, ThreadPoolId thread_pool_id, QueryStatusPtr process_list_element) { backup_coordination->setStage(Stage::BUILDING_FILE_INFOS, "", /* sync = */ true); - backup_coordination->addFileInfos(::DB::buildFileInfosForBackupEntries(backup_entries, backup->getBaseBackup(), read_settings, getThreadPool(ThreadPoolId::BACKUP), process_list_element)); + backup_coordination->addFileInfos(::DB::buildFileInfosForBackupEntries(backup_entries, backup->getBaseBackup(), read_settings, getThreadPool(thread_pool_id), process_list_element)); } @@ -752,12 +778,13 @@ void BackupsWorker::writeBackupEntries( const OperationID & backup_id, std::shared_ptr backup_coordination, bool is_internal_backup, + ThreadPoolId thread_pool_id, QueryStatusPtr process_list_element) { LOG_TRACE(log, "{}, num backup entries={}", Stage::WRITING_BACKUP, backup_entries.size()); backup_coordination->setStage(Stage::WRITING_BACKUP, "", /* sync = */ true); - auto file_infos = backup_coordination->getFileInfos(); + const auto & file_infos = backup_coordination->getFileInfos(); if (file_infos.size() != backup_entries.size()) { throw Exception( @@ -771,7 +798,7 @@ void BackupsWorker::writeBackupEntries( std::atomic_bool failed = false; bool always_single_threaded = !backup->supportsWritingInMultipleThreads(); - auto & thread_pool = getThreadPool(ThreadPoolId::BACKUP); + auto & thread_pool = getThreadPool(thread_pool_id); std::vector writing_order; if (test_randomize_order) diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h index 2fb856a6103c..efa36ad05106 100644 --- a/src/Backups/BackupsWorker.h +++ b/src/Backups/BackupsWorker.h @@ -99,11 +99,13 @@ class BackupsWorker bool on_cluster, const ClusterPtr & cluster); + enum class ThreadPoolId : uint8_t; + /// Builds file infos for specified backup entries. - void buildFileInfosForBackupEntries(const BackupPtr & backup, const BackupEntries & backup_entries, const ReadSettings & read_settings, std::shared_ptr backup_coordination, QueryStatusPtr process_list_element); + void buildFileInfosForBackupEntries(const BackupPtr & backup, const BackupEntries & backup_entries, const ReadSettings & read_settings, std::shared_ptr backup_coordination, ThreadPoolId thread_pool_id, QueryStatusPtr process_list_element); /// Write backup entries to an opened backup. - void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, const BackupOperationID & backup_id, std::shared_ptr backup_coordination, bool is_internal_backup, QueryStatusPtr process_list_element); + void writeBackupEntries(BackupMutablePtr backup, BackupEntries && backup_entries, const BackupOperationID & backup_id, std::shared_ptr backup_coordination, bool is_internal_backup, ThreadPoolId thread_pool_id, QueryStatusPtr process_list_element); std::pair startRestoring(const ASTPtr & query, ContextMutablePtr context); struct RestoreStarter; @@ -145,7 +147,6 @@ class BackupsWorker void setNumFilesAndSize(const BackupOperationID & id, size_t num_files, UInt64 total_size, size_t num_entries, UInt64 uncompressed_size, UInt64 compressed_size, size_t num_read_files, UInt64 num_read_bytes); - enum class ThreadPoolId : uint8_t; ThreadPool & getThreadPool(ThreadPoolId thread_pool_id); /// Waits for some time if `test_inject_sleep` is true. diff --git a/src/Backups/IBackup.h b/src/Backups/IBackup.h index b73891e2b466..15879cd6ecdd 100644 --- a/src/Backups/IBackup.h +++ b/src/Backups/IBackup.h @@ -115,9 +115,11 @@ class IBackup : public std::enable_shared_from_this virtual std::unique_ptr readFile(const String & file_name, const SizeAndChecksum & size_and_checksum) const = 0; /// Copies a file from the backup to a specified destination disk. Returns the number of bytes written. - virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const = 0; + /// When `sync` is true the destination file's contents are fsynced before this call returns, so a + /// restored part can be made as durable as an inserted one (see MergeTreeData::restorePartFromBackup). + virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, bool sync) const = 0; - virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) const = 0; + virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode, bool sync) const = 0; /// Puts a new entry to the backup. virtual void writeFile(const BackupFileInfo & file_info, BackupEntryPtr entry) = 0; diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index 48503cdde64f..98d7db184387 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -109,7 +109,10 @@ class IBackupCoordination /// Adds file information. /// If specified checksum+size are new for this IBackupContentsInfo the function sets `is_data_file_required`. virtual void addFileInfos(BackupFileInfos && file_infos) = 0; - virtual BackupFileInfos getFileInfos() const = 0; + /// Returns the file infos of the current host by reference to avoid copying them (a backup can contain millions). + /// The reference is valid until the coordination is destroyed. It must only be called after file collection has + /// finished (i.e. no more addFileInfos()), because the referenced storage is immutable only after preparation. + virtual const BackupFileInfos & getFileInfos() const = 0; /// Iterates the file infos of all hosts in place, without copying them into a vector /// (a backup can contain millions). diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index d97434b4b0c5..56e6df256b53 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -1187,11 +1187,18 @@ void ColumnObject::updateHashWithValue(size_t n, SipHash & hash) const void ColumnObject::updateHashWithValueRange(size_t begin, size_t end, SipHash & hash) const { + /// Typed paths are always in the same order for all instances of the same Object type, + /// so there is no need to hash the paths themselves. for (const auto & path : sorted_typed_paths) typed_paths.find(path)->second->updateHashWithValueRange(begin, end, hash); + /// Dynamic paths may differ, so we hash the paths together with values. for (const auto & path : sorted_dynamic_paths) + { + hash.update(path.size()); + hash.update(path); dynamic_paths.find(path)->second->updateHashWithValueRange(begin, end, hash); + } shared_data->updateHashWithValueRange(begin, end, hash); } @@ -1487,28 +1494,31 @@ void ColumnObject::protect() void ColumnObject::forEachMutableSubcolumn(DB::IColumn::MutableColumnCallback callback) { - for (auto & [_, column] : typed_paths) - callback(column); - for (auto & [path, column] : dynamic_paths) + for (const auto & path : sorted_typed_paths) + callback(typed_paths.find(path)->second); + for (const auto & path : sorted_dynamic_paths) { - callback(column); - dynamic_paths_ptrs[path] = assert_cast(column.get()); + auto it = dynamic_paths.find(path); + callback(it->second); + dynamic_paths_ptrs[it->first] = assert_cast(it->second.get()); } callback(shared_data); } void ColumnObject::forEachMutableSubcolumnRecursively(DB::IColumn::RecursiveMutableColumnCallback callback) { - for (auto & [_, column] : typed_paths) + for (const auto & path : sorted_typed_paths) { + auto & column = typed_paths.find(path)->second; callback(*column); column->forEachMutableSubcolumnRecursively(callback); } - for (auto & [path, column] : dynamic_paths) + for (const auto & path : sorted_dynamic_paths) { - callback(*column); - column->forEachMutableSubcolumnRecursively(callback); - dynamic_paths_ptrs[path] = assert_cast(column.get()); + auto it = dynamic_paths.find(path); + callback(*it->second); + it->second->forEachMutableSubcolumnRecursively(callback); + dynamic_paths_ptrs[it->first] = assert_cast(it->second.get()); } callback(*shared_data); shared_data->forEachMutableSubcolumnRecursively(callback); @@ -1516,23 +1526,25 @@ void ColumnObject::forEachMutableSubcolumnRecursively(DB::IColumn::RecursiveMuta void ColumnObject::forEachSubcolumn(DB::IColumn::ColumnCallback callback) const { - for (const auto & [_, column] : typed_paths) - callback(column); - for (const auto & [path, column] : dynamic_paths) - callback(column); + for (const auto & path : sorted_typed_paths) + callback(typed_paths.find(path)->second); + for (const auto & path : sorted_dynamic_paths) + callback(dynamic_paths.find(path)->second); callback(shared_data); } void ColumnObject::forEachSubcolumnRecursively(DB::IColumn::RecursiveColumnCallback callback) const { - for (const auto & [_, column] : typed_paths) + for (const auto & path : sorted_typed_paths) { + const auto & column = typed_paths.find(path)->second; callback(*column); column->forEachSubcolumnRecursively(callback); } - for (const auto & [path, column] : dynamic_paths) + for (const auto & path : sorted_dynamic_paths) { + const auto & column = dynamic_paths.find(path)->second; callback(*column); column->forEachSubcolumnRecursively(callback); } diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 1acaec6d4f1f..b787916b3b42 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -178,6 +178,10 @@ class ColumnUnique final : public COWHelper getOrFindValueIndex(std::string_view value) const override { + /// The reserved prefix slots are not in the reverse index, so match the default value here. + if (auto index = getNestedTypeDefaultValueIndex(); getRawColumnPtr()->getDataAt(index) == value) + return index; + if (std::optional res = reverse_index.getIndex(value); res) return res; diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp index ba27feec6e86..f0a82555f6a9 100644 --- a/src/Columns/ColumnVariant.cpp +++ b/src/Columns/ColumnVariant.cpp @@ -27,6 +27,16 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; extern const int SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; + extern const int INCORRECT_DATA; +} + +static void checkDiscriminatorValue(ColumnVariant::Discriminator discr, size_t num_variants, bool allow_logical_error) +{ + if (discr != ColumnVariant::NULL_DISCRIMINATOR && discr >= num_variants) + throw Exception( + allow_logical_error ? ErrorCodes::LOGICAL_ERROR : ErrorCodes::INCORRECT_DATA, + "Invalid discriminator value {} (num_variants = {})", + static_cast(discr), num_variants); } std::string ColumnVariant::getName() const @@ -844,6 +854,8 @@ void ColumnVariant::deserializeAndInsertFromArena(ReadBuffer & in, const IColumn Discriminator global_discr = 0; readBinaryLittleEndian(global_discr, in); + checkDiscriminatorValue(global_discr, variants.size(), /* allow_logical_error= */ false); + Discriminator local_discr = localDiscriminatorByGlobal(global_discr); getLocalDiscriminators().push_back(local_discr); if (local_discr == NULL_DISCRIMINATOR) @@ -864,6 +876,8 @@ void ColumnVariant::skipSerializedInArena(ReadBuffer & in) const if (global_discr == NULL_DISCRIMINATOR) return; + checkDiscriminatorValue(global_discr, variants.size(), /* allow_logical_error= */ true); + variants[localDiscriminatorByGlobal(global_discr)]->skipSerializedInArena(in); } @@ -1957,12 +1971,12 @@ void ColumnVariant::fixDynamicStructure() variant->fixDynamicStructure(); } -void ColumnVariant::validateState() const +void ColumnVariant::validateState(bool allow_logical_error) const { const auto & local_discriminators_data = getLocalDiscriminators(); const auto & offsets_data = getOffsets(); if (local_discriminators_data.size() != offsets_data.size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of discriminators and offsets should be equal, but {} and {} were given", local_discriminators_data.size(), offsets_data.size()); + throw Exception(allow_logical_error ? ErrorCodes::LOGICAL_ERROR : ErrorCodes::INCORRECT_DATA, "Size of discriminators and offsets should be equal, but {} and {} were given", local_discriminators_data.size(), offsets_data.size()); VectorWithMemoryTracking actual_variant_sizes(variants.size()); for (size_t i = 0; i != variants.size(); ++i) @@ -1974,16 +1988,17 @@ void ColumnVariant::validateState() const auto local_discr = local_discriminators_data[i]; if (local_discr != NULL_DISCRIMINATOR) { + checkDiscriminatorValue(local_discr, variants.size(), allow_logical_error); ++expected_variant_sizes[local_discr]; if (offsets_data[i] >= actual_variant_sizes[local_discr]) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Offset at position {} is {}, but variant {} ({}) has size {}", i, offsets_data[i], static_cast(local_discr), variants[local_discr]->getName(), variants[local_discr]->size()); + throw Exception(allow_logical_error ? ErrorCodes::LOGICAL_ERROR : ErrorCodes::INCORRECT_DATA, "Offset at position {} is {}, but variant {} ({}) has size {}", i, offsets_data[i], static_cast(local_discr), variants[local_discr]->getName(), variants[local_discr]->size()); } } for (size_t i = 0; i != variants.size(); ++i) { if (variants[i]->size() != expected_variant_sizes[i]) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Variant {} ({}) has size {}, but expected {}", i, variants[i]->getName(), variants[i]->size(), expected_variant_sizes[i]); + throw Exception(allow_logical_error ? ErrorCodes::LOGICAL_ERROR : ErrorCodes::INCORRECT_DATA, "Variant {} ({}) has size {}, but expected {}", i, variants[i]->getName(), variants[i]->size(), expected_variant_sizes[i]); } } diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h index b4891c9162d4..e6a38bd458f8 100644 --- a/src/Columns/ColumnVariant.h +++ b/src/Columns/ColumnVariant.h @@ -365,7 +365,7 @@ class ColumnVariant final : public COWHelper, Colum bool hasStatistics() const override; void takeOrCalculateStatisticsFrom(const VectorWithMemoryTracking & source_columns) override; - void validateState() const; + void validateState(bool allow_logical_error = true) const; private: void insertFromImpl(const IColumn & src_, size_t n, const VectorWithMemoryTracking * global_discriminators_mapping); diff --git a/src/Columns/getLeastSuperColumn.cpp b/src/Columns/getLeastSuperColumn.cpp index de71ce584319..d07f4670a379 100644 --- a/src/Columns/getLeastSuperColumn.cpp +++ b/src/Columns/getLeastSuperColumn.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -14,8 +15,25 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +static bool containsAggregateStateColumn(const IColumn & column) +{ + if (typeid_cast(&column)) + return true; + + bool found = false; + column.forEachSubcolumn([&](const auto & subcolumn) { found = found || containsAggregateStateColumn(*subcolumn); }); + return found; +} + static bool sameConstants(const IColumn & a, const IColumn & b) { + /// Aggregate-state values cannot be compared as `Field`: the comparison throws when the + /// aggregate function type names differ, and they may legitimately differ between `UNION` + /// branches when the functions have the same state representation (e.g. `quantileState` + /// and `quantilesState(0.9)`). Don't save constness for them. + if (containsAggregateStateColumn(assert_cast(a).getDataColumn())) + return false; + return assert_cast(a).getField() == assert_cast(b).getField(); } diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index 3d484c2d6bac..27e7389ebe85 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -95,6 +95,7 @@ struct HashMethodSingleLowCardinalityColumn : public SingleColumnMethod using FindResult = columns_hashing_impl::FindResultImpl; static constexpr bool has_cheap_key_calculation = Base::has_cheap_key_calculation; + static constexpr bool has_cheap_key_holder = Base::has_cheap_key_holder; static constexpr bool has_pre_computed_hashes = Base::has_pre_computed_hashes; static HashMethodContextPtr createContext(const HashMethodContextSettings & settings) @@ -360,6 +361,13 @@ struct HashMethodSerialized } static constexpr bool has_cheap_key_calculation = false; + /// `getKeyHolder` serializes every key column for the row. With `prealloc = false` that means a + /// fresh `serializeKeysToPoolContiguous` into the arena; with `prealloc = true` and batch + /// serialization disabled it means a per-row heap allocation plus the same serialization. This is + /// the dominant cost of the aggregation, so `Aggregator` must not pay it twice per row to + /// prefetch. When the keys *are* batch-serialized upfront this method prefetches on its own, + /// using `precomputed_hashes` below, which needs no second `getKeyHolder` call. + static constexpr bool has_cheap_key_holder = false; static constexpr bool has_pre_computed_hashes = prealloc; ColumnRawPtrs key_columns; diff --git a/src/Common/ColumnsHashing/HashMethod.h b/src/Common/ColumnsHashing/HashMethod.h index e9926231b9f2..6412c1d5ed25 100644 --- a/src/Common/ColumnsHashing/HashMethod.h +++ b/src/Common/ColumnsHashing/HashMethod.h @@ -29,6 +29,41 @@ static inline UInt128 ALWAYS_INLINE hash128( /// NOLINT return hash.get128(); } +/** Hash methods declare two independent prefetch predicates. They are not interchangeable, and the + * places that read them are disjoint. + * + * `has_cheap_key_holder` - read by `Aggregator::executeImpl`. + * + * Is it acceptable to call `getKeyHolder(row)` a second time for the same row purely to issue a + * software prefetch? The aggregation prefetch pipeline runs + * + * auto && key_holder = state.getKeyHolder(i + look_ahead, pool); + * data.prefetch(std::move(key_holder)); + * + * ahead of the `emplaceKey`/`findKey` loop, so the look-ahead row's key holder is built once for + * the prefetch and once again when that row is actually processed. Hiding a cache miss is only a + * win when that duplicated work is cheaper than the miss. + * + * `true` means `getKeyHolder` reads the key in place: an unaligned load, a `packFixed`, or a + * `string_view` over the column's own memory. Rebuilding it costs a handful of instructions. + * + * `false` means `getKeyHolder` materializes the key - serializing every key column into the arena, + * or hashing every key column through virtual `IColumn` calls. For those methods building the key + * *is* the dominant cost of the aggregation, so paying it twice per row costs far more than the + * miss it hides. + * + * Note this is deliberately not "is the hash cheap". Hashing a string is not cheap, but a prefetch + * has to hash the key by definition, and `HashMethodString` still profits because building its key + * holder is free. + * + * `has_cheap_key_calculation` - read by the JOIN probe loop, via `join_prefetch_supported` in + * HashJoinMethodsImpl.h (the `KeyGetterForType` aliases in HashJoin/KeyGetter.h resolve to these + * same hash methods). It is the stricter "the whole key calculation, hashing included, is cheap", + * and is left as it was: the JOIN probe loop has its own cost balance, which this file's + * aggregation-side reasoning says nothing about. Only the aggregator reads + * `has_cheap_key_holder`. + */ + /// For the case when there is one numeric key. /// UInt8/16/32/64 for any type with corresponding bit width. template @@ -44,6 +79,8 @@ struct HashMethodOneNumber : public columns_hashing_impl::HashMethodBase< using Base = columns_hashing_impl::HashMethodBase; static constexpr bool has_cheap_key_calculation = true; + /// An unaligned load from the column's own memory. + static constexpr bool has_cheap_key_holder = true; static constexpr bool has_pre_computed_hashes = false; const char * vec; @@ -112,6 +149,8 @@ struct HashMethodOneNumberInRange : public columns_hashing_impl::HashMethodBase< static constexpr bool has_range_check = true; static constexpr bool has_cheap_key_calculation = true; + /// An unaligned load from the column's own memory. + static constexpr bool has_cheap_key_holder = true; const char * vec; FieldType min_key{}; @@ -175,6 +214,8 @@ struct HashMethodString : public columns_hashing_impl::HashMethodBase< using Base = columns_hashing_impl::HashMethodBase; static constexpr bool has_cheap_key_calculation = false; + /// A `string_view` over the column's own chars; the arena copy only happens on persist. + static constexpr bool has_cheap_key_holder = true; static constexpr bool has_pre_computed_hashes = false; const IColumn::Offset * offsets; @@ -235,6 +276,8 @@ struct HashMethodFixedString : public columns_hashing_impl::HashMethodBase< using Base = columns_hashing_impl::HashMethodBase; static constexpr bool has_cheap_key_calculation = false; + /// A `string_view` over the column's own chars; the arena copy only happens on persist. + static constexpr bool has_cheap_key_holder = true; static constexpr bool has_pre_computed_hashes = false; size_t n; @@ -307,6 +350,8 @@ struct HashMethodKeysFixed static constexpr bool has_low_cardinality = has_low_cardinality_; static constexpr bool has_cheap_key_calculation = true; + /// `packFixed` copies a few fixed-width fields into the key; no allocation. + static constexpr bool has_cheap_key_holder = true; static constexpr bool has_pre_computed_hashes = false; LowCardinalityKeys low_cardinality_keys; @@ -481,6 +526,8 @@ struct HashMethodHashed using Base = columns_hashing_impl::HashMethodBase; static constexpr bool has_cheap_key_calculation = false; + /// `hash128` SipHashes every key column through a virtual `IColumn::updateHashWithValue`. + static constexpr bool has_cheap_key_holder = false; static constexpr bool has_pre_computed_hashes = false; ColumnRawPtrs key_columns; diff --git a/src/Common/Crypto/X509Certificate.cpp b/src/Common/Crypto/X509Certificate.cpp index e66c7bee4d6f..a2009ddf3f4f 100644 --- a/src/Common/Crypto/X509Certificate.cpp +++ b/src/Common/Crypto/X509Certificate.cpp @@ -19,6 +19,9 @@ extern const int BAD_ARGUMENTS; X509Certificate::X509Certificate(X509 * cert_) : certificate(cert_) { + /// Every accessor dereferences the certificate, so a null pointer here turns into a segfault later. + if (!certificate) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot create a certificate from a null pointer"); } X509Certificate::operator X509 *() const diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 12dbc03ee70a..1ed80d3223b3 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -85,6 +85,7 @@ static struct InitFiu REGULAR(distributed_cache_fail_request_in_the_middle_of_request_always) \ REGULAR(file_cache_stall_free_space_ratio_keeping_thread) \ PAUSEABLE(file_cache_pause_before_do_eviction) \ + PAUSEABLE(file_segment_pause_before_write) \ REGULAR(file_cache_simulate_evicting_segment) \ REGULAR(cache_filesystem_failure) \ REGULAR(file_segment_range_writer_partial_write_then_network_error) \ @@ -210,6 +211,7 @@ static struct InitFiu ONCE(database_iceberg_gcs) \ REGULAR(rmt_delay_execute_drop_range) \ REGULAR(rmt_delay_commit_part) \ + PAUSEABLE_ONCE(rmt_pause_before_commit_local_part) \ ONCE(local_object_storage_network_error_during_remove) \ REGULAR(lightweight_show_tables) \ REGULAR(smt_part_update_duplicated_part) \ @@ -219,6 +221,8 @@ static struct InitFiu ONCE(oom_canary_force_oom_evidence) \ PAUSEABLE(truncate_database_tables_pause) \ REGULAR(datalake_try_get_table_return_nullptr) \ + REGULAR(datalake_simulate_missing_table_state) \ + REGULAR(datalake_get_tables_throw) \ PAUSEABLE_ONCE(drop_database_before_exclusive_ddl_lock) \ REGULAR(storage_merge_tree_background_schedule_merge_fail) \ REGULAR(patch_parts_reverse_column_order) \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 87e11d082281..aa7d152c9cd0 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -47,6 +47,8 @@ M(FailedInsertQuery, "Same as FailedQuery, but only for INSERT queries.", ValueType::Number) \ M(FailedAsyncInsertQuery, "Number of failed ASYNC INSERT queries.", ValueType::Number) \ M(ASTFuzzerQueries, "Number of fuzzed queries attempted by the server-side AST fuzzer.", ValueType::Number) \ + M(ASTFuzzerSkippedBackupRestore, "Number of fuzzed BACKUP/RESTORE queries the server-side AST fuzzer skipped instead of executing.", ValueType::Number) \ + M(ASTFuzzerSkippedReplicatedDDLInternal, "Number of times the server-side AST fuzzer skipped fuzzing because an internal replicated-database DDL execution (a live ZooKeeperMetadataTransaction) was in flight on the context.", ValueType::Number) \ M(QueryTimeMicroseconds, "Total time of all queries.", ValueType::Microseconds) \ M(SelectQueryTimeMicroseconds, "Total time of SELECT queries.", ValueType::Microseconds) \ M(InsertQueryTimeMicroseconds, "Total time of INSERT queries.", ValueType::Microseconds) \ @@ -1072,6 +1074,7 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheCheckCorrectness, "Number of times FileCache::assertCacheCorrectness was called", ValueType::Number) \ M(FilesystemCacheCheckCorrectnessMicroseconds, "How much time does FileCache::assertCacheCorrectness takes", ValueType::Microseconds) \ M(FileSegmentWaitMicroseconds, "Wait on DOWNLOADING state", ValueType::Microseconds) \ + M(FileSegmentWaitTimeouts, "Number of times waiting on a DOWNLOADING file segment timed out (see `filesystem_cache_wait_for_concurrent_download_timeout_milliseconds`)", ValueType::Number) \ M(FileSegmentCompleteMicroseconds, "Duration of FileSegment::complete() in filesystem cache", ValueType::Microseconds) \ M(FileSegmentLockMicroseconds, "Lock file segment time", ValueType::Microseconds) \ M(FileSegmentWriteMicroseconds, "File segment write() time", ValueType::Microseconds) \ @@ -1680,6 +1683,8 @@ The server successfully detected this situation and will download merged part fr M(JemallocFailedDeallocationSampleTracking, "Total number of times tracking of jemalloc deallocation sample failed", ValueType::Number) \ \ M(LoadedStatisticsMicroseconds, "Elapsed time of loading statistics from parts", ValueType::Microseconds) \ + M(SelectivityEstimatorInSetNotBuilt, "Number of `IN` conditions the selectivity estimator could not analyse because the set was not built yet, and it must not run the subquery to fill it", ValueType::Number) \ + M(SelectivityEstimatorInSetEstimatedFromSize, "Number of `IN` conditions whose selectivity was estimated from the size and bounds of the set instead of its exact ranges, because the set exceeds `statistics_max_set_size_for_exact_selectivity_estimation`", ValueType::Number) \ \ M(RuntimeDataflowStatisticsInputBytes, "Collected statistics on the number of bytes replicas would read if the query was executed with parallel replicas", ValueType::Number) \ M(RuntimeDataflowStatisticsOutputBytes, "Collected statistics on the number of bytes replicas would send to the initiator if the query was executed with parallel replicas", ValueType::Number) \ diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index acf67bfd5c89..1d8bcd8db0d9 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -587,22 +588,74 @@ void StackTrace::tryCapture() constexpr std::pair replacements[] = {{"::__1", ""}, {"std::basic_string, std::allocator>", "String"}}; -// Demangle @c symbol_name if it's not from __functional header (as such functions don't provide any useful -// information but pollute stack traces). -// Replace parts from @c replacements with shorter aliases -static String collapseDemangledNames(std::optional file, String symbol_name) +/// The type-erasing wrappers of `std::function`, spelled as they appear after @c replacements dropped +/// the `::__1` ABI namespace. These are the frames whose demangled names are pure noise: they spell out +/// the whole captured type and say nothing that the surrounding frames do not already say. Everything +/// else keeps its name, including a `std::` symbol that merely happens to live in a `__functional` +/// header (`std::hash`, `std::less`, ...) - such a frame is where the code really is, so its name is +/// the only useful part of it. In particular, the generic invocation helpers (`std::invoke`, +/// `std::__invoke`, `std::mem_fn`) are deliberately not here: they are not `std::function`-specific, +/// and a frame's name can name the callable they dispatch to - a single-frame prefix match cannot tell +/// a `std::function` trampoline from a direct use, so they keep their names. +constexpr std::string_view std_function_plumbing[] = { + "std::__function::", /// `__func`, `__value_func`, `__alloc_func`, `__policy_func`, `__policy_invoker` +}; + +/// The members of `std::function` itself that carry the noise: the type-erasing call operator, and the +/// constructors, the assignment operators and the destructor, which copy, move and destroy the captured +/// callable. Every other member (`swap`, `target`, `target_type`, `operator bool`, ...) does work of its +/// own and is a normal frame, so it keeps its name. The constructor is spelled both as +/// `function(std::function<` (the copy and move constructors - spelled with the argument so that the +/// default constructor `function()` and `function(std::nullptr_t)`, which merely create an empty object +/// and have short, informative names, are not caught) and as `function<` (the constructor taking a +/// callable, which is a function template, so its own template arguments follow the name: +/// `function(MyCallable&&)`); the assignment operator likewise as +/// `operator=(std::function<` (the copy and move assignment; `operator=(std::nullptr_t)` just resets the +/// object and stays visible) and as `operator=<` (the callable-taking overload: +/// `operator=(MyCallable&&)`). +constexpr std::string_view std_function_noisy_members[] + = {"operator()", "function(std::function<", "function<", "~function(", "operator=(std::function<", "operator=<"}; + +static bool isStdFunctionPlumbing(const String & symbol_name) { - if (symbol_name.empty()) - return "?"; + if (std::ranges::any_of(std_function_plumbing, [&](std::string_view prefix) { return symbol_name.starts_with(prefix); })) + return true; + + constexpr std::string_view std_function = "std::function<"; + if (!symbol_name.starts_with(std_function)) + return false; - if (file.has_value()) + /// Skip the template argument list to reach the member name: the signature of the callable can nest + /// its own `<` and `>`, so the closing bracket is the one that brings the depth back to zero. + size_t depth = 1; + size_t pos = std_function.size(); + for (; pos < symbol_name.size() && depth != 0; ++pos) { - std::string_view file_copy = file.value(); - if (auto trim_pos = file_copy.find_last_of('/'); trim_pos != std::string_view::npos) - file_copy.remove_suffix(file_copy.size() - trim_pos); - if (file_copy.ends_with("functional")) - return "?"; + if (symbol_name[pos] == '<') + ++depth; + else if (symbol_name[pos] == '>') + --depth; } + if (depth != 0) + return false; + + std::string_view member{symbol_name}; + member.remove_prefix(pos); + if (!member.starts_with("::")) + return false; + member.remove_prefix(2); + + return std::ranges::any_of(std_function_noisy_members, [&](std::string_view noisy) { return member.starts_with(noisy); }); +} + +// Hide the name of `std::function` plumbing frames (the `__func`/`__value_func`/`__policy_func` +// trampolines from libc++'s `__functional` headers): their demangled names are huge - they spell out +// the whole captured lambda type - and they say nothing that the surrounding frames don't already say. +// Replace parts from @c replacements with shorter aliases +String StackTrace::collapseDemangledNames(std::optional file, String symbol_name) +{ + if (symbol_name.empty()) + return "?"; // TODO myrrc surely there is a written version already for better in place search&replace for (auto [needle, to] : replacements) @@ -615,6 +668,22 @@ static String collapseDemangledNames(std::optional file, Strin } } + /// The file of a frame is the source line the *instruction* maps to, which is not necessarily + /// where the enclosing function is defined: a compiler-generated or inlined `std::function` + /// operation puts a line-table entry pointing into `__functional` in the middle of an ordinary + /// function. Requiring the symbol to name the plumbing as well keeps the frame of such a function + /// named - it is the only useful part of the frame, and dropping it left `trace_full` in + /// `system.crash_log` with a bare `?` for the frame that actually crashed. This is much more + /// likely in a ThinLTO build, where `std::function` calls are inlined across translation units. + if (file.has_value() && isStdFunctionPlumbing(symbol_name)) + { + std::string_view file_copy = file.value(); + if (auto trim_pos = file_copy.find_last_of('/'); trim_pos != std::string_view::npos) + file_copy.remove_suffix(file_copy.size() - trim_pos); + if (file_copy.ends_with("functional")) + return "?"; + } + return symbol_name; } @@ -675,7 +744,7 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s } if (frame.symbol.has_value()) - out << collapseDemangledNames(frame.file, frame.symbol.value()); + out << StackTrace::collapseDemangledNames(frame.file, frame.symbol.value()); else out << "?"; diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 3f90cd8bcd05..c3bccf8bd7e6 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -78,6 +78,11 @@ class StackTrace /// Please note: addresses are also available in the system.stack_trace and system.trace_log tables. static void setShowAddresses(bool show); + /// Renders the demangled name of a frame for display: shortens well-known libc++ spellings, and returns + /// "?" for frames whose name carries no information. @param file is the source location of the frame. + /// Public only so that it can be unit tested; use @c toStringEveryLine to format a stack trace. + static String collapseDemangledNames(std::optional file, String symbol_name); + protected: void tryCapture(); diff --git a/src/Common/WKB.cpp b/src/Common/WKB.cpp index b74c0e57a8a4..dac897aed622 100644 --- a/src/Common/WKB.cpp +++ b/src/Common/WKB.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -153,6 +154,10 @@ static MultiPolygon readMultiPolygonWKB(ReadBuffer & in_buffer, GeometricObject parseWKBFormat(ReadBuffer & in_buffer, UInt32 max_element_count) { + /// Multi* geometries contain nested geometries, and the nesting is not bounded by the element + /// count checks: every level can hold a single element. Guard the recursion against a stack overflow. + checkStackSize(); + UInt32 limit = effectiveLimit(max_element_count); char little_endian = 0; diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index 75cbe1083d86..b83816f3d367 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -1093,9 +1093,70 @@ size_t ZooKeeperErrorResponse::sizeImpl() const return Coordination::size(error); } +ZooKeeperRequestPtr ZooKeeperRequest::cloneForMulti(const ACLs &) const +{ + throw Exception::fromMessage(Error::ZBADARGUMENTS, "Illegal command as part of multi ZooKeeper request"); +} + +ZooKeeperRequestPtr ZooKeeperCreateRequest::cloneForMulti(const ACLs & default_acls) const +{ + auto result = std::make_shared(*this); + if (result->acls.empty()) + result->acls = default_acls; + return result; +} + +std::optional ZooKeeperMultiRequest::getOperationType(OpNum op_num) +{ + switch (op_num) + { + case OpNum::Create: + case OpNum::Create2: + case OpNum::CreateTTL: + case OpNum::CreateIfNotExists: + case OpNum::Remove: + case OpNum::TryRemove: + case OpNum::RemoveRecursive: + case OpNum::Set: + case OpNum::Check: + case OpNum::CheckNotExists: + case OpNum::CheckStat: + return OperationType::Write; + + case OpNum::Get: + case OpNum::Exists: + case OpNum::SimpleList: + case OpNum::List: + case OpNum::FilteredList: + case OpNum::FilteredListWithStatsAndData: + case OpNum::ListRecursive: + return OperationType::Read; + + case OpNum::Close: + case OpNum::Error: + case OpNum::GetACL: + case OpNum::SetACL: + case OpNum::Sync: + case OpNum::Heartbeat: + case OpNum::Multi: + case OpNum::Reconfig: + case OpNum::CheckWatch: + case OpNum::RemoveWatch: + case OpNum::MultiRead: + case OpNum::Auth: + case OpNum::SetWatch: + case OpNum::SetWatch2: + case OpNum::AddWatch: + case OpNum::SessionID: + return std::nullopt; + } +} + void ZooKeeperMultiRequest::checkOperationType(OperationType type) { - chassert(!operation_type.has_value() || *operation_type == type); + if (operation_type.has_value() && *operation_type != type) + throw Exception::fromMessage(Error::ZBADARGUMENTS, "Cannot mix read and write commands in a multi ZooKeeper request"); + operation_type = type; } @@ -1114,67 +1175,18 @@ ZooKeeperMultiRequest::ZooKeeperMultiRequest(std::span(generic_request.get())) - { - checkOperationType(Write); - auto create = std::make_shared(*concrete_request_create); - if (create->acls.empty()) - create->acls = default_acls; - requests.push_back(create); - } - else if (const auto * concrete_request_remove = dynamic_cast(generic_request.get())) - { - checkOperationType(Write); - requests.push_back(std::make_shared(*concrete_request_remove)); - } - else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) - { - checkOperationType(Write); - requests.push_back(std::make_shared(*concrete_request_remove_recursive)); - } - else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) - { - checkOperationType(Write); - requests.push_back(std::make_shared(*concrete_request_set)); - } - else if (const auto * concrete_request_check = dynamic_cast(generic_request.get())) - { - checkOperationType(Write); - requests.push_back(std::make_shared(*concrete_request_check)); - } - else if (const auto * concrete_request_get = dynamic_cast(generic_request.get())) - { - checkOperationType(Read); - requests.push_back(std::make_shared(*concrete_request_get)); - } - else if (const auto * concrete_request_exists = dynamic_cast(generic_request.get())) - { - checkOperationType(Read); - requests.push_back(std::make_shared(*concrete_request_exists)); - } - else if (const auto * concrete_request_simple_list = dynamic_cast(generic_request.get())) - { - checkOperationType(Read); - requests.push_back(std::make_shared(*concrete_request_simple_list)); - } - else if (const auto * concrete_request_list_recursive = dynamic_cast(generic_request.get())) - { - checkOperationType(Read); - requests.push_back(std::make_shared(*concrete_request_list_recursive)); - } - else if (const auto * concrete_request_list = dynamic_cast(generic_request.get())) - { - checkOperationType(Read); - if (const auto * with_stats = dynamic_cast(concrete_request_list)) - requests.push_back(std::make_shared(*with_stats)); - else - requests.push_back(std::make_shared(*concrete_request_list)); - } - else + const auto * zk_request = dynamic_cast(generic_request.get()); + if (!zk_request) + throw Exception::fromMessage(Error::ZBADARGUMENTS, "Illegal command as part of multi ZooKeeper request"); + + const auto type = getOperationType(zk_request->getOpNum()); + if (!type) throw Exception::fromMessage(Error::ZBADARGUMENTS, "Illegal command as part of multi ZooKeeper request"); + + checkOperationType(*type); + requests.push_back(zk_request->cloneForMulti(default_acls)); } } @@ -1249,6 +1261,13 @@ void ZooKeeperMultiRequest::readImpl(ReadBuffer & in, RequestValidator request_v ZooKeeperRequestPtr request = ZooKeeperRequestFactory::instance().get(op_num); request->readImpl(in); + + const auto type = getOperationType(request->getOpNum()); + if (!type) + throw Exception::fromMessage(Error::ZBADARGUMENTS, "Illegal command as part of multi ZooKeeper request"); + + checkOperationType(*type); + if (request_validator) request_validator(*request); requests.push_back(request); diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index d98fa27b56a4..50223f8c1cb6 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -88,6 +88,7 @@ struct ZooKeeperRequest : virtual Request virtual ZooKeeperResponsePtr makeResponse() const = 0; virtual bool isReadRequest() const = 0; + virtual std::shared_ptr cloneForMulti(const ACLs & default_acls) const; virtual void createLogElements(LogElements & elems) const; }; @@ -259,6 +260,7 @@ struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return false; } + ZooKeeperRequestPtr cloneForMulti(const ACLs & default_acls) const override; size_t bytesSize() const override { return CreateRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); } @@ -320,6 +322,10 @@ struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return false; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return RemoveRequest::bytesSize() + sizeof(xid); } @@ -355,6 +361,10 @@ struct ZooKeeperRemoveRecursiveRequest final : RemoveRecursiveRequest, ZooKeeper ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return false; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return RemoveRecursiveRequest::bytesSize() + sizeof(xid); } }; @@ -382,6 +392,10 @@ struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return true; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return ExistsRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); } }; @@ -411,6 +425,10 @@ struct ZooKeeperGetRequest final : GetRequest, ZooKeeperRequest ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return true; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return GetRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); } }; @@ -439,6 +457,10 @@ struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest std::string toStringImpl(bool short_format) const override; ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return false; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return SetRequest::bytesSize() + sizeof(xid); } @@ -469,6 +491,10 @@ struct ZooKeeperListRequest : ListRequest, ZooKeeperRequest std::string toStringImpl(bool short_format) const override; ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return true; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return ListRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); } }; @@ -477,6 +503,10 @@ struct ZooKeeperSimpleListRequest final : ZooKeeperListRequest { OpNum getOpNum() const override { return OpNum::SimpleList; } ZooKeeperResponsePtr makeResponse() const override; + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } }; struct ZooKeeperFilteredListRequest : ZooKeeperListRequest @@ -488,6 +518,10 @@ struct ZooKeeperFilteredListRequest : ZooKeeperListRequest size_t sizeImpl() const override; void readImpl(ReadBuffer & in) override; std::string toStringImpl(bool short_format) const override; + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return ZooKeeperListRequest::bytesSize() + sizeof(list_request_type); } }; @@ -505,6 +539,10 @@ struct ZooKeeperFilteredListWithStatsAndDataRequest final : ZooKeeperFilteredLis void readImpl(ReadBuffer & in) override; std::string toStringImpl(bool short_format) const override; ZooKeeperResponsePtr makeResponse() const override; + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return ZooKeeperFilteredListRequest::bytesSize() + sizeof(with_stat) + sizeof(with_data); } }; @@ -553,6 +591,10 @@ struct ZooKeeperCheckRequest : CheckRequest, ZooKeeperRequest ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return true; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return CheckRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); } @@ -785,6 +827,7 @@ struct ZooKeeperMultiRequest final : MultiRequest, ZooKeepe std::optional operation_type; private: + static std::optional getOperationType(OpNum op_num); void checkOperationType(OperationType type); }; @@ -876,6 +919,10 @@ struct ZooKeeperListRecursiveRequest final : ListRecursiveRequest, ZooKeeperRequ ZooKeeperResponsePtr makeResponse() const override; bool isReadRequest() const override { return true; } + ZooKeeperRequestPtr cloneForMulti(const ACLs &) const override + { + return std::make_shared(*this); + } size_t bytesSize() const override { return ListRecursiveRequest::bytesSize() + sizeof(xid); } }; diff --git a/src/Common/ZooKeeper/tests/gtest_zookeeper.cpp b/src/Common/ZooKeeper/tests/gtest_zookeeper.cpp index 5a989e5932f6..6f3e9a07e1f5 100644 --- a/src/Common/ZooKeeper/tests/gtest_zookeeper.cpp +++ b/src/Common/ZooKeeper/tests/gtest_zookeeper.cpp @@ -1,3 +1,8 @@ +#include +#include + +#include + #include #include @@ -13,3 +18,20 @@ TEST(ZooKeeperTest, TestMatchPath) ASSERT_EQ(matchPath("/path", "/path/"), PathMatchResult::EXACT); ASSERT_EQ(matchPath("/path/", "/path"), PathMatchResult::EXACT); } + +TEST(ZooKeeperTest, MultiRequestRejectsCloseSubrequest) +{ + using namespace Coordination; + + WriteBufferFromOwnString out; + write(OpNum::Close, out); + write(false, out); + write(-1, out); + write(OpNum::Error, out); + write(true, out); + write(-1, out); + + auto request = ZooKeeperRequestFactory::instance().get(OpNum::Multi); + ReadBufferFromString in(out.str()); + EXPECT_THROW(request->readImpl(in), Coordination::Exception); +} diff --git a/src/Common/parseRemoteDescription.cpp b/src/Common/parseRemoteDescription.cpp index 7ec5996a3f9e..f147bfc29874 100644 --- a/src/Common/parseRemoteDescription.cpp +++ b/src/Common/parseRemoteDescription.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -55,6 +56,10 @@ static bool parseNumber(const String & description, size_t l, size_t r, size_t & std::vector parseRemoteDescription( const String & description, size_t l, size_t r, char separator, size_t max_addresses, const String & func_name) { + /// Nested braces are parsed recursively, and `max_addresses` bounds the number of generated + /// addresses, not the nesting depth: `{{{{...,...}}}}` recurses once per level. + checkStackSize(); + std::vector res; std::vector cur; diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index 4959fb535647..64dd24bba970 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -155,6 +155,7 @@ namespace DB M(SESSION_CLEANUP, "SessionCleanup") \ M(SEND_TO_SHELL_CMD, "SendToShellCmd") \ M(SIGNAL_LISTENER, "SignalListnr") \ + M(SNAPSHOT_ASYNC, "SnapshotAsync") \ M(SSH_HANDLER, "SSHHandler") \ M(SUGGEST, "Suggest") \ M(SYSTEM_LOG_FLUSH, "SystemLogFlush") \ diff --git a/src/Common/tests/gtest_stack_trace_collapse_names.cpp b/src/Common/tests/gtest_stack_trace_collapse_names.cpp new file mode 100644 index 000000000000..098d96abccb1 --- /dev/null +++ b/src/Common/tests/gtest_stack_trace_collapse_names.cpp @@ -0,0 +1,124 @@ +#include + +#include + +namespace +{ + +/// A frame inside libc++'s `std::function` plumbing. Its name spells out the whole captured type and +/// tells nothing that the neighbouring frames do not, so only "?" is displayed for it. +const String std_function_trampoline + = "std::__1::__function::__func>, bool)::$_0, " + "std::__1::allocator>, bool)::$_0>, void ()>::operator()()"; + +const String execute_query = "DB::executeQuery(std::__1::basic_string_view>, " + "std::__1::shared_ptr, DB::QueryFlags, DB::QueryProcessingStage::Enum)"; + +constexpr std::string_view function_h = "./contrib/llvm-project/libcxx/include/__functional/function.h"; +constexpr std::string_view invoke_h = "./contrib/llvm-project/libcxx/include/__functional/invoke.h"; +constexpr std::string_view hash_h = "./contrib/llvm-project/libcxx/include/__functional/hash.h"; +constexpr std::string_view execute_query_cpp = "./src/Interpreters/executeQuery.cpp"; + +} + +TEST(StackTraceCollapseNames, HidesStdFunctionPlumbing) +{ + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, std_function_trampoline), "?"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::__function::__value_func::operator()() const"), "?"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::operator()() const"), "?"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function)>::function(std::__1::function)> const&)"), "?"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::~function()"), "?"); + /// The constructor that takes a callable is a function template, so its own template arguments follow + /// the member name - this is the "construct from a lambda" frame, the most common one of them all. + EXPECT_EQ( + StackTrace::collapseDemangledNames( + function_h, "std::__1::function::function(DB::AsyncLoader::AsyncLoader()::$_0&&)"), + "?"); + /// The assignment operators do the same type-erasing work: the copy and move assignment, and the + /// callable-taking overload, which is a function template just like the corresponding constructor - + /// this is the "assign a lambda" frame of `f = [capture] { ... };`. + EXPECT_EQ( + StackTrace::collapseDemangledNames( + function_h, "std::__1::function::operator=(std::__1::function&&)"), + "?"); + EXPECT_EQ( + StackTrace::collapseDemangledNames( + function_h, "std::__1::function::operator=(DB::AsyncLoader::AsyncLoader()::$_0&&)"), + "?"); +} + +/// Only the type erasure of `std::function` is noise. Its other members do work of their own, so a frame +/// of one of them names the code that is actually running and must keep its name. +TEST(StackTraceCollapseNames, KeepsOrdinaryStdFunctionMembers) +{ + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::swap(std::__1::function&)"), + "std::function::swap(std::function&)"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::target_type() const"), + "std::function::target_type() const"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::operator bool() const"), + "std::function::operator bool() const"); + /// Unlike the copy and move assignment, assigning `nullptr` does not copy a captured callable around: + /// it just resets the object, and its name is short and says exactly that, so it stays visible. + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::operator=(std::nullptr_t)"), + "std::function::operator=(std::nullptr_t)"); + /// The same goes for construction of an empty `std::function`: the default constructor and the + /// `nullptr` one do not type-erase a callable, and their names are just as short and informative. + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::function()"), + "std::function::function()"); + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, "std::__1::function::function(std::nullptr_t)"), + "std::function::function(std::nullptr_t)"); +} + +/// The file of a frame is the source line the faulting instruction maps to, and an ordinary function can +/// have individual instructions attributed to a libc++ `__functional` header - an inlined `std::function` +/// operation, or compiler-generated code reported with line 0. Such a frame must keep its own name: it is +/// the only useful part of the frame. Getting this wrong left the frame that actually crashed displayed as +/// a bare `?` in `system.crash_log` and in the fatal log, in builds with ThinLTO enabled. +TEST(StackTraceCollapseNames, KeepsOrdinaryFunctionAttributedToFunctionalHeader) +{ + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, execute_query), + "DB::executeQuery(std::basic_string_view>, std::shared_ptr, " + "DB::QueryFlags, DB::QueryProcessingStage::Enum)"); +} + +/// Not every symbol that lives in a `__functional` header is `std::function` plumbing: libc++ puts +/// `std::hash`, `std::less`, `std::identity` and friends there too, as well as the generic invocation +/// helpers `std::invoke` / `std::__invoke` / `std::mem_fn`, which are used far beyond `std::function`. +/// A frame of one of those is where the code really is, and its name can name the callable it +/// dispatches to, so it must keep its name - only the type-erasing wrappers of `std::function` are noise. +TEST(StackTraceCollapseNames, KeepsMeaningfulStdSymbolFromFunctionalHeader) +{ + EXPECT_EQ( + StackTrace::collapseDemangledNames( + hash_h, "std::__1::hash, std::__1::allocator>>::operator()"), + "std::hash::operator()"); + EXPECT_EQ(StackTrace::collapseDemangledNames(hash_h, "std::__1::__murmur2_or_cityhash::operator()"), + "std::__murmur2_or_cityhash::operator()"); + EXPECT_EQ(StackTrace::collapseDemangledNames(invoke_h, "std::__1::__invoke(DB::AsyncLoader::Pool&)"), + "std::__invoke(DB::AsyncLoader::Pool&)"); + EXPECT_EQ(StackTrace::collapseDemangledNames(invoke_h, "std::__1::invoke(void (&)(int), int&&)"), + "std::invoke(void (&)(int), int&&)"); +} + +TEST(StackTraceCollapseNames, ShortensKnownSpellings) +{ + /// `::__1` is dropped, which also turns the libc++ spelling of `std::string` into the one that is + /// then collapsed to `String`. + EXPECT_EQ(StackTrace::collapseDemangledNames(execute_query_cpp, "DB::f(std::__1::vector)"), "DB::f(std::vector)"); + EXPECT_EQ( + StackTrace::collapseDemangledNames( + execute_query_cpp, + "DB::f(std::__1::basic_string, std::__1::allocator> const&)"), + "DB::f(String const&)"); +} + +TEST(StackTraceCollapseNames, EmptyAndMissingFile) +{ + EXPECT_EQ(StackTrace::collapseDemangledNames(function_h, ""), "?"); + EXPECT_EQ(StackTrace::collapseDemangledNames(std::nullopt, ""), "?"); + EXPECT_EQ(StackTrace::collapseDemangledNames(std::nullopt, "DB::f()"), "DB::f()"); + /// A frame with no file at all is not suppressed - there is nothing to recognise it by. + EXPECT_NE(StackTrace::collapseDemangledNames(std::nullopt, std_function_trampoline), "?"); +} diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index 1e91b3408bbd..b32d51ef27e4 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -164,6 +164,10 @@ static void readHeaderAndGetCodecAndSize( throw Exception(ErrorCodes::TOO_LARGE_SIZE_COMPRESSED, "Too large size_compressed_without_checksum: {}. " "Most likely corrupted data.", size_compressed_without_checksum); + if (size_decompressed > DBMS_MAX_DECOMPRESSED_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_SIZE_COMPRESSED, "Too large size_decompressed: {}. " + "Most likely corrupted data.", size_decompressed); + if (size_compressed_without_checksum < header_size) throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, "Can't decompress data: " "the compressed data size ({}, this should include header size) is less than the header size ({})", diff --git a/src/Compression/CompressionInfo.h b/src/Compression/CompressionInfo.h index aab45dc9f36e..89a3f9969b12 100644 --- a/src/Compression/CompressionInfo.h +++ b/src/Compression/CompressionInfo.h @@ -6,6 +6,13 @@ constexpr uint64_t DBMS_MAX_COMPRESSED_SIZE = 0x40000000ULL; /// 1GB +/** The decompressed size of a block is taken from the block header, and the buffer for it is + * allocated before the block is decompressed, so it has to be bounded as well: a block that + * declares a huge decompressed size makes the reader allocate that much from a tiny payload. + * Blocks are written with `max_compress_block_size`, which is 1 MB by default. + */ +constexpr uint64_t DBMS_MAX_DECOMPRESSED_SIZE = 0x40000000ULL; /// 1GB + /** one byte for method, 4 bytes for compressed size, 4 bytes for uncompressed size */ constexpr uint8_t COMPRESSED_BLOCK_HEADER_SIZE = 9; diff --git a/src/Compression/LZ4_decompress_faster.cpp b/src/Compression/LZ4_decompress_faster.cpp index cd1c2c1d960a..dd529c8f1b3e 100644 --- a/src/Compression/LZ4_decompress_faster.cpp +++ b/src/Compression/LZ4_decompress_faster.cpp @@ -639,9 +639,14 @@ bool decompress( size_t dest_size, [[maybe_unused]] PerformanceStatistics & statistics) { - if (source_size == 0 || dest_size == 0) + if (dest_size == 0) return true; + /// There is nothing to decompress from, but the caller expects `dest_size` bytes to be written, + /// and would otherwise hand out the previous contents of the destination buffer. + if (source_size == 0) + return false; + /// When a specific method is forced, always use it regardless of block size. /// The size threshold below only applies to the adaptive bandit algorithm /// where timing very small blocks would add too much noise. diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 303fe0b2f25c..1a138a83be4b 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -1226,6 +1226,9 @@ using AliasMap = std::unordered_map; /** Find setting index by name. Returns -1 if not found. */ \ size_t find(std::string_view name) const; \ \ + /** Find setting index by its byte offset within Data (as stored in SettingIndex). Returns -1 if not found. */ \ + size_t findByOffset(size_t data_offset) const; \ + \ /* Metadata accessors (by index) */ \ const String & getName(size_t index) const { return field_infos[index].name; } \ std::string_view getPath(size_t index) const { return field_infos[index].path; } \ @@ -1333,6 +1336,7 @@ using AliasMap = std::unordered_map; \ std::vector field_infos; /* Metadata for all settings */ \ std::unordered_map name_to_index_map; /* Fast name -> index lookup */ \ + std::unordered_map offset_to_index_map; /* Fast data offset -> index lookup */ \ /* Canonical default-constructed instance. Used to reset individual settings to their */ \ /* declared defaults via a typed copy (see resetValueToDefault) and to read the default */ \ /* string representation (see getDefaultValueString). Initialized once via the tag */ \ @@ -1483,11 +1487,12 @@ using AliasMap = std::unordered_map; LIST_OF_SETTINGS_WITHOUT_PATH_MACRO(IMPLEMENT_SETTINGS_TRAITS_, IMPLEMENT_SETTINGS_TRAITS_) \ LIST_OF_SETTINGS_WITH_PATH_MACRO(IMPLEMENT_SETTINGS_TRAITS_WITH_PATH_, IMPLEMENT_SETTINGS_TRAITS_WITH_PATH_) \ _Pragma("clang diagnostic pop") \ - /* Build name -> index map for fast lookups */ \ + /* Build name -> index and data offset -> index maps for fast lookups */ \ for (size_t i = 0, size = res.field_infos.size(); i < size; ++i) \ { \ const auto & info = res.field_infos[i]; \ res.name_to_index_map.emplace(info.name, i); \ + res.offset_to_index_map.emplace(info.data_offset, i); \ } \ return res; \ }(); \ @@ -1503,6 +1508,14 @@ using AliasMap = std::unordered_map; return it->second; \ return static_cast(-1); \ } \ + \ + size_t SETTINGS_TRAITS_NAME::Accessor::findByOffset(size_t data_offset) const \ + { \ + auto it = offset_to_index_map.find(data_offset); \ + if (it != offset_to_index_map.end()) \ + return it->second; \ + return static_cast(-1); \ + } \ /// Generate a FieldInfo entry for a setting without a config path. diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 25d20278a0f4..7c2d53da1c61 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -1,8 +1,13 @@ #include #include +#include #include +#include +#include #include #include +#include +#include #include #include #include @@ -62,6 +67,168 @@ static const IColumn * getActualColumn(const IColumn * column) return actual_column; } +/// Compares the structure of two columns. Aggregate-state columns whose functions have the same +/// state representation (e.g. `quantileState` and `quantilesState(0.9)`) are compatible even +/// though their names differ, and this relaxation must apply at any nesting depth: an expression +/// over a `UNION` of such states can wrap them into another column (e.g. into a `Tuple`), and the +/// per-branch headers then differ only by the aggregate function nested inside. For all other +/// columns the comparison is as strict as comparing full column names. +static bool haveCompatibleColumnStructure(const IColumn & actual, const IColumn & expected) +{ + /// A `Sparse` column is structurally interchangeable with the full column of the same type it + /// wraps, and this holds at any depth, not only at the top level: one branch can materialize a + /// nested subcolumn (`recursiveRemoveSparse`) while another keeps it sparse. Unwrap `Sparse` on + /// either side independently, mirroring the top-level unwrap in `checkColumnStructure`. + if (const auto * actual_sparse = typeid_cast(&actual)) + return haveCompatibleColumnStructure(actual_sparse->getValuesColumn(), expected); + if (const auto * expected_sparse = typeid_cast(&expected)) + return haveCompatibleColumnStructure(actual, expected_sparse->getValuesColumn()); + + const auto * actual_agg = typeid_cast(&actual); + const auto * expected_agg = typeid_cast(&expected); + if (actual_agg && expected_agg) + return actual_agg->getAggregateFunction()->haveSameStateRepresentation(*expected_agg->getAggregateFunction()); + + if (typeid(actual) != typeid(expected)) + return false; + + /// `Variant` is compositional too: its type equality (checked before this point) already + /// fixes the set and the global order of alternatives, so the only thing that can differ + /// between two structurally-equal-typed `Variant` columns is an aggregate state nested + /// inside an alternative, which is exactly what we want to relax. But the local order of + /// the nested variant columns (the order `forEachSubcolumn` iterates them in) is a property + /// of a particular column, not of the type, and `getName` lists the variants in the global + /// order — so compare the alternatives pairwise by global discriminator, like `getName` does. + if (const auto * actual_variant = typeid_cast(&actual)) + { + const auto & expected_variant = assert_cast(expected); + const size_t num_variants = actual_variant->getNumVariants(); + if (num_variants != expected_variant.getNumVariants()) + return false; + + for (size_t global_discr = 0; global_discr < num_variants; ++global_discr) + if (!haveCompatibleColumnStructure( + actual_variant->getVariantByGlobalDiscriminator(global_discr), + expected_variant.getVariantByGlobalDiscriminator(global_discr))) + return false; + + return true; + } + + /// `Replicated` is compositional too, but only its nested column is structural: the internal + /// indexes column is a variable-width encoding detail (`UInt8` .. `UInt64`, widened lazily) + /// that `getName` does not include, yet `forEachSubcolumn` exposes — so descending into all + /// subcolumns would make the check stricter than the name comparison and reject a valid + /// runtime block against its header. Compare only the nested column, like + /// `ColumnReplicated::structureEquals` does. + if (const auto * actual_replicated = typeid_cast(&actual)) + { + const auto & expected_replicated = assert_cast(expected); + return haveCompatibleColumnStructure(*actual_replicated->getNestedColumn(), *expected_replicated.getNestedColumn()); + } + + /// Descend only into the plain container columns whose name is a pure composition of the + /// nested column names, so that for everything else the comparison stays exactly as strict + /// as comparing full column names. `Dynamic`/`Object` are deliberately left strict because + /// their nested structure is not fixed by the type. + const bool is_compositional = typeid_cast(&actual) || typeid_cast(&actual) + || typeid_cast(&actual) || typeid_cast(&actual) + || typeid_cast(&actual); + + if (!is_compositional) + return actual.getName() == expected.getName(); + + std::vector actual_children; + std::vector expected_children; + actual.forEachSubcolumn([&](const auto & subcolumn) { actual_children.push_back(subcolumn.get()); }); + expected.forEachSubcolumn([&](const auto & subcolumn) { expected_children.push_back(subcolumn.get()); }); + + if (actual_children.size() != expected_children.size()) + return false; + + for (size_t i = 0; i < actual_children.size(); ++i) + if (!haveCompatibleColumnStructure(*actual_children[i], *expected_children[i])) + return false; + + return true; +} + +static bool haveCompatibleConstantValues(const Field & actual, const Field & expected, bool strict_aggregate_states); + +static bool haveCompatibleConstantValueVectors(const FieldVector & actual, const FieldVector & expected, bool strict_aggregate_states) +{ + if (actual.size() != expected.size()) + return false; + + for (size_t i = 0; i < actual.size(); ++i) + if (!haveCompatibleConstantValues(actual[i], expected[i], strict_aggregate_states)) + return false; + + return true; +} + +/// Compares two constant values, relaxing only the aggregate-state leaves: the `Field` comparison +/// of aggregate states throws when the aggregate function type names differ, even when the states +/// are compatible by `haveSameStateRepresentation` (which the type and column structure checks +/// have already established at this point). For such leaves compare only the serialized state, so +/// that genuinely different constants — including a differing non-aggregate element next to a +/// compatible aggregate state inside the same `Tuple` — are still reported as a mismatch. +/// +/// The relaxation is only sound while the type of a value is fully determined by the column type, +/// which is not the case under `Variant`, `Dynamic` and `JSON` — see `typeCanHideTheValueType`. +/// For those, `strict_aggregate_states` also requires the aggregate function names to be equal +/// (compared field by field, because `Field::operator ==` throws for differing names). +static bool haveCompatibleConstantValues(const Field & actual, const Field & expected, bool strict_aggregate_states) +{ + if (actual.getType() != expected.getType()) + return false; + + switch (actual.getType()) + { + case Field::Types::AggregateFunctionState: + { + const auto & actual_state = actual.safeGet(); + const auto & expected_state = expected.safeGet(); + if (strict_aggregate_states && actual_state.name != expected_state.name) + return false; + return actual_state.data == expected_state.data; + } + case Field::Types::Array: + return haveCompatibleConstantValueVectors(actual.safeGet(), expected.safeGet(), strict_aggregate_states); + case Field::Types::Tuple: + return haveCompatibleConstantValueVectors(actual.safeGet(), expected.safeGet(), strict_aggregate_states); + case Field::Types::Map: + return haveCompatibleConstantValueVectors(actual.safeGet(), expected.safeGet(), strict_aggregate_states); + default: + return actual == expected; + } +} + +/// Whether a value of this type is converted to a `Field` that no longer tells which type the +/// value actually has. A `Variant` flattens a row to the `Field` of its active alternative +/// (`ColumnVariant::operator []`) and `DataTypeVariant::equals` allows several aggregate-state +/// alternatives that are compatible by state representation; `Dynamic` and `JSON` similarly store +/// values of types that are not fixed by the column type. Two values on different alternatives can +/// then produce equal `Field`s although the alternative itself is a part of the value and is +/// observable (e.g. by `variantType`), so the aggregate-state relaxation must not apply inside them. +/// +/// Only these three types are checked, not `IDataType::hasDynamicSubcolumns`: the latter is also +/// true for a plain `Map`, which merely exposes the `m.keys` and `m.values` virtual subcolumns while +/// the type of every value it holds is still fixed by the declared `Map(K, V)`. +static bool typeCanHideTheValueType(const IDataType & type) +{ + if (isVariant(type) || isDynamic(type) || isObject(type)) + return true; + + bool result = false; + type.forEachChild([&](const IDataType & child) + { + result = result || typeCanHideTheValueType(child); + }); + + return result; +} + template static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, const ColumnWithTypeAndName & expected, std::string_view context_description, bool allow_materialize, int code) @@ -97,19 +264,7 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con expected_column = getActualColumn(expected_column); } - const auto * actual_column_maybe_agg = typeid_cast(actual_column); - const auto * expected_column_maybe_agg = typeid_cast(expected_column); - - if (actual_column_maybe_agg && expected_column_maybe_agg) - { - if (!actual_column_maybe_agg->getAggregateFunction()->haveSameStateRepresentation(*expected_column_maybe_agg->getAggregateFunction())) - return onError(code, - "Block structure mismatch in {} stream: different columns:\n{}\n{}", - context_description, - actual.dumpStructure(), - expected.dumpStructure()); - } - else if (actual_column->getName() != expected_column->getName()) + if (!haveCompatibleColumnStructure(*actual_column, *expected_column)) { return onError(code, "Block structure mismatch in {} stream: different columns:\n{}\n{}", @@ -125,7 +280,10 @@ static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, con Field actual_value = assert_cast(*actual.column).getField(); Field expected_value = assert_cast(*expected.column).getField(); - if (actual_value != expected_value) + /// The types are already checked to be equal at this point. + const bool strict_aggregate_states = actual.type && typeCanHideTheValueType(*actual.type); + + if (!haveCompatibleConstantValues(actual_value, expected_value, strict_aggregate_states)) return onError(code, "Block structure mismatch in {} stream: different values of constants in column '{}': actual: {}, expected: {}", context_description, diff --git a/src/Core/Defines.h b/src/Core/Defines.h index b351507566e8..82d00163228a 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -147,6 +147,13 @@ static constexpr auto DEFAULT_REMOVE_SHARED_RECURSIVE_FILE_LIMIT = 1000uz; static constexpr auto DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS = 1'000'000uz; -static constexpr auto DEFAULT_NATIVE_BINARY_MAX_NUM_ROWS = 1'000'000'000'000uz; +/// The row count of a block is read from the wire before the block data, and the bulk +/// deserialization resizes the column to it before reading, so a block header declaring a huge row +/// count allocates that much from a payload that may be a few bytes. A block of a billion rows is +/// already two orders of magnitude larger than anything ClickHouse produces. +/// +/// Not `uz`: the value does not fit into `size_t` on 32-bit platforms, and it is compared against +/// a row count read from the wire as `UInt64`. +static constexpr auto DEFAULT_NATIVE_BINARY_MAX_NUM_ROWS = 1'000'000'000ULL; } diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 6276bd58f068..10aaa505a3cf 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -310,7 +310,7 @@ bool Field::operator<= (const Field & rhs) const { static constexpr int nan_direction_hint = 1; /// Put NaN at the end Float64 f1 = get(); - Float64 f2 = get(); + Float64 f2 = rhs.get(); return FloatCompareHelper::less(f1, f2, nan_direction_hint) || FloatCompareHelper::equals(f1, f2, nan_direction_hint); } diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 0d8374948fce..0b2397190264 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -176,8 +177,17 @@ class MessageTransport template std::unique_ptr receiveWithPayloadSize(Int32 payload_size) { + if (payload_size < 0) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Negative payload size {} received from client", payload_size); + std::unique_ptr message = std::make_unique(payload_size); - message->deserialize(*in); + + /// The message is parsed with a buffer limited to the declared payload size, so that parsing + /// cannot read past the end of the message. Otherwise a client could declare a small message + /// and then stream data without a terminator, making the parser consume it without a bound. + LimitReadBuffer limited_in(*in, {.read_no_more = static_cast(payload_size)}); + message->deserialize(limited_in); return message; } @@ -441,7 +451,9 @@ class StartupMessage : FirstMessage parameters.insert({std::move(parameter_name), std::move(parameter_value)}); - if (payload_size < 0) + /// `payload_size` is the declared size of the message and never changes, so the check + /// has to be made against the remaining size instead. + if (ps < 0) { throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, "Size of payload is larger than one declared in the message of type {}.", diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 5d0f3cc34cc2..56940bd62472 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -2758,6 +2758,9 @@ Try using an index if there is a subquery or a table expression on the right sid )", 0) \ DECLARE(UInt64, use_index_for_in_with_subqueries_max_values, 0, R"( The maximum size of the set in the right-hand side of the IN operator to use table index for filtering. It allows to avoid performance degradation and higher memory usage due to the preparation of additional data structures for large queries. Zero means no limit. +)", 0) \ + DECLARE(UInt64, statistics_max_set_size_for_exact_selectivity_estimation, 10000, R"( +The maximum size of the set in the right-hand side of the `IN` operator for which the selectivity estimator derives the exact ranges covered by the set. Deriving them costs a `Field` per element, a sort, and one statistics probe per element, which for a large set dominates query planning. Above this limit the estimator instead derives the selectivity from the size of the set and its bounding range, which is a single linear pass over the set without the sort or the per-element statistics probes. Zero means no limit. )", 0) \ DECLARE(Bool, analyze_index_with_space_filling_curves, true, R"( If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)` or `ORDER BY hilbertEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis. @@ -4592,7 +4595,7 @@ Possible values: - 1 — Optimization enabled. )", 0) \ DECLARE(Bool, optimize_trivial_group_by_limit_query, true, R"( -Enables or disables the optimization of a trivial query `SELECT key_expr FROM table GROUP BY key_expr LIMIT n` (with no aggregate functions in the projection, no `HAVING`/`ORDER BY`/`LIMIT BY`/window clauses, and no `GROUP BY` modifiers) by setting `max_rows_to_group_by = n + offset` with `group_by_overflow_mode = 'any'`. The aggregation stops once `n + offset` distinct keys are produced. +Enables or disables the optimization of a trivial query `SELECT key_expr FROM table GROUP BY key_expr LIMIT n` (with no aggregate functions, window functions or `arrayJoin` in the projection, no `HAVING`/`ORDER BY`/`QUALIFY`/`LIMIT BY`/`DISTINCT`/window clauses, and no `GROUP BY` modifiers) by setting `max_rows_to_group_by = n + offset` with `group_by_overflow_mode = 'any'`. The aggregation stops once `n + offset` distinct keys are produced. The optimization is suppressed when the user has explicitly set `group_by_overflow_mode` to a non-`any` value (to preserve their explicit `throw`/`break` contract), and when the user has already set a tighter `max_rows_to_group_by` (the optimization would be a no-op). @@ -6627,6 +6630,9 @@ Limit on size of a single batch of file segments that a read buffer can request )", 0) \ DECLARE(UInt64, filesystem_cache_reserve_space_wait_lock_timeout_milliseconds, 1000, R"( Wait time to lock cache for space reservation in filesystem cache +)", 0) \ + DECLARE(UInt64, filesystem_cache_wait_for_concurrent_download_timeout_milliseconds, 1000, R"( +Maximum time to wait for a file segment which is being downloaded to the filesystem cache by a concurrent query. When the timeout is reached, the read bypasses the filesystem cache for that range and reads directly from remote storage, while the concurrent download continues to fill the cache. Value `0` means do not wait at all: bypass the cache immediately if the needed range is not downloaded yet. Lowering this value bounds the tail latency of cache-hit reads which would otherwise wait for another query's download pace at the cost of additional requests to remote storage. )", 0) \ DECLARE(Bool, filesystem_cache_prefer_bigger_buffer_size, true, R"( Prefer bigger buffer size if filesystem cache is enabled to avoid writing small file segments which deteriorate cache performance. On the other hand, enabling this setting might increase memory usage. @@ -7659,7 +7665,7 @@ Cloud default value: `1`. Allow new query analyzer. )", IMPORTANT, enable_analyzer) \ DECLARE(Bool, analyzer_compatibility_join_using_top_level_identifier, false, R"( -Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`). +Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`). Aliases defined on subexpressions inside the SELECT list are also considered (for example, in `SELECT uniqExact(a + 1 AS b) FROM t1 JOIN t2 USING (b)` the join is performed by `t1.a + 1 = t2.b`). When the matching alias is defined on a subexpression inside the SELECT list rather than as a top-level alias, parallel replicas are disabled for the query. For queries sent to remote servers (`Distributed` tables, the `remote` table function), such a query is rejected with an exception only when the identifier cannot be resolved on the remote server at all; if the alias shadows a real column of the left table, the remote server joins by that column instead, so the results may differ from local execution. )", 0) \ DECLARE(Bool, analyzer_compatibility_allow_compound_identifiers_in_unflatten_nested, true, R"( Allow to add compound identifiers to nested. This is a compatibility setting because it changes the query result. When disabled, `SELECT a.b.c FROM table ARRAY JOIN a` does not work, and `SELECT a FROM table` does not include `a.b.c` column into `Nested a` result. @@ -7677,6 +7683,20 @@ Possible values: - 0 - `FINAL` applies only to the table it is specified on. - 1 - `FINAL` on the left-most table of a JOIN is applied to all joined tables. +)", 0) \ + DECLARE(Bool, analyzer_compatibility_multiple_joins_qualify_column_names, false, R"( +When enabled and the `FROM` clause of a query contains two or more `JOIN`s (comma-separated tables count; `ARRAY JOIN` does not), the analyzer names result columns the way the old analyzer's multiple-joins rewrite did: +- columns produced by expanding `*`, `
.*` or `COLUMNS('')` get names of the form `.` (the qualifier is the table expression's alias if it has one, otherwise the table name without the database, otherwise the CTE name; columns of a joined subquery without an alias are left unqualified). Two kinds of column keep their bare name because they belong to the join rather than to a single table expression: a column produced by `ARRAY JOIN`, and a key merged by `JOIN ... USING`. Outer references such as `SELECT ll.arr` or `SELECT ll.k` therefore do not resolve in those two shapes; +- the identifier-list form `COLUMNS(col1, col2)` is not a matcher expansion: each column keeps the name exactly as its identifier was written, so `COLUMNS(x)` produces `x` and `COLUMNS(a.x)` produces `a.x`; +- an unaliased column reference in the `SELECT` list keeps its name exactly as written (e.g. `SELECT a.x` produces a column named `a.x` even when `x` is unambiguous). + +This makes outer queries that reference such columns by their qualified names work, for example: + +```sql +SELECT ll.Date FROM (SELECT * FROM t AS ll LEFT JOIN t1 ON ll.k = t1.k LEFT JOIN t2 ON ll.k = t2.k); +``` + +Takes effect only when the analyzer is enabled (`enable_analyzer = 1`). )", 0) \ DECLARE(Bool, enable_identifier_resolve_cache, true, R"( Enable the identifier resolution cache in the query analyzer. The cache shares resolved alias nodes to prevent AST explosion when the same alias is referenced multiple times. Set to false to disable caching if incorrect results are suspected. @@ -7842,6 +7862,9 @@ As each series represents a node in Keeper, it is recommended to have no more th )", 0) \ DECLARE(Bool, use_hive_partitioning, true, R"( When enabled, ClickHouse will detect Hive-style partitioning in path (`/name=value/`) in file-like table engines [File](/sql-reference/table-functions/file#hive-style-partitioning)/[S3](/sql-reference/table-functions/s3#hive-style-partitioning)/[URL](/sql-reference/table-functions/url#hive-style-partitioning)/[HDFS](/sql-reference/table-functions/hdfs#hive-style-partitioning)/[AzureBlobStorage](/sql-reference/table-functions/azureBlobStorage#hive-style-partitioning) and will allow to use partition columns as virtual columns in the query. These virtual columns will have the same names as in the partitioned path, but starting with `_`. +)", 0) \ + DECLARE(Bool, throw_on_hive_partitioning_resolution_failure, false, R"( +Throw an exception instead of logging a warning when Hive-style partitioning detection for an object storage table fails to list the storage. When disabled, the query runs without the Hive partition columns, which may change its result. )", 0) \ DECLARE(UInt64, parallel_hash_join_threshold, 100'000, R"( When hash-based join algorithm is applied, this threshold helps to decide between using `hash` and `parallel_hash` (only if estimation of the right table size is available). @@ -8536,20 +8559,20 @@ Initial delay in milliseconds before the first retry of a failed AI function API If true (default), an AI function call that fails permanently after exhausting all retries aborts the query with an exception. If false, the failed row receives the default value for the column type (empty string for String) and processing continues. )", EXPERIMENTAL) \ DECLARE(UInt64, ai_function_max_input_tokens_per_query, 1000000, R"( -Maximum total input (prompt) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by one call's worth of input tokens, since the number of input tokens of a call are not known in advance. Set to 0 to disable. +Maximum total input (prompt) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of input tokens per in-flight request, since a call's input tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. This limit is only enforced for providers that report a `usage` object in their response (OpenAI, Anthropic, vLLM). Providers that omit token usage (notably HuggingFace TEI) cause the counter to stay at 0 — use `ai_function_max_api_calls_per_query` instead to bound such calls. )", EXPERIMENTAL) \ DECLARE(UInt64, ai_function_max_output_tokens_per_query, 500000, R"( -Maximum total output (completion) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by one call's worth of output tokens, since the number of output tokens of a call are not known in advance. Set to 0 to disable. +Maximum total output (completion) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of output tokens per in-flight request, since a call's output tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. This limit is only enforced for providers that report a `usage` object in their response (OpenAI, Anthropic, vLLM). It does not apply to embedding functions (notably aiEmbed), which never produce output tokens. )", EXPERIMENTAL) \ DECLARE(UInt64, ai_function_max_api_calls_per_query, 0, R"( -Maximum number of HTTP requests that AI functions may dispatch per query. Set to 0 to disable. +Maximum number of HTTP requests that AI functions may dispatch per query. Enforced independently by each server and query fragment: within one execution context it is an exact cap shared by every AI function, block, and thread there, but a distributed query (across shards or parallel-replica fragments) may dispatch up to this many requests per shard/fragment. It must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. )", EXPERIMENTAL) \ DECLARE(Bool, ai_function_throw_on_quota_exceeded, true, R"( -If true (default), exceeding an AI function quota limit (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`, or `ai_function_max_api_calls_per_query`) aborts the query with an exception. If false, remaining rows receive the default value for the column type (empty string for String). +If true (default), exceeding an AI function quota limit (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`, or `ai_function_max_api_calls_per_query`) aborts the query with an exception. If false, remaining rows receive the default value for the column type (empty string for String). Like the quota limits, this must be set in the top-level query - a sub-query `SETTINGS` override is ignored. )", EXPERIMENTAL) \ DECLARE(NonZeroUInt64, ai_function_embedding_max_batch_size, 100, R"( Maximum number of texts to include in a single HTTP request made by `aiEmbed`. Texts are grouped into batches of this size to reduce API call overhead. For example, 500 unique texts with a batch size of 100 result in 5 HTTP requests. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 008ba54f43d2..29cb4aa7b766 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -46,6 +46,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() addSettingsChanges(settings_changes_history, "26.6", { + {"analyzer_compatibility_multiple_joins_qualify_column_names", false, false, "New compatibility setting. When enabled, the analyzer mimics the old analyzer's qualified result column names for queries whose FROM clause has two or more JOINs."}, {"analyzer_compatibility_apply_final_to_all_joined_tables", true, false, "Fixed a bug in the analyzer where FINAL on the left-most table of a JOIN was incorrectly applied to the other joined tables as well. previous_value=true so `compatibility` with versions before 26.6 restores the old behavior."}, {"analyzer_compatibility_allow_non_aggregate_in_having", false, false, "New compatibility setting. When enabled, the new analyzer mimics the legacy `HAVING`-to-`WHERE` rewrite for non-aggregate AND-conjuncts instead of raising `NOT_AN_AGGREGATE`."}, {"reserve_memory", 0, 0, "New setting to reserve memory for specific workload before starting a query."}, @@ -104,6 +105,9 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, {"export_merge_tree_partition_max_retries", 3, 3, "Obsolete and ignored: export partition tasks now retry retryable failures until the task timeout and fail immediately on non-retryable errors, instead of using a fixed retry budget"}, + {"filesystem_cache_wait_for_concurrent_download_timeout_milliseconds", 60000, 1000, "New setting to bound how long a read waits for a file segment being downloaded to the filesystem cache by a concurrent query; on timeout the read bypasses the cache instead of waiting indefinitely. The previous value 60000 corresponds to the old behavior (one full 60 s wait cycle on the downloader)."}, + {"throw_on_hive_partitioning_resolution_failure", false, false, "New setting to fail the query when Hive-style partitioning detection for an object storage table cannot list the storage. Disabled here to keep the pre-existing behavior of running without the Hive partition columns, and enabled by default from 26.8."}, + {"statistics_max_set_size_for_exact_selectivity_estimation", 0, 10000, "New setting to bound the cost of estimating the selectivity of `IN` with a large set: above the limit the estimator uses the size of the set and its bounding range instead of the exact ranges. Before 26.8 the estimation was uncapped, so the previous value is 0 (no limit) and `compatibility` with an earlier version restores the exact ranges for sets of any size."}, }); addSettingsChanges(settings_changes_history, "26.5", @@ -1341,6 +1345,7 @@ const VersionToSettingsChangesMap & getMergeTreeSettingsChangesHistory() {"text_index_posting_list_block_size", 1048576, 1048576, "New setting"}, {"text_index_posting_list_codec", "none", "none", "New setting"}, {"allow_experimental_text_index_positions", false, false, "New setting"}, + {"text_index_serialization_version", "v0_initial", "v1_with_codec", "New setting. Controls the on-disk format version of text indexes. Reverts to 'v0_initial' under older compatibility so that newer servers keep writing the previous format that older servers can read during a rolling upgrade."}, {"materialize_projections_on_insert", true, true, "New setting"}, {"materialize_projections_on_merge", false, false, "New setting"}, {"shared_merge_tree_inactive_replica_cutoff_seconds", 0, 0, "New setting which controls for how long an inactive replica is taken into account by the background cleanup (0 means two ZooKeeper session timeouts)"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index a5bd46940a30..e84926119387 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -457,6 +457,13 @@ IMPLEMENT_SETTING_ENUM( {{"none", TextIndexPostingListCodec::None}, {"bitpacking", TextIndexPostingListCodec::Bitpacking}}) +IMPLEMENT_SETTING_ENUM( + MergeTreeTextIndexSerializationVersion, + ErrorCodes::BAD_ARGUMENTS, + {{"v0_initial", MergeTreeTextIndexSerializationVersion::V0_Initial}, + {"v1_with_codec", MergeTreeTextIndexSerializationVersion::V1_WithCodec}, + {"v2_with_positions", MergeTreeTextIndexSerializationVersion::V2_WithPositions}}) + IMPLEMENT_SETTING_ENUM( MergeTreePartMinMaxIndexColumns, ErrorCodes::BAD_ARGUMENTS, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index ebf65cc5b39f..a17f433ba6ec 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -512,6 +512,17 @@ enum class TextIndexPostingListCodec : uint8_t DECLARE_SETTING_ENUM(TextIndexPostingListCodec) +/// On-disk serialization format version of text indexes. +/// These are the on-disk version numbers and must remain stable. +enum class MergeTreeTextIndexSerializationVersion : uint8_t +{ + V0_Initial = 0, + V1_WithCodec = 1, + V2_WithPositions = 2, +}; + +DECLARE_SETTING_ENUM(MergeTreeTextIndexSerializationVersion) + /// NOTE: Part level min-max index depends on strict columns order. /// That means if you want to add new columns segment to index - it will not be materialized until /// previous segment will be materialized in all data parts via mutation or merge. diff --git a/src/Core/tests/gtest_block_map_aggregate_state_constant.cpp b/src/Core/tests/gtest_block_map_aggregate_state_constant.cpp new file mode 100644 index 000000000000..1eec93e5601c --- /dev/null +++ b/src/Core/tests/gtest_block_map_aggregate_state_constant.cpp @@ -0,0 +1,72 @@ +#include + +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// A single-row constant `Map(String, state_type)` holding one entry with the given key and a +/// default (empty) aggregate state as its value. +ColumnWithTypeAndName makeConstantMapOfAggregateState(const DataTypePtr & state_type, const String & key) +{ + auto state_column = state_type->createColumn(); + state_column->insertDefault(); + + Field state_field; + state_column->get(0, state_field); + + auto map_type = std::make_shared(std::make_shared(), state_type); + auto map_column = map_type->createColumn(); + map_column->insert(Map{Tuple{key, state_field}}); + + return ColumnWithTypeAndName{ColumnConst::create(std::move(map_column), 1), map_type, "m"}; +} + +Block makeBlock(ColumnWithTypeAndName column) +{ + return Block{std::move(column)}; +} + +} + +/// Aggregate states whose functions have the same state representation are compatible, and for +/// constants the comparison of their values is relaxed to the serialized state, because the +/// function names are allowed to differ. This relaxation has to work through plain containers such +/// as `Map`: the type of every value a `Map` holds is fixed by the declared `Map(K, V)`, so a `Map` +/// cannot hide which type a nested value has (unlike `Variant`, `Dynamic` and `JSON`). +GTEST_TEST(BlockStructure, ConstantMapRelaxesTheNestedAggregateState) +{ + tryRegisterAggregateFunctions(); + + auto quantile_type = DataTypeFactory::instance().get("AggregateFunction(quantile(0.5), UInt8)"); + auto quantiles_type = DataTypeFactory::instance().get("AggregateFunction(quantiles(0.9), UInt8)"); + + Block quantile = makeBlock(makeConstantMapOfAggregateState(quantile_type, "k")); + Block quantiles = makeBlock(makeConstantMapOfAggregateState(quantiles_type, "k")); + + EXPECT_TRUE(blocksHaveEqualStructure(quantile, quantiles)); + EXPECT_TRUE(blocksHaveEqualStructure(quantiles, quantile)); +} + +/// Everything else inside the constant `Map` is still compared strictly: a compatible aggregate +/// state next to a differing key is a different constant. +GTEST_TEST(BlockStructure, ConstantMapKeepsComparingTheOtherValues) +{ + tryRegisterAggregateFunctions(); + + auto quantile_type = DataTypeFactory::instance().get("AggregateFunction(quantile(0.5), UInt8)"); + auto quantiles_type = DataTypeFactory::instance().get("AggregateFunction(quantiles(0.9), UInt8)"); + + Block quantile = makeBlock(makeConstantMapOfAggregateState(quantile_type, "k")); + Block quantiles = makeBlock(makeConstantMapOfAggregateState(quantiles_type, "other")); + + EXPECT_FALSE(blocksHaveEqualStructure(quantile, quantiles)); + EXPECT_FALSE(blocksHaveEqualStructure(quantiles, quantile)); +} diff --git a/src/Core/tests/gtest_block_nested_sparse_structure.cpp b/src/Core/tests/gtest_block_nested_sparse_structure.cpp new file mode 100644 index 000000000000..78e017235724 --- /dev/null +++ b/src/Core/tests/gtest_block_nested_sparse_structure.cpp @@ -0,0 +1,62 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// A sparse column of `size` default `UInt64` values. +ColumnPtr makeSparseUInt64(size_t size) +{ + auto values = ColumnUInt64::create(); + values->insertDefault(); /// The value at position 0 is the default one. + return ColumnSparse::create(std::move(values), ColumnUInt64::create(), size); +} + +Block makeTupleBlock(ColumnPtr nested, DataTypePtr nested_type) +{ + auto type = std::make_shared(DataTypes{std::move(nested_type)}); + return Block{ColumnWithTypeAndName{ColumnTuple::create(Columns{std::move(nested)}), std::move(type), "t"}}; +} + +} + +/// A `Sparse` column is interchangeable with the full column it wraps at any nesting depth, +/// not only at the top level: one branch of a query can materialize a nested subcolumn while +/// another one keeps it sparse. +GTEST_TEST(BlockStructure, NestedSparseIsCompatibleWithFull) +{ + const size_t size = 3; + auto uint64_type = std::make_shared(); + + Block sparse = makeTupleBlock(makeSparseUInt64(size), uint64_type); + Block full = makeTupleBlock(ColumnUInt64::create(size, 0), uint64_type); + + EXPECT_TRUE(blocksHaveEqualStructure(sparse, full)); + EXPECT_TRUE(blocksHaveEqualStructure(full, sparse)); + EXPECT_TRUE(blocksHaveEqualStructure(sparse, sparse)); +} + +/// The nested comparison stays strict for everything else: unwrapping `Sparse` must not make +/// structurally different nested columns compare equal. +GTEST_TEST(BlockStructure, NestedSparseIsNotCompatibleWithAnotherColumn) +{ + const size_t size = 3; + auto uint64_type = std::make_shared(); + + Block sparse = makeTupleBlock(makeSparseUInt64(size), uint64_type); + /// The declared type is the same, only the column inside the tuple is a different one. + Block other = makeTupleBlock(ColumnString::create(), uint64_type); + + EXPECT_FALSE(blocksHaveEqualStructure(sparse, other)); + EXPECT_FALSE(blocksHaveEqualStructure(other, sparse)); +} diff --git a/src/Core/tests/gtest_block_variant_aggregate_state_constant.cpp b/src/Core/tests/gtest_block_variant_aggregate_state_constant.cpp new file mode 100644 index 000000000000..984caf5162dd --- /dev/null +++ b/src/Core/tests/gtest_block_variant_aggregate_state_constant.cpp @@ -0,0 +1,67 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// A constant `Variant` of a single row whose active alternative is `global_discriminator`. +/// The value of the alternative is a default (empty) aggregate state. +ColumnPtr makeConstantVariantOfAggregateState(const DataTypeVariant & type, size_t global_discriminator) +{ + MutableColumns variants; + for (const auto & alternative : type.getVariants()) + variants.push_back(alternative->createColumn()); + + variants[global_discriminator]->insertDefault(); + + auto local_discriminators = ColumnVariant::ColumnDiscriminators::create(); + local_discriminators->insertValue(static_cast(global_discriminator)); + + auto offsets = ColumnVariant::ColumnOffsets::create(); + offsets->insertValue(0); + + /// The local order of the variants is the global one, so no discriminators mapping is needed. + auto variant = ColumnVariant::create(std::move(local_discriminators), std::move(offsets), std::move(variants)); + return ColumnConst::create(std::move(variant), 1); +} + +Block makeBlock(const ColumnPtr & column, const DataTypePtr & type) +{ + return Block{ColumnWithTypeAndName{column, type, "v"}}; +} + +} + +/// Aggregate states whose functions have the same state representation are compatible, and for +/// constants the comparison of their values is relaxed to the serialized state, because the +/// function names are allowed to differ. Under a `Variant` this relaxation must not apply: the +/// `Field` of a `Variant` value is the value of its active alternative and no longer says which +/// alternative it is, so two constants on different alternatives with the same serialized state +/// must not be reported as the same value. +GTEST_TEST(BlockStructure, ConstantVariantKeepsTheActiveAggregateStateAlternative) +{ + tryRegisterAggregateFunctions(); + + auto quantile_type = DataTypeFactory::instance().get("AggregateFunction(quantile(0.5), UInt8)"); + auto quantiles_type = DataTypeFactory::instance().get("AggregateFunction(quantiles(0.9), UInt8)"); + auto variant_type = std::make_shared(DataTypes{quantile_type, quantiles_type}); + + Block first = makeBlock(makeConstantVariantOfAggregateState(*variant_type, 0), variant_type); + Block second = makeBlock(makeConstantVariantOfAggregateState(*variant_type, 1), variant_type); + + EXPECT_FALSE(blocksHaveEqualStructure(first, second)); + EXPECT_FALSE(blocksHaveEqualStructure(second, first)); + + /// The same alternative on both sides is still the same constant. + EXPECT_TRUE(blocksHaveEqualStructure(first, makeBlock(makeConstantVariantOfAggregateState(*variant_type, 0), variant_type))); + EXPECT_TRUE(blocksHaveEqualStructure(second, makeBlock(makeConstantVariantOfAggregateState(*variant_type, 1), variant_type))); +} diff --git a/src/Core/tests/gtest_field.cpp b/src/Core/tests/gtest_field.cpp index 96964fe67b6c..b7014ff50656 100644 --- a/src/Core/tests/gtest_field.cpp +++ b/src/Core/tests/gtest_field.cpp @@ -1,6 +1,8 @@ #include #include +#include + using namespace DB; GTEST_TEST(Field, FromBool) @@ -110,3 +112,113 @@ GTEST_TEST(Field, DeeplyNestedCopyAndDestroyDoesNotOverflowStack) ASSERT_EQ(src.getType(), Field::Types::Object); } } + + +GTEST_TEST(Field, CompareFloat64) +{ + const Field one{Float64(1.0)}; + const Field two{Float64(2.0)}; + const Field one_again{Float64(1.0)}; + + ASSERT_TRUE(one < two); + ASSERT_FALSE(two < one); + ASSERT_FALSE(one < one_again); + + ASSERT_TRUE(one <= two); + ASSERT_FALSE(two <= one); + ASSERT_TRUE(one <= one_again); + + ASSERT_TRUE(two > one); + ASSERT_FALSE(one > two); + ASSERT_FALSE(one > one_again); + + ASSERT_TRUE(two >= one); + ASSERT_FALSE(one >= two); + ASSERT_TRUE(one >= one_again); + + ASSERT_TRUE(one == one_again); + ASSERT_FALSE(one == two); + ASSERT_TRUE(one != two); + + /// The same for integers, to make sure the Float64 branch is not the odd one out. + ASSERT_FALSE(Field(Int64(2)) <= Field(Int64(1))); + ASSERT_FALSE(Field(Int64(1)) >= Field(Int64(2))); +} + + +GTEST_TEST(Field, CompareFloat64NaN) +{ + /// NaN is ordered after every number (nan_direction_hint == 1) and is equal to itself. + const Field nan{std::numeric_limits::quiet_NaN()}; + const Field nan_again{std::numeric_limits::quiet_NaN()}; + const Field inf{std::numeric_limits::infinity()}; + const Field one{Float64(1.0)}; + + ASSERT_TRUE(one < nan); + ASSERT_TRUE(inf < nan); + ASSERT_FALSE(nan < one); + ASSERT_FALSE(nan < nan_again); + + ASSERT_TRUE(one <= nan); + ASSERT_FALSE(nan <= one); + ASSERT_TRUE(nan <= nan_again); + + ASSERT_TRUE(nan > one); + ASSERT_FALSE(one > nan); + ASSERT_FALSE(nan > nan_again); + + ASSERT_TRUE(nan >= one); + ASSERT_FALSE(one >= nan); + ASSERT_TRUE(nan >= nan_again); + + ASSERT_TRUE(nan == nan_again); + ASSERT_FALSE(nan == one); +} + + +GTEST_TEST(Field, CompareDifferentTypes) +{ + /// Fields of different types are ordered by Types::Which before any value comparison, + /// so values don't matter across types; operator== / != short-circuit on differing Which. + const Field i{Int64(999)}; /// Which::Int64 == 2 + const Field s{String("a")}; /// Which::String == 16 + + ASSERT_TRUE(i < s); + ASSERT_FALSE(s < i); + + ASSERT_TRUE(i <= s); + ASSERT_FALSE(s <= i); + + ASSERT_TRUE(s > i); + ASSERT_TRUE(s >= i); + ASSERT_FALSE(i >= s); + + ASSERT_FALSE(i == s); + ASSERT_TRUE(i != s); + ASSERT_FALSE(Field(Int64(1)) == Field(UInt64(1))); /// same value, different Which + ASSERT_TRUE(Field(Int64(1)) != Field(UInt64(1))); +} + + +GTEST_TEST(Field, CompareUUID) +{ + /// UUID is a StrongTypedef with operator< but no operator<=, so the <= / >= + /// branches compare toUnderType(); pin that the two forms stay consistent. + const Field one{UUID(UInt128(1))}; + const Field two{UUID(UInt128(2))}; + const Field one_again{UUID(UInt128(1))}; + + ASSERT_TRUE(one < two); + ASSERT_FALSE(two < one); + + ASSERT_TRUE(one <= two); + ASSERT_FALSE(two <= one); + ASSERT_TRUE(one <= one_again); + + ASSERT_TRUE(two >= one); + ASSERT_FALSE(one >= two); + ASSERT_TRUE(one >= one_again); + + ASSERT_TRUE(one == one_again); + ASSERT_FALSE(one == two); +} diff --git a/src/DataTypes/DataTypeDynamic.cpp b/src/DataTypes/DataTypeDynamic.cpp index 94921b2aee21..a36fb5452645 100644 --- a/src/DataTypes/DataTypeDynamic.cpp +++ b/src/DataTypes/DataTypeDynamic.cpp @@ -1011,14 +1011,17 @@ std::unique_ptr DataTypeDynamic::getDynamicSubcolumnDa variant_column.getLocalDiscriminatorsPtr(), "", *discriminator, - variant_column.localDiscriminatorByGlobal(*discriminator)); + variant_column.localDiscriminatorByGlobal(*discriminator), + variant_column.getNumVariants()); else creator = std::make_unique( variant_column.getLocalDiscriminatorsPtr(), "", *discriminator, variant_column.localDiscriminatorByGlobal(*discriminator), - make_subcolumn_nullable); + make_subcolumn_nullable, + nullptr, + variant_column.getNumVariants()); res->column = creator->create(res->column); } /// Check if requested type was extracted from shared variant. In this case we should use diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index 5779fa732cb7..be466eb1aec3 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -392,7 +392,9 @@ The serialization layer computes which bucket the requested key belongs to and r When the full map is read (e.g., `SELECT m`), all buckets are read and reassembled into the original map. This is slower than `basic` serialization due to the overhead of reading and merging multiple substreams. :::note -The order of keys within a map value may differ from the original insertion order when using `with_buckets` serialization. Keys are distributed across buckets by hash and are reassembled in bucket order, not insertion order. With `basic` serialization, the key order from inserted maps is preserved. +Since version 26.8, `with_buckets` serialization preserves the original key order: an additional `bucket_indexes` substream records which bucket every key-value pair was taken from, so the map is reassembled in the order it was written instead of in bucket order. + +Parts written by earlier versions do not contain that substream. Their maps are still reassembled in bucket order, and the original key order cannot be restored for them because it was never stored on disk — rewriting such a part (by a merge or `OPTIMIZE FINAL`) freezes the bucket order it currently has instead of recovering the insertion order. With `basic` serialization, the key order from inserted maps has always been preserved. ::: The bucket count can vary between parts. When parts with different bucket counts are merged, the new part's bucket count is recalculated from the merged statistics. Parts with `basic` and `with_buckets` serialization can coexist in the same table and are merged transparently. diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 7082129870fc..5d4a8d8e15ce 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -834,9 +834,9 @@ bool ISerialization::isVariantSubcolumn(const SubstreamPath & substream_path) bool ISerialization::tryToChangeStreamFileNameSettingsForNotFoundStream(const ISerialization::SubstreamPath & substream_path, ISerialization::StreamFileNameSettings & stream_file_name_settings) { - if (isVariantSubcolumn(substream_path) && stream_file_name_settings.escape_variant_substreams) + if (isVariantSubcolumn(substream_path)) { - stream_file_name_settings.escape_variant_substreams = false; + stream_file_name_settings.escape_variant_substreams = !stream_file_name_settings.escape_variant_substreams; return true; } diff --git a/src/DataTypes/Serializations/SerializationDynamicElement.cpp b/src/DataTypes/Serializations/SerializationDynamicElement.cpp index 4e07d4931d24..737fa21de45d 100644 --- a/src/DataTypes/Serializations/SerializationDynamicElement.cpp +++ b/src/DataTypes/Serializations/SerializationDynamicElement.cpp @@ -119,9 +119,11 @@ void SerializationDynamicElement::deserializeBinaryBulkStatePrefix( { settings.path.push_back(Substream::DynamicData); if (is_null_map_subcolumn) - dynamic_element_state->variant_serialization = SerializationVariantElementNullMap::create(dynamic_element_name, *global_discr); + dynamic_element_state->variant_serialization = SerializationVariantElementNullMap::create( + dynamic_element_name, *global_discr, variant_type.getVariants().size()); else - dynamic_element_state->variant_serialization = SerializationVariantElement::create(nested_serialization, dynamic_element_name, *global_discr); + dynamic_element_state->variant_serialization = SerializationVariantElement::create( + nested_serialization, dynamic_element_name, *global_discr, variant_type.getVariants().size()); dynamic_element_state->variant_serialization->deserializeBinaryBulkStatePrefix(settings, dynamic_element_state->variant_element_state, cache); dynamic_element_state->read_from_shared_variant = false; settings.path.pop_back(); @@ -135,7 +137,8 @@ void SerializationDynamicElement::deserializeBinaryBulkStatePrefix( dynamic_element_state->variant_serialization = SerializationVariantElement::create( shared_variant_serialization, ColumnDynamic::getSharedVariantTypeName(), - *shared_variant_global_discr); + *shared_variant_global_discr, + variant_type.getVariants().size()); dynamic_element_state->variant_serialization->deserializeBinaryBulkStatePrefix(settings, dynamic_element_state->variant_element_state, cache); dynamic_element_state->read_from_shared_variant = true; settings.path.pop_back(); diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index 6674e902cb8f..ab3109d902f6 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -1409,6 +1409,15 @@ void SerializationMap::deserializeBinaryBulkWithMultipleStreams( /// otherwise fall back to bucket-ascending order (old parts without the index stream). else { + /// The `bucket_indexes` stream is a flat array with one entry per key-value pair, so the + /// number of entries that belong to the first `rows_offset` rows is not known in advance + /// and those entries cannot be skipped on their own. Read the skipped rows together with + /// the requested ones, reassemble the whole range in the original order and drop the + /// prefix afterwards, so that the index stream stays in sync with the bucket streams. + const bool reorder_with_skipped_rows = map_state->has_bucket_index && rows_offset != 0; + const size_t buckets_rows_offset = reorder_with_skipped_rows ? 0 : rows_offset; + const size_t buckets_limit = reorder_with_skipped_rows ? rows_offset + limit : limit; + VectorWithMemoryTracking map_buckets(buckets_info_state->buckets); for (size_t bucket = 0; bucket != buckets_info_state->buckets; ++bucket) { @@ -1416,7 +1425,7 @@ void SerializationMap::deserializeBinaryBulkWithMultipleStreams( settings.path.back().bucket = bucket; map_buckets[bucket] = column_map.cloneEmpty(); ColumnPtr nested_ptr = assert_cast(*map_buckets[bucket]).getNestedColumnPtr(); - nested_serialization->deserializeBinaryBulkWithMultipleStreams(nested_ptr, rows_offset, limit, settings, map_state->bucket_nested_states[bucket], cache); + nested_serialization->deserializeBinaryBulkWithMultipleStreams(nested_ptr, buckets_rows_offset, buckets_limit, settings, map_state->bucket_nested_states[bucket], cache); settings.path.pop_back(); } @@ -1439,7 +1448,17 @@ void SerializationMap::deserializeBinaryBulkWithMultipleStreams( bucket_index_column, 0, total_kv_pairs, settings, map_state->bucket_index_state, cache); settings.path.pop_back(); - collectMapFromBucketsWithOrder(map_buckets, *bucket_index_column, column_map); + if (reorder_with_skipped_rows) + { + auto whole_range_column = column_map.cloneEmpty(); + collectMapFromBucketsWithOrder(map_buckets, *bucket_index_column, *whole_range_column); + if (whole_range_column->size() > rows_offset) + column_map.insertRangeFrom(*whole_range_column, rows_offset, whole_range_column->size() - rows_offset); + } + else + { + collectMapFromBucketsWithOrder(map_buckets, *bucket_index_column, column_map); + } } else { diff --git a/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp b/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp index 1b3958e0c2de..d54a8a9d72c7 100644 --- a/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp +++ b/src/DataTypes/Serializations/SerializationMapKeysOrValues.cpp @@ -340,13 +340,22 @@ void SerializationMapKeysOrValues::deserializeBinaryBulkWithMultipleStreams( /// otherwise fall back to bucket-ascending order (old parts without the index stream). else { + /// The `bucket_indexes` stream is a flat array with one entry per key-value pair, so the + /// number of entries that belong to the first `rows_offset` rows is not known in advance + /// and those entries cannot be skipped on their own. Read the skipped rows together with + /// the requested ones, reassemble the whole range in the original order and drop the + /// prefix afterwards, so that the index stream stays in sync with the bucket streams. + const bool reorder_with_skipped_rows = map_keys_or_values_with_buckets_state->has_bucket_index && rows_offset != 0; + const size_t buckets_rows_offset = reorder_with_skipped_rows ? 0 : rows_offset; + const size_t buckets_limit = reorder_with_skipped_rows ? rows_offset + limit : limit; + VectorWithMemoryTracking keys_or_values_buckets(buckets_info_state_concrete->buckets); for (size_t bucket = 0; bucket != buckets_info_state_concrete->buckets; ++bucket) { settings.path.push_back(Substream::Bucket); settings.path.back().bucket = bucket; keys_or_values_buckets[bucket] = column->cloneEmpty(); - keys_or_values_serialization->deserializeBinaryBulkWithMultipleStreams(keys_or_values_buckets[bucket], rows_offset, limit, settings, map_keys_or_values_with_buckets_state->bucket_keys_or_values_states[bucket], cache); + keys_or_values_serialization->deserializeBinaryBulkWithMultipleStreams(keys_or_values_buckets[bucket], buckets_rows_offset, buckets_limit, settings, map_keys_or_values_with_buckets_state->bucket_keys_or_values_states[bucket], cache); settings.path.pop_back(); } @@ -368,7 +377,17 @@ void SerializationMapKeysOrValues::deserializeBinaryBulkWithMultipleStreams( bucket_index_column, 0, total_kv_pairs, settings, map_keys_or_values_with_buckets_state->bucket_index_state, cache); settings.path.pop_back(); - collectMapKeysOrValuesFromBucketsWithOrder(keys_or_values_buckets, *bucket_index_column, *column->assumeMutable()); + if (reorder_with_skipped_rows) + { + auto whole_range_column = column->cloneEmpty(); + collectMapKeysOrValuesFromBucketsWithOrder(keys_or_values_buckets, *bucket_index_column, *whole_range_column); + if (whole_range_column->size() > rows_offset) + column->assumeMutable()->insertRangeFrom(*whole_range_column, rows_offset, whole_range_column->size() - rows_offset); + } + else + { + collectMapKeysOrValuesFromBucketsWithOrder(keys_or_values_buckets, *bucket_index_column, *column->assumeMutable()); + } } else { diff --git a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp index 40eac1ad895e..ff23034d609f 100644 --- a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp +++ b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp @@ -146,11 +146,19 @@ void SerializationObjectSharedData::enumerateStreams( else addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataStructure); - addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataData); - addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataPathsMarks); - addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataSubstreams); - addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataSubstreamsMarks); - addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataPathsSubstreamsMetadata); + /// When deserialize state is present, it means the whole shared data will be read + /// via deserializeBinaryBulkWithMultipleStreams, which only uses Structure + Copy streams. + /// Per-bucket Data/PathsMarks/Substreams/SubstreamsMarks/PathsSubstreamsMetadata are only + /// needed when writing or reading individual paths via SerializationObjectSharedDataPath (separate class). + /// Skip them to avoid unnecessary mark file loads and file opens during prefetching. + if (!shared_data_state) + { + addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataData); + addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataPathsMarks); + addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataSubstreams); + addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataSubstreamsMarks); + addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataPathsSubstreamsMetadata); + } if (settings.use_specialized_prefixes_and_suffixes_substreams) addSubstreamAndCallCallback(settings.path, callback, Substream::ObjectSharedDataStructureSuffix); diff --git a/src/DataTypes/Serializations/SerializationReplicated.cpp b/src/DataTypes/Serializations/SerializationReplicated.cpp index af341454d40c..60a94a485575 100644 --- a/src/DataTypes/Serializations/SerializationReplicated.cpp +++ b/src/DataTypes/Serializations/SerializationReplicated.cpp @@ -14,6 +14,41 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; +} + +namespace +{ + +/// Validate that every deserialized index is within [0, num_elements) so that later +/// ColumnReplicated accessors don't dereference nested_column[index] out of bounds. +void checkDeserializedIndexes(const IColumn & indexes, size_t size_of_indexes_type, size_t num_elements) +{ + auto check = [&](auto type) + { + using IndexType = decltype(type); + const auto & indexes_data = assert_cast &>(indexes).getData(); + for (auto index : indexes_data) + { + if (index >= num_elements) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Invalid index {} in ColumnReplicated in Native format: it must be less than the number of elements ({})", + static_cast(index), num_elements); + } + }; + + switch (size_of_indexes_type) + { + case sizeof(UInt8): check(UInt8{}); break; + case sizeof(UInt16): check(UInt16{}); break; + case sizeof(UInt32): check(UInt32{}); break; + case sizeof(UInt64): check(UInt64{}); break; + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size of index type for ColumnReplicated: {}", size_of_indexes_type); + } +} + } @@ -241,8 +276,6 @@ void SerializationReplicated::deserializeBinaryBulkWithMultipleStreams( throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected size of index type for ColumnReplicated: {}", UInt32(size_of_indexes_type)); } - column_replicated.getIndexes().attachIndexes(std::move(indexes)); - settings.path.push_back(Substream::ReplicatedElements); auto * elements_stream = settings.getter(settings.path); settings.path.pop_back(); @@ -252,7 +285,22 @@ void SerializationReplicated::deserializeBinaryBulkWithMultipleStreams( size_t num_elements = 0; readVarUInt(num_elements, *elements_stream); + + checkDeserializedIndexes(*indexes, size_of_indexes_type, num_elements); + column_replicated.getIndexes().attachIndexes(std::move(indexes)); + nested->deserializeBinaryBulkWithMultipleStreams(column_replicated.getNestedColumn(), 0, num_elements, settings, state, cache); + + /// Bulk readers of primitive types (e.g. `SerializationNumber::deserializeBinaryBulk`) short-read on EOF + /// instead of throwing, so a truncated elements stream would otherwise leave the nested column smaller + /// than num_elements while already-validated indexes still reference the missing rows. `NativeReader` + /// only checks `column->size()`, which for `ColumnReplicated` is the index count, not the nested column + /// size, so this must be verified explicitly here. + if (column_replicated.getNestedColumn()->size() != num_elements) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Cannot read all elements of ColumnReplicated in Native format: read {} of {}", + column_replicated.getNestedColumn()->size(), num_elements); } void SerializationReplicated::serializeBinary(const Field & field, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/DataTypes/Serializations/SerializationString.cpp b/src/DataTypes/Serializations/SerializationString.cpp index 8d48ff11b09f..1e256303930d 100644 --- a/src/DataTypes/Serializations/SerializationString.cpp +++ b/src/DataTypes/Serializations/SerializationString.cpp @@ -171,13 +171,12 @@ try UInt64 size = 0; readVarUInt(size, istr); - static constexpr size_t max_string_size = 16_GiB; /// Arbitrary value to prevent logical errors and overflows, but large enough. - if (size > max_string_size) + if (size > SerializationString::MAX_STRING_SIZE) throw Exception( ErrorCodes::TOO_LARGE_STRING_SIZE, "Too large string size: {}. The maximum is: {}.", size, - max_string_size); + SerializationString::MAX_STRING_SIZE); offset += size; if (unlikely(offset > data.size())) diff --git a/src/DataTypes/Serializations/SerializationString.h b/src/DataTypes/Serializations/SerializationString.h index 0cfb1f6fb74e..7f2d91cb649a 100644 --- a/src/DataTypes/Serializations/SerializationString.h +++ b/src/DataTypes/Serializations/SerializationString.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -23,6 +24,9 @@ class SerializationString final : public ISerialization explicit SerializationString(MergeTreeStringSerializationVersion version_ = MergeTreeStringSerializationVersion::SINGLE_STREAM); public: + /// Arbitrary guard against absurd sizes from corrupted input, large enough for any real string. + static constexpr size_t MAX_STRING_SIZE = 16_GiB; + static UInt128 getHash(MergeTreeStringSerializationVersion version_); static SerializationPtr create(MergeTreeStringSerializationVersion version_ = MergeTreeStringSerializationVersion::SINGLE_STREAM); diff --git a/src/DataTypes/Serializations/SerializationVariant.cpp b/src/DataTypes/Serializations/SerializationVariant.cpp index 92e56e1731c2..bc9ab8802a0f 100644 --- a/src/DataTypes/Serializations/SerializationVariant.cpp +++ b/src/DataTypes/Serializations/SerializationVariant.cpp @@ -34,6 +34,16 @@ namespace ErrorCodes extern const int INCORRECT_DATA; } +/// Validate that a discriminator value is within bounds (< num_variants) or is NULL_DISCRIMINATOR. +/// Throws INCORRECT_DATA in native format (untrusted input) or LOGICAL_ERROR otherwise. +static void checkDiscriminatorValue(ColumnVariant::Discriminator discr, size_t num_variants, bool native_format) +{ + if (discr != ColumnVariant::NULL_DISCRIMINATOR && discr >= num_variants) + throw Exception( + native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, + "Invalid discriminator value {} (num_variants = {})", + static_cast(discr), num_variants); +} UInt128 SerializationVariant::getHash(const VariantSerializations & variant_serializations_, const String & variant_name_) { @@ -138,7 +148,9 @@ void SerializationVariant::enumerateStreams( variant_names[i], i, column_variant ? column_variant->localDiscriminatorByGlobal(i) : i, - make_subcolumn_nullable); + make_subcolumn_nullable, + nullptr, + variant_serializations.size()); auto variant_data = SubstreamData(variant_serializations[i]) .withType(type) @@ -166,7 +178,8 @@ void SerializationVariant::enumerateStreams( if (!canExtractedSubcolumnsBeInsideNullable(variant_types[i])) continue; - settings.path.back().creator = std::make_shared(local_discriminators, variant_names[i], i, column_variant ? column_variant->localDiscriminatorByGlobal(i) : i); + settings.path.back().creator = std::make_shared( + local_discriminators, variant_names[i], i, column_variant ? column_variant->localDiscriminatorByGlobal(i) : i, variant_serializations.size()); settings.path.push_back(Substream::VariantElementNullMap); settings.path.back().variant_element_name = variant_names[i]; settings.path.back().data = null_map_data; @@ -573,7 +586,7 @@ void SerializationVariant::deserializeBinaryBulkWithMultipleStreams( { auto variant_pair = deserializeCompactDiscriminators( col.getLocalDiscriminatorsPtr(), rows_offset, limit, discriminators_stream, settings.continuous_reading, - *discriminators_state); + *discriminators_state, settings); variant_rows_offsets = variant_pair.first; variant_limits = variant_pair.second; @@ -621,7 +634,10 @@ void SerializationVariant::deserializeBinaryBulkWithMultipleStreams( { ColumnVariant::Discriminator discr = discriminators_data[i]; if (discr != ColumnVariant::NULL_DISCRIMINATOR) + { + checkDiscriminatorValue(discr, variant_rows_offsets.size(), settings.native_format); ++variant_rows_offsets[discr]; + } } } } @@ -644,7 +660,10 @@ void SerializationVariant::deserializeBinaryBulkWithMultipleStreams( { ColumnVariant::Discriminator discr = discriminators_data[i]; if (discr != ColumnVariant::NULL_DISCRIMINATOR) + { + checkDiscriminatorValue(discr, variant_limits.size(), settings.native_format); ++variant_limits[discr]; + } } } @@ -743,7 +762,7 @@ void SerializationVariant::deserializeBinaryBulkWithMultipleStreams( } settings.path.pop_back(); - col.validateState(); + col.validateState(/*allow_logical_error=*/ !settings.native_format); } std::pair, std::vector> SerializationVariant::deserializeCompactDiscriminators( @@ -752,7 +771,8 @@ std::pair, std::vector> SerializationVariant::deseri size_t limit, ReadBuffer * stream, bool continuous_reading, - DeserializeBinaryBulkStateVariantDiscriminators & state) const + DeserializeBinaryBulkStateVariantDiscriminators & state, + const DeserializeBinaryBulkSettings & settings) const { auto & discriminators = assert_cast(*discriminators_column->assumeMutable()); auto & discriminators_data = discriminators.getData(); @@ -774,7 +794,7 @@ std::pair, std::vector> SerializationVariant::deseri if (stream->eof()) return {variant_rows_offsets, variant_limits}; - readDiscriminatorsGranuleStart(state, stream); + readDiscriminatorsGranuleStart(state, stream, variant_serializations.size(), settings); } size_t limit_in_granule = std::min(limit, state.remaining_rows_in_granule); @@ -807,14 +827,20 @@ std::pair, std::vector> SerializationVariant::deseri { ColumnVariant::Discriminator discr = discriminators_data[i]; if (discr != ColumnVariant::NULL_DISCRIMINATOR) + { + checkDiscriminatorValue(discr, variant_rows_offsets.size(), settings.native_format); ++variant_rows_offsets[discr]; + } } for (size_t i = start + skipped_rows; i != discriminators_data.size(); ++i) { ColumnVariant::Discriminator discr = discriminators_data[i]; if (discr != ColumnVariant::NULL_DISCRIMINATOR) + { + checkDiscriminatorValue(discr, variant_limits.size(), settings.native_format); ++variant_limits[discr]; + } } rows_offset -= skipped_rows; @@ -827,7 +853,11 @@ std::pair, std::vector> SerializationVariant::deseri return {variant_rows_offsets, variant_limits}; } -void SerializationVariant::readDiscriminatorsGranuleStart(DeserializeBinaryBulkStateVariantDiscriminators & state, DB::ReadBuffer * stream) +void SerializationVariant::readDiscriminatorsGranuleStart( + DeserializeBinaryBulkStateVariantDiscriminators & state, + ReadBuffer * stream, + size_t num_variants, + const DeserializeBinaryBulkSettings & settings) { UInt64 granule_size = 0; readVarUInt(granule_size, *stream); @@ -839,7 +869,10 @@ void SerializationVariant::readDiscriminatorsGranuleStart(DeserializeBinaryBulkS state.granule_format = static_cast(granule_format); if (granule_format == CompactDiscriminatorsGranuleFormat::COMPACT) + { readBinaryLittleEndian(state.compact_discr, *stream); + checkDiscriminatorValue(state.compact_discr, num_variants, settings.native_format); + } } void SerializationVariant::addVariantElementToPath(DB::ISerialization::SubstreamPath & path, size_t i) const diff --git a/src/DataTypes/Serializations/SerializationVariant.h b/src/DataTypes/Serializations/SerializationVariant.h index 09b2c6daae56..3cd2585cc71d 100644 --- a/src/DataTypes/Serializations/SerializationVariant.h +++ b/src/DataTypes/Serializations/SerializationVariant.h @@ -206,9 +206,16 @@ class SerializationVariant final : public ISerialization size_t limit, ReadBuffer * stream, bool continuous_reading, - DeserializeBinaryBulkStateVariantDiscriminators & state) const; + DeserializeBinaryBulkStateVariantDiscriminators & state, + const DeserializeBinaryBulkSettings & settings) const; - static void readDiscriminatorsGranuleStart(DeserializeBinaryBulkStateVariantDiscriminators & state, ReadBuffer * stream); + /// Reads the compact-discriminators granule header and validates the compact discriminator + /// against num_variants when num_variants > 0. + static void readDiscriminatorsGranuleStart( + DeserializeBinaryBulkStateVariantDiscriminators & state, + ReadBuffer * stream, + size_t num_variants, + const DeserializeBinaryBulkSettings & settings); /// Shared implementation for Escaped and Raw text deserialization. /// Checks for NULL representation in the raw buffer before escape processing diff --git a/src/DataTypes/Serializations/SerializationVariantElement.cpp b/src/DataTypes/Serializations/SerializationVariantElement.cpp index 1779b9af463f..9586fb7a564e 100644 --- a/src/DataTypes/Serializations/SerializationVariantElement.cpp +++ b/src/DataTypes/Serializations/SerializationVariantElement.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -UInt128 SerializationVariantElement::getHash(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_) +UInt128 SerializationVariantElement::getHash(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_, size_t num_variants_) { SipHash hash; hash.update("VariantElement"); @@ -24,14 +24,19 @@ UInt128 SerializationVariantElement::getHash(const SerializationPtr & nested_, c hash.update(variant_element_name_.size()); hash.update(variant_element_name_); hash.update(variant_discriminator_); + hash.update(num_variants_); return hash.get128(); } -SerializationPtr SerializationVariantElement::create(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_) +SerializationPtr SerializationVariantElement::create( + const SerializationPtr & nested_, + const String & variant_element_name_, + ColumnVariant::Discriminator variant_discriminator_, + size_t num_variants_) { if (!nested_->supportsPooling()) - return std::shared_ptr(new SerializationVariantElement(nested_, variant_element_name_, variant_discriminator_)); - return ISerialization::pooled(getHash(nested_, variant_element_name_, variant_discriminator_), [&] { return new SerializationVariantElement(nested_, variant_element_name_, variant_discriminator_); }); + return std::shared_ptr(new SerializationVariantElement(nested_, variant_element_name_, variant_discriminator_, num_variants_)); + return ISerialization::pooled(getHash(nested_, variant_element_name_, variant_discriminator_, num_variants_), [&] { return new SerializationVariantElement(nested_, variant_element_name_, variant_discriminator_, num_variants_); }); } struct SerializationVariantElement::DeserializeBinaryBulkStateVariantElement : public ISerialization::DeserializeBinaryBulkState @@ -171,6 +176,8 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams( discriminators_stream, settings.continuous_reading, variant_element_state->discriminators_state, + settings, + num_variants, this); variant_rows_offset = variant_pair.first; @@ -314,6 +321,8 @@ std::pair SerializationVariantElement::deserializeCompactDiscrim DB::ReadBuffer * stream, bool continuous_reading, DeserializeBinaryBulkStatePtr & discriminators_state_, + const DeserializeBinaryBulkSettings & settings, + size_t num_variants, const ISerialization * serialization) { auto * discriminators_state = checkAndGetState(discriminators_state_, serialization); @@ -337,7 +346,8 @@ std::pair SerializationVariantElement::deserializeCompactDiscrim if (stream->eof()) return {variant_rows_offset, variant_limit}; - SerializationVariant::readDiscriminatorsGranuleStart(*discriminators_state, stream); + SerializationVariant::readDiscriminatorsGranuleStart( + *discriminators_state, stream, num_variants, settings); } size_t limit_in_granule = std::min(limit, discriminators_state->remaining_rows_in_granule); @@ -401,13 +411,15 @@ SerializationVariantElement::VariantSubcolumnCreator::VariantSubcolumnCreator( ColumnVariant::Discriminator global_variant_discriminator_, ColumnVariant::Discriminator local_variant_discriminator_, bool make_nullable_, - const ColumnPtr & null_map_) + const ColumnPtr & null_map_, + size_t num_variants_) : local_discriminators(local_discriminators_) , null_map(null_map_) , variant_element_name(variant_element_name_) , global_variant_discriminator(global_variant_discriminator_) , local_variant_discriminator(local_variant_discriminator_) , make_nullable(make_nullable_) + , num_variants(num_variants_) { } @@ -419,7 +431,7 @@ DataTypePtr SerializationVariantElement::VariantSubcolumnCreator::create(const D SerializationPtr SerializationVariantElement::VariantSubcolumnCreator::create(const SerializationPtr & prev, const DataTypePtr &) const { - return SerializationVariantElement::create(prev, variant_element_name, global_variant_discriminator); + return SerializationVariantElement::create(prev, variant_element_name, global_variant_discriminator, num_variants); } ColumnPtr SerializationVariantElement::VariantSubcolumnCreator::create(const DB::ColumnPtr & prev) const diff --git a/src/DataTypes/Serializations/SerializationVariantElement.h b/src/DataTypes/Serializations/SerializationVariantElement.h index cbec912f4dcc..4f479fdfffe7 100644 --- a/src/DataTypes/Serializations/SerializationVariantElement.h +++ b/src/DataTypes/Serializations/SerializationVariantElement.h @@ -18,17 +18,29 @@ class SerializationVariantElement final : public SerializationWrapper /// we need its type name and global discriminator. String variant_element_name; ColumnVariant::Discriminator variant_discriminator; - - SerializationVariantElement(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_) + /// Total number of variants in the Variant type; used for bounds-checking + /// compact discriminators read from the wire. + size_t num_variants; + + SerializationVariantElement( + const SerializationPtr & nested_, + const String & variant_element_name_, + ColumnVariant::Discriminator variant_discriminator_, + size_t num_variants_) : SerializationWrapper(nested_) , variant_element_name(variant_element_name_) , variant_discriminator(variant_discriminator_) + , num_variants(num_variants_) { } public: - static UInt128 getHash(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_); - static SerializationPtr create(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_); + static UInt128 getHash(const SerializationPtr & nested_, const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_, size_t num_variants_); + static SerializationPtr create( + const SerializationPtr & nested_, + const String & variant_element_name_, + ColumnVariant::Discriminator variant_discriminator_, + size_t num_variants_ = 0); size_t allocatedBytes() const override; void enumerateStreams( @@ -74,6 +86,7 @@ class SerializationVariantElement final : public SerializationWrapper const ColumnVariant::Discriminator global_variant_discriminator; const ColumnVariant::Discriminator local_variant_discriminator; bool make_nullable; + size_t num_variants; public: VariantSubcolumnCreator( @@ -82,7 +95,8 @@ class SerializationVariantElement final : public SerializationWrapper ColumnVariant::Discriminator global_variant_discriminator_, ColumnVariant::Discriminator local_variant_discriminator_, bool make_nullable_, - const ColumnPtr & null_map_ = nullptr); + const ColumnPtr & null_map_ = nullptr, + size_t num_variants_ = 0); DataTypePtr create(const DataTypePtr & prev) const override; ColumnPtr create(const ColumnPtr & prev) const override; @@ -102,6 +116,8 @@ class SerializationVariantElement final : public SerializationWrapper ReadBuffer * stream, bool continuous_reading, DeserializeBinaryBulkStatePtr & discriminators_state_, + const DeserializeBinaryBulkSettings & settings, + size_t num_variants, const ISerialization * serialization); void addVariantToPath(SubstreamPath & path) const; diff --git a/src/DataTypes/Serializations/SerializationVariantElementNullMap.cpp b/src/DataTypes/Serializations/SerializationVariantElementNullMap.cpp index 5647229894ba..98dc0efe24fa 100644 --- a/src/DataTypes/Serializations/SerializationVariantElementNullMap.cpp +++ b/src/DataTypes/Serializations/SerializationVariantElementNullMap.cpp @@ -38,19 +38,23 @@ struct DeserializeBinaryBulkStateVariantElementNullMap : public ISerialization:: }; -UInt128 SerializationVariantElementNullMap::getHash(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_) +UInt128 SerializationVariantElementNullMap::getHash(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_, size_t num_variants_) { SipHash hash; hash.update("VariantElementNullMap"); hash.update(variant_element_name_.size()); hash.update(variant_element_name_); hash.update(variant_discriminator_); + hash.update(num_variants_); return hash.get128(); } -SerializationPtr SerializationVariantElementNullMap::create(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_) +SerializationPtr SerializationVariantElementNullMap::create( + const String & variant_element_name_, + ColumnVariant::Discriminator variant_discriminator_, + size_t num_variants_) { - return ISerialization::pooled(getHash(variant_element_name_, variant_discriminator_), [&] { return new SerializationVariantElementNullMap(variant_element_name_, variant_discriminator_); }); + return ISerialization::pooled(getHash(variant_element_name_, variant_discriminator_, num_variants_), [&] { return new SerializationVariantElementNullMap(variant_element_name_, variant_discriminator_, num_variants_); }); } void SerializationVariantElementNullMap::enumerateStreams( @@ -152,6 +156,8 @@ void SerializationVariantElementNullMap::deserializeBinaryBulkWithMultipleStream discriminators_stream, settings.continuous_reading, variant_element_null_map_state->discriminators_state, + settings, + num_variants, this); variant_limit = variant_pair.second; @@ -201,11 +207,13 @@ SerializationVariantElementNullMap::VariantNullMapSubcolumnCreator::VariantNullM const ColumnPtr & local_discriminators_, const String & variant_element_name_, ColumnVariant::Discriminator global_variant_discriminator_, - ColumnVariant::Discriminator local_variant_discriminator_) + ColumnVariant::Discriminator local_variant_discriminator_, + size_t num_variants_) : local_discriminators(local_discriminators_) , variant_element_name(variant_element_name_) , global_variant_discriminator(global_variant_discriminator_) , local_variant_discriminator(local_variant_discriminator_) + , num_variants(num_variants_) { } @@ -216,7 +224,7 @@ DataTypePtr SerializationVariantElementNullMap::VariantNullMapSubcolumnCreator:: SerializationPtr SerializationVariantElementNullMap::VariantNullMapSubcolumnCreator::create(const DB::SerializationPtr &, const DataTypePtr &) const { - return SerializationVariantElementNullMap::create(variant_element_name, global_variant_discriminator); + return SerializationVariantElementNullMap::create(variant_element_name, global_variant_discriminator, num_variants); } ColumnPtr SerializationVariantElementNullMap::VariantNullMapSubcolumnCreator::create(const DB::ColumnPtr &) const diff --git a/src/DataTypes/Serializations/SerializationVariantElementNullMap.h b/src/DataTypes/Serializations/SerializationVariantElementNullMap.h index c2c78cadafa4..740a5eb40bcc 100644 --- a/src/DataTypes/Serializations/SerializationVariantElementNullMap.h +++ b/src/DataTypes/Serializations/SerializationVariantElementNullMap.h @@ -25,15 +25,23 @@ class SerializationVariantElement; class SerializationVariantElementNullMap final : public SimpleTextSerialization { private: - SerializationVariantElementNullMap(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_) - : variant_element_name(variant_element_name_), variant_discriminator(variant_discriminator_) + SerializationVariantElementNullMap( + const String & variant_element_name_, + ColumnVariant::Discriminator variant_discriminator_, + size_t num_variants_) + : variant_element_name(variant_element_name_) + , variant_discriminator(variant_discriminator_) + , num_variants(num_variants_) { } public: - static UInt128 getHash(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_); + static UInt128 getHash(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_, size_t num_variants_); - static SerializationPtr create(const String & variant_element_name_, ColumnVariant::Discriminator variant_discriminator_); + static SerializationPtr create( + const String & variant_element_name_, + ColumnVariant::Discriminator variant_discriminator_, + size_t num_variants_ = 0); size_t allocatedBytes() const override; @@ -85,12 +93,14 @@ class SerializationVariantElementNullMap final : public SimpleTextSerialization const String variant_element_name; const ColumnVariant::Discriminator global_variant_discriminator; const ColumnVariant::Discriminator local_variant_discriminator; + size_t num_variants; VariantNullMapSubcolumnCreator( const ColumnPtr & local_discriminators_, const String & variant_element_name_, ColumnVariant::Discriminator global_variant_discriminator_, - ColumnVariant::Discriminator local_variant_discriminator_); + ColumnVariant::Discriminator local_variant_discriminator_, + size_t num_variants_ = 0); DataTypePtr create(const DataTypePtr & prev) const override; ColumnPtr create(const ColumnPtr & prev) const override; @@ -109,6 +119,9 @@ class SerializationVariantElementNullMap final : public SimpleTextSerialization /// we need variant element type name and global discriminator. String variant_element_name; ColumnVariant::Discriminator variant_discriminator; + /// Total number of variants in the Variant type; used for bounds-checking + /// compact discriminators read from the wire. + size_t num_variants; }; diff --git a/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp b/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp new file mode 100644 index 000000000000..1b162cfb85c2 --- /dev/null +++ b/src/DataTypes/Serializations/tests/gtest_map_bucketed_serialization.cpp @@ -0,0 +1,168 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +using namespace DB; + +namespace +{ + +constexpr size_t NUM_BUCKETS = 4; + +/// A set of in-memory substreams, keyed by the substream name that a real data part would use +/// as a file name. It plays the role of the on-disk streams of a single column. +using Streams = std::map; + +DataTypePtr getMapType() +{ + return DataTypeFactory::instance().get("Map(String, UInt64)"); +} + +SerializationPtr getBucketedSerialization(const DataTypePtr & type) +{ + SerializationInfoSettings info_settings; + info_settings.map_serialization_version = MergeTreeMapSerializationVersion::WITH_BUCKETS; + return type->getSerialization(info_settings); +} + +/// A `Map` column where every row has a different number of key-value pairs and the keys of a +/// single row land in several buckets in an order that is not the bucket order. Both properties +/// are needed to notice a bucket index stream that is out of sync with the bucket streams. +ColumnPtr makeTestColumn(const DataTypePtr & type, size_t num_rows) +{ + auto column = type->createColumn(); + for (size_t row = 0; row != num_rows; ++row) + { + Map map; + for (size_t i = 0; i != 1 + row % 7; ++i) + map.push_back(Tuple{"key_" + std::to_string(row) + "_" + std::to_string(i), UInt64(row * 100 + i)}); + column->insert(map); + } + return std::move(column); +} + +Streams serializeWithBuckets(const DataTypePtr & type, const IColumn & column) +{ + std::map> buffers; + + ISerialization::SerializeBinaryBulkSettings settings; + settings.write_statistics = ISerialization::SerializeBinaryBulkSettings::StatisticsMode::PREFIX; + settings.max_buckets_in_map = NUM_BUCKETS; + settings.map_buckets_strategy = MergeTreeMapBucketsStrategy::CONSTANT; + settings.getter = [&](const ISerialization::SubstreamPath & path) -> WriteBuffer * + { + auto name = ISerialization::getSubcolumnNameForStream(path); + auto it = buffers.find(name); + if (it == buffers.end()) + it = buffers.emplace(name, std::make_unique()).first; + return it->second.get(); + }; + + auto serialization = getBucketedSerialization(type); + ISerialization::SerializeBinaryBulkStatePtr state; + serialization->serializeBinaryBulkStatePrefix(column, settings, state); + serialization->serializeBinaryBulkWithMultipleStreams(column, 0, column.size(), settings, state); + serialization->serializeBinaryBulkStateSuffix(settings, state); + + Streams streams; + for (auto & [name, buffer] : buffers) + { + buffer->finalize(); + streams[name] = buffer->str(); + } + return streams; +} + +/// Reads `limit` rows starting from `rows_offset` out of the serialized streams. +/// `serialization` is either the whole-column serialization or a subcolumn one. +ColumnPtr deserializeRange( + const SerializationPtr & serialization, const DataTypePtr & column_type, const Streams & streams, size_t rows_offset, size_t limit) +{ + std::map> buffers; + + ISerialization::DeserializeBinaryBulkSettings settings; + settings.getter = [&](const ISerialization::SubstreamPath & path) -> ReadBuffer * + { + auto name = ISerialization::getSubcolumnNameForStream(path); + auto stream_it = streams.find(name); + if (stream_it == streams.end()) + return nullptr; + + auto it = buffers.find(name); + if (it == buffers.end()) + it = buffers.emplace(name, std::make_unique(stream_it->second)).first; + return it->second.get(); + }; + settings.check_stream_exists_callback = [&](const ISerialization::SubstreamPath & path) + { + return streams.contains(ISerialization::getSubcolumnNameForStream(path)); + }; + + ISerialization::DeserializeBinaryBulkStatePtr state; + serialization->deserializeBinaryBulkStatePrefix(settings, state, nullptr); + + ColumnPtr column = column_type->createColumn(); + serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_offset, limit, settings, state, nullptr); + return column; +} + +void assertRangesEqual(const IColumn & expected, size_t expected_offset, const IColumn & actual) +{ + ASSERT_EQ(expected.size() - expected_offset, actual.size()); + for (size_t row = 0; row != actual.size(); ++row) + ASSERT_EQ(expected[expected_offset + row], actual[row]) << "at row " << row; +} + +} + +/// The `bucket_indexes` stream that restores the original key order holds one entry per key-value +/// pair, so it cannot be positioned by a number of rows. Reading a range that starts in the middle +/// of a granule (`rows_offset > 0`) must still return exactly the rows of that range, in the +/// original key order, and not the bucket indexes of the skipped rows. +TEST(MapBucketedSerialization, ReadWithRowsOffset) +{ + auto type = getMapType(); + constexpr size_t num_rows = 20; + auto column = makeTestColumn(type, num_rows); + auto streams = serializeWithBuckets(type, *column); + + /// More than one bucket is what makes the bucket index stream necessary in the first place. + ASSERT_TRUE(streams.contains("bucket_indexes")); + + auto serialization = getBucketedSerialization(type); + for (size_t rows_offset = 0; rows_offset != num_rows; ++rows_offset) + { + auto result = deserializeRange(serialization, type, streams, rows_offset, num_rows - rows_offset); + assertRangesEqual(*column, rows_offset, *result); + } +} + +/// The same holds for the `keys` subcolumn, which reassembles the keys of all buckets on its own. +TEST(MapBucketedSerialization, ReadKeysSubcolumnWithRowsOffset) +{ + auto type = getMapType(); + constexpr size_t num_rows = 20; + auto column = makeTestColumn(type, num_rows); + auto streams = serializeWithBuckets(type, *column); + + auto keys_type = type->getSubcolumnType("keys"); + auto keys_serialization = type->getSubcolumnSerialization("keys", getBucketedSerialization(type)); + auto expected_keys = deserializeRange(keys_serialization, keys_type, streams, 0, num_rows); + + for (size_t rows_offset = 1; rows_offset != num_rows; ++rows_offset) + { + auto result = deserializeRange(keys_serialization, keys_type, streams, rows_offset, num_rows - rows_offset); + assertRangesEqual(*expected_keys, rows_offset, *result); + } +} diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index 012b9de7cb43..d404e3a4eb65 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -28,6 +28,7 @@ static inline std::unordered_map SETTINGS_TO_HIDE = /// AWS credentials {"aws_access_key_id", DEFAULT_MASKING_RULE}, {"aws_secret_access_key", DEFAULT_MASKING_RULE}, + {"aws_external_id", DEFAULT_MASKING_RULE}, /// OneLake credentials {"onelake_client_secret", DEFAULT_MASKING_RULE}, {"onelake_bearer_token", DEFAULT_MASKING_RULE}, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index ba2fc89c7799..6f2a608398f6 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include #include @@ -107,6 +108,7 @@ namespace Setting extern const SettingsBool parallel_replicas_for_cluster_engines; extern const SettingsString cluster_for_parallel_replicas; extern const SettingsBool database_datalake_require_metadata_access; + extern const SettingsBool show_data_lake_catalogs_in_system_tables; } @@ -129,6 +131,7 @@ namespace FailPoints { extern const char lightweight_show_tables[]; extern const char datalake_try_get_table_return_nullptr[]; + extern const char datalake_get_tables_throw[]; } DatabaseDataLake::DatabaseDataLake( @@ -141,7 +144,7 @@ DatabaseDataLake::DatabaseDataLake( bool lazy_init) : IDatabase(database_name_) , url(url_) - , settings(settings_) + , database_settings(std::make_unique(settings_)) , database_engine_definition(database_engine_definition_) , table_engine_definition(table_engine_definition_) , log(getLogger("DatabaseDataLake(" + database_name_ + ")")) @@ -160,6 +163,9 @@ DatabaseDataLake::DatabaseDataLake( void DatabaseDataLake::validateSettings() { + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + if (settings[DatabaseDataLakeSetting::catalog_type].value == DB::DatabaseDataLakeCatalogType::GLUE) { if (settings[DatabaseDataLakeSetting::region].value.empty()) @@ -191,6 +197,9 @@ void DatabaseDataLake::initialize() const { /// Caller holds `catalog_mutex`: this runs either from the constructor (CREATE, eager) /// or from `getCatalog` on first access (ATTACH, lazy). + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + if (settings[DatabaseDataLakeSetting::catalog_type].value == DatabaseDataLakeCatalogType::NONE) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unspecified catalog type"); @@ -363,18 +372,29 @@ std::shared_ptr DatabaseDataLake::getCatalog() const if (!catalog_impl) initialize(); + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + catalog_impl->setVendedCredentialsCacheTTL( std::chrono::seconds(settings[DatabaseDataLakeSetting::vended_credentials_cache_ttl].value)); return catalog_impl; } +void DatabaseDataLake::resetCatalog() const +{ + catalog_impl = nullptr; +} + StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration( DatabaseDataLakeStorageType type, DataLakeStorageSettingsPtr storage_settings) const { /// TODO: add tests for azure, local storage types. + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + auto catalog = getCatalog(); switch (catalog->getCatalogType()) { @@ -546,6 +566,9 @@ StorageObjectStorageConfigurationPtr DatabaseDataLake::getConfiguration( std::string DatabaseDataLake::getStorageEndpointForTable(const DataLake::TableMetadata & table_metadata) const { + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + auto endpoint_from_settings = settings[DatabaseDataLakeSetting::storage_endpoint].value; if (endpoint_from_settings.empty()) return table_metadata.getLocation(); @@ -570,6 +593,9 @@ StoragePtr DatabaseDataLake::tryGetTable(const String & name, ContextPtr context StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr context_, bool lightweight, bool ignore_if_not_iceberg) const { + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + auto catalog = getCatalog(); auto table_metadata = DataLake::TableMetadata().withSchema().withLocation().withDataLakeSpecificProperties(); if (settings[DatabaseDataLakeSetting::force_add_bucket]) @@ -700,11 +726,12 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con auto rest_catalog = std::static_pointer_cast(catalog); if (!rest_catalog) throw Exception(ErrorCodes::LOGICAL_ERROR, "Catalog is not equals to one lake"); + const auto auth = rest_catalog->getStateSnapshot(); azure_configuration->setInitializationAsOneLake( - rest_catalog->getClientId(), - rest_catalog->getClientSecret(), - rest_catalog->getTenantId(), - rest_catalog->getBearerToken(), + auth->client_id, + auth->client_secret, + auth->tenant_id, + auth->bearer_token, settings[DatabaseDataLakeSetting::onelake_use_blob_endpoint].value ); #else @@ -803,10 +830,17 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( /// It must not fail on case of some datalake error. try { + fiu_do_on(FailPoints::datalake_get_tables_throw, + { + throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); + }); + iceberg_tables = getCatalog()->getTables(); } catch (...) { + if (context_->getSettingsRef()[Setting::show_data_lake_catalogs_in_system_tables]) + throw; tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -886,7 +920,7 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( } std::vector DatabaseDataLake::getLightweightTablesIterator( - ContextPtr /*context_*/, + ContextPtr context_, const FilterByNameFunction & filter_by_table_name, bool /*skip_not_loaded*/) const { @@ -897,10 +931,17 @@ std::vector DatabaseDataLake::getLightweightTablesItera /// It must not fail on case of some datalake error. try { + fiu_do_on(FailPoints::datalake_get_tables_throw, + { + throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); + }); + iceberg_tables = getCatalog()->getTables(); } catch (...) { + if (context_->getSettingsRef()[Setting::show_data_lake_catalogs_in_system_tables]) + throw; tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -953,11 +994,93 @@ void DatabaseDataLake::checkDatabase() const LOG_TEST(log, "Database '{}' is OK", getDatabaseName()); } +void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr /*query_context*/) +{ + const auto current_settings = database_settings.get(); + + /// This check in some sense duplicate check in ICatalog, because it's a valid case when + /// catalog can be unitilized here, and we actually use alter to "resurrect it". For example provide + /// proper credentials with settings. + DataLake::CatalogSettingsAlterValidatorFactory::instance().validate(*current_settings, settings_changes); + + auto new_settings = std::make_unique(*current_settings); + new_settings->applyChanges(settings_changes); + + ASTPtr new_engine_definition; + { + std::lock_guard lock(mutex); + new_engine_definition = database_engine_definition->clone(); + } + auto * storage = new_engine_definition->as(); + if (!storage) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine definition of database {} is not a storage AST", getDatabaseName()); + + if (storage->settings) + { + auto & stored_changes = storage->settings->changes; + for (const auto & change : settings_changes) + { + /// cleanup duplicates + std::erase_if(stored_changes, [&](const auto & prev) { return prev.name == change.name; }); + stored_changes.push_back(change); + } + } + else + { + auto storage_settings_ast = make_intrusive(); + storage_settings_ast->is_standalone = false; + storage_settings_ast->changes = settings_changes; + storage->set(storage->settings, storage_settings_ast); + } + + std::shared_ptr local_catalog_snapshot; + { + std::lock_guard lock(catalog_mutex); + local_catalog_snapshot = catalog_impl; + } + + /// Prepare the new catalog state without publishing it: validation, the eager token + /// fetch and the config reload may throw, and then nothing has changed yet. + DataLake::ICatalog::PreparedSettingsChangesPtr prepared_catalog_changes; + if (local_catalog_snapshot) + prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes); + + /// Persist the new metadata before publishing anything: if the write fails, the live + /// state is untouched and matches the old metadata on disk. The create query is built + /// from the patched definition because the live one is not swapped yet. + auto new_create_query = make_intrusive(); + new_create_query->setDatabase(getDatabaseName()); + new_create_query->set(new_create_query->storage, new_engine_definition); + new_create_query->uuid = db_uuid; + DatabaseCatalog::instance().updateMetadataFile(getDatabaseName(), new_create_query); + + /// Publish. Nothing below throws. + if (local_catalog_snapshot) + local_catalog_snapshot->commitSettingsChanges(std::move(prepared_catalog_changes)); + database_settings.set(std::move(new_settings)); + { + std::lock_guard lock(mutex); + database_engine_definition = new_engine_definition; + } + if (!local_catalog_snapshot) + { + /// The catalog was not built when the ALTER started. If a concurrent query + /// built it meanwhile, it used the old settings: drop it so the next access + /// rebuilds it with the new ones. Also clear a recorded construction failure + /// (e.g. credentials lost on RESTORE) for the same reason. + std::lock_guard lock(catalog_mutex); + resetCatalog(); + } +} + ASTPtr DatabaseDataLake::getCreateTableQueryImpl( const String & name, ContextPtr context_, bool throw_on_error) const { + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + auto catalog = getCatalog(); auto table_metadata = DataLake::TableMetadata().withLocation().withSchema(); if (settings[DatabaseDataLakeSetting::force_add_bucket]) diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index bd5fc6ac44f0..fc67aad2b133 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace DB @@ -67,6 +68,8 @@ class DatabaseDataLake final : public IDatabase, WithContext const String & name, bool /*sync*/) override; + void applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr query_context) override; + std::shared_ptr getCatalog() const; protected: ASTPtr getCreateDatabaseQueryImpl() const override TSA_REQUIRES(mutex); @@ -76,9 +79,9 @@ class DatabaseDataLake final : public IDatabase, WithContext /// Iceberg Catalog url. const std::string url; /// SETTINGS from CREATE query. - const DatabaseDataLakeSettings settings; + MultiVersion database_settings; /// Database engine definition taken from initial CREATE DATABASE query. - const ASTPtr database_engine_definition; + ASTPtr database_engine_definition TSA_GUARDED_BY(mutex); const ASTPtr table_engine_definition; const LoggerPtr log; /// Crendetials to authenticate Iceberg Catalog. @@ -97,6 +100,9 @@ class DatabaseDataLake final : public IDatabase, WithContext /// front. Guarded by `catalog_mutex` because lazy initialization can race concurrent readers. void initialize() const TSA_REQUIRES(catalog_mutex); + /// Drop the cached catalog so the next `getCatalog` rebuilds it from the current settings. + void resetCatalog() const TSA_REQUIRES(catalog_mutex); + StorageObjectStorageConfigurationPtr getConfiguration( DatabaseDataLakeStorageType type, DataLakeStorageSettingsPtr storage_settings) const; diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index 216d32290c24..b4cf78f25a33 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -14,6 +14,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; + extern const int LOGICAL_ERROR; } #define DATABASE_ICEBERG_RELATED_SETTINGS(DECLARE, ALIAS) \ @@ -110,4 +111,13 @@ SettingsChanges DatabaseDataLakeSettings::allChanged() const return changes; } +const String & DatabaseDataLakeSettings::getSettingName(DatabaseDataLakeSettingsString setting) +{ + const auto & accessor = DatabaseDataLakeSettingsTraits::Accessor::instance(); + const size_t index = accessor.findByOffset(setting.offset); + if (index == static_cast(-1)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown database DataLake setting"); + return accessor.getName(index); +} + } diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.h b/src/Databases/DataLake/DatabaseDataLakeSettings.h index be2813a44de1..bf67af80cc87 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.h +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.h @@ -71,6 +71,10 @@ struct DatabaseDataLakeSettings SettingsChanges allChanged() const; + /// Name of the setting referenced by its `DatabaseDataLakeSetting::*` index, + /// so catalog code can match `SettingsChanges` entries without magic strings. + static const String & getSettingName(DatabaseDataLakeSettingsString setting); + private: std::unique_ptr impl; }; diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index a8dda240e7ce..a9cbb54e05f3 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -640,7 +640,7 @@ String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_lo } } -void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name) const +void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/) const { Aws::Glue::Model::CreateDatabaseRequest create_request; Aws::Glue::Model::DatabaseInput db_input; @@ -649,7 +649,14 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name) cons ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); - glue_client->CreateDatabase(create_request); + auto outcome = glue_client->CreateDatabase(create_request); + if (!outcome.IsSuccess() && outcome.GetError().GetErrorType() != Aws::Glue::GlueErrors::ALREADY_EXISTS) + { + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, + "Exception calling CreateDatabase for namespace {}: {}", + namespace_name, outcome.GetError().GetMessage()); + } } void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*metadata_content*/) const @@ -659,8 +666,6 @@ void GlueCatalog::createTable(const String & namespace_name, const String & tabl "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - createNamespaceIfNotExists(namespace_name); - Aws::Glue::Model::CreateTableRequest request; request.SetDatabaseName(namespace_name); diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 8d10ba0c8667..4d7a1bb1fc9c 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -68,6 +68,8 @@ class GlueCatalog final : public ICatalog, private DB::WithContext void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; bool updateSchema( @@ -96,8 +98,6 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & glue_column_type); private: - void createNamespaceIfNotExists(const String & namespace_name) const; - std::unique_ptr glue_client; const LoggerPtr log; std::shared_ptr credentials_provider; diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 62cf44930225..ddeeaa25afbf 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -15,6 +16,11 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; } +namespace DB::DatabaseDataLakeSetting +{ + extern const DatabaseDataLakeSettingsDatabaseDataLakeCatalogType catalog_type; +} + namespace DB::FailPoints { extern const char database_iceberg_gcs[]; @@ -340,6 +346,11 @@ void ICatalog::createTable(const String & /*namespace_name*/, const String & /*t throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createTable is not implemented"); } +void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/) const +{ + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createNamespaceIfNotExists is not implemented"); +} + bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); @@ -362,4 +373,41 @@ void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*tab throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "dropTable is not implemented"); } +ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges(const DB::SettingsChanges & /*changes*/) +{ + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Settings of a catalog of this type cannot be altered"); +} + +void ICatalog::commitSettingsChanges(PreparedSettingsChangesPtr /*prepared*/) +{ + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes were prepared for a catalog that cannot commit them"); +} + +CatalogSettingsAlterValidatorFactory & CatalogSettingsAlterValidatorFactory::instance() +{ + static CatalogSettingsAlterValidatorFactory factory; + return factory; +} + +void CatalogSettingsAlterValidatorFactory::registerValidator(DB::DatabaseDataLakeCatalogType catalog_type, Validator validator) +{ + if (!validators.emplace(catalog_type, std::move(validator)).second) + throw DB::Exception( + DB::ErrorCodes::LOGICAL_ERROR, + "Settings alter validator for catalog type '{}' is already registered", + DB::SettingFieldDatabaseDataLakeCatalogType(catalog_type).toString()); +} + +void CatalogSettingsAlterValidatorFactory::validate(const DB::DatabaseDataLakeSettings & current_settings, const DB::SettingsChanges & changes) const +{ + const auto it = validators.find(current_settings[DB::DatabaseDataLakeSetting::catalog_type].value); + if (it == validators.end()) + throw DB::Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "ALTER MODIFY SETTING is not supported for catalog type '{}'", + current_settings[DB::DatabaseDataLakeSetting::catalog_type].toString()); + + it->second(current_settings, changes); +} + } diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index f77bfcff0405..9cb18c15177b 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -11,12 +11,16 @@ #include #include +#include +#include + namespace DB { class Context; using ContextPtr = std::shared_ptr; +struct DatabaseDataLakeSettings; } namespace DataLake @@ -26,6 +30,24 @@ using StorageType = DB::DatabaseDataLakeStorageType; StorageType parseStorageTypeFromLocation(const std::string & location); StorageType parseStorageTypeFromString(const std::string &type); +/// Registry of `ALTER DATABASE ... MODIFY SETTING` validators. Each catalog that +/// supports altering settings registers its own validator; catalog types without +/// a registered validator do not support altering settings and get `NOT_IMPLEMENTED`. +class CatalogSettingsAlterValidatorFactory +{ +public: + using Validator = std::function; + + static CatalogSettingsAlterValidatorFactory & instance(); + + void registerValidator(DB::DatabaseDataLakeCatalogType catalog_type, Validator validator); + + void validate(const DB::DatabaseDataLakeSettings & current_settings, const DB::SettingsChanges & changes) const; + +private: + std::unordered_map validators; +}; + struct DataLakeSpecificProperties { std::string iceberg_metadata_file_location; @@ -189,9 +211,14 @@ class ICatalog /// E.g. one of S3, Azure, Local, HDFS. virtual std::optional getStorageType() const = 0; - /// Creates new table in catalog. + /// Creates new table in catalog. Callers must ensure the namespace exists before + /// writing any table files to storage: a catalog that shares its storage view with + /// the data refuses to create a namespace over a plain directory those files create. virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const; + /// Creates the namespace unless it already exists. + virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location) const; + /// Updates metadata in catalog. virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const; @@ -227,6 +254,28 @@ class ICatalog virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} + /// Result of `prepareSettingsChanges`: the new catalog state built off to the side, + /// ready to be published by `commitSettingsChanges`. + struct PreparedSettingsChanges + { + virtual ~PreparedSettingsChanges() = default; + }; + using PreparedSettingsChangesPtr = std::unique_ptr; + + /// Validate `ALTER DATABASE ... MODIFY SETTING` changes and build the new catalog + /// state without publishing anything (may throw, may do network I/O). The state + /// becomes visible only after `commitSettingsChanges`, so the caller can persist + /// the changes in between and abandon the prepared state on failure. + virtual PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes); + + /// Publish the state built by `prepareSettingsChanges`. Must not fail. + virtual void commitSettingsChanges(PreparedSettingsChangesPtr prepared); + + void applySettingsChanges(const DB::SettingsChanges & changes) + { + commitSettingsChanges(prepareSettingsChanges(changes)); + } + protected: /// Name of the warehouse, /// which is sometimes also called "catalog name". diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index a2d2dd04c080..cffce96f082d 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -248,9 +248,10 @@ void PaimonRestCatalog::createAuthHeaders( String date_time = get_or_default(headers_map, DLF_DATE_HEADER_KEY, fmt::format(AUTH_DATE_TIME_FORMATTER, *utc_tm)); String date = date_time.substr(0, 8); generate_sign_headers(data, date_time, std::nullopt); - String authorization - = token->dlf_generated_authorization.empty() ? get_authorization(date, date_time) : token->dlf_generated_authorization; - token->dlf_generated_authorization = authorization; + /// The DLF v4 signature covers the canonical request (method, resource path, query + /// parameters and signed headers), so it must be computed for every request anew: + /// a signature from an earlier request is invalid for any other one. + String authorization = get_authorization(date, date_time); headers_map.emplace(DLF_AUTHORIZATION_HEADER_KEY, authorization); current_headers.clear(); for (const auto & entry : headers_map) @@ -301,22 +302,8 @@ DB::ReadWriteBufferFromHTTPPtr PaimonRestCatalog::createReadBuffer( .create(credentials); }; - bool refresh_token = true; LOG_TRACE(log, "Requesting endpoint: {}", endpoint); - try - { - return create_buffer(); - } - catch (DB::HTTPException & e) - { - if (e.code() == Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED && refresh_token && token->token_provider == "dlf") - { - refresh_token = false; - token->dlf_generated_authorization = ""; - return create_buffer(); - } - throw; - } + return create_buffer(); } void PaimonRestCatalog::forEachDatabase(DB::Strings & databases, StopCondition stop_condition, ExecuteFunc execute_func) const @@ -458,7 +445,7 @@ bool PaimonRestCatalog::existsTable(const String & database_name, const String & } catch (const DB::HTTPException & e) { - if (e.code() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) + if (e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) { return false; } @@ -573,7 +560,7 @@ bool PaimonRestCatalog::tryGetTableMetadata(const String & database_name, const } catch (const DB::HTTPException & e) { - if (e.code() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) + if (e.getHTTPStatus() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) { return false; } diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index c81722c63964..9aab6b815bdf 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -59,7 +59,6 @@ struct PaimonToken const String bearer_token; const String dlf_access_key_id; const String dlf_access_key_secret; - mutable String dlf_generated_authorization; explicit PaimonToken(const String & bearer_token_) : token_provider("bearer") diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 58bf459992f0..c98fcf48279b 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -18,6 +18,7 @@ #if USE_AVRO #include +#include #include #include @@ -58,6 +59,7 @@ #include #include #include +#include namespace DB::ErrorCodes @@ -106,6 +108,16 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogDropTableMicroseconds; } +namespace DB::DatabaseDataLakeSetting +{ + extern const DatabaseDataLakeSettingsString catalog_credential; + extern const DatabaseDataLakeSettingsString auth_header; + extern const DatabaseDataLakeSettingsString onelake_tenant_id; + extern const DatabaseDataLakeSettingsString onelake_bearer_token; + extern const DatabaseDataLakeSettingsString onelake_client_id; + extern const DatabaseDataLakeSettingsString onelake_client_secret; +} + namespace DataLake { @@ -447,17 +459,19 @@ RestCatalog::RestCatalog( , oauth_server_use_request_body(oauth_server_use_request_body_) , allowed_namespaces(namespaces_) { + CatalogState initial_state; if (!catalog_credential_.empty()) { - std::tie(client_id, client_secret) = parseCatalogCredential(catalog_credential_); + std::tie(initial_state.client_id, initial_state.client_secret) = parseCatalogCredential(catalog_credential_); update_token_if_expired = true; } else if (!auth_header_.empty()) { - auth_header = parseAuthHeader(auth_header_); - validateAuthHeaders(auth_header.value()); + initial_state.auth_header = parseAuthHeader(auth_header_); + validateAuthHeaders(initial_state.auth_header.value()); } - config = loadConfig(); + initial_state.config = loadConfig(initial_state); + state.set(std::make_unique(std::move(initial_state))); } RestCatalog::RestCatalog( @@ -480,7 +494,7 @@ RestCatalog::RestCatalog( } -RestCatalog::Config RestCatalog::loadConfig() +RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; @@ -489,7 +503,7 @@ RestCatalog::Config RestCatalog::loadConfig() { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); - auto buf = createReadBuffer(CONFIG_ENDPOINT, params); + auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */{}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -536,6 +550,7 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const } DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( + const CatalogState & catalog_state, bool update_token, const String & /*method*/, const Poco::URI & /*url*/, @@ -549,20 +564,24 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. - if (auth_header.has_value()) + if (catalog_state.auth_header.has_value()) { - return DB::HTTPHeaderEntries{auth_header.value()}; + return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } /// Option 2: user provided grant_type, client_id and client_secret. /// We would make OAuthClientCredentialsRequest /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 - if (!client_id.empty()) + if (!catalog_state.client_id.empty()) { + /// The cached token may have been minted with other credentials than the ones in + /// `catalog_state` (e.g. right after `ALTER DATABASE ... MODIFY SETTING`); then the + /// request fails with 401/403 and is retried with `update_token = true`, fetching + /// a token with the snapshot's credentials. auto current = access_token.get(); if (!current || update_token) { - access_token.set(std::make_unique(retrieveAccessToken())); + access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); current = access_token.get(); } @@ -586,36 +605,246 @@ OneLakeCatalog::OneLakeCatalog( const std::string & namespaces_, DB::ContextPtr context_) : RestCatalog(warehouse_, base_url_, auth_scope_, oauth_server_uri_, oauth_server_use_request_body_, namespaces_, context_) - , tenant_id(onelake_tenant_id) { + CatalogState initial_state; + initial_state.tenant_id = onelake_tenant_id; if (!bearer_token_.empty()) { /// Pre-obtained token scoped to https://storage.azure.com. Used for both catalog header /// and Azure Blob access. Does not support refresh. - bearer_token = bearer_token_; - auth_header = DB::HTTPHeaderEntry("Authorization", "Bearer " + bearer_token); - validateAuthHeaders(auth_header.value()); + initial_state.bearer_token = bearer_token_; + initial_state.auth_header = DB::HTTPHeaderEntry("Authorization", "Bearer " + bearer_token_); + validateAuthHeaders(initial_state.auth_header.value()); } else { - client_id = onelake_client_id; - client_secret = onelake_client_secret; + initial_state.client_id = onelake_client_id; + initial_state.client_secret = onelake_client_secret; update_token_if_expired = true; - // Get token before loading config so getAuthHeaders() can work - if (!client_id.empty() && !client_secret.empty()) + } + initial_state.config = loadConfig(initial_state); + state.set(std::make_unique(std::move(initial_state))); +} + +void RestCatalog::validateSettingsChangesImpl( + const DB::SettingsChanges & changes, + const std::unordered_set & alterable_settings, + const std::string & auth_mode_description) +{ + for (const auto & change : changes) + { + if (alterable_settings.empty()) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "Setting `{}` cannot be altered for a {}: the database was created without authentication settings", + change.name, + auth_mode_description); + + if (!alterable_settings.contains(change.name)) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "Setting `{}` cannot be altered for a {} " + "(the authentication mode is fixed when the database is created; " + "alterable settings are: {})", + change.name, + auth_mode_description, + fmt::join(alterable_settings, ", ")); + + if (change.value.getType() != DB::Field::Types::String) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Setting `{}` must be a string", change.name); + + if (change.value.safeGet().empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Setting `{}` cannot be set to an empty value", change.name); + } +} + +void RestCatalog::validateSettingsChanges(const DB::SettingsChanges & changes, bool credential_mode, bool header_mode) +{ + static const std::unordered_set credential_mode_settings = { + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::catalog_credential)}; + static const std::unordered_set header_mode_settings = { + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::auth_header)}; + static const std::unordered_set no_auth_settings = {}; + + if (credential_mode) + validateSettingsChangesImpl(changes, credential_mode_settings, "REST catalog with catalog credential authentication"); + else if (header_mode) + validateSettingsChangesImpl(changes, header_mode_settings, "REST catalog with auth header authentication"); + else + validateSettingsChangesImpl(changes, no_auth_settings, "REST catalog"); +} + +struct RestCatalog::PreparedAuthChanges : ICatalog::PreparedSettingsChanges +{ + std::unique_ptr new_state; + /// Set only when the OAuth credentials changed. + std::unique_ptr new_access_token; +}; + +ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const DB::SettingsChanges & changes) +{ + const auto old_state = state.get(); + CatalogState new_state = *old_state; + + auto prepared = std::make_unique(); + std::optional new_auth_headers; + applySettingsChangesToState(changes, *old_state, new_state, new_auth_headers, prepared->new_access_token); + + /// The config was loaded with the old credentials; the new ones may resolve the + /// warehouse to a different prefix or base location, so reload it before publishing. + new_state.config = loadConfig(new_state, new_auth_headers); + prepared->new_state = std::make_unique(std::move(new_state)); + return prepared; +} + +void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr prepared) +{ + auto * prepared_auth = dynamic_cast(prepared.get()); + if (!prepared_auth || !prepared_auth->new_state) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); + + state.set(std::move(prepared_auth->new_state)); + if (prepared_auth->new_access_token) + access_token.set(std::move(prepared_auth->new_access_token)); +} + +void RestCatalog::applySettingsChangesToState( + const DB::SettingsChanges & changes, + const CatalogState & old_state, + CatalogState & new_state, + std::optional & new_auth_headers, + std::unique_ptr & new_access_token) +{ + const bool credential_mode = !old_state.client_id.empty(); + const bool header_mode = old_state.auth_header.has_value(); + + validateSettingsChanges(changes, credential_mode, header_mode); + + for (const auto & change : changes) + { + if (change.name == DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::catalog_credential)) + { + std::tie(new_state.client_id, new_state.client_secret) = parseCatalogCredential(change.value.safeGet()); + } + else if (change.name == DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::auth_header)) { - access_token.set(std::make_unique(retrieveAccessToken())); + new_state.auth_header = parseAuthHeader(change.value.safeGet()); + validateAuthHeaders(new_state.auth_header.value()); } + else + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected setting `{}` after validation", change.name); + } + + if (credential_mode && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) + { + /// Eagerly fetch a token with the not-yet-published credentials: wrong credentials + /// fail the ALTER right here, and the config reload authenticates with that token + /// instead of the cached one. + new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + new_access_token->token}}; } - config = loadConfig(); } -String OneLakeCatalog::getBearerToken() const +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( + const CatalogState & catalog_state, + bool update_token, + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body) const +{ + auto headers = RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body); + headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); + return headers; +} + +void OneLakeCatalog::validateSettingsChanges(const DB::SettingsChanges & changes, bool bearer_mode) { - return bearer_token; + static const std::unordered_set bearer_mode_settings = { + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_tenant_id), + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_bearer_token)}; + static const std::unordered_set client_mode_settings = { + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_tenant_id), + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_client_id), + DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_client_secret)}; + + RestCatalog::validateSettingsChangesImpl( + changes, + bearer_mode ? bearer_mode_settings : client_mode_settings, + bearer_mode ? "OneLake catalog with bearer token authentication" : "OneLake catalog with client credentials authentication"); } -AccessToken RestCatalog::retrieveAccessToken() const +void OneLakeCatalog::applySettingsChangesToState( + const DB::SettingsChanges & changes, + const CatalogState & old_state, + CatalogState & new_state, + std::optional & new_auth_headers, + std::unique_ptr & new_access_token) +{ + const bool bearer_mode = !old_state.bearer_token.empty(); + + validateSettingsChanges(changes, bearer_mode); + + for (const auto & change : changes) + { + if (change.name == DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_tenant_id)) + new_state.tenant_id = change.value.safeGet(); + else if (change.name == DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_bearer_token)) + new_state.bearer_token = change.value.safeGet(); + else if (change.name == DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_client_id)) + new_state.client_id = change.value.safeGet(); + else if (change.name == DB::DatabaseDataLakeSettings::getSettingName(DB::DatabaseDataLakeSetting::onelake_client_secret)) + new_state.client_secret = change.value.safeGet(); + else + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected setting `{}` after validation", change.name); + } + + if (bearer_mode) + { + new_state.auth_header = DB::HTTPHeaderEntry("Authorization", "Bearer " + new_state.bearer_token); + validateAuthHeaders(new_state.auth_header.value()); + } + else if (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret) + { + /// Eagerly fetch a token with the not-yet-published credentials: wrong credentials + /// fail the ALTER right here, and the config reload authenticates with that token + /// instead of the cached one. + new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + new_access_token->token}}; + } +} + +namespace +{ + +[[maybe_unused]] const bool rest_settings_alter_validator_registered = [] +{ + CatalogSettingsAlterValidatorFactory::instance().registerValidator( + DB::DatabaseDataLakeCatalogType::ICEBERG_REST, + [](const DB::DatabaseDataLakeSettings & current_settings, const DB::SettingsChanges & changes) + { + const bool credential_mode = !current_settings[DB::DatabaseDataLakeSetting::catalog_credential].value.empty(); + const bool header_mode = !current_settings[DB::DatabaseDataLakeSetting::auth_header].value.empty(); + RestCatalog::validateSettingsChanges(changes, credential_mode, header_mode); + }); + return true; +}(); + +[[maybe_unused]] const bool onelake_settings_alter_validator_registered = [] +{ + CatalogSettingsAlterValidatorFactory::instance().registerValidator( + DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE, + [](const DB::DatabaseDataLakeSettings & current_settings, const DB::SettingsChanges & changes) + { + const bool bearer_mode = !current_settings[DB::DatabaseDataLakeSetting::onelake_bearer_token].value.empty(); + OneLakeCatalog::validateSettingsChanges(changes, bearer_mode); + }); + return true; +}(); + +} + +AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; @@ -676,7 +905,9 @@ AccessToken RestCatalog::retrieveAccessToken() const request.set("Accept", "application/json"); std::ostream & os = session->sendRequest(request); - out_stream_callback(os); + /// The query-parameters flavor of the request has no body. + if (out_stream_callback) + out_stream_callback(os); Poco::Net::HTTPResponse response; std::istream & rs = session->receiveResponse(response); @@ -729,15 +960,18 @@ BigLakeCatalog::BigLakeCatalog( { access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); } - config = loadConfig(); + CatalogState initial_state; + initial_state.config = loadConfig(initial_state); + state.set(std::make_unique(std::move(initial_state))); } DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( + const CatalogState & catalog_state, bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/) const + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body) const { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -769,7 +1003,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } - return RestCatalog::getAuthHeaders(update_token); + return RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body); } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -892,15 +1126,18 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const std::optional RestCatalog::getStorageType() const { - if (config.default_base_location.empty()) + const auto state_snapshot = state.get(); + if (state_snapshot->config.default_base_location.empty()) return std::nullopt; - return parseStorageTypeFromLocation(config.default_base_location); + return parseStorageTypeFromLocation(state_snapshot->config.default_base_location); } DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( + const CatalogState & catalog_state, const std::string & endpoint, const Poco::URI::QueryParameters & params, - const DB::HTTPHeaderEntries & headers) const + const DB::HTTPHeaderEntries & headers, + const std::optional & auth_headers) const { const auto & context = getContext(); @@ -911,7 +1148,9 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token) { - auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}); + auto result_headers = auth_headers + ? *auth_headers + : getAuthHeaders(catalog_state, update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1067,6 +1306,8 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace) const { + const auto state_snapshot = state.get(); + Poco::URI::QueryParameters base_params; if (!base_namespace.empty()) base_params = createParentNamespaceParams(base_namespace); @@ -1093,7 +1334,7 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); - auto buf = createReadBuffer(config.prefix / NAMESPACES_ENDPOINT, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1226,6 +1467,8 @@ DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limi throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", base_namespace); + const auto state_snapshot = state.get(); + auto encoded_namespace = encodeNamespaceForURI(base_namespace); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encoded_namespace / "tables"; @@ -1249,7 +1492,7 @@ DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limi ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); - auto buf = createReadBuffer(config.prefix / endpoint, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, params); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -1350,12 +1593,14 @@ bool RestCatalog::tryGetTableMetadata( { return getTableMetadataImpl(namespace_name, table_name, context_, result); } - catch (const DB::Exception & ex) + catch (const DB::HTTPException & ex) { - if (ex.code() == DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED) - throw; - LOG_DEBUG(log, "tryGetTableMetadata response: {}", ex.what()); - return false; + if (ex.getHTTPStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) + { + LOG_DEBUG(log, "Table {}.{} does not exist: {}", namespace_name, table_name, ex.displayText()); + return false; + } + throw; } } @@ -1488,13 +1733,14 @@ bool RestCatalog::getTableMetadataImpl( } } + const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; String json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); - auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); if (buf->eof()) { @@ -1593,7 +1839,7 @@ bool RestCatalog::getTableMetadataImpl( return true; } -void RestCatalog::sendRequest(const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const +void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const { std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM if (request_body) @@ -1617,7 +1863,7 @@ void RestCatalog::sendRequest(const String & endpoint, Poco::JSON::Object::Ptr r DB::HTTPHeaderEntries extra_headers; extra_headers.emplace_back("Content-Type", "application/json"); - DB::HTTPHeaderEntries headers = getAuthHeaders(/* update_token = */ true, method, url, extra_headers, body_str); + DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, /* update_token = */ true, method, url, extra_headers, body_str); headers.emplace_back("Content-Type", "application/json"); auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -1639,7 +1885,24 @@ void RestCatalog::sendRequest(const String & endpoint, Poco::JSON::Object::Ptr r void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location) const { - const std::string endpoint = (base_url / config.prefix / NAMESPACES_ENDPOINT).generic_string(); + const auto state_snapshot = state.get(); + + /// Check existence first: creation may be denied to a principal that is still + /// allowed to use a pre-provisioned namespace. + const std::string check_endpoint + = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name)).generic_string(); + try + { + sendRequest(*state_snapshot, check_endpoint, /* request_body */ nullptr, Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); + return; + } + catch (const DB::HTTPException & e) + { + if (e.getHTTPStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND) + throw; + } + + const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT).generic_string(); Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; { @@ -1657,11 +1920,13 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); - sendRequest(endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body); } - catch (...) + catch (const DB::HTTPException & e) { - DB::tryLogCurrentException(log); + /// Lost the race to a concurrent creator. + if (e.getHTTPStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_CONFLICT) + throw; } } @@ -1671,9 +1936,8 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - createNamespaceIfNotExists(namespace_name, metadata_content->getValue("location")); - - const std::string endpoint = (base_url / config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables").generic_string(); + const auto state_snapshot = state.get(); + const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables").generic_string(); Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; request_body->set("name", table_name); @@ -1705,7 +1969,7 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); - sendRequest(endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body); } catch (const DB::HTTPException & ex) { @@ -1722,7 +1986,8 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t "REST catalog does not support metadata-only updates without a snapshot " "(required for EXPIRE SNAPSHOTS)"); - const std::string endpoint = (base_url / config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); + const auto state_snapshot = state.get(); + const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); @@ -1730,7 +1995,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); - sendRequest(endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body); } catch (const DB::HTTPException & ex) { @@ -1757,14 +2022,15 @@ bool RestCatalog::updateSchema( { fiu_do_on(DB::FailPoints::iceberg_alter_catalog_update_schema_fail, { return false; }); - const std::string endpoint = (base_url / config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); + const auto state_snapshot = state.get(); + const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateSchemaRequestBody( namespace_name, table_name, metadata, new_schema, previous_schema_id, new_last_column_id); try { - sendRequest(endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body); } catch (const DB::HTTPException & ex) { @@ -1792,8 +2058,9 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); + const auto state_snapshot = state.get(); const std::string endpoint - = (base_url / config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() + = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() + "?purgeRequested=False"; Poco::JSON::Object::Ptr request_body = nullptr; @@ -1801,7 +2068,7 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2008,6 +2275,7 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal DB::HTTPHeaderEntries headers; headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); + const auto state_snapshot = state.get(); const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; @@ -2016,7 +2284,7 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); - auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); if (buf->eof()) { diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 4eb33d1045ab..a3fa794fcc30 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -15,6 +15,7 @@ #include #include #include +#include #include namespace DB @@ -106,23 +107,8 @@ class RestCatalog : public ICatalog, public DB::WithContext ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; - String getClientId() const { return client_id; } - String getClientSecret() const { return client_secret; } - void setVendedCredentialsCacheTTL(std::chrono::seconds ttl) override { vended_credentials_cache_ttl.store(ttl, std::memory_order_relaxed); } -protected: - RestCatalog( - const std::string & warehouse_, - const std::string & base_url_, - const std::string & auth_scope_, - const std::string & oauth_server_uri_, - bool oauth_server_use_request_body_, - const std::string & namespaces_, - DB::ContextPtr context_); - - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const; - struct Config { /// Prefix is a path of the catalog endpoint, @@ -135,19 +121,55 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string toString() const; }; + /// Credentials together with the catalog configuration they resolve to + /// (the /v1/config response depends on the credentials), published as one + /// atomic snapshot so readers never see a torn combination of them. + struct CatalogState + { + std::optional auth_header; + std::string client_id; + std::string client_secret; + std::string tenant_id; + std::string bearer_token; + Config config; + }; + using CatalogStateVersion = MultiVersion::Version; + + CatalogStateVersion getStateSnapshot() const { return state.get(); } + + ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes) override; + + void commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr prepared) override; + + /// Check that we actually support these settings alter + static void validateSettingsChangesImpl( + const DB::SettingsChanges & changes, + const std::unordered_set & alterable_settings, + const std::string & auth_mode_description); + + /// `credential_mode` means the catalog authenticates with `catalog_credential`, + /// `header_mode` with `auth_header`. The mode is fixed when the database is created. + static void validateSettingsChanges(const DB::SettingsChanges & changes, bool credential_mode, bool header_mode); + +protected: + RestCatalog( + const std::string & warehouse_, + const std::string & base_url_, + const std::string & auth_scope_, + const std::string & oauth_server_uri_, + bool oauth_server_use_request_body_, + const std::string & namespaces_, + DB::ContextPtr context_); + + void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + const std::filesystem::path base_url; const LoggerPtr log; - /// Catalog configuration settings from /v1/config endpoint. - Config config; - - /// Auth headers of format: "Authorization": " " - std::optional auth_header; + MultiVersion state{std::make_unique()}; /// Parameters for OAuth (common for REST catalog). bool update_token_if_expired = false; - std::string client_id; - std::string client_secret; std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; @@ -187,10 +209,14 @@ class RestCatalog : public ICatalog, public DB::WithContext Poco::Net::HTTPBasicCredentials credentials{}; + /// `catalog_state` is the snapshot the caller derived the endpoint from, so that one + /// request never mixes the endpoint of one state version with the auth of another. DB::ReadWriteBufferFromHTTPPtr createReadBuffer( + const CatalogState & catalog_state, const std::string & endpoint, const Poco::URI::QueryParameters & params = {}, - const DB::HTTPHeaderEntries & headers = {}) const; + const DB::HTTPHeaderEntries & headers = {}, + const std::optional & auth_headers = std::nullopt) const; Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; @@ -218,8 +244,13 @@ class RestCatalog : public ICatalog, public DB::WithContext TableMetadata & result, bool allow_credentials_cache = true) const; - Config loadConfig(); + /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. + Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); + /// `method`, `url`, `extra_headers` and `body` describe the request being authenticated. They are + /// used by catalogs that sign the request itself (AWS SigV4 in `S3TablesCatalog`); catalogs that + /// authenticate with a token or a static header ignore them. virtual DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, bool update_token, const String & method = {}, const Poco::URI & url = {}, @@ -231,6 +262,7 @@ class RestCatalog : public ICatalog, public DB::WithContext static void parseCatalogConfigurationSettings(const Poco::JSON::Object::Ptr & object, Config & result); void sendRequest( + const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method = Poco::Net::HTTPRequest::HTTP_POST, @@ -246,7 +278,21 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, const VendedStorageCredentials & parsed) const; - AccessToken retrieveAccessToken() const; + AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; + + struct PreparedAuthChanges; + + /// Hook for `prepareSettingsChanges`: validate `changes` and apply them to `new_state`, + /// building the new auth artifacts, without publishing anything. When the OAuth + /// credentials change, the eagerly fetched token goes into `new_access_token` and + /// `new_auth_headers`, so that wrong credentials fail the ALTER right here and the + /// config reload authenticates with the new token instead of the cached one. + virtual void applySettingsChangesToState( + const DB::SettingsChanges & changes, + const CatalogState & old_state, + CatalogState & new_state, + std::optional & new_auth_headers, + std::unique_ptr & new_access_token); }; class OneLakeCatalog : public RestCatalog @@ -270,15 +316,26 @@ class OneLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE; } - String getTenantId() const { return tenant_id; } + DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}) const override; - String getBearerToken() const; + /// `bearer_mode` means the catalog authenticates with `onelake_bearer_token`, + /// otherwise with the `onelake_client_id` + `onelake_client_secret` pair. + /// The mode is fixed when the database is created. + static void validateSettingsChanges(const DB::SettingsChanges & changes, bool bearer_mode); protected: - /// Parameters for OneLake OAuth. - const std::string tenant_id; - /// Set from `onelake_bearer_token`. - String bearer_token; + void applySettingsChangesToState( + const DB::SettingsChanges & changes, + const CatalogState & old_state, + CatalogState & new_state, + std::optional & new_auth_headers, + std::unique_ptr & new_access_token) override; }; class BigLakeCatalog : public RestCatalog @@ -303,6 +360,7 @@ class BigLakeCatalog : public RestCatalog } DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, bool update_token, const String & method = {}, const Poco::URI & url = {}, diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 07cd7e723da3..39b779607061 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -113,14 +113,17 @@ S3TablesCatalog::S3TablesCatalog( Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, /* urlEscapePath = */ false); - config = loadConfig(); + CatalogState initial_state; + initial_state.config = loadConfig(initial_state); - if (config.prefix.empty()) + if (initial_state.config.prefix.empty()) { String encoded_warehouse; Poco::URI::encode(warehouse_, "", encoded_warehouse); - config.prefix = encoded_warehouse; + initial_state.config.prefix = encoded_warehouse; } + + state.set(std::make_unique(std::move(initial_state))); } /// S3 Tables only supports a single level of namespaces (no nesting), @@ -216,8 +219,9 @@ ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfiguratio void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name) const { + const auto state_snapshot = state.get(); const std::string endpoint - = (base_url / config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() + = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() + "?purgeRequested=True"; Poco::JSON::Object::Ptr request_body = nullptr; @@ -225,7 +229,7 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -237,6 +241,7 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( + const CatalogState & /*catalog_state*/, bool /*update_token*/, const String & method, const Poco::URI & url, diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index aff432c1b679..32d296a73bca 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -47,6 +47,7 @@ class S3TablesCatalog final : public RestCatalog protected: DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, bool update_token, const String & method = {}, const Poco::URI & url = {}, diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index e385d8b0629b..9e5c3a7f86a8 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,8 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; } } @@ -52,6 +55,14 @@ void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & bod response.send() << body; } +void writeError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) +{ + response.setStatus(status); + response.setContentType("application/json"); + response.setContentLength(body.size()); + response.send() << body; +} + std::string getRawPath(const std::string & uri) { const auto query_pos = uri.find('?'); @@ -80,6 +91,12 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler return; } + if (path == "/v1/oauth/tokens") + { + writeJSON(response, R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); + return; + } + if (path == "/v1/namespaces") { const auto parent = getParent(params); @@ -121,6 +138,24 @@ class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler return; } + if (path == "/v1/namespaces/namespace/tables/table_a") + { + writeJSON(response, R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); + return; + } + + if (path == "/v1/namespaces/namespace/tables/missing_table") + { + writeError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); + return; + } + + if (path == "/v1/namespaces/namespace/tables/unauthorized_table") + { + writeError(response, Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); + return; + } + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected request to fake Iceberg REST catalog: {}", request.getURI()); } @@ -184,6 +219,19 @@ class RestCatalogTestServer std::unique_ptr server; }; +void expectThrowsCode(std::function fn, int expected_code) +{ + try + { + fn(); + FAIL() << "expected DB::Exception with code " << expected_code; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), expected_code); + } +} + bool restCatalogEmpty(CatalogShape shape) { RestCatalogTestServer server(shape); @@ -221,4 +269,182 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) EXPECT_TRUE(restCatalogEmpty(CatalogShape::Empty)); } +TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) +{ + RestCatalogTestServer server(CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"", + /* auth_scope */"", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* namespaces */"*", + context); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "id:secret"); + expectThrowsCode([&] { catalog.applySettingsChanges(changes); }, DB::ErrorCodes::BAD_ARGUMENTS); +} + +TEST(RestCatalog, ApplySettingsChangesCredentialMode) +{ + RestCatalogTestServer server(CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"client-1:secret-1", + /* auth_scope */"scope", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* namespaces */"*", + context); + + EXPECT_EQ(catalog.getStateSnapshot()->client_id, "client-1"); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client-2:secret-2"); + catalog.applySettingsChanges(changes); + + const auto snapshot = catalog.getStateSnapshot(); + EXPECT_EQ(snapshot->client_id, "client-2"); + EXPECT_EQ(snapshot->client_secret, "secret-2"); + + DB::SettingsChanges mode_switch; + mode_switch.emplace_back("auth_header", "Authorization: Bearer token"); + expectThrowsCode([&] { catalog.applySettingsChanges(mode_switch); }, DB::ErrorCodes::BAD_ARGUMENTS); + + DB::SettingsChanges unknown_setting; + unknown_setting.emplace_back("warehouse", "other"); + expectThrowsCode([&] { catalog.applySettingsChanges(unknown_setting); }, DB::ErrorCodes::BAD_ARGUMENTS); + + /// Malformed credential (no `:` separator) fails the ALTER atomically. + DB::SettingsChanges malformed; + malformed.emplace_back("catalog_credential", "no-separator"); + expectThrowsCode([&] { catalog.applySettingsChanges(malformed); }, DB::ErrorCodes::BAD_ARGUMENTS); + EXPECT_EQ(catalog.getStateSnapshot()->client_id, "client-2"); +} + +TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) +{ + RestCatalogTestServer server(CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"", + /* auth_scope */"", + /* auth_header */"Authorization: Bearer token-1", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* namespaces */"*", + context); + + DB::SettingsChanges changes; + changes.emplace_back("auth_header", "Authorization: Bearer token-2"); + catalog.applySettingsChanges(changes); + + const auto snapshot = catalog.getStateSnapshot(); + ASSERT_TRUE(snapshot->auth_header.has_value()); + EXPECT_EQ(snapshot->auth_header->value, " Bearer token-2"); + + DB::SettingsChanges mode_switch; + mode_switch.emplace_back("catalog_credential", "id:secret"); + expectThrowsCode([&] { catalog.applySettingsChanges(mode_switch); }, DB::ErrorCodes::BAD_ARGUMENTS); +} + +TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) +{ + RestCatalogTestServer server(CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + OneLakeCatalog catalog( + "warehouse", + server.getUrl(), + /* onelake_tenant_id */"tenant-1", + /* onelake_client_id */"", + /* onelake_client_secret */"", + /* bearer_token */"token-1", + /* auth_scope */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* namespaces */"*", + context); + + const auto snapshot_before = catalog.getStateSnapshot(); + EXPECT_EQ(snapshot_before->tenant_id, "tenant-1"); + EXPECT_EQ(snapshot_before->bearer_token, "token-1"); + ASSERT_TRUE(snapshot_before->auth_header.has_value()); + EXPECT_EQ(snapshot_before->auth_header->value, "Bearer token-1"); + + DB::SettingsChanges changes; + changes.emplace_back("onelake_bearer_token", "token-2"); + changes.emplace_back("onelake_tenant_id", "tenant-2"); + catalog.applySettingsChanges(changes); + + const auto snapshot_after = catalog.getStateSnapshot(); + EXPECT_EQ(snapshot_after->tenant_id, "tenant-2"); + EXPECT_EQ(snapshot_after->bearer_token, "token-2"); + ASSERT_TRUE(snapshot_after->auth_header.has_value()); + EXPECT_EQ(snapshot_after->auth_header->value, "Bearer token-2"); + + EXPECT_EQ(snapshot_before->tenant_id, "tenant-1"); + EXPECT_EQ(snapshot_before->bearer_token, "token-1"); + + DB::SettingsChanges mode_switch; + mode_switch.emplace_back("onelake_tenant_id", "tenant-3"); + mode_switch.emplace_back("onelake_client_id", "client-1"); + expectThrowsCode([&] { catalog.applySettingsChanges(mode_switch); }, DB::ErrorCodes::BAD_ARGUMENTS); + EXPECT_EQ(catalog.getStateSnapshot()->tenant_id, "tenant-2"); + + DB::SettingsChanges unknown_setting; + unknown_setting.emplace_back("warehouse", "other"); + expectThrowsCode([&] { catalog.applySettingsChanges(unknown_setting); }, DB::ErrorCodes::BAD_ARGUMENTS); + + DB::SettingsChanges empty_value; + empty_value.emplace_back("onelake_bearer_token", ""); + expectThrowsCode([&] { catalog.applySettingsChanges(empty_value); }, DB::ErrorCodes::BAD_ARGUMENTS); +} + +TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) +{ + RestCatalogTestServer server(CatalogShape::TopLevelTable); + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + + RestCatalog catalog( + "warehouse", + server.getUrl(), + /* catalog_credential */"", + /* auth_scope */"", + /* auth_header */"", + /* oauth_server_uri */"", + /* oauth_server_use_request_body */false, + /* namespaces */"*", + context); + + TableMetadata existing; + EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); + EXPECT_TRUE(catalog.existsTable("namespace", "table_a")); + + TableMetadata missing; + EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); + EXPECT_FALSE(catalog.existsTable("namespace", "missing_table")); + + TableMetadata unauthorized; + EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table"), DB::HTTPException); +} + #endif diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index 6e16a62227f6..5bd0032957d3 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -696,7 +696,8 @@ Block mergeBlockWithPipe( /** * Returns ColumnVector data as PaddedPodArray. - * If column is constant parameter backup_storage is used to store values. + * If the column has to be converted to a full one, parameter backup_storage is used to store values, + * because the converted column may not be owned by anything that outlives this call. */ /// TODO: Remove template @@ -705,7 +706,6 @@ static const PaddedPODArray & getColumnVectorData( const ColumnPtr column, PaddedPODArray & backup_storage) { - bool is_const_column = isColumnConst(*column); auto full_column = removeSpecialRepresentations(column->convertToFullColumnIfConst()); auto vector_col = checkAndGetColumn>(full_column.get()); @@ -717,12 +717,13 @@ static const PaddedPODArray & getColumnVectorData( TypeName); } - if (is_const_column) + /// A different pointer means a conversion happened (Const, Sparse or ColumnReplicated; a Tuple + /// never reaches here because the check above requires a ColumnVector), so the data may live + /// only in a column owned by `full_column` and die at return: copy it. An unconverted column is + /// kept alive by `column` itself. + if (full_column.get() != column.get()) { - // With type conversion and const columns we need to use backup storage here - auto & data = vector_col->getData(); - backup_storage.assign(data); - + backup_storage.assign(vector_col->getData()); return backup_storage; } diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp index 7ced03ed87c9..9abfe5c54527 100644 --- a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp +++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp @@ -164,11 +164,15 @@ void AsynchronousBoundedReadBuffer::prefetch(Priority priority) last_prefetch_info.submit_time = std::chrono::system_clock::now(); last_prefetch_info.priority = priority; + /// prefetch_buffer is reused for the new prefetch, invalidating the retained data. + prefetch_result.reset(); + /// Don't allocate any buffers if page cache is in use, the cache has its own buffers (PageCacheCell). if (!use_page_cache) prefetch_buffer.resize(buffer_size); prefetch_future = readAsync(prefetch_buffer.data(), buffer_size, priority); + prefetch_pending.store(true, std::memory_order_release); ProfileEvents::increment(ProfileEvents::RemoteFSPrefetches); } @@ -256,6 +260,7 @@ bool AsynchronousBoundedReadBuffer::nextImpl() } prefetch_future = {}; + prefetch_pending.store(false, std::memory_order_relaxed); prefetch_buffer.swap(memory); if (enable_prefetches_log) @@ -449,6 +454,7 @@ void AsynchronousBoundedReadBuffer::resetPrefetch(FilesystemPrefetchState state) auto result = prefetch_future.get(); prefetch_future = {}; + prefetch_pending.store(false, std::memory_order_relaxed); last_prefetch_info = {}; ProfileEvents::increment(ProfileEvents::RemoteFSPrefetchedBytes, result.size); @@ -472,27 +478,35 @@ size_t AsynchronousBoundedReadBuffer::readBigAt(char * to, size_t n, size_t rang if (!impl->supportsReadAt()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method readBigAt() is not implemented for a given implementation"); - /// A small-object initial prefetch may be in flight even though the consumer reads via positioned - /// reads (e.g. a small Parquet/ORC/Arrow file read through an object storage table function). - /// readBigAt() and the sequential prefetch must not run against impl concurrently, so consume the - /// prefetch first: serve the part of the requested range that the prefetch covers straight from the - /// prefetched buffer, and read only the missing suffix (if any) directly. - if (prefetch_future.valid()) + /// An in-flight prefetch must be consumed before reading from impl. Concurrent readBigAt + /// callers race on this, hence the mutex; the atomic makes the already-consumed case lock-free. + /// The prefetched data is retained in prefetch_result, so all subsequent readBigAt calls + /// (not only the consuming one) serve their range from it when covered. + if (prefetch_pending.load(std::memory_order_acquire)) { - IAsynchronousReader::Result result; + std::lock_guard lock(prefetch_future_mutex); + if (prefetch_future.valid()) { - ProfileEventTimeIncrement watch(ProfileEvents::AsynchronousRemoteReadWaitMicroseconds); - CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; - result = prefetch_future.get(); + { + ProfileEventTimeIncrement watch(ProfileEvents::AsynchronousRemoteReadWaitMicroseconds); + CurrentMetrics::Increment metric_increment{CurrentMetrics::AsynchronousReadWait}; + prefetch_result = prefetch_future.get(); + } + prefetch_future = {}; + last_prefetch_info = {}; + /// Publishes prefetch_result: readers of a false prefetch_pending see it fully written. + prefetch_pending.store(false, std::memory_order_release); } - prefetch_future = {}; - last_prefetch_info = {}; + } + if (prefetch_result) + { + const auto & result = *prefetch_result; const size_t prefetched_bytes = result.size - result.offset; const size_t prefetch_end = result.file_offset_of_buffer_end; const size_t prefetch_begin = prefetch_end - prefetched_bytes; - /// Serve the prefix of the range that the prefetch covers. + /// Serve the prefix of the range that the prefetched data covers. if (prefetched_bytes != 0 && range_begin >= prefetch_begin && range_begin < prefetch_end) { const size_t from_prefetch = std::min(n, prefetch_end - range_begin); @@ -518,9 +532,6 @@ size_t AsynchronousBoundedReadBuffer::readBigAt(char * to, size_t n, size_t rang return from_prefetch + impl->readBigAt(to + from_prefetch, n - from_prefetch, range_begin + from_prefetch, suffix_progress); } - - /// The prefetched range does not cover the head of the request; drop it and read directly. - ProfileEvents::increment(ProfileEvents::RemoteFSCancelledPrefetches); } return impl->readBigAt(to, n, range_begin, progress_callback); diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.h b/src/Disks/IO/AsynchronousBoundedReadBuffer.h index 46a279e8af79..ac98720306cc 100644 --- a/src/Disks/IO/AsynchronousBoundedReadBuffer.h +++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include #include @@ -53,10 +55,9 @@ class AsynchronousBoundedReadBuffer : public ReadBufferFromFileBase /// Used only for unit test. const ImplPtr & getImpl() { return impl; } - /// NOTE: readBigAt() does not use the async logic of AsynchronousBoundedReadBuffer; it calls impl's - /// (when supported), which is possible because readBigAt is asynchronous on its own. If a (small-object) - /// initial prefetch is in flight it is consumed first: the requested range is served from the prefetched - /// buffer when covered, otherwise the prefetch is dropped and the read falls back to impl. + /// NOTE: readBigAt does not use the async logic of AsynchronousBoundedReadBuffer; it calls impl's + /// (when supported). An in-flight prefetch is consumed first (readBigAt must not run against impl + /// concurrently with it) and its data is retained: readBigAt calls serve from it when covered. bool supportsReadAt() override { return impl->supportsReadAt(); } size_t readBigAt(char * to, size_t n, size_t range_begin, const std::function & progress_callback) const override; @@ -80,6 +81,14 @@ class AsynchronousBoundedReadBuffer : public ReadBufferFromFileBase Memory<> prefetch_buffer; /// mutable: a pending prefetch may be consumed from the const readBigAt(). mutable std::future prefetch_future; + /// Guards consumption of prefetch_future from readBigAt, which may be called concurrently. + /// The sequential interface must not be called in parallel with readBigAt, so it takes no locks. + mutable std::mutex prefetch_future_mutex; + /// Lock-free check for readBigAt whether a prefetch is in flight. + mutable std::atomic prefetch_pending{false}; + /// A prefetch consumed by readBigAt, retained so that any readBigAt call can serve data from it. + /// Immutable once published (by the store to prefetch_pending); reset by the sequential prefetch. + mutable std::optional prefetch_result; /// When using userspace page cache, we directly use memory owned by the cache instead of /// allocating our own buffers. diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index e711a2dfd30a..1f4911f126f0 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include @@ -131,6 +133,7 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile( std::optional CachedOnDiskReadBufferFromFile::tryGetFileSize() { + std::lock_guard lock(file_size_mutex); if (file_size.has_value()) return file_size; @@ -477,7 +480,18 @@ CachedOnDiskReadBufferFromFile::createReadFromFileSegmentState( return create(ReadType::CACHED); } - download_state = file_segment.wait(offset); + download_state = file_segment.wait( + offset, info_.cache_settings.wait_for_concurrent_download_timeout_milliseconds); + + if (download_state == FileSegment::State::DOWNLOADING && !canStartFromCache(offset, file_segment)) + { + LOG_TEST( + log, "Bypassing cache because waiting for a concurrent download did not succeed within the timeout. " + "File segment info: {}", file_segment.getInfoForLog()); + + return create(ReadType::REMOTE_FS_READ_BYPASS_CACHE); + } + continue; } case FileSegment::State::DOWNLOADED: @@ -1533,6 +1547,10 @@ size_t CachedOnDiskReadBufferFromFile::readBigAt( size_t range_begin, const std::function & progress_callback) const { + /// Use the mutex-protected getter, not the lazily initialized file_size member: + /// readBigAt may run concurrently with the sequential read path. + const size_t object_size = const_cast(*this).getFileSize(); + ReadInfo current_info( info.cache_key, info.source_file_path, info.implementation_buffer_creator, info.use_external_buffer, info.cache_settings, info.local_fs_buffer_size, @@ -1554,7 +1572,7 @@ size_t CachedOnDiskReadBufferFromFile::readBigAt( info.cache_key, /* offset */range_begin, /* size */n, - file_size.value(), + object_size, create_settings, /* batch_size */0, origin); @@ -1572,7 +1590,6 @@ size_t CachedOnDiskReadBufferFromFile::readBigAt( bool cancelled = false; bool implementation_buffer_can_be_reused = false; ReadFromFileSegmentStatePtr current_state; - auto object_size = const_cast(*this).getFileSize(); SCOPE_EXIT({ if (current_info.file_segments->empty()) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index ee2022864399..e3636c42ca57 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -250,6 +251,10 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase ReadFromFileSegmentStatePtr state; ReadInfo info; + /// Guards the lazily initialized file_size: tryGetFileSize may be called from readBigAt + /// concurrently with the sequential read path. + mutable std::mutex file_size_mutex; + size_t first_offset = 0; String nextimpl_step_log_info; diff --git a/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp b/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp index 0d68fd36105d..d3969279c46f 100644 --- a/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp +++ b/src/Disks/tests/gtest_asynchronous_bounded_read_buffer.cpp @@ -1,17 +1,28 @@ #include +#include +#include #include #include #include #include #include #include +#include +#include +#include #include +#include using namespace DB; namespace fs = std::filesystem; +namespace ProfileEvents +{ + extern const Event RemoteFSPrefetchedReads; +} + class AsynchronousBoundedReadBufferTest : public ::testing::TestWithParam { public: @@ -84,3 +95,138 @@ TEST_F(AsynchronousBoundedReadBufferTest, setReadUntilPosition) EXPECT_EQ(try_read(15), ""); } } + +TEST_F(AsynchronousBoundedReadBufferTest, concurrentReadBigAtWithPrefetch) +{ + /// readBigAt may be called concurrently (e.g. by ParallelReadBuffer), possibly while an initial + /// prefetch is in flight. Concurrent calls used to race on consuming prefetch_future. + + String contents; + contents.reserve(100000); + for (size_t i = 0; i < 100000; ++i) + contents += static_cast('a' + i % 26); + + const String file_path = makeTempFile(contents); + ThreadPoolRemoteFSReader remote_fs_reader(4, 0); + + constexpr size_t num_threads = 4; + constexpr size_t num_iterations = 500; + /// Smaller than the file, so the prefetch is usually still in flight when the reads run. + constexpr size_t buffer_size = 16384; + + const ProfileEvents::Count prefetched_reads_before = ProfileEvents::global_counters[ProfileEvents::RemoteFSPrefetchedReads]; + + for (size_t iteration = 0; iteration < num_iterations; ++iteration) + { + AsynchronousBoundedReadBuffer read_buffer( + createReadBufferFromFileBase(file_path, ReadSettings{}), remote_fs_reader, + buffer_size, /* min_bytes_for_seek */ 0, + Priority{0}, /* page_cache_block_size */ 0, /* enable_prefetches_log */ false); + + read_buffer.prefetch(Priority{0}); + + std::atomic ready{0}; + std::array errors; + std::vector threads; + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t] + { + /// Barrier, to maximize the chance that the readBigAt calls overlap. + ready.fetch_add(1); + while (ready.load() < num_threads) + ; + + try + { + /// Threads read different, partially overlapping ranges. + const size_t offset = (t < 2) ? 1000 * (t + 1) : 20000 * t; + const size_t count = 30000; + String buf(count, 0); + size_t total = 0; + while (total < count) + { + size_t read = read_buffer.readBigAt(buf.data() + total, count - total, offset + total, nullptr); + if (read == 0) + break; + total += read; + } + if (total != count) + errors[t] = fmt::format("short read: {} instead of {}", total, count); + else if (memcmp(buf.data(), contents.data() + offset, count) != 0) + errors[t] = "read data does not match file contents"; + } + catch (...) + { + errors[t] = getCurrentExceptionMessage(true); + } + }); + } + + for (auto & thread : threads) + thread.join(); + + for (size_t t = 0; t < num_threads; ++t) + ASSERT_EQ(errors[t], "") << "thread " << t << ", iteration " << iteration; + } + + /// The prefetched data is retained after being consumed, so in every iteration both threads + /// whose ranges start inside it must have been served from it, not only the consuming one. + const auto prefetched_reads = ProfileEvents::global_counters[ProfileEvents::RemoteFSPrefetchedReads] - prefetched_reads_before; + EXPECT_GE(prefetched_reads, 2 * num_iterations); +} + +TEST_F(AsynchronousBoundedReadBufferTest, readBigAtFromRetainedPrefetch) +{ + String contents; + contents.reserve(100000); + for (size_t i = 0; i < 100000; ++i) + contents += static_cast('a' + i % 26); + + const String file_path = makeTempFile(contents); + ThreadPoolRemoteFSReader remote_fs_reader(4, 0); + + /// Smaller than the file, so the prefetch covers only its prefix. + constexpr size_t buffer_size = 16384; + + AsynchronousBoundedReadBuffer read_buffer( + createReadBufferFromFileBase(file_path, ReadSettings{}), remote_fs_reader, + buffer_size, /* min_bytes_for_seek */ 0, + Priority{0}, /* page_cache_block_size */ 0, /* enable_prefetches_log */ false); + + read_buffer.prefetch(Priority{0}); + + const ProfileEvents::Count prefetched_reads_before = ProfileEvents::global_counters[ProfileEvents::RemoteFSPrefetchedReads]; + auto prefetched_reads = [&] { return ProfileEvents::global_counters[ProfileEvents::RemoteFSPrefetchedReads] - prefetched_reads_before; }; + + auto read_at = [&](size_t offset, size_t count) + { + String buf(count, 0); + size_t total = 0; + while (total < count) + { + size_t read = read_buffer.readBigAt(buf.data() + total, count - total, offset + total, nullptr); + if (read == 0) + break; + total += read; + } + EXPECT_EQ(total, count); + EXPECT_EQ(buf, contents.substr(offset, count)); + }; + + /// A read past the prefetched range consumes the prefetch, but retains its data. + read_at(50000, 10000); + EXPECT_EQ(prefetched_reads(), 0); + + /// Reads inside the prefetched range are served from the retained data, each of them. + read_at(0, 10000); + EXPECT_EQ(prefetched_reads(), 1); + read_at(5000, 5000); + EXPECT_EQ(prefetched_reads(), 2); + + /// A read crossing the end of the prefetched range: the head is served from the retained + /// data, the suffix is read directly. + read_at(10000, 20000); + EXPECT_EQ(prefetched_reads(), 3); +} diff --git a/src/Formats/CapnProtoSchema.cpp b/src/Formats/CapnProtoSchema.cpp index d3828695ebcc..977f07070a5a 100644 --- a/src/Formats/CapnProtoSchema.cpp +++ b/src/Formats/CapnProtoSchema.cpp @@ -9,11 +9,19 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include #include +#include +#include + namespace DB { @@ -27,8 +35,130 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace +{ + +/// The Cap'n Proto schema parser is a recursive-descent parser without a depth limit of its own, +/// so a schema with deeply nested type expressions, such as `List(List(List(...)))`, exhausts the +/// thread stack while it is being lexed - before ClickHouse gets a chance to look at the result. +/// Bound the nesting of the schema text before handing it over to the parser. +/// +/// A schema may `import` other schema files, and the parser reads them with the same recursive +/// parser, so the whole import graph has to be checked, not just the entry file. Imports with an +/// absolute path are resolved against the schema directory (the only import root we pass to the +/// parser), and relative ones against the directory of the importing file, exactly as Cap'n Proto +/// does it. +void checkCapnProtoSchemaNestingDepth(const String & schema_directory, const String & schema_path) +{ + const std::filesystem::path root = schema_directory; + + std::vector to_visit; + to_visit.push_back(root / schema_path); + + std::unordered_set visited; + + while (!to_visit.empty()) + { + const std::filesystem::path path = to_visit.back(); + to_visit.pop_back(); + + std::error_code error; + const std::filesystem::path canonical_path = std::filesystem::weakly_canonical(path, error); + if (error) + continue; /// A malformed path is reported with a proper message by the parser below. + + if (!visited.emplace(canonical_path.string()).second) + continue; + + if (!std::filesystem::is_regular_file(canonical_path, error)) + continue; /// A missing file is reported with a proper message by the parser below. + + ReadBufferFromFile in(canonical_path); + String content; + readStringUntilEOF(content, in); + + size_t depth = 0; + /// The last identifier seen before the current position, to recognize `import "..."`. + String last_word; + /// Whether `last_word` has already been terminated by whitespace. + bool last_word_finished = false; + + for (size_t i = 0; i < content.size(); ++i) + { + const char c = content[i]; + + /// Cap'n Proto has line comments and text literals; brackets inside them are not nesting. + if (c == '#') + { + last_word.clear(); + last_word_finished = false; + i = content.find('\n', i); + if (i == String::npos) + break; + } + else if (c == '"') + { + String literal; + for (++i; i < content.size() && content[i] != '"'; ++i) + { + if (content[i] == '\\' && i + 1 < content.size()) + ++i; + literal += content[i]; + } + + if (last_word == "import" && !literal.empty()) + { + if (literal[0] == '/') + to_visit.push_back(root / literal.substr(1)); + else + to_visit.push_back(canonical_path.parent_path() / literal); + } + + last_word.clear(); + last_word_finished = false; + } + else if (isWordCharASCII(c)) + { + if (last_word_finished) + { + last_word.clear(); + last_word_finished = false; + } + last_word += c; + } + else if (isWhitespaceASCII(c)) + { + /// Whitespace separates `import` from the path, so it must not forget the keyword. + last_word_finished = !last_word.empty(); + } + else + { + last_word.clear(); + last_word_finished = false; + + if (c == '(' || c == '[' || c == '{') + { + ++depth; + if (depth > DBMS_DEFAULT_MAX_PARSER_DEPTH) + throw Exception(ErrorCodes::CANNOT_PARSE_CAPN_PROTO_SCHEMA, + "The CapnProto schema is nested too deeply: the limit is {}", DBMS_DEFAULT_MAX_PARSER_DEPTH); + } + else if (c == ')' || c == ']' || c == '}') + { + if (depth > 0) + --depth; + } + } + } + } +} + +} + capnp::StructSchema CapnProtoSchemaParser::getMessageSchema(const FormatSchemaInfo & schema_info) { + checkCapnProtoSchemaNestingDepth(schema_info.schemaDirectory(), schema_info.schemaPath()); + capnp::ParsedSchema schema; try { @@ -74,6 +204,8 @@ bool checkIfStructIsNamedUnion(const capnp::StructSchema & struct_schema) /// Get full name of type for better exception messages. String getCapnProtoFullTypeName(const capnp::Type & type) { + checkStackSize(); + static const std::map capnp_simple_type_names = { {capnp::schema::Type::Which::BOOL, "Bool"}, @@ -171,6 +303,8 @@ namespace DataTypePtr getDataTypeFromCapnProtoType(const capnp::Type & capnp_type, bool skip_unsupported_fields) { + checkStackSize(); + switch (capnp_type.which()) { case capnp::schema::Type::INT8: diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index 93841a6dea91..70c750eb2e27 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -1938,55 +1938,107 @@ class ObjectJSONNode : public JSONExtractTreeNode std::vector> & paths_and_values_for_shared_data, size_t current_size, String & error, - bool is_root) const + bool is_root, + bool skip_typed_path_check = false) const { if (shouldSkipPath(current_path, insert_settings)) return true; - if (element.isObject() && (!typed_path_nodes.contains(current_path) || (format_settings.json.type_json_allow_duplicated_key_with_literal_and_nested_object && hasTypedPathWithPrefix(current_path + ".")))) + if (element.isObject() && (skip_typed_path_check || !typed_path_nodes.contains(current_path) || (format_settings.json.type_json_allow_duplicated_key_with_literal_and_nested_object && hasTypedPathWithPrefix(current_path + ".")))) { - std::unordered_map> visited_keys; + /// First pass: collect the set of distinct element types (LITERAL/OBJECT) per key. + /// This lets us detect all duplicates upfront so we can: + /// - throw errors immediately for invalid duplicates (before any data is inserted), + /// - handle both orderings of literal+object duplicates in the main loop + /// (e.g. {"a":42,"a":{"b":42}} and {"a":{"b":42},"a":42} are both valid). + std::unordered_map> key_element_types; for (auto [key, value] : element.getObject()) { - String path = current_path; - if (!is_root) - path.append("."); - if (insert_settings.escape_dots_in_json_keys) - path += escapeDotInJSONKey(String(key)); + auto value_element_type = getJSONElementType(value); + auto & types = key_element_types[key]; + + if (types.empty()) + { + /// First occurrence of this key — just record its type. + types.insert(value_element_type); + continue; + } + + /// Duplicate key detected. Decide whether to allow or reject. + if (types.contains(value_element_type)) + { + /// Same-type duplicate (e.g. two literals or two objects for the same key). + /// This is never allowed by type_json_allow_duplicated_key_with_literal_and_nested_object + /// (which only allows literal+object pairs), so skip or throw. + if (format_settings.json.type_json_skip_duplicated_paths) + continue; + + error = fmt::format("Duplicate path found during parsing JSON object: {}. You can enable setting " + "type_json_skip_duplicated_paths to skip duplicated paths during insert", + buildChildPath(current_path, key, insert_settings, is_root)); + return false; + } + + /// Different-type duplicate (one literal, one object for the same key). + if (format_settings.json.type_json_allow_duplicated_key_with_literal_and_nested_object) + { + /// Setting enabled — record the second type so the main loop processes both. + types.insert(value_element_type); + } else - path += key; + { + /// Setting disabled — this is an error unless we can skip. + if (format_settings.json.type_json_skip_duplicated_paths) + continue; + + error = fmt::format( + "Duplicate path found during parsing JSON object: {}. You can enable setting " + "type_json_skip_duplicated_paths to skip duplicated paths during insert or setting " + "type_json_allow_duplicated_key_with_literal_and_nested_object to allow duplicated " + "path with literal and nested object", + buildChildPath(current_path, key, insert_settings, is_root)); + return false; + } + } - auto it = visited_keys.find(key); + /// Second pass: process each key-value pair in original order. + /// All invalid duplicates have already been rejected in the first pass, + /// so here we only need to skip already-processed (key, type) combinations. + std::unordered_map> visited_keys; + for (auto [key, value] : element.getObject()) + { + String path = buildChildPath(current_path, key, insert_settings, is_root); auto value_element_type = getJSONElementType(value); + auto it = visited_keys.find(key); if (it != visited_keys.end()) { - if (format_settings.json.type_json_allow_duplicated_key_with_literal_and_nested_object) - { - /// We can't have duplicated key with the same type (literal/object). - if (it->second.contains(value_element_type)) - { - if (format_settings.json.type_json_skip_duplicated_paths) - continue; - error = fmt::format("Duplicate path found during parsing JSON object: {}. You can enable setting type_json_skip_duplicated_paths to skip duplicated paths during insert", path); - return false; - } - - it->second.insert(value_element_type); - } - else - { - if (format_settings.json.type_json_skip_duplicated_paths) - continue; - error = fmt::format("Duplicate path found during parsing JSON object: {}. You can enable setting type_json_skip_duplicated_paths to skip duplicated paths during insert or setting type_json_allow_duplicated_key_with_literal_and_nested_object to allow duplicated path with literal and nested object", path); - return false; - } + /// We have seen this key before. Skip if: + /// - we already processed a value with this element type for this key + /// (same-type duplicate allowed by type_json_skip_duplicated_paths), or + /// - the first pass rejected this type for this key (different-type duplicate + /// that was skipped because type_json_allow_duplicated_key_with_literal_and_nested_object + /// is disabled but type_json_skip_duplicated_paths is enabled). + if (it->second.contains(value_element_type) || !key_element_types[key].contains(value_element_type)) + continue; + it->second.insert(value_element_type); } else { visited_keys[key].insert(value_element_type); } - if (!traverseAndInsert(column_object, value, path, insert_settings, format_settings, paths_and_values_for_shared_data, current_size, error, false)) + /// When a key has both literal and object values (key_element_types has 2 types), + /// and the key corresponds to a typed path with a non-nested type (e.g. Int32, String — + /// not JSON/Map/Tuple that naturally parse objects), the object value should NOT be + /// inserted into the typed path. Instead, pass skip_typed_path_check=true so the + /// recursive call enters object traversal and sends the object's children to + /// dynamic/shared data. The literal value will fill the typed path via normal recursion. + bool skip_typed = key_element_types[key].size() > 1 + && value_element_type == JSONElementType::OBJECT + && typed_path_nodes.contains(path) + && !canParseObjectValue(typed_paths_types.at(path)); + + if (!traverseAndInsert(column_object, value, path, insert_settings, format_settings, paths_and_values_for_shared_data, current_size, error, false, skip_typed)) return false; } @@ -2063,6 +2115,19 @@ class ObjectJSONNode : public JSONExtractTreeNode return true; } + /// Build the full path for a child key within the current object. + String buildChildPath(const String & current_path, std::string_view key, const JSONExtractInsertSettings & insert_settings, bool is_root) const + { + String path = current_path; + if (!is_root) + path.append("."); + if (insert_settings.escape_dots_in_json_keys) + path += escapeDotInJSONKey(String(key)); + else + path += key; + return path; + } + bool shouldSkipPath(const String & path, const JSONExtractInsertSettings & insert_settings) const { if (paths_to_skip.contains(path)) @@ -2373,6 +2438,16 @@ class ObjectJSONNode : public JSONExtractTreeNode return JSONElementType::LITERAL; } } + + /// Check if the type of a typed path can naturally parse a JSON object value. + /// Types like JSON, Map, Tuple represent structured data and should receive + /// the object value when there is a literal+object duplicate key. + /// Scalar types (Int, String, etc.) should receive the literal value instead. + bool canParseObjectValue(const DataTypePtr & type) const + { + auto id = removeNullable(removeLowCardinality(type))->getTypeId(); + return id == TypeIndex::Object || id == TypeIndex::Map || id == TypeIndex::Tuple; + } }; } diff --git a/src/Formats/NativeReader.cpp b/src/Formats/NativeReader.cpp index d9eee412c1d1..ebafe5c00cba 100644 --- a/src/Formats/NativeReader.cpp +++ b/src/Formats/NativeReader.cpp @@ -177,6 +177,13 @@ Block NativeReader::read() if (columns == 0 && header.empty() && rows != 0) throw Exception(ErrorCodes::INCORRECT_DATA, "Zero columns but {} rows in Native format.", rows); + /// `rows` comes from the block header, and the limit it is checked against is deliberately + /// generous, so it must not be used to preallocate the columns: a header declaring a huge row + /// count would reserve that much per column before a single byte of column data is read. + /// Reserving is only an optimization here - deserialization appends to the column anyway - so + /// bound it by a plausible block size and let the column grow past that on its own. + const size_t rows_to_reserve = std::min(rows, DEFAULT_INSERT_BLOCK_SIZE); + for (size_t i = 0; i < columns; ++i) { if (use_index) @@ -221,14 +228,14 @@ Block NativeReader::read() serialization = column.type->getSerialization(*info); auto new_column = column.type->createColumn(*serialization); - new_column->reserve(rows); + new_column->reserve(rows_to_reserve); read_column = std::move(new_column); } else { serialization = column.type->getDefaultSerialization(); auto new_column = column.type->createColumn(*serialization); - new_column->reserve(rows); + new_column->reserve(rows_to_reserve); read_column = std::move(new_column); } diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 22a5e8a487e1..76c01b5b7885 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -140,7 +140,9 @@ namespace // Should we omit null values (zero for numbers / empty string for strings) while storing them. bool shouldSkipZeroOrEmpty(const FieldDescriptor & field_descriptor, bool google_wrappers_special_treatment = false) { - if (!field_descriptor.is_optional()) + /// `FieldDescriptor::is_optional` was removed along with the rest of the label accessors; + /// a singular field is one that is neither repeated nor required. + if (field_descriptor.is_repeated() || field_descriptor.is_required()) return false; if (field_descriptor.containing_type()->options().map_entry()) return false; diff --git a/src/Functions/AI/AIQuotaTracker.cpp b/src/Functions/AI/AIQuotaTracker.cpp index 82760d8b7307..374e22d37037 100644 --- a/src/Functions/AI/AIQuotaTracker.cpp +++ b/src/Functions/AI/AIQuotaTracker.cpp @@ -9,22 +9,11 @@ namespace ErrorCodes extern const int LIMIT_EXCEEDED; } -bool AIQuotaTracker::checkQuotas() +bool AIQuotaTracker::quotasExceededLocked() { if (quota_exceeded) return true; - if (max_api_calls > 0 && api_calls >= max_api_calls) - { - if (throw_on_quota_exceeded) - throw Exception(ErrorCodes::LIMIT_EXCEEDED, - "AI API call limit reached: {} calls made, maximum: {}. " - "This is controlled by the 'ai_function_max_api_calls_per_query' setting", - api_calls, max_api_calls); - quota_exceeded = true; - return true; - } - if (max_input_tokens > 0 && input_tokens >= max_input_tokens) { if (throw_on_quota_exceeded) @@ -50,13 +39,44 @@ bool AIQuotaTracker::checkQuotas() return false; } -void AIQuotaTracker::recordAttempt() +bool AIQuotaTracker::checkQuotas() +{ + std::lock_guard lock(mutex); + return quotasExceededLocked(); +} + +bool AIQuotaTracker::recordApiCall() { - ++api_calls; + std::lock_guard lock(mutex); + + /// Don't start a new request once any quota is known-exhausted (e.g. another thread's response + /// just pushed the token budget over), even though the API-call count itself is still under its + /// own limit. This keeps token overshoot to the requests already in flight at that moment. + if (quotasExceededLocked()) + return false; + + if (max_api_calls == 0) /// 0 disables the API-call limit. + return true; + + if (api_calls < max_api_calls) + { + ++api_calls; + return true; + } + + if (throw_on_quota_exceeded) + throw Exception(ErrorCodes::LIMIT_EXCEEDED, + "AI API call limit reached: {} calls made, maximum: {}. " + "This is controlled by the 'ai_function_max_api_calls_per_query' setting", + api_calls, max_api_calls); + + quota_exceeded = true; + return false; } void AIQuotaTracker::recordTokens(UInt64 in_tokens, UInt64 out_tokens) { + std::lock_guard lock(mutex); input_tokens += in_tokens; output_tokens += out_tokens; } diff --git a/src/Functions/AI/AIQuotaTracker.h b/src/Functions/AI/AIQuotaTracker.h index d07209087c44..515821b0f9f6 100644 --- a/src/Functions/AI/AIQuotaTracker.h +++ b/src/Functions/AI/AIQuotaTracker.h @@ -3,9 +3,19 @@ #include #include +#include + namespace DB { +/// Tracks AI-function quota usage for one query. A single instance is shared by every AI function +/// call in the query context (owned by the query `Context`) and updated concurrently from the +/// pipeline threads, so the counters are guarded by `mutex`. It is per query-execution context: a +/// distributed query has one per shard/fragment (each makes its own `Context`), so the limits bound +/// each server independently rather than the query globally. +/// +/// The API-call limit is a hard cap within a context, while token limits are best effort (we only +/// know the usage after the call returns). class AIQuotaTracker { public: @@ -18,15 +28,21 @@ class AIQuotaTracker , throw_on_quota_exceeded(throw_on_quota_exceeded_) {} - /// Check all quotas, return true if any quota is met or exceeded, false otherwise. Should be called before issuing API call. + /// Check the token quotas (and the sticky exceeded flag). Returns true if a limit is met or + /// exceeded, false otherwise. Should be called before issuing an API call. The API-call limit is + /// enforced separately by `recordApiCall`. bool checkQuotas(); - /// Count one outbound API call against the request quota. Should be called before each provider request - /// (including retries), so a misbehaving endpoint can't bypass `ai_function_max_api_calls_per_query`. - void recordAttempt(); + /// Count one outbound API call against the request quota, only while under the limit. Should be + /// called before each provider request (including retries), so a misbehaving + /// endpoint can't bypass `ai_function_max_api_calls_per_query`. Returns true if the call is within + /// the limit (the caller may dispatch), false once the per-query API-call limit is reached or any + /// quota is already exhausted (so no new request starts after the token budget is known-spent); + /// throws when `throw_on_quota_exceeded`. Exact: `api_calls` never exceeds the limit. + bool recordApiCall(); /// Record token usage on a successful response. Tokens are only billed by the provider when the call succeeds, - /// so this is kept separate from `recordAttempt` and called only after the response is parsed. + /// so this is kept separate and called only after the response is parsed. void recordTokens(UInt64 in_tokens, UInt64 out_tokens); @@ -36,10 +52,16 @@ class AIQuotaTracker const UInt64 max_api_calls; const bool throw_on_quota_exceeded; - bool quota_exceeded = false; - UInt64 input_tokens = 0; - UInt64 output_tokens = 0; - UInt64 api_calls = 0; + std::mutex mutex; + bool quota_exceeded TSA_GUARDED_BY(mutex) = false; + UInt64 input_tokens TSA_GUARDED_BY(mutex) = 0; + UInt64 output_tokens TSA_GUARDED_BY(mutex) = 0; + UInt64 api_calls TSA_GUARDED_BY(mutex) = 0; + + /// The sticky-flag + token-limit check, assuming `mutex` is held. Sets the sticky flag (or throws, + /// per `throw_on_quota_exceeded`) when a token quota is met. Shared by `checkQuotas` and + /// `recordApiCall` so a call is never started once a quota is known-exhausted. + bool quotasExceededLocked() TSA_REQUIRES(mutex); }; } diff --git a/src/Functions/CastOverloadResolver.cpp b/src/Functions/CastOverloadResolver.cpp index b75ec14d5408..2a310f5fbb5d 100644 --- a/src/Functions/CastOverloadResolver.cpp +++ b/src/Functions/CastOverloadResolver.cpp @@ -414,4 +414,9 @@ SELECT accurateCastOrNull('abc', 'UInt32') factory.registerFunction("accurateCastOrNull", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::accurateOrNull, false, {}); }, accurateCastOrNull_documentation); } +FunctionOverloadResolverPtr createCastOverloadResolver(ContextPtr context, CastType cast_type, std::optional diagnostic) +{ + return CastOverloadResolverImpl::create(context, cast_type, false, std::move(diagnostic)); +} + } diff --git a/src/Functions/CastOverloadResolver.h b/src/Functions/CastOverloadResolver.h index 3446f4aaafce..3e36e61d3cf9 100644 --- a/src/Functions/CastOverloadResolver.h +++ b/src/Functions/CastOverloadResolver.h @@ -30,4 +30,6 @@ struct CastDiagnostic FunctionBasePtr createInternalCast(ColumnWithTypeAndName from, DataTypePtr to, CastType cast_type, std::optional diagnostic, ContextPtr context); +FunctionOverloadResolverPtr createCastOverloadResolver(ContextPtr context, CastType cast_type, std::optional diagnostic); + } diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index bc972dea744a..fa8e96f521f8 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -46,10 +46,6 @@ namespace Setting extern const SettingsUInt64 ai_function_max_retries; extern const SettingsUInt64 ai_function_retry_initial_delay_ms; extern const SettingsBool ai_function_throw_on_error; - extern const SettingsUInt64 ai_function_max_input_tokens_per_query; - extern const SettingsUInt64 ai_function_max_output_tokens_per_query; - extern const SettingsUInt64 ai_function_max_api_calls_per_query; - extern const SettingsBool ai_function_throw_on_quota_exceeded; extern const SettingsString ai_function_text_default_credentials; } @@ -376,11 +372,8 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; - AIQuotaTracker quota( - settings[Setting::ai_function_max_input_tokens_per_query].value, - settings[Setting::ai_function_max_output_tokens_per_query].value, - settings[Setting::ai_function_max_api_calls_per_query].value, - settings[Setting::ai_function_throw_on_quota_exceeded].value); + /// Shared across every AI function call in the query + auto quota_tracker = getContext()->getAIQuotaTracker(); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); @@ -403,7 +396,7 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, continue; } - if (quota.checkQuotas()) + if (quota_tracker->checkQuotas()) { result_col->insertDefault(); ++rows_skipped; @@ -416,10 +409,9 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) { - /// Enforce the API-call quota before every provider request, including retries, so a flaky - /// endpoint can't dispatch more than `ai_function_max_api_calls_per_query` requests per query. - /// Kept outside the `try` so a `throw_on_quota_exceeded` throw is not caught by the retry handler. - if (quota.checkQuotas()) + /// Reserve an API-call slot before each request; this also performs a quota check. + /// Kept outside the `try` so a `throw_on_quota_exceeded` exception isn't caught by the retry handler. + if (!quota_tracker->recordApiCall()) break; try @@ -433,13 +425,11 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, ai_request.max_tokens = max_tokens; ai_request.function_name = getName(); - /// update api_calls/quotas before call so failed calls are still added to total ++total_api_calls; - quota.recordAttempt(); auto ai_response = provider->call(ai_request, timeouts); - quota.recordTokens(ai_response.input_tokens, ai_response.output_tokens); + quota_tracker->recordTokens(ai_response.input_tokens, ai_response.output_tokens); total_input_tokens += ai_response.input_tokens; total_output_tokens += ai_response.output_tokens; diff --git a/src/Functions/FunctionShowCertificate.cpp b/src/Functions/FunctionShowCertificate.cpp index 05514210b11f..cdee65c786cb 100644 --- a/src/Functions/FunctionShowCertificate.cpp +++ b/src/Functions/FunctionShowCertificate.cpp @@ -1,6 +1,7 @@ #include "config.h" #include +#include #include #include @@ -17,6 +18,7 @@ #if USE_SSL #include #include + #include #endif namespace DB @@ -30,6 +32,27 @@ namespace ErrorCodes namespace { +#if USE_SSL +/// The certificate that the server currently serves to clients. +/// It is not always the certificate of the default SSL context: when certificates are provisioned +/// dynamically (the `` configuration), that context has no certificate at all, and only +/// `CertificateReloader` knows the certificate in use. +std::optional getServerCertificate() +{ + auto served_certificate = CertificateReloader::instance().getCertificate(Poco::Net::SSLManager::CFG_SERVER_PREFIX); + if (served_certificate) + return served_certificate; + + X509 * context_certificate = SSL_CTX_get0_certificate(Poco::Net::SSLManager::instance().defaultServerContext()->sslContext()); + if (!context_certificate) + return {}; + + /// `SSL_CTX_get0_certificate` does not transfer the ownership, and `X509` is reference counted. + X509_up_ref(context_certificate); + return X509Certificate(context_certificate); +} +#endif + // showCertificate() class FunctionShowCertificate final : public IFunction { @@ -71,15 +94,11 @@ class FunctionShowCertificate final : public IFunction if (input_rows_count) { #if USE_SSL - std::unique_ptr x509_cert; + std::optional x509_cert; if (!certificate.empty()) - x509_cert = std::make_unique(certificate); - - if (!x509_cert) - { - const auto * server_context_cert = SSL_CTX_get0_certificate(Poco::Net::SSLManager::instance().defaultServerContext()->sslContext()); - x509_cert = std::make_unique(X509_dup(server_context_cert)); - } + x509_cert.emplace(certificate); + else + x509_cert = getServerCertificate(); if (x509_cert) { @@ -143,6 +162,7 @@ REGISTER_FUNCTION(ShowCertificate) { FunctionDocumentation::Description description = R"( Shows information about the current server's Secure Sockets Layer (SSL) certificate if it has been configured. +An empty map is returned if the server has no certificate, for example, when the certificate is provisioned with ACME and has not been issued yet. See [Configuring TLS](/guides/sre/tls/configuring-tls) for more information on how to configure ClickHouse to use OpenSSL certificates to validate connections. )"; FunctionDocumentation::Syntax syntax = "showCertificate()"; diff --git a/src/Functions/FunctionTopKFilter.cpp b/src/Functions/FunctionTopKFilter.cpp index afb6d3fe099f..bdf343beb8a3 100644 --- a/src/Functions/FunctionTopKFilter.cpp +++ b/src/Functions/FunctionTopKFilter.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -14,6 +16,32 @@ namespace DB { +namespace +{ + +/// `Tuple` comparison functions reject an empty `Tuple` nested inside another `Tuple`, +/// while the column comparison path supports it. Other composite types have their +/// own comparison implementations, so only descend through `Tuple` and `Nullable`. +bool hasEmptyTuple(const DataTypePtr & type) +{ + const auto * nullable_type = typeid_cast(type.get()); + const auto & nested_type = nullable_type ? nullable_type->getNestedType() : type; + const auto * tuple_type = typeid_cast(nested_type.get()); + if (!tuple_type) + return false; + + if (tuple_type->getElements().empty()) + return true; + + for (const auto & element_type : tuple_type->getElements()) + if (hasEmptyTuple(element_type)) + return true; + + return false; +} + +} + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -82,7 +110,7 @@ class FunctionTopKFilter final : public IFunction auto current_threshold = threshold_tracker->getValue(); auto data_type = arguments[0].type; - if (collator || data_type->isNullable() || isDynamic(data_type) || isVariant(data_type)) + if (collator || data_type->isNullable() || isDynamic(data_type) || isVariant(data_type) || hasEmptyTuple(data_type)) return executeGeneral(arguments[0], current_threshold, data_type, input_rows_count); return executeVectorized(arguments[0], current_threshold, data_type, input_rows_count); @@ -107,7 +135,7 @@ class FunctionTopKFilter final : public IFunction return elem_compare->execute(args, elem_compare->getResultType(), input_rows_count, false); } - /// General path for Nullable and/or collation-aware types. + /// General path for `Nullable`, collation-aware, and non-vectorizable `Tuple` types. ColumnPtr executeGeneral( const ColumnWithTypeAndName & argument, const Field & current_threshold, diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 85a55e328516..a43dd43236ab 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -4091,9 +4091,6 @@ struct ToDateTimeMonotonicity } }; -/** The monotonicity for the `toString` function is mainly determined for test purposes. - * It is doubtful that anyone is looking to optimize queries with conditions `toString(CounterID) = 34`. - */ struct ToStringMonotonicity { static bool has() { return true; } @@ -4120,13 +4117,36 @@ struct ToStringMonotonicity return {.is_monotonic = true, .is_always_monotonic = true, .is_strict = true}; } - /// `toString` function is monotonous if the argument is Date or Date32 or DateTime or String, or non-negative numbers with the same number of symbols. - if (checkDataTypes(type_ptr)) - return positive; + /// `toString(String)` is the identity. + if (checkDataTypes(type_ptr)) + return {.is_monotonic = true, .is_always_monotonic = true, .is_strict = true}; + + /// `Date` is formatted as a zero-padded `YYYY-MM-DD` of a fixed width independently of the time zone, + /// and the whole type range falls into the years 1970-2149, so the order is preserved exactly. + if (checkDataTypes(type_ptr)) + return {.is_monotonic = true, .is_always_monotonic = true, .is_strict = true}; + + /// The same holds for `Date32`, except that day numbers out of the type range are saturated + /// to `0000-01-01` and `9999-12-31` when formatted, which makes the transformation non-injective. + if (checkDataTypes(type_ptr)) + return {.is_monotonic = true, .is_always_monotonic = true}; + + /// `DateTime` is formatted in the time zone of the type, and local time decreases when the clocks are + /// turned back, so the order is preserved only if the time zone never changes its offset. + if (checkAndGetDataType(type_ptr)) + { + return not_monotonic; + } + + /// `Time` and `Time64` are formatted with a sign and a variable number of digits for hours, + /// so, for example, `'99:00:00'` is greater than `'100:00:00'` as a string. + if (checkDataTypes(type_ptr)) + return not_monotonic; if (left.isNull() || right.isNull()) return {}; + /// `toString` is monotonous for non-negative numbers with the same number of symbols. if (left.getType() == Field::Types::UInt64 && right.getType() == Field::Types::UInt64) { diff --git a/src/Functions/LowCardinalityExecutionHelpers.h b/src/Functions/LowCardinalityExecutionHelpers.h index be0cae0c0120..44e85d171c37 100644 --- a/src/Functions/LowCardinalityExecutionHelpers.h +++ b/src/Functions/LowCardinalityExecutionHelpers.h @@ -5,9 +5,12 @@ #include #include #include +#include #include #include +#include #include +#include #include #include @@ -15,6 +18,25 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_CONVERT_TYPE; + extern const int CANNOT_PARSE_BOOL; + extern const int CANNOT_PARSE_DATE; + extern const int CANNOT_PARSE_DATETIME; + extern const int CANNOT_PARSE_IPV4; + extern const int CANNOT_PARSE_IPV6; + extern const int CANNOT_PARSE_NUMBER; + extern const int CANNOT_PARSE_TEXT; + extern const int CANNOT_PARSE_UUID; + extern const int DECIMAL_OVERFLOW; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NOT_IMPLEMENTED; + extern const int TOO_LARGE_STRING_SIZE; + extern const int UNKNOWN_ELEMENT_OF_ENUM; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +} + namespace LowCardinalityExecutionHelpers { @@ -165,6 +187,58 @@ inline ColumnPtr dictionaryMatchesForSelectedIndexes( return sparse_matches; } +/// Is [code] a cast declining its input, rather than a fault of the caller? Anything else (a memory +/// limit, a logical error, a cancellation) is not an answer about the value and must propagate. +inline bool isConstantCastDecline(int code) +{ + return code == ErrorCodes::CANNOT_CONVERT_TYPE + || code == ErrorCodes::CANNOT_PARSE_BOOL + || code == ErrorCodes::CANNOT_PARSE_DATE + || code == ErrorCodes::CANNOT_PARSE_DATETIME + || code == ErrorCodes::CANNOT_PARSE_IPV4 + || code == ErrorCodes::CANNOT_PARSE_IPV6 + || code == ErrorCodes::CANNOT_PARSE_NUMBER + || code == ErrorCodes::CANNOT_PARSE_TEXT + || code == ErrorCodes::CANNOT_PARSE_UUID + || code == ErrorCodes::DECIMAL_OVERFLOW + || code == ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT + || code == ErrorCodes::NOT_IMPLEMENTED + || code == ErrorCodes::TOO_LARGE_STRING_SIZE + || code == ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM + || code == ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +} + +/// Did [value] survive the cast that produced [image]? The cast alone cannot report loss, since it +/// truncates UInt64(256) to UInt8(0) and succeeds, so compare the two in the type they meet in, where +/// neither side's padding is a difference. +inline bool targetTypeRepresentsValue( + const ColumnPtr & value, const DataTypePtr & value_type, const ColumnPtr & image, const DataTypePtr & image_type) +{ + try + { + /// Without a common type the pair only compares as numbers, so [value_type] is where they meet. + const auto common_type = tryGetLeastSupertype(DataTypes{value_type, image_type}); + const auto compare_type = common_type ? makeNullable(common_type) : makeNullable(value_type); + + const auto restored = castColumnAccurateOrNull({image, image_type, ""}, compare_type); + if (restored->empty() || restored->isNullAt(0)) + return false; + + const auto original = castColumnAccurateOrNull({value, value_type, ""}, compare_type); + if (original->empty() || original->isNullAt(0)) + return false; + + return accurateEquals((*restored)[0], (*original)[0]); + } + catch (const Exception & e) + { + if (!isConstantCastDecline(e.code())) + throw; + + return false; + } +} + /// Returns false if the constant value is not present in the dictionary. If the constant is NULL, /// returns true and sets [dictionary_index] to the default LC null index, matching the existing /// Array(LowCardinality) index-function behavior. @@ -183,13 +257,32 @@ inline __attribute__((always_inline)) bool dictionaryIndexForConstant( return true; auto value_type_without_low_cardinality = recursiveRemoveLowCardinality(value_type); + auto original_value = value; + auto cast_type = target_type; value = castColumn({value, value_type_without_low_cardinality, ""}, target_type); if (value->isNullable()) + { value = assert_cast(*value).getNestedColumnPtr(); + cast_type = removeNullable(cast_type); + } - std::string_view elem = value->getDataAt(0); - if (auto maybe_index = low_cardinality_data.getDictionary().getOrFindValueIndex(elem)) + const auto & dictionary = low_cardinality_data.getDictionary(); + + auto find_in_dictionary = [&](std::string_view elem) -> std::optional + { + /// The default slot holds its value whether or not any row references it, and the cast above + /// narrows without reporting loss, so UInt64(256) reaches it as UInt8(0). Answering from that + /// slot requires the constant to have survived the cast; one that did not equals no element. + if (elem == dictionary.getNestedNotNullableColumn()->getDataAt(dictionary.getNestedTypeDefaultValueIndex()) + && !target_type->equals(*value_type_without_low_cardinality) + && !targetTypeRepresentsValue(original_value, value_type_without_low_cardinality, value, cast_type)) + return {}; + + return dictionary.getOrFindValueIndex(elem); + }; + + if (auto maybe_index = find_in_dictionary(value->getDataAt(0))) { dictionary_index = *maybe_index; return true; diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index 5d54732639cb..55785fc166e6 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -45,10 +45,6 @@ namespace Setting extern const SettingsUInt64 ai_function_max_retries; extern const SettingsUInt64 ai_function_retry_initial_delay_ms; extern const SettingsBool ai_function_throw_on_error; - extern const SettingsUInt64 ai_function_max_input_tokens_per_query; - extern const SettingsUInt64 ai_function_max_output_tokens_per_query; - extern const SettingsUInt64 ai_function_max_api_calls_per_query; - extern const SettingsBool ai_function_throw_on_quota_exceeded; extern const SettingsNonZeroUInt64 ai_function_embedding_max_batch_size; extern const SettingsString ai_function_embedding_default_credentials; } @@ -142,11 +138,8 @@ class FunctionAiEmbed final : public IFunction bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; size_t max_batch_size = static_cast(settings[Setting::ai_function_embedding_max_batch_size].value); - AIQuotaTracker quota( - settings[Setting::ai_function_max_input_tokens_per_query].value, - settings[Setting::ai_function_max_output_tokens_per_query].value, - settings[Setting::ai_function_max_api_calls_per_query].value, - settings[Setting::ai_function_throw_on_quota_exceeded].value); + /// Shared across every AI function call in the query + auto quota_tracker = getContext()->getAIQuotaTracker(); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); @@ -197,7 +190,7 @@ class FunctionAiEmbed final : public IFunction for (size_t batch_start = 0; batch_start < live_rows.size(); batch_start += max_batch_size) { - if (quota.checkQuotas()) + if (quota_tracker->checkQuotas()) { rows_skipped += live_rows.size() - batch_start; break; @@ -218,20 +211,17 @@ class FunctionAiEmbed final : public IFunction bool batch_ok = false; for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) { - /// Enforce the API-call quota before every provider request, including retries, so a flaky - /// endpoint can't dispatch more than `ai_function_max_api_calls_per_query` requests per query. - /// Kept outside the `try` so a `throw_on_quota_exceeded` throw is not caught by the retry handler. - if (quota.checkQuotas()) + /// Reserve an API-call slot before each request; this also performs a quota check. + /// Kept outside the `try` so a `throw_on_quota_exceeded` exception isn't caught by the retry handler. + if (!quota_tracker->recordApiCall()) break; try { - /// update api_calls/quotas before call so failed calls are still added to total ++total_api_calls; - quota.recordAttempt(); ai_embedding_response = provider->embed(ai_embedding_request, timeouts); total_input_tokens += ai_embedding_response.input_tokens; - quota.recordTokens(ai_embedding_response.input_tokens, 0); + quota_tracker->recordTokens(ai_embedding_response.input_tokens, 0); batch_ok = true; break; } diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index 39ca6e9f2e91..986b8dcb3aa2 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -857,6 +857,14 @@ class FunctionArrayIndex final : public IFunction const auto & array_type = assert_cast(*arguments[0].type); const auto target_type = recursiveRemoveLowCardinality(array_type.getNestedType()); + /// A float zero equals two byte-distinct dictionary entries, -0.0 and 0.0, and a single index + /// cannot denote both, so leave a zero needle to the path that compares values. The needle + /// type is narrowed only so that reading it as a float is total. + const auto needle_type = removeNullable(recursiveRemoveLowCardinality(arguments[1].type)); + if (isFloat(removeNullable(target_type)) && (isNumber(needle_type) || isEnum(needle_type)) + && !right_const->isNullAt(0) && right_const->getDataColumnPtr()->getFloat64(0) == 0.0) + return nullptr; + UInt64 index = 0; UInt64 left_size = arguments[0].column->size(); ResultColumnPtr col_result = ResultColumnType::create(); diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp index f6276f1da708..be57742d5bfa 100644 --- a/src/Functions/castOrDefault.cpp +++ b/src/Functions/castOrDefault.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -12,15 +11,18 @@ #include #include #include +#include #include +#include #include -#include #include #include #include +#include #include +#include namespace DB { @@ -36,6 +38,20 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +/// Row-wise source null map (ColumnUInt8, 1 = source value is NULL), or nullptr +/// when the column cannot hold NULLs. Dynamic/Variant encode NULLs with their +/// null discriminator instead of a separate null map. +static ColumnPtr getSourceNullMap(const IColumn & source_column) +{ + if (const auto * source_nullable = checkAndGetColumn(&source_column)) + return source_nullable->getNullMapColumnPtr(); + if (const auto * source_dynamic = checkAndGetColumn(&source_column)) + return source_dynamic->getVariantColumn().createNullMap(); + if (const auto * source_variant = checkAndGetColumn(&source_column)) + return source_variant->createNullMap(); + return nullptr; +} + class FunctionCastOrDefault final : public IFunction { public: @@ -46,7 +62,11 @@ class FunctionCastOrDefault final : public IFunction return std::make_shared(context); } - explicit FunctionCastOrDefault(ContextPtr context_) : keep_nullable(context_->getSettingsRef()[Setting::cast_keep_nullable]) { } + explicit FunctionCastOrDefault(ContextPtr context_) + : keep_nullable(context_->getSettingsRef()[Setting::cast_keep_nullable]) + , cast_or_null_resolver(createCastOverloadResolver(context_, CastType::accurateOrNull, {})) + { + } String getName() const override { return name; } @@ -78,9 +98,23 @@ class FunctionCastOrDefault final : public IFunction getName(), arguments[1].type->getName()); - DataTypePtr result_type = DataTypeFactory::instance().get(type_column_typed->getValue()); + /// Delegate type determination to the cast resolver. This ensures that + /// DataTypeValidationSettings and timezone substitution are applied + /// consistently between getReturnTypeImpl and executeImpl. + ColumnsWithTypeAndName cast_args{arguments[0], arguments[1]}; + DataTypePtr result_type = removeNullable(cast_or_null_resolver->getReturnType(cast_args)); + + /// The resolver uses CastType::accurateOrNull which wraps non-Nullable + /// targets in Nullable (to detect cast failures via NULL). We strip that + /// wrapper above. But when the user explicitly requested a Nullable target + /// type, the resolver didn't add the Nullable wrapper — the target was + /// already Nullable — so removeNullable incorrectly stripped the + /// user-requested Nullable. Restore it. + auto user_target_type = DataTypeFactory::instance().get(type_column_typed->getValue()); + if (user_target_type->isNullable()) + result_type = makeNullable(result_type); - if (keep_nullable && arguments.front().type->isNullable()) + if (keep_nullable && canContainNull(*arguments.front().type) && result_type->canBeInsideNullable()) result_type = makeNullable(result_type); if (arguments.size() == 3) @@ -128,20 +162,35 @@ class FunctionCastOrDefault final : public IFunction auto non_const_column_to_cast = column_to_cast.column->convertToFullColumnIfConst(); ColumnWithTypeAndName column_to_cast_non_const{non_const_column_to_cast, column_to_cast.type, column_to_cast.name}; - auto cast_result = castColumnAccurateOrNull(column_to_cast_non_const, return_type); + ColumnsWithTypeAndName cast_args + { + column_to_cast_non_const, + { + DataTypeString().createColumnConst(non_const_column_to_cast->size(), return_type->getName()), + std::make_shared(), + "" + } + }; + auto probe_type = cast_or_null_resolver->getReturnType(cast_args); + auto cast_func = cast_or_null_resolver->build(cast_args); + auto cast_result = cast_func->execute(cast_args, probe_type, non_const_column_to_cast->size(), false); + auto cast_result_full = cast_result->convertToFullColumnIfLowCardinality(); + auto cast_null_map_column = getSourceNullMap(*cast_result_full); + auto source_column_full = non_const_column_to_cast->convertToFullColumnIfLowCardinality(); + auto source_null_map_column = getSourceNullMap(*source_column_full); + const auto * cast_null_map_data = cast_null_map_column + ? &assert_cast(*cast_null_map_column).getData() + : nullptr; + const auto * source_null_map_data = source_null_map_column + ? &assert_cast(*source_null_map_column).getData() + : nullptr; + if (!cast_null_map_data) + return cast_result; - const auto & cast_result_nullable = assert_cast(*cast_result); - const auto & null_map_data = cast_result_nullable.getNullMapData(); - size_t null_map_data_size = null_map_data.size(); - const auto & nested_column = cast_result_nullable.getNestedColumn(); auto result = return_type->createColumn(); - result->reserve(null_map_data_size); + result->reserve(cast_result->size()); - ColumnNullable * result_nullable = nullptr; - if (result->isNullable()) - result_nullable = assert_cast(&*result); - - size_t start_insert_index = 0; + const auto * cast_result_nullable = checkAndGetColumn(cast_result.get()); Field default_value; ColumnPtr default_column; @@ -160,19 +209,24 @@ class FunctionCastOrDefault final : public IFunction default_value = return_type->getDefault(); } - for (size_t i = 0; i < null_map_data_size; ++i) + /// For a Nullable cast result and a non-Nullable target the values live in the nested column. + const IColumn & cast_values = cast_result_nullable && !result->isNullable() + ? cast_result_nullable->getNestedColumn() + : *cast_result; + + const bool return_type_can_contain_null = canContainNull(*return_type); + const size_t rows = cast_result->size(); + size_t start_insert_index = 0; + + for (size_t i = 0; i < rows; ++i) { - bool is_current_index_null = null_map_data[i]; - if (!is_current_index_null) + const bool is_source_null = source_null_map_data && (*source_null_map_data)[i]; + const bool cast_failed = (*cast_null_map_data)[i] && (!is_source_null || !return_type_can_contain_null); + if (!cast_failed) continue; if (i != start_insert_index) - { - if (result_nullable) - result_nullable->insertRangeFromNotNullable(nested_column, start_insert_index, i - start_insert_index); - else - result->insertRangeFrom(nested_column, start_insert_index, i - start_insert_index); - } + result->insertRangeFrom(cast_values, start_insert_index, i - start_insert_index); if (default_column) result->insertFrom(*default_column, i); @@ -182,20 +236,15 @@ class FunctionCastOrDefault final : public IFunction start_insert_index = i + 1; } - if (null_map_data_size != start_insert_index) - { - if (result_nullable) - result_nullable->insertRangeFromNotNullable(nested_column, start_insert_index, null_map_data_size - start_insert_index); - else - result->insertRangeFrom(nested_column, start_insert_index, null_map_data_size - start_insert_index); - } + if (rows != start_insert_index) + result->insertRangeFrom(cast_values, start_insert_index, rows - start_insert_index); return result; } private: - bool keep_nullable; + FunctionOverloadResolverPtr cast_or_null_resolver; }; class FunctionCastOrDefaultTyped final : public IFunction diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index a0b385d982e3..5d6dfa9ddb41 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -913,6 +913,39 @@ class FunctionFormatDateTimeImpl final : public IFunction return true; } + class TimeZoneCache + { + public: + const DateLUTImpl & getOrSet(std::string_view time_zone_name) + { + for (const Entry & cached_time_zone : cached_time_zones) + { + if (cached_time_zone.time_zone == nullptr) + break; + if (cached_time_zone.name == time_zone_name) + return *cached_time_zone.time_zone; + } + + /// insert new entry or replace existing entry + const DateLUTImpl & time_zone = DateLUT::instance(time_zone_name); + cached_time_zones[next] = {String(time_zone_name), &time_zone}; + next = (next + 1) % CACHE_SIZE; + return time_zone; + } + + private: + constexpr static size_t CACHE_SIZE = 4; + + struct Entry + { + String name; + const DateLUTImpl * time_zone = nullptr; + }; + using CacheTable = std::array; + CacheTable cached_time_zones; + size_t next = 0; /// position to insert the next entry at + }; + const bool mysql_M_is_month_name; const bool mysql_f_prints_single_zero; const bool mysql_f_prints_scale_number_of_digits; @@ -1142,14 +1175,19 @@ class FunctionFormatDateTimeImpl final : public IFunction auto * begin = reinterpret_cast(res_data.data()); auto * pos = begin; + + /// Non-const time zone arguments are resolved per row. This is done under an expensive mutex in DateLUT. + /// Lots of queries use only a handful of different time zones. As an optimization, cache the resolved time zones. + TimeZoneCache time_zone_cache; + for (size_t i = 0; i < input_rows_count; ++i) { if (!const_time_zone_column && arguments.size() > 2) { - if (!arguments[2].column.get()->getDataAt(i).empty()) - time_zone = &DateLUT::instance(arguments[2].column.get()->getDataAt(i)); - else + std::string_view time_zone_name = arguments[2].column->getDataAt(i); + if (time_zone_name.empty()) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Provided time zone must be non-empty"); + time_zone = &time_zone_cache.getOrSet(time_zone_name); } if constexpr (std::is_same_v) { diff --git a/src/Functions/formatRow.cpp b/src/Functions/formatRow.cpp index e9ad56af715b..3e4b45d99b85 100644 --- a/src/Functions/formatRow.cpp +++ b/src/Functions/formatRow.cpp @@ -93,7 +93,16 @@ class FunctionFormatRow final : public IFunction row_output_format->finalize(); if (no_newline) { - if (buffer.position() != buffer.buffer().begin() && buffer.position()[-1] == '\n') + /// Strip a single trailing newline, but only when this row actually emitted at least one byte. + /// `buffer.count()` is the absolute number of bytes written; the current row starts at the + /// previous row's end offset (0 for the first row). Comparing against it prevents rewinding into + /// the previous row when this row is empty, which would make `offsets` non-monotonic and cause a + /// `size_t` underflow in `ColumnString::sizeAt`. The check against `buffer.buffer().begin()` + /// additionally keeps the position within the current working buffer so `--buffer.position()` + /// never moves the cursor before it. + const size_t row_start = i == 0 ? 0 : offsets[i - 1]; + if (buffer.count() > row_start && buffer.position() > buffer.buffer().begin() + && buffer.position()[-1] == '\n') --buffer.position(); } diff --git a/src/Functions/jsonMergePatch.cpp b/src/Functions/jsonMergePatch.cpp index 4b8972c1e885..0a9ddc2f5fc3 100644 --- a/src/Functions/jsonMergePatch.cpp +++ b/src/Functions/jsonMergePatch.cpp @@ -29,6 +29,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int ILLEGAL_COLUMN; + extern const int TOO_DEEP_RECURSION; } namespace @@ -43,6 +44,40 @@ namespace using TrackedWriter = rapidjson::Writer, rapidjson::UTF8, RapidJSONMemoryTrackerAllocator>; + /// Parsing is iterative (see RAPIDJSON_PARSE_DEFAULT_FLAGS above), but copying, merging and + /// serializing a document all recurse over its tree, so a valid but deeply nested document + /// would exhaust the thread stack. Reject such documents right after parsing; the check itself + /// is iterative. + constexpr size_t max_json_merge_patch_depth = 1000; + + void checkJSONDepth(const TrackedValue & root) + { + VectorWithMemoryTracking> to_visit; + to_visit.emplace_back(&root, 1); + + while (!to_visit.empty()) + { + const auto [value, depth] = to_visit.back(); + to_visit.pop_back(); + + if (depth > max_json_merge_patch_depth) + throw Exception(ErrorCodes::TOO_DEEP_RECURSION, + "Too deep nesting in a JSON document passed to function JSONMergePatch: the limit is {}", + max_json_merge_patch_depth); + + if (value->IsObject()) + { + for (auto it = value->MemberBegin(); it != value->MemberEnd(); ++it) + to_visit.emplace_back(&it->value, depth + 1); + } + else if (value->IsArray()) + { + for (const auto * it = value->Begin(); it != value->End(); ++it) + to_visit.emplace_back(&*it, depth + 1); + } + } + } + // select JSONMergePatch('{"a":1}','{"name": "joey"}','{"name": "tom"}','{"name": "zoey"}'); // || // \/ @@ -113,6 +148,8 @@ namespace if (!document.IsObject()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong JSON string to merge. Expected JSON object"); + + checkJSONDepth(document); }; const bool is_first_const = isColumnConst(*arguments[0].column); diff --git a/src/Functions/tests/gtest_conversion_monotonic.cpp b/src/Functions/tests/gtest_conversion_monotonic.cpp index 26335c3f3013..7336c5df4bd8 100644 --- a/src/Functions/tests/gtest_conversion_monotonic.cpp +++ b/src/Functions/tests/gtest_conversion_monotonic.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include using namespace DB; @@ -74,3 +75,25 @@ TEST(ConversionMonotonic, toStringLowCardinalityFixedString) ASSERT_EQ(monotonicity.is_always_monotonic, true); ASSERT_EQ(monotonicity.is_strict, true); } + +TEST(ConversionMonotonic, toStringString) +{ + const auto monotonicity = ToStringMonotonicity::get(DataTypeString(), {}, {}); + + ASSERT_EQ(monotonicity.is_monotonic, true); + ASSERT_EQ(monotonicity.is_positive, true); + ASSERT_EQ(monotonicity.is_always_monotonic, true); + ASSERT_EQ(monotonicity.is_strict, true); +} + +TEST(ConversionMonotonic, toStringLowCardinalityString) +{ + DataTypeLowCardinality low_cardinality_string_type(std::make_shared()); + + const auto monotonicity = ToStringMonotonicity::get(low_cardinality_string_type, {}, {}); + + ASSERT_EQ(monotonicity.is_monotonic, true); + ASSERT_EQ(monotonicity.is_positive, true); + ASSERT_EQ(monotonicity.is_always_monotonic, true); + ASSERT_EQ(monotonicity.is_strict, true); +} diff --git a/src/Functions/tests/gtest_monotonic.cpp b/src/Functions/tests/gtest_monotonic.cpp index 9ae0e1f07167..1cfc29c76908 100644 --- a/src/Functions/tests/gtest_monotonic.cpp +++ b/src/Functions/tests/gtest_monotonic.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -80,3 +81,29 @@ TEST(Monotonicity, Coalesce) { testNullWrapperMonotonicity("coalesce", {std::make_shared(), std::make_shared()}); } + +TEST(Monotonicity, ToNullable) +{ + /// `toNullable` only wraps the value, so it is strictly increasing on the whole range of any argument type. + const DataTypes argument_types = { + std::make_shared(), + std::make_shared(), + makeNullable(std::make_shared()), + std::make_shared(std::make_shared()), + }; + + for (const auto & argument_type : argument_types) + { + SCOPED_TRACE(argument_type->getName()); + + auto function_base = buildFunction("toNullable", {argument_type}); + ASSERT_TRUE(function_base->hasInformationAboutMonotonicity()); + + const auto monotonicity = function_base->getMonotonicityForRange(*argument_type, Field{}, Field{}); + + ASSERT_TRUE(monotonicity.is_monotonic); + ASSERT_TRUE(monotonicity.is_positive); + ASSERT_TRUE(monotonicity.is_always_monotonic); + ASSERT_TRUE(monotonicity.is_strict); + } +} diff --git a/src/Functions/toNullable.cpp b/src/Functions/toNullable.cpp index cc4ccffcfd51..0b23e5a13c34 100644 --- a/src/Functions/toNullable.cpp +++ b/src/Functions/toNullable.cpp @@ -52,6 +52,12 @@ class FunctionToNullable final : public IFunction { return makeNullableOrLowCardinalityNullable(arguments[0].column); } + bool hasInformationAboutMonotonicity() const override { return true; } + + Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override + { + return { .is_monotonic = true, .is_positive = true, .is_always_monotonic = true, .is_strict = true }; + } #if USE_EMBEDDED_COMPILER bool isCompilableImpl(const DataTypes & arguments, const DataTypePtr &) const override { return canBeNativeType(arguments[0]); } diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 5b1c09b55f49..c139cc0f20cb 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -2041,6 +2041,29 @@ bool trySkipJSONField(ReadBuffer & buf, std::string_view name_of_field, const Fo } +/// The same as `readStringBinary`, but the string grows as the bytes arrive instead of being resized +/// to the declared size first, so that a size declared by the peer cannot become an allocation on +/// its own when the payload never follows. +static void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size = DEFAULT_MAX_STRING_SIZE) +{ + size_t size = 0; + readVarUInt(size, buf); + + if (size > max_string_size) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Too large string size."); + + s.clear(); + while (s.size() < size) + { + if (buf.eof()) + throwReadAfterEOF(); + + const size_t bytes_to_copy = std::min(size - s.size(), buf.available()); + s.append(buf.position(), bytes_to_copy); + buf.position() += bytes_to_copy; + } +} + Exception readException(ReadBuffer & buf, const String & additional_message, bool remote_exception) { int code = 0; @@ -2050,9 +2073,11 @@ Exception readException(ReadBuffer & buf, const String & additional_message, boo bool has_nested = false; /// Obsolete readBinaryLittleEndian(code, buf); - readBinary(name, buf); - readBinary(message, buf); - readBinary(stack_trace, buf); + /// This is the first thing read from a server during the handshake, and the sizes of these + /// strings come from the other side, so read them without preallocating the declared size. + readStringBinaryGrowing(name, buf); + readStringBinaryGrowing(message, buf); + readStringBinaryGrowing(stack_trace, buf); readBinary(has_nested, buf); WriteBufferFromOwnString out; diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index d74844481674..bcfd8d09a1e2 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -111,6 +111,9 @@ struct FilesystemCacheSettings /// the sister `DistributedCacheSettings::prefer_bigger_buffer_size`. bool prefer_bigger_buffer_size = true; size_t reserve_space_wait_lock_timeout_milliseconds = 1000; + /// How long a read may wait for a file segment which is being downloaded by a concurrent query + /// before bypassing the cache and reading directly from remote storage. + size_t wait_for_concurrent_download_timeout_milliseconds = 1000; size_t max_download_size_per_query = (128UL * 1024 * 1024 * 1024); bool skip_download_if_exceeds_per_query_cache_write_limit = true; bool enable_log = false; diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index c866f76ad3f8..c6d540533e4e 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -1232,11 +1233,46 @@ static ColumnWithTypeAndName executeActionForPartialResult( { try { + /// Do not fold when an argument's type differs from the type this node was resolved + /// for. The comparison is exact, not wrapper-stripped: `isNullable` derives its value + /// from the argument type alone, so a wrapper-only difference would give a wrong value + /// with an unchanged result type. + bool argument_types_drifted = false; + const auto & expected_argument_types = node->function_base->getArgumentTypes(); + if (expected_argument_types.size() == arguments.size()) + { + for (size_t i = 0; i < arguments.size(); ++i) + { + if (!arguments[i].type || !expected_argument_types[i]) + continue; + if (!arguments[i].type->equals(*expected_argument_types[i])) + { + argument_types_drifted = true; + break; + } + } + } + + if (argument_types_drifted) + { + /// An empty column of the declared type keeps header computation going; with one row + /// the column stays null, because callers read any non-null output as definitive. + /// `DataTypeFunction` (a captured lambda) is the one type here that cannot be + /// instantiated - it inherits `IDataTypeDummy::createColumn`, which throws + /// `NOT_IMPLEMENTED` - so it is left null too. (`DataTypeNothing` and `DataTypeSet` + /// share that base but do override `createColumn`.) + if (input_rows_count == 0 && !typeid_cast(res_column.type.get())) + res_column.column = res_column.type->createColumn(); + break; + } + if (only_constant_arguments) res_column.column = node->function->execute(arguments, res_column.type, input_rows_count, true); else res_column.column = node->function_base->getConstantResultForNonConstArguments(arguments, res_column.type); + /// Arguments did not drift (checked above), so a result-type mismatch here is a genuine + /// function contract violation rather than an EXCHANGE TABLES race. if (res_column.column && !columnMatchesType(*res_column.column, *res_column.type)) throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -1263,6 +1299,14 @@ static ColumnWithTypeAndName executeActionForPartialResult( case ActionsDAG::ActionType::ARRAY_JOIN: { auto key = arguments.at(0); + + /// Carry the ACTUAL nested type, as the `ALIAS` case below does and as + /// `ExpressionActions::executeAction` does here: the declared `result_type` is stale, and + /// keeping it would hide the difference from the `FUNCTION` check above. Runs before the + /// early exits, which leave the column null but still hand the TYPE to the parent. + if (const auto & key_array_type = getArrayJoinDataType(key.type)) + res_column.type = key_array_type->getNestedType(); + if (!key.column) break; @@ -1300,7 +1344,13 @@ static ColumnWithTypeAndName executeActionForPartialResult( case ActionsDAG::ActionType::ALIAS: { + /// Carry the argument's ACTUAL type, not just its column, as + /// `ExpressionActions::executeAction` does: an alias never changes a value, so copying a + /// drifted column under the stale declared type would hide the difference from the + /// `FUNCTION` check above and a function behind the alias would still be executed on it. res_column.column = arguments.at(0).column; + if (arguments.at(0).type) + res_column.type = arguments.at(0).type; break; } @@ -2868,7 +2918,10 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsForFilter(const std::string & co dumpDAG()); std::unordered_set split_nodes = {node}; - auto res = split(split_nodes); + /// The filter name may also be an input name. Two same-named outputs of different structure in the + /// first half would break the Block invariant, so let split() rename the promoted node and repair + /// the second half. The mapping carries the final name of the filter node. + auto res = split(split_nodes, /*create_split_nodes_mapping=*/ true, /*avoid_duplicate_inputs=*/ true); return res; } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 801000e53407..d576b2418501 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -974,9 +974,11 @@ void NO_INLINE Aggregator::executeImpl( if (!no_more_keys) { /// Prefetching doesn't make sense for small hash tables, because they fit in caches entirely. - /// Enable prefetch for all key types including strings — the adaptive PrefetchingHelper - /// handles variable hash computation cost by measuring actual iteration latency. - const bool prefetch = params.enable_prefetch + /// It also doesn't make sense when building the key holder is expensive: the look-ahead + /// below calls `getKeyHolder` a second time for every row, so a method that materializes + /// its key there (e.g. serializing all key columns) would pay its dominant per-row cost + /// twice - far more than the cache miss the prefetch hides. See `has_cheap_key_holder`. + const bool prefetch = State::has_cheap_key_holder && params.enable_prefetch && (method.data.getBufferSizeInBytes() > min_bytes_for_prefetch); #if USE_EMBEDDED_COMPILER @@ -3049,8 +3051,9 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl( AggregatedDataVariantsPtr & res = non_empty_data[0]; bool no_more_keys = false; - /// Enable prefetch for all key types including strings — the adaptive PrefetchingHelper - /// handles variable hash computation cost by measuring actual iteration latency. + /// Enabled for all key types: unlike `executeImplBatch`, the merge path prefetches by the hash + /// already stored in the source cell (`mergeToViaEmplace`), so it never rebuilds a key and + /// `has_cheap_key_holder` does not apply here. const bool prefetch = params.enable_prefetch && (getDataVariant(*res).data.getBufferSizeInBytes() > min_bytes_for_prefetch); @@ -3136,8 +3139,9 @@ void NO_INLINE Aggregator::mergeBucketImpl( /// We merge all aggregation results to the first. AggregatedDataVariantsPtr & res = data[0]; - /// Enable prefetch for all key types including strings — the adaptive PrefetchingHelper - /// handles variable hash computation cost by measuring actual iteration latency. + /// Enabled for all key types: unlike `executeImplBatch`, the merge path prefetches by the hash + /// already stored in the source cell (`mergeToViaEmplace`), so it never rebuilds a key and + /// `has_cheap_key_holder` does not apply here. const bool prefetch = params.enable_prefetch && (Method::Data::NUM_BUCKETS * getDataVariant(*res).data.impls[bucket].getBufferSizeInBytes() > min_bytes_for_prefetch); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 88207be6ba60..068842f3c5f5 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -586,7 +586,7 @@ AsynchronousInsertQueue::PushResult AsynchronousInsertQueue::pushDataChunk(ASTPt } if (inserted) - it->second = shard.queue.emplace(now + timeout_ms, Container{key, std::make_unique(timeout_ms)}).first; + it->second = shard.queue.emplace(now + timeout_ms, Container{key, std::make_unique(timeout_ms)}); auto queue_it = it->second; auto & data = queue_it->second.data; diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 073a0128d6b6..5007f121d431 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -235,7 +235,8 @@ class AsynchronousInsertQueue : public WithContext /// Ordered container /// Key is a timestamp of the first insert into batch. /// Used to detect for how long the batch is active, so we can dump it by timer. - using Queue = std::map; + /// Must be a multimap: two queries with different keys may theoretically compute the same deadline. + using Queue = std::multimap; using QueueIterator = Queue::iterator; using QueueIteratorByKey = std::unordered_map; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index a9188f90d31a..5f74515c80e6 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -103,6 +103,7 @@ #include #include #include +#include #include #include #include @@ -285,6 +286,10 @@ namespace DB { namespace Setting { + extern const SettingsUInt64 ai_function_max_input_tokens_per_query; + extern const SettingsUInt64 ai_function_max_output_tokens_per_query; + extern const SettingsUInt64 ai_function_max_api_calls_per_query; + extern const SettingsBool ai_function_throw_on_quota_exceeded; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; extern const SettingsFloat ast_fuzzer_runs; extern const SettingsUInt64 automatic_parallel_replicas_mode; @@ -300,6 +305,7 @@ namespace Setting extern const SettingsBool enable_blob_storage_log_for_read_operations; extern const SettingsUInt64 filesystem_cache_max_download_size; extern const SettingsUInt64 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds; + extern const SettingsUInt64 filesystem_cache_wait_for_concurrent_download_timeout_milliseconds; extern const SettingsUInt64 filesystem_cache_segments_batch_size; extern const SettingsBool filesystem_cache_allow_background_download; extern const SettingsBool filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage; @@ -1518,12 +1524,27 @@ DatabaseAndTable Context::getOrCacheStorage(const StorageID & id, std::function< if (auto it = shard.set.find(id); it != shard.set.end()) { DatabaseAndTable storage = DatabaseCatalog::instance().tryGetByUUID(it->uuid); - if (storage.second) + /// The cache is keyed by qualified name only (see `StorageCache::Shard::set`), so a hit can + /// carry a UUID that no longer matches the name we are resolving. Return the cached storage + /// only if it is still fresh. Otherwise the entry is stale and must not be reused: + /// - the table no longer exists by its UUID (e.g. a refreshable materialized view's inner + /// table was dropped and recreated), or + /// - the UUID still exists but the name was reassigned to a different table by a rename or + /// exchange within the same query. This happens during `CREATE OR REPLACE`, which creates a + /// temporary table, populates it (caching the temporary name -> temporary UUID here), then + /// atomically swaps it with the target via `EXCHANGE`. After the swap the temporary name + /// refers to the old table that is about to be dropped, but the cache would still hand out + /// the new (now live) table - so dropping by the temporary name would shut down the live + /// table instead and break it (e.g. detaching a materialized view from its source), or + /// - the caller asked for a specific UUID but the cached entry resolves to a different one + /// (a same-name replacement); returning it would silently substitute the wrong table + /// instead of letting the fresh lookup report `UNKNOWN_TABLE`/`TABLE_UUID_MISMATCH`. + /// In all cases remove the stale entry and fall through to a fresh lookup by name. + if (storage.second + && storage.second->getStorageID().getQualifiedName() == id.getQualifiedName() + && (!id.hasUUID() || it->uuid == id.uuid)) return storage; - /// The table was cached but no longer exists by its UUID - /// (e.g. refreshable materialized view's inner table was dropped and recreated). - /// Remove the stale entry and fall through to a fresh lookup by name. shard.set.erase(it); } @@ -2341,10 +2362,10 @@ ClassifierPtr Context::getWorkloadClassifier() const return classifier; } -void Context::releaseWorkloadResources() const +void Context::releaseQuerySlot() const { if (auto elem = getProcessListElementSafe()) - elem->releaseWorkloadResources(); + elem->releaseQuerySlot(); } String Context::getMergeWorkload() const @@ -3338,6 +3359,12 @@ void Context::checkSettingsConstraints(const SettingsChanges & changes, SettingS doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } +void Context::checkSettingsConstraintsForSettingsReset(const std::vector & names, SettingSource source) +{ + SharedLockGuard lock(mutex); + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.checkResetToDefault(*settings, names, source); +} + void Context::checkSettingsConstraints(SettingsChanges & changes, SettingSource source) { SharedLockGuard lock(mutex); @@ -8060,6 +8087,8 @@ ReadSettings Context::getReadSettings() const res.filesystem_cache_settings.segments_batch_size = settings_ref[Setting::filesystem_cache_segments_batch_size]; res.filesystem_cache_settings.reserve_space_wait_lock_timeout_milliseconds = settings_ref[Setting::filesystem_cache_reserve_space_wait_lock_timeout_milliseconds]; + res.filesystem_cache_settings.wait_for_concurrent_download_timeout_milliseconds + = settings_ref[Setting::filesystem_cache_wait_for_concurrent_download_timeout_milliseconds]; res.filesystem_cache_settings.allow_background_download = settings_ref[Setting::filesystem_cache_allow_background_download]; res.filesystem_cache_settings.allow_background_download_for_metadata_files_in_packed_storage = settings_ref[Setting::filesystem_cache_enable_background_download_for_metadata_files_in_packed_storage]; @@ -8254,6 +8283,24 @@ ReverseLookupCache & Context::getReverseLookupCache() const return *query_context->reverse_lookup_cache; } +AIQuotaTrackerPtr Context::getAIQuotaTracker() const +{ + auto query_context = getQueryContext(); + + const auto & settings_ref = query_context->getSettingsRef(); + + std::lock_guard lock(query_context->mutex); + if (!query_context->ai_quota_tracker) + { + query_context->ai_quota_tracker = std::make_shared( + settings_ref[Setting::ai_function_max_input_tokens_per_query], + settings_ref[Setting::ai_function_max_output_tokens_per_query], + settings_ref[Setting::ai_function_max_api_calls_per_query], + settings_ref[Setting::ai_function_throw_on_quota_exceeded]); + } + return query_context->ai_quota_tracker; +} + void Context::setRuntimeFilterLookup(const RuntimeFilterLookupPtr & filter_lookup) { runtime_filter_lookup = filter_lookup; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 1c63f988afe7..5a9943af651d 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -283,6 +283,9 @@ using PreparedSetsCachePtr = std::shared_ptr; class ReverseLookupCache; using ReverseLookupCachePtr = std::shared_ptr; +class AIQuotaTracker; +using AIQuotaTrackerPtr = std::shared_ptr; + /// IRuntimeFilterLookup stores and finds per-query join runtime-filter handles under (random) names. /// Runtime filters optimize some JOINs by building a filter from the right side and pre-filtering the left side. struct IRuntimeFilterLookup; @@ -630,6 +633,9 @@ class ContextData /// This is a per query cache and not shared across queries. mutable ReverseLookupCachePtr reverse_lookup_cache; + /// AI-function quota usage for the current query, shared by every AI function call in it. + mutable AIQuotaTrackerPtr ai_quota_tracker; + /// this is a mode of parallel replicas where we set parallel_replicas_count and parallel_replicas_offset /// and generate specific filters on the replicas (e.g. when using parallel replicas with sample key) /// if we already use a different mode of parallel replicas we want to disable this mode @@ -910,7 +916,10 @@ class Context: public ContextData, public std::enable_shared_from_this /// Resource management related ResourceManagerPtr getResourceManager() const; ClassifierPtr getWorkloadClassifier() const; - void releaseWorkloadResources() const; + /// Release the query slot early so the client can reuse it for its next query. + /// Only the query slot is released, not the memory reservation: pipeline threads still hold raw + /// pointers to it, so it is released later by `BlockIO::onFinish` after the pipeline is finalized. + void releaseQuerySlot() const; String getMergeWorkload() const; void setMergeWorkload(const String & value); String getLicenseFile() const; @@ -1163,6 +1172,7 @@ class Context: public ContextData, public std::enable_shared_from_this void checkSettingsConstraints(const SettingChange & change, SettingSource source); void checkSettingsConstraints(const SettingsChanges & changes, SettingSource source); void checkSettingsConstraints(SettingsChanges & changes, SettingSource source); + void checkSettingsConstraintsForSettingsReset(const std::vector & names, SettingSource source); void clampToSettingsConstraints(SettingsChanges & changes, SettingSource source); void checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; @@ -1869,6 +1879,8 @@ class Context: public ContextData, public std::enable_shared_from_this ReverseLookupCache & getReverseLookupCache() const; + AIQuotaTrackerPtr getAIQuotaTracker() const; + /// IRuntimeFilterLookup stores and finds per-query join runtime-filter handles by (random) names, /// used to optimize some JOINs by early pre-filtering the left side with a filter built from the right. void setRuntimeFilterLookup(const RuntimeFilterLookupPtr & filter_lookup); diff --git a/src/Interpreters/FileCache/FileSegment.cpp b/src/Interpreters/FileCache/FileSegment.cpp index 72189755c404..9ee85886d6eb 100644 --- a/src/Interpreters/FileCache/FileSegment.cpp +++ b/src/Interpreters/FileCache/FileSegment.cpp @@ -22,6 +22,7 @@ namespace fs = std::filesystem; namespace ProfileEvents { extern const Event FileSegmentWaitMicroseconds; + extern const Event FileSegmentWaitTimeouts; extern const Event FileSegmentCompleteMicroseconds; extern const Event FileSegmentLockMicroseconds; extern const Event FileSegmentWriteMicroseconds; @@ -50,6 +51,7 @@ namespace ErrorCodes namespace FailPoints { extern const char cache_filesystem_failure[]; + extern const char file_segment_pause_before_write[]; } String toString(FileSegmentKind kind) @@ -393,6 +395,9 @@ void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) void FileSegment::write(char * from, size_t size, size_t offset_in_file) { + /// Keeps the segment in DOWNLOADING state, for testing the concurrent download wait timeout. + FailPointInjection::pauseFailPoint(FailPoints::file_segment_pause_before_write); + ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWriteMicroseconds); auto file_segment_path = getPath(); DownloadState * download = nullptr; @@ -528,7 +533,7 @@ void FileSegment::write(char * from, size_t size, size_t offset_in_file) chassert(getCurrentWriteOffset() == offset_in_file + size); } -FileSegment::State FileSegment::wait(size_t offset) +FileSegment::State FileSegment::wait(size_t offset, size_t timeout_ms) { OpenTelemetry::SpanHolder span("FileSegment::wait"); span.addAttribute("clickhouse.key", key().toString()); @@ -550,11 +555,23 @@ FileSegment::State FileSegment::wait(size_t offset) chassert(!getDownloaderUnlocked(lk).empty()); chassert(!isDownloaderUnlocked(lk)); - [[maybe_unused]] const auto ok = cv.wait_for(lk, std::chrono::seconds(60), [&, this]() + auto downloaded = [&, this]() { return download_state != State::DOWNLOADING || offset < getCurrentWriteOffset(); - }); - /// chassert(ok); + }; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (true) + { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) + { + ProfileEvents::increment(ProfileEvents::FileSegmentWaitTimeouts); + break; + } + const auto slice = std::min(std::chrono::seconds(1), deadline - now); + if (cv.wait_for(lk, slice, downloaded)) + break; + } } return download_state; diff --git a/src/Interpreters/FileCache/FileSegment.h b/src/Interpreters/FileCache/FileSegment.h index 6cb660402978..f06327b7fe4c 100644 --- a/src/Interpreters/FileCache/FileSegment.h +++ b/src/Interpreters/FileCache/FileSegment.h @@ -122,8 +122,7 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). DownloaderId getDownloader() const; - /// Wait for the change of state from DOWNLOADING to any other. - State wait(size_t offset); + State wait(size_t offset, size_t timeout_ms = 60000); bool isDownloaded() const; diff --git a/src/Interpreters/FileCache/QueryLimit.cpp b/src/Interpreters/FileCache/QueryLimit.cpp index a84f5dc0c293..f537c8bae0d7 100644 --- a/src/Interpreters/FileCache/QueryLimit.cpp +++ b/src/Interpreters/FileCache/QueryLimit.cpp @@ -24,21 +24,46 @@ FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::tryGetQueryContext(con if (!isQueryInitialized()) return nullptr; + std::lock_guard lock(query_map_mutex); auto query_iter = query_map.find(std::string(CurrentThread::getQueryId())); return (query_iter == query_map.end()) ? nullptr : query_iter->second; } -void FileCacheQueryLimit::removeQueryContext(const std::string & query_id, const CachePriorityGuard::WriteLock &) +FileCacheQueryLimit::QueryContextPtr +FileCacheQueryLimit::removeQueryContext(const std::string & query_id, QueryContextPtr & context, const CachePriorityGuard::WriteLock &) { - auto query_iter = query_map.find(query_id); - if (query_iter == query_map.end()) + QueryContextPtr doomed; { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Attempt to release query context that does not exist (query_id: {})", - query_id); + std::lock_guard lock(query_map_mutex); + + auto query_iter = query_map.find(query_id); + const bool owns_map_entry = query_iter != query_map.end() && query_iter->second == context; + + /// Drop this holder's own reference to the context under the lock, then decide. use_count() + /// is not a synchronization primitive, so the decision must be made after every reference + /// change to the context is serialized by this mutex (which also guards getOrSetQueryContext). + /// Deciding before dropping the reference (or dropping it outside the lock) is a TOCTOU: + /// two holders releasing at once can both observe the shared count and both skip the erase, + /// orphaning the map entry, or one can erase while the other is being revived (see #109508). + context.reset(); + + if (owns_map_entry && query_iter->second.use_count() == 1) + { + /// The reference this holder held is gone and the map entry is now the sole owner, so + /// this was the last holder. Extract the pointer instead of erasing in place so the + /// QueryContext (its records map and per-query priority queue) is destroyed by the + /// caller after the cache write lock is released, not under it. Otherwise a query that + /// touched many segments frees all of that state while holding cache->lockCache(), + /// blocking unrelated reserve/eviction work for the duration of teardown. + doomed = std::move(query_iter->second); + query_map.erase(query_iter); + } + /// If owns_map_entry is false, the entry was already removed or re-created for a newer holder + /// via getOrSetQueryContext; another live holder now owns it, so leave it in place. If the + /// map entry is not the sole owner, another holder for the same query_id is still alive and + /// the context must stay so the per-query limit keeps being enforced. } - query_map.erase(query_iter); + return doomed; } FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::getOrSetQueryContext( @@ -49,6 +74,7 @@ FileCacheQueryLimit::QueryContextPtr FileCacheQueryLimit::getOrSetQueryContext( if (query_id.empty()) return nullptr; + std::lock_guard lock(query_map_mutex); auto [it, inserted] = query_map.emplace(query_id, nullptr); if (inserted) { @@ -125,12 +151,19 @@ FileCacheQueryLimit::QueryContextHolder::QueryContextHolder( FileCacheQueryLimit::QueryContextHolder::~QueryContextHolder() { - /// If only the query_map and the current holder hold the context_query, - /// the query has been completed and the query_context is released. - if (context && context.use_count() == 2) + /// The last-holder decision (and the drop of this holder's reference) must happen inside + /// removeQueryContext under the cache write lock, not here: dropping the reference or deciding + /// outside the lock races with revival via getOrSetQueryContext and can leak or orphan the entry. + /// context is only set when the per-query download limit is enabled, so this is a no-op otherwise. + if (context) { - auto lock = cache->lockCache(); - query_limit->removeQueryContext(query_id, lock); + /// When this is the last holder, removeQueryContext hands the context back so it is destroyed + /// here, after the cache lock scope has ended, rather than under cache->lockCache(). + QueryContextPtr doomed; + { + auto lock = cache->lockCache(); + doomed = query_limit->removeQueryContext(query_id, context, lock); + } } } diff --git a/src/Interpreters/FileCache/QueryLimit.h b/src/Interpreters/FileCache/QueryLimit.h index e0f60402b898..98716a6fc6d9 100644 --- a/src/Interpreters/FileCache/QueryLimit.h +++ b/src/Interpreters/FileCache/QueryLimit.h @@ -2,6 +2,8 @@ #include #include +#include + namespace DB { struct ReadSettings; @@ -21,7 +23,11 @@ class FileCacheQueryLimit const FilesystemCacheSettings & settings, const CachePriorityGuard::WriteLock &); - void removeQueryContext(const std::string & query_id, const CachePriorityGuard::WriteLock &); + /// Releases this holder's reference to the query context and, when it was the last holder, + /// removes the map entry and returns the now-orphaned context so the caller can destroy it + /// after releasing the cache write lock (see ~QueryContextHolder). Returns nullptr when the + /// context is still owned by another live holder. + QueryContextPtr removeQueryContext(const std::string & query_id, QueryContextPtr & context, const CachePriorityGuard::WriteLock &); class QueryContext { @@ -77,6 +83,11 @@ class FileCacheQueryLimit private: using QueryContextMap = std::unordered_map; QueryContextMap query_map; + /// query_map is reached under two different cache locks: reads (tryGetQueryContext) run under + /// CacheStateGuard while writes (getOrSetQueryContext/removeQueryContext) run under + /// CachePriorityGuard, so neither cache lock serializes access to the map by itself. This + /// dedicated leaf mutex is the single lock that actually guards query_map. + mutable std::mutex query_map_mutex; }; using FileCacheQueryLimitPtr = std::unique_ptr; diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index 669bb98706ea..cbc8c271fe7f 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -381,6 +381,8 @@ static HashJoin::Type chooseMethod(JoinKind kind, const ColumnRawPtrs & key_colu if (keys_size == 1 && key_columns[0]->isNumeric()) { size_t size_of_field = key_columns[0]->sizeOfValueIfFixed(); + /// The loop above bails out before assigning `key_sizes` for a `LowCardinality` column. + key_sizes[0] = size_of_field; if (size_of_field == 1) return Type::key8; if (size_of_field == 2) diff --git a/src/Interpreters/InsertDeduplication.cpp b/src/Interpreters/InsertDeduplication.cpp index c259bf7fb323..c6a55109854b 100644 --- a/src/Interpreters/InsertDeduplication.cpp +++ b/src/Interpreters/InsertDeduplication.cpp @@ -163,21 +163,16 @@ DeduplicationInfo::Ptr DeduplicationInfo::filterToPartition(const PaddedPODArray return cloneSelf(); /// Attributing tokens to partitions walks each token's row range over the selector, which is - /// only possible while the offsets still describe the block that was split. Behind an `Alias` - /// hop over a row-count-changing view the deduplication info is re-anchored to the view-output - /// chunks and there is no mapping from the tokens' source rows to the selector anymore. Refuse - /// loudly instead of reading out of the selector's bounds (see filterImpl). - if (row_to_partition.size() != getRows()) - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Cannot attribute {} deduplication tokens to the partitions of the insert: the deduplication info " - "describes {} rows, but the block was split into partitions over {} rows because a materialized view " - "with a row-count-changing inner query was processed before a table with the `Alias` engine. " - "Debug: {}", - getCount(), - getRows(), - row_to_partition.size(), - debug()); + /// only valid at the direct insert destination, where the offsets still describe exactly the + /// block that was split. A materialized-view (or `Alias`-hop) target may have changed the row + /// count in its inner query, so there is no mapping from the tokens' source rows to the + /// view-output selector: keep every token in every partition instead. A repeated token may + /// still be deduplicated per partition through the cached data hashes. + if (level == Level::VIEW) + return cloneSelf(); + + /// At the direct destination the offsets describe the split block, so the walk is in bounds. + chassert(row_to_partition.size() == getRows()); /// Keep only tokens that have at least one row in this partition. std::set absent_offsets; diff --git a/src/Interpreters/InsertDependenciesBuilder.cpp b/src/Interpreters/InsertDependenciesBuilder.cpp index c1ab8004fe0b..af09d672b2c3 100644 --- a/src/Interpreters/InsertDependenciesBuilder.cpp +++ b/src/Interpreters/InsertDependenciesBuilder.cpp @@ -946,7 +946,7 @@ VectorWithMemoryTracking InsertDependenciesBuilder::createChainWithDepend { auto & chain = result_chains.emplace_back(std::move(processor_list)); chain.attachResources(std::move(resources)); - chain.setNumThreads(init_context->getSettingsRef()[Setting::max_threads]); + chain.setNumThreads(getViewProcessingNumThreads()); chain.setConcurrencyControl(init_context->getSettingsRef()[Setting::use_concurrency_control]); } @@ -995,7 +995,7 @@ Chain InsertDependenciesBuilder::createChainWithDependencies() const result.addSink(std::make_shared(output_headers.at(root_view))); } - result.setNumThreads(init_context->getSettingsRef()[Setting::max_threads]); + result.setNumThreads(getViewProcessingNumThreads()); result.setConcurrencyControl(init_context->getSettingsRef()[Setting::use_concurrency_control]); result.addInsertDependenciesBuilder(shared_from_this()); @@ -1721,6 +1721,15 @@ bool InsertDependenciesBuilder::isViewsInvolved() const } +size_t InsertDependenciesBuilder::getViewProcessingNumThreads() const +{ + const auto & settings = init_context->getSettingsRef(); + if (settings[Setting::parallel_view_processing] || !isViewsInvolved()) + return static_cast(settings[Setting::max_threads]); + return 1; +} + + StorageIDMaybeEmpty InsertDependenciesBuilder::DependencyPath::parent(size_t inheritance) const { if (path.size() > inheritance) diff --git a/src/Interpreters/InsertDependenciesBuilder.h b/src/Interpreters/InsertDependenciesBuilder.h index a230bb8392ce..5b60f5b93301 100644 --- a/src/Interpreters/InsertDependenciesBuilder.h +++ b/src/Interpreters/InsertDependenciesBuilder.h @@ -120,6 +120,9 @@ class InsertDependenciesBuilder : public std::enable_shared_from_this #include #include +#include #include #include #include @@ -482,7 +483,14 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) validateReplicatedDatabaseSegments(segments, database); if (auto lightweight_result = tryRewriteToLightweightUpdate(segments, table, getContext(), query_ptr)) + { + /// The patch part is committed while the pipeline runs, so the share lock must outlive this + /// function: otherwise a concurrent DROP can clear the data parts index under the sink. + QueryPlanResourceHolder update_resources; + update_resources.table_locks.emplace_back(std::move(table_lock)); + lightweight_result->pipeline.addResources(std::move(update_resources)); return std::move(lightweight_result.value()); + } return runCommandSegments(segments, table, getContext()); } diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 81c275ae7269..e52f836b04d2 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -265,7 +265,8 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS else if (query.kind == ASTDropQuery::Kind::Drop) context_->checkAccess(drop_storage, table_id); - ddl_guard->releaseTableLock(); + if (ddl_guard) + ddl_guard->releaseTableLock(); table.reset(); query_to_send.if_empty = false; @@ -867,6 +868,8 @@ void InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind kind, ContextPtr drop_query->kind = kind; drop_query->sync = sync; drop_query->if_exists = true; + /// The DDLGuard for this exact name is already held above, and it is not recursive. + drop_query->no_ddl_lock = need_ddl_guard; ASTPtr ast_drop_query = drop_query; /// FIXME We have to use global context to execute DROP query for inner table /// to avoid "Not enough privileges" error if current user has only DROP VIEW ON mat_view_name privilege diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 0cbed9ba56b1..2322a4ba6824 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -68,6 +68,7 @@ namespace Setting extern const SettingsBool distributed_foreground_insert; extern const SettingsBool insert_null_as_default; extern const SettingsBool optimize_trivial_insert_select; + extern const SettingsBool parallel_view_processing; extern const SettingsDeduplicateInsertSelectMode deduplicate_insert_select; extern const SettingsMaxThreads max_threads; extern const SettingsUInt64 max_insert_threads; @@ -532,6 +533,9 @@ QueryPipeline InterpreterInsertQuery::addInsertToSelectPipeline(ASTInsertQuery & pipeline.addChains(std::move(sink_chains)); pipeline.setMaxThreads(max_threads); + // Cap to 1 when parallel_view_processing=0. Pipe::max_parallel_streams is a watermark that + // resize() does not lower, so limitMaxThreads is needed even after resize(sink_stream_size). + pipeline.limitMaxThreads(insert_dependencies->getViewProcessingNumThreads()); pipeline.setSinks([&](const SharedHeader & cur_header, QueryPipelineBuilder::StreamType) -> ProcessorPtr { @@ -834,7 +838,9 @@ QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query // Pipeline ceiling: simple upper bound on parallelism. Actual slot grants are // demand-driven by lazy ConcurrencyControl / CPULeaseAllocation, so a wide ceiling // does not translate into reserved-but-unused slots. - pipeline.setNumThreads(max_threads); + // max_threads is already memory-adjusted; use it for the parallel case to preserve that adjustment. + const bool serial_views = !settings[Setting::parallel_view_processing] && insert_dependencies->isViewsInvolved(); + pipeline.setNumThreads(serial_views ? 1 : max_threads); pipeline.setConcurrencyControl(settings[Setting::use_concurrency_control]); if (query.hasInlinedData() && !async_insert) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 5beabf325223..6800a277c17c 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -66,6 +66,20 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( ASTSelectWithUnionQuery * ast = query_ptr->as(); bool require_full_header = ast->hasNonDefaultUnionMode(); + /// INTERSECT/EXCEPT children always return their full header (they ignore + /// required_result_column_names), so the whole UNION must keep the full header too. + if (!require_full_header) + { + for (const auto & select : ast->list_of_selects->children) + { + if (select->as()) + { + require_full_header = true; + break; + } + } + } + const Settings & settings = context->getSettingsRef(); if (options.subquery_depth == 0 && (settings[Setting::limit] > 0 || settings[Setting::offset] > 0)) settings_limit_offset_needed = true; diff --git a/src/Interpreters/InterpreterSetQuery.cpp b/src/Interpreters/InterpreterSetQuery.cpp index 50e20b7ffa9c..3eda5a05449b 100644 --- a/src/Interpreters/InterpreterSetQuery.cpp +++ b/src/Interpreters/InterpreterSetQuery.cpp @@ -29,6 +29,8 @@ BlockIO InterpreterSetQuery::execute() { const auto & ast = query_ptr->as(); getContext()->checkSettingsConstraints(ast.changes, SettingSource::QUERY); + /// Checked before anything is applied, so that a violation leaves the whole statement without effect. + getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, SettingSource::QUERY); auto session_context = getContext()->getSessionContext(); session_context->applySettingsChanges(ast.changes); session_context->addQueryParameters(NameToNameMap{ast.query_parameters.begin(), ast.query_parameters.end()}); @@ -41,7 +43,10 @@ void InterpreterSetQuery::executeForCurrentContext(bool ignore_setting_constrain { const auto & ast = query_ptr->as(); if (!ignore_setting_constraints) + { getContext()->checkSettingsConstraints(ast.changes, SettingSource::QUERY); + getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, SettingSource::QUERY); + } getContext()->applySettingsChanges(ast.changes); getContext()->resetSettingsToDefaultValue(ast.default_settings); } diff --git a/src/Interpreters/InterpreterUpdateQuery.cpp b/src/Interpreters/InterpreterUpdateQuery.cpp index a690e2047aa5..473ef4877655 100644 --- a/src/Interpreters/InterpreterUpdateQuery.cpp +++ b/src/Interpreters/InterpreterUpdateQuery.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,13 @@ BlockIO InterpreterUpdateQuery::execute() BlockIO res; res.pipeline = table->updateLightweight(commands, getContext()); res.pipeline.addStorageHolder(table); + + /// The patch part is committed while the pipeline runs, so the share lock must outlive this + /// function: otherwise a concurrent DROP can clear the data parts index under the sink. + QueryPlanResourceHolder update_resources; + update_resources.table_locks.emplace_back(std::move(table_lock)); + res.pipeline.addResources(std::move(update_resources)); + return res; } diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index e28c8bfd5d03..53b47cbcebd8 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -99,6 +99,19 @@ static bool equals(const DataTypes & lhs, const DataTypes & rhs) } +SetPtr FutureSet::getOrderedSetIfAlreadyBuilt(const ContextPtr & context) +{ + /// Only `FutureSetFromSubquery` can be unbuilt at this point, and its `buildOrderedSetInplace` + /// returns straight away once `get` is non-null, so nothing is executed here. Its other early + /// exit - adopting the set of `external_table_set` - is deliberately not reproduced: on a set + /// that is not built yet, that branch is precisely what runs the `GLOBAL IN` subquery. + if (!get()) + return nullptr; + + return buildOrderedSetInplace(context); +} + + FutureSetFromStorage::FutureSetFromStorage(Hash hash_, ASTPtr ast_, SetPtr set_, std::optional storage_id_) : hash(hash_), ast(std::move(ast_)), storage_id(std::move(storage_id_)), set(std::move(set_)) {} SetPtr FutureSetFromStorage::get() const { return set; } diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index 6e06ef9517a2..dd3d6f346aad 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -60,6 +60,13 @@ class FutureSet /// If possible, return set with stored elements useful for PK analysis. virtual SetPtr buildOrderedSetInplace(const ContextPtr & context) = 0; + /// The same, but never runs the subquery that fills the set: returns null if it is not built yet. + /// Its only caller is `ConditionSelectivityEstimator`, which wants a single selectivity number; + /// every other caller consumes the elements to prune or read data and so is entitled to build. + /// A cost model that executes a subquery gives planning a side effect of unbounded cost, for a + /// result the plan may end up not needing at all, so any further consult-only caller belongs here. + SetPtr getOrderedSetIfAlreadyBuilt(const ContextPtr & context); + using Hash = CityHash_v1_0_2::uint128; virtual Hash getHash() const = 0; diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 9a88ae2a2cd7..9380e42767ce 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -529,10 +529,20 @@ QueryStatus::QueryStatus( void QueryStatus::releaseWorkloadResources() { - memory_reservation.reset(); + releaseMemoryReservation(); + releaseQuerySlot(); +} + +void QueryStatus::releaseQuerySlot() +{ query_slot.reset(); } +void QueryStatus::releaseMemoryReservation() +{ + memory_reservation.reset(); +} + QueryStatus::~QueryStatus() { #if !defined(NDEBUG) diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index dca4faeb6c1d..0eaaade534d5 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -295,6 +295,15 @@ class QueryStatus : public WithContext /// Manually release all acquired workload resources. void releaseWorkloadResources(); + + /// Release the query slot only. Safe to call while the query pipeline is still running: + /// pipeline threads do not access the query slot. + void releaseQuerySlot(); + + /// Release the memory reservation only. MUST NOT be called while the query pipeline is still + /// running: pipeline threads hold raw pointers to `MemoryReservation` (see `WorkloadResources` + /// in `PipelineExecutor`) and would race with its destruction. + void releaseMemoryReservation(); }; using QueryStatusPtr = std::shared_ptr; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 1ac5c8aa6050..142ef0477321 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -520,6 +520,8 @@ void ThreadStatus::initPerformanceCounters() performance_counters.resetCounters(); memory_tracker.resetCounters(); memory_tracker.setDescription("Thread"); + progress_in.reset(); + progress_out.reset(); // query_start_time.nanoseconds cannot be used here since RUsageCounters expect CLOCK_MONOTONIC *last_rusage = RUsageCounters::current(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index f77a34ca8e80..4e01b60465fa 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -116,6 +117,8 @@ namespace ProfileEvents extern const Event InsertQueryTimeMicroseconds; extern const Event OtherQueryTimeMicroseconds; extern const Event ASTFuzzerQueries; + extern const Event ASTFuzzerSkippedBackupRestore; + extern const Event ASTFuzzerSkippedReplicatedDDLInternal; } namespace CurrentMetrics @@ -2027,6 +2030,22 @@ static void executeASTFuzzerQueries(const ASTPtr & ast, const ContextMutablePtr if (!any_query && !isReadOnlyQuery(ast)) return; + /// Do not fuzz while an internal replicated-DDL execution is in flight on `context`. + /// DatabaseReplicatedDDLWorker re-executes a committed DDL entry whose serialized settings still + /// carry ast_fuzzer_runs, so the fuzzer would fire again on the entry's live, single-shot + /// ZooKeeperMetadataTransaction. A fuzzed follow-up DDL then either adds ops to the already-executed + /// txn (ZooKeeperMetadataTransaction::addOp throws "Cannot add ZooKeeper operation because query is + /// executed") or, because is_replicated_database_internal makes shouldReplicateQuery() route it to a + /// local commit, reaches DatabaseReplicated::commit* with no txn while the DDL worker is active and + /// trips the `!ddl_worker->isCurrentlyActive() || txn` assertion. Both are LOGICAL_ERRORs that abort + /// debug/sanitizer builds. The initiating client query is fuzzed normally; only this redundant + /// re-fuzz during log replay is skipped. + if (context->getClientInfo().is_replicated_database_internal || context->getZooKeeperMetadataTransaction()) + { + ProfileEvents::increment(ProfileEvents::ASTFuzzerSkippedReplicatedDDLInternal); + return; + } + size_t num_runs = static_cast(ast_fuzzer_runs_value); double fractional = ast_fuzzer_runs_value - static_cast(num_runs); if (fractional > 0) @@ -2076,6 +2095,19 @@ static void executeASTFuzzerQueries(const ASTPtr & ast, const ContextMutablePtr fuzzed_query_params = fuzzer->getLastQueryParameters(); } + /// Skip fuzzed `BACKUP` / `RESTORE` queries. An async `RESTORE`/`BACKUP` returns from + /// `executeQuery` immediately while `BackupsWorker` keeps the query context alive and its + /// background workers read it via `Context::createCopy` under the shared `Context::mutex`. + /// The per-iteration cleanup below would then mutate that escaped context without holding + /// the mutex, reintroducing the very `merge_tree_transaction` data race this code avoids. + /// Checked first (before the depth/format/length guards below), and counted, so the skip + /// is attributable to the query type alone regardless of those other early-continue paths. + if (fuzzed_ast->as()) + { + ProfileEvents::increment(ProfileEvents::ASTFuzzerSkippedBackupRestore); + continue; + } + /// Skip deeply nested ASTs to avoid stack overflow during formatting or execution. try { @@ -2109,10 +2141,6 @@ static void executeASTFuzzerQueries(const ASTPtr & ast, const ContextMutablePtr ProfileEvents::increment(ProfileEvents::ASTFuzzerQueries); LOG_TRACE(logger, "Fuzzed query: {}", fuzzed_query); - /// Reset the transaction (if any), it is stored in session and local context (see InterpreterTransactionControlQuery::executeBegin()) - context->getQueryContext()->getSessionContext()->setCurrentTransaction(NO_TRANSACTION_PTR); - context->setCurrentTransaction(NO_TRANSACTION_PTR); - /// Declare contexts outside try block so we can reset transactions on all paths. /// MergeTreeTransactionHolder destructor calls rollbackTransaction (noexcept), /// which uses getCurrentExceptionCode with bare `throw;` - that only works @@ -2132,6 +2160,16 @@ static void executeASTFuzzerQueries(const ASTPtr & ast, const ContextMutablePtr { fuzz_session_context = Context::createCopy(context); fuzz_session_context->makeSessionContext(); + /// Reset the transaction (if any) on the fuzz session context to isolate + /// fuzzed queries from the caller's transaction state. The transaction pointer + /// was copied as a shared_ptr in the copy constructor (see `InterpreterTransactionControlQuery::executeBegin` + /// which stores it in both session and query contexts). + /// We clear it on the copy, not on the parent `context`: mutating the caller's + /// `merge_tree_transaction` races with concurrent readers of the same `Context` + /// (e.g. `RESTORE ASYNC` background workers calling `Context::createCopy` under + /// the shared `Context::mutex`), and it also has the surprising side effect of + /// silently clearing the user's active transaction on the caller session. + fuzz_session_context->setCurrentTransaction(NO_TRANSACTION_PTR); fuzz_context = Context::createCopy(fuzz_session_context); fuzz_context->makeQueryContext(); @@ -2557,10 +2595,6 @@ void executeQuery( /// 2. When handling HTTP requests, in `HTTPHandler::processQuery`, there is `query_finish_callback` which is invoked before `onFinish`. /// It releases the session and finalizes the output. The client might use the same session to query other queries. Hence, the transaction must be committed before `query_finish_callback`. /// Refer: https://github.com/ClickHouse/ClickHouse/issues/80428 - /// - /// It must also be committed before the AST fuzzer runs: the fuzzer resets the transaction stored - /// in the session and query contexts (see executeASTFuzzerQueries), which would otherwise leave the - /// executor's running flag set while `context->getCurrentTransaction()` is already gone. if (implicit_tcl_executor->transactionRunning()) implicit_tcl_executor->commit(context); @@ -2598,8 +2632,10 @@ void executeQuery( throw; } - /// We release query slot here to make sure client can safely reuse the slot with his next query, otherwise it will be released too late by BlockIO. - context->releaseWorkloadResources(); + /// We release the query slot here to make sure the client can safely reuse the slot with his next query, otherwise it will be released too late by BlockIO. + /// Only the query slot is released here, not the memory reservation: pipeline threads still hold raw pointers to it, + /// so it is released later by `streams.onFinish()` after the pipeline has been finalized. + context->releaseQuerySlot(); /// The order is important here: /// - first we save the finish_time that will be used for the entry in query_log/opentelemetry_span_log.finish_time_us diff --git a/src/Interpreters/tests/gtest_actions_dag_partial_result_type_drift.cpp b/src/Interpreters/tests/gtest_actions_dag_partial_result_type_drift.cpp new file mode 100644 index 000000000000..7b1f562e33fe --- /dev/null +++ b/src/Interpreters/tests/gtest_actions_dag_partial_result_type_drift.cpp @@ -0,0 +1,359 @@ +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int NOT_FOUND_COLUMN_IN_BLOCK; +} + +namespace +{ + +/// `ActionsDAG::updateHeader` matches header columns to DAG input nodes BY NAME ONLY +/// (see `matchInputPositionsToHeader`), so a header column whose type differs from the type +/// the DAG's function nodes were resolved for is bound to those nodes without any type check. +/// Historically such a mismatch was produced by a concurrent `EXCHANGE TABLES` swapping the +/// underlying table between analysis and header computation (that root cause was fixed +/// separately by restoring the per-query storage-cache pinning). These tests do not depend on +/// any particular source of the mismatch: they inject it directly, which makes the path +/// deterministic and free of any race. +ActionsDAG makeDagOverInput(const String & column_name, const DataTypePtr & resolved_type, const String & function_name) +{ + ActionsDAG dag; + const auto & input = dag.addInput(column_name, resolved_type); + + auto function = FunctionFactory::instance().get(function_name, getContext().context); + const auto & function_node = dag.addFunction(function, {&input}, function_name + "_result"); + + dag.getOutputs().clear(); + dag.getOutputs().push_back(&function_node); + return dag; +} + +Block headerWith(const String & column_name, const DataTypePtr & type) +{ + return Block{ColumnWithTypeAndName{type->createColumn(), type, column_name}}; +} + +} + +/// A strict function (arithmetic-like) resolved for one base type and then handed a header column +/// of another base type used to execute anyway during partial evaluation, tripping a +/// `LOGICAL_ERROR` inside the function body. In debug and sanitizer builds `LOGICAL_ERROR` is an +/// assertion failure (`handle_error_code` in `src/Common/Exception.cpp` calls +/// `abortOnFailedAssertion`), so this aborted the whole server process during query planning. +/// With the do-not-fold guard the drifted argument is simply not folded and header computation +/// completes, yielding the node's declared result type. +TEST(ActionsDAGPartialResultTypeDrift, BaseTypeDriftDoesNotAbortHeaderComputation) +{ + tryRegisterFunctions(); + + /// Resolved for `Float64` (as `03915_exchange_tables_race` does), header arrives as `Int256`. + auto dag = makeDagOverInput("n", std::make_shared(), "negate"); + const auto expected_result_type = dag.getOutputs().front()->result_type->getName(); + + Block drifted_header = headerWith("n", std::make_shared()); + + Block result; + ASSERT_NO_THROW(result = dag.updateHeader(drifted_header)); + + ASSERT_TRUE(result.has("negate_result")); + const auto & folded = result.getByName("negate_result"); + /// The declared result type is known from analysis and must be reported unchanged: the guard + /// skips only the fold, it must not alter the computed header. + EXPECT_EQ(folded.type->getName(), expected_result_type); + /// Pin the shape of the zero-row fallback as well, since the type alone does not distinguish it + /// from a null column or from a fabricated default constant. It must be a real but EMPTY and + /// NON-CONSTANT column: a constant would present a definitive folded value to header consumers, + /// which is exactly what skipping the fold is meant to avoid. + ASSERT_NE(folded.column, nullptr); + EXPECT_EQ(folded.column->size(), 0u); + EXPECT_FALSE(isColumnConst(*folded.column)); +} + +/// Wrapper-only drift (`String` -> `Nullable(String)`). This is the case a wrapper-stripped type +/// comparison would miss, and it has a distinct failure mode: `materialize` returns its argument +/// type, so on drift it produced a column whose type differed from the node's resolved result +/// type and tripped the `columnMatchesType` check ("Unexpected return type from materialize"), +/// which is again a `LOGICAL_ERROR` and therefore an abort in debug/sanitizer builds. +TEST(ActionsDAGPartialResultTypeDrift, WrapperOnlyDriftDoesNotAbortHeaderComputation) +{ + tryRegisterFunctions(); + + auto dag = makeDagOverInput("s", std::make_shared(), "materialize"); + const auto expected_result_type = dag.getOutputs().front()->result_type->getName(); + + Block drifted_header = headerWith("s", std::make_shared(std::make_shared())); + + Block result; + ASSERT_NO_THROW(result = dag.updateHeader(drifted_header)); + + ASSERT_TRUE(result.has("materialize_result")); + EXPECT_EQ(result.getByName("materialize_result").type->getName(), expected_result_type); +} + +/// An alias between the input and the function must not hide the drift. An alias never changes a +/// value, so its declared `result_type` is a copy of the type its argument was resolved for; if the +/// partial evaluator copies a drifted column while keeping that stale declared type, the +/// argument-type check sees matching declared types and the function is executed on a mismatched +/// column anyway - the same `LOGICAL_ERROR` abort, just one node further away. +TEST(ActionsDAGPartialResultTypeDrift, DriftBehindAliasDoesNotAbortHeaderComputation) +{ + tryRegisterFunctions(); + + ActionsDAG dag; + const auto & input = dag.addInput("n", std::make_shared()); + const auto & alias = dag.addAlias(input, "n_alias"); + auto function = FunctionFactory::instance().get("negate", getContext().context); + const auto & function_node = dag.addFunction(function, {&alias}, "negate_result"); + dag.getOutputs().clear(); + dag.getOutputs().push_back(&function_node); + + const auto expected_result_type = function_node.result_type->getName(); + + Block drifted_header = headerWith("n", std::make_shared()); + + Block result; + ASSERT_NO_THROW(result = dag.updateHeader(drifted_header)); + + ASSERT_TRUE(result.has("negate_result")); + EXPECT_EQ(result.getByName("negate_result").type->getName(), expected_result_type); +} + +/// `ARRAY_JOIN` must not hide the drift either. It extracts the nested column of its argument, so +/// keeping the declared nested type from analysis while extracting a drifted one would let a +/// downstream function be executed on a mismatched column, exactly as an alias would. +TEST(ActionsDAGPartialResultTypeDrift, DriftBehindArrayJoinDoesNotAbortHeaderComputation) +{ + tryRegisterFunctions(); + + ActionsDAG dag; + const auto & input = dag.addInput("a", std::make_shared(std::make_shared())); + const auto & array_join = dag.addArrayJoin(input, "a_array_join"); + auto function = FunctionFactory::instance().get("negate", getContext().context); + const auto & function_node = dag.addFunction(function, {&array_join}, "negate_result"); + dag.getOutputs().clear(); + dag.getOutputs().push_back(&function_node); + + const auto expected_result_type = function_node.result_type->getName(); + + auto drifted_type = std::make_shared(std::make_shared()); + Block drifted_header = headerWith("a", drifted_type); + + Block result; + ASSERT_NO_THROW(result = dag.updateHeader(drifted_header)); + + ASSERT_TRUE(result.has("negate_result")); + EXPECT_EQ(result.getByName("negate_result").type->getName(), expected_result_type); +} + +/// Drift must not produce a WRONG folded value either, which no type check can catch on its own: +/// `isNullable` derives its result from the argument type alone, so its result type stays `UInt8` +/// whether or not the argument drifted. Resolved for `String` it folds to 0; handed a +/// `Nullable(String)` column it would fold to 1 - a definitive but wrong value. +/// +/// This is also the `input_rows_count == 1` contract. `evaluatePartialResult` with one row is used +/// by the optimizer callers (JOIN rewrites, shard skipping, virtual-column path extraction), which +/// treat any non-null column as a definitive folded value, so on drift the column must stay null and +/// route them through their "unknown value" path rather than hand them a fabricated 1. +TEST(ActionsDAGPartialResultTypeDrift, WrapperOnlyDriftFoldsNoValueForOneRowCallers) +{ + tryRegisterFunctions(); + + auto dag = makeDagOverInput("s", std::make_shared(), "isNullable"); + + auto drifted_type = std::make_shared(std::make_shared()); + ActionsDAG::IntermediateExecutionResult node_to_column; + node_to_column[dag.getInputs().front()] + = ColumnWithTypeAndName{drifted_type->createColumnConstWithDefaultValue(1), drifted_type, "s"}; + + ColumnsWithTypeAndName result; + ASSERT_NO_THROW( + result = ActionsDAG::evaluatePartialResult(node_to_column, dag.getOutputs(), /* input_rows_count= */ 1, {})); + + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result.front().column, nullptr) + << "a drifted argument must not fold to a definitive value for the one-row callers"; +} + +/// The zero-row fallback builds an empty column of the node's declared result type, but one result +/// type cannot be instantiated: `DataTypeFunction` inherits `IDataTypeDummy::createColumn`, which +/// throws `NOT_IMPLEMENTED`. (`DataTypeNothing` and `DataTypeSet` derive from the same base but do +/// override `createColumn`, so they are built normally.) A captured lambda is an ordinary `FUNCTION` +/// node whose result type is `DataTypeFunction`, so if one of its captured arguments is bound to a +/// differently-typed header column, skipping the fold must not turn into that error - the column is +/// simply left null. +TEST(ActionsDAGPartialResultTypeDrift, DriftOnNonInstantiableResultTypeIsSkippedCleanly) +{ + tryRegisterFunctions(); + + /// The lambda body is `c`, i.e. it just returns its captured argument. + ActionsDAG lambda_dag; + const auto & captured = lambda_dag.addInput("c", std::make_shared()); + lambda_dag.getOutputs().clear(); + lambda_dag.getOutputs().push_back(&captured); + + auto capture = std::make_shared( + std::move(lambda_dag), + ExpressionActionsSettings(getContext().context), + Names{"c"}, + NamesAndTypesList{}, + std::make_shared(), + "c", + /* allow_constant_folding= */ true); + + ActionsDAG dag; + const auto & input = dag.addInput("c", std::make_shared()); + const auto & capture_node = dag.addFunction(capture, {&input}, "lambda_result"); + dag.getOutputs().clear(); + dag.getOutputs().push_back(&capture_node); + + ASSERT_TRUE(typeid_cast(capture_node.result_type.get())) + << "this case is only meaningful while a captured lambda's result type is DataTypeFunction"; + + Block drifted_header = headerWith("c", std::make_shared()); + + Block result; + ASSERT_NO_THROW(result = dag.updateHeader(drifted_header)); + ASSERT_TRUE(result.has("lambda_result")); + EXPECT_EQ(result.getByName("lambda_result").column, nullptr); +} + +/// A captured lambda consumed by a higher-order function is the case that matters in practice: the +/// capture is a CHILD of `arrayMap`, not a terminal output. Its column stays null (see above), so the +/// consuming function reports a recoverable `NOT_FOUND_COLUMN_IN_BLOCK`. That is the improvement being +/// pinned here: executing the stale capture instead raises +/// `Cannot capture column N because it has incompatible type`, a `LOGICAL_ERROR` that aborts the +/// server in debug and sanitizer builds. +TEST(ActionsDAGPartialResultTypeDrift, DriftInCapturedLambdaUnderArrayMapIsRecoverable) +{ + tryRegisterFunctions(); + + /// Lambda `x -> x + c`, capturing `c`. + ActionsDAG lambda_dag; + const auto & captured = lambda_dag.addInput("c", std::make_shared()); + const auto & lambda_argument = lambda_dag.addInput("x", std::make_shared()); + auto plus = FunctionFactory::instance().get("plus", getContext().context); + const auto & body = lambda_dag.addFunction(plus, {&lambda_argument, &captured}, "body"); + lambda_dag.getOutputs().clear(); + lambda_dag.getOutputs().push_back(&body); + + auto capture = std::make_shared( + std::move(lambda_dag), + ExpressionActionsSettings(getContext().context), + Names{"c"}, + NamesAndTypesList{{"x", std::make_shared()}}, + std::make_shared(), + "body", + /* allow_constant_folding= */ true); + + auto array_type = std::make_shared(std::make_shared()); + + ActionsDAG dag; + const auto & array_input = dag.addInput("arr", array_type); + const auto & captured_input = dag.addInput("c", std::make_shared()); + const auto & capture_node = dag.addFunction(capture, {&captured_input}, "lambda"); + auto array_map = FunctionFactory::instance().get("arrayMap", getContext().context); + const auto & mapped = dag.addFunction(array_map, {&capture_node, &array_input}, "mapped"); + dag.getOutputs().clear(); + dag.getOutputs().push_back(&mapped); + + auto drifted_type = std::make_shared(); + Block drifted_header{ + ColumnWithTypeAndName{array_type->createColumn(), array_type, "arr"}, + ColumnWithTypeAndName{drifted_type->createColumn(), drifted_type, "c"}}; + + try + { + dag.updateHeader(drifted_header); + FAIL() << "a drifted capture cannot produce a usable header"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK); + EXPECT_EQ(e.message().find("Cannot capture column"), std::string::npos) + << "the stale capture must not be executed: " << e.message(); + } +} + +/// The one-row path needs the `ARRAY_JOIN` type carried too. There the branch leaves the column null +/// (arrayJoin changes the row count, so it is skipped for non-header evaluation) but the TYPE is still +/// handed to the parent. If that were the stale declared type, a wrapper-sensitive folder such as +/// `isNullable` would see no difference and fold a definitive `0` for the optimizer callers instead of +/// remaining unknown. +TEST(ActionsDAGPartialResultTypeDrift, DriftBehindArrayJoinFoldsNoValueForOneRowCallers) +{ + tryRegisterFunctions(); + + ActionsDAG dag; + const auto & input = dag.addInput("a", std::make_shared(std::make_shared())); + const auto & array_join = dag.addArrayJoin(input, "a_array_join"); + auto function = FunctionFactory::instance().get("isNullable", getContext().context); + const auto & function_node = dag.addFunction(function, {&array_join}, "isNullable_result"); + dag.getOutputs().clear(); + dag.getOutputs().push_back(&function_node); + + auto drifted_type = std::make_shared(std::make_shared(std::make_shared())); + ActionsDAG::IntermediateExecutionResult node_to_column; + node_to_column[dag.getInputs().front()] + = ColumnWithTypeAndName{drifted_type->createColumnConstWithDefaultValue(1), drifted_type, "a"}; + + ColumnsWithTypeAndName result; + ASSERT_NO_THROW( + result = ActionsDAG::evaluatePartialResult( + node_to_column, dag.getOutputs(), /* input_rows_count= */ 1, {.allow_unknown_function_arguments = true})); + + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result.front().column, nullptr) + << "a differently-typed array-join argument must not fold to a definitive value"; +} + +/// A header column whose type did NOT drift must still be constant-folded exactly as before, so the +/// guard cannot be satisfied by disabling folding altogether. `updateHeader` is on the planning path +/// of every query, so this pins that the fix is scoped to the drifted case. +/// +/// The oracle is the folded VALUE, not just the result type: a type-only assertion would also hold +/// if folding were disabled unconditionally, because the guard's fallback produces an empty column +/// of the same declared type. A constant argument makes the fold observable - it yields a +/// `ColumnConst` carrying the computed value, which the do-not-fold path never produces. +TEST(ActionsDAGPartialResultTypeDrift, MatchingTypeStillFoldsConstant) +{ + tryRegisterFunctions(); + + ActionsDAG dag; + const auto & constant = dag.addColumn( + ColumnConst::create(ColumnVector::create(1, 42), 1), + std::make_shared(), + "c"); + auto function = FunctionFactory::instance().get("negate", getContext().context); + const auto & function_node = dag.addFunction(function, {&constant}, "negate_result"); + dag.getOutputs().clear(); + dag.getOutputs().push_back(&function_node); + + Block result; + ASSERT_NO_THROW(result = dag.updateHeader(Block{})); + + ASSERT_TRUE(result.has("negate_result")); + const auto & folded = result.getByName("negate_result"); + ASSERT_NE(folded.column, nullptr) << "a non-drifted constant argument must still be folded"; + ASSERT_TRUE(isColumnConst(*folded.column)) << "the fold must produce a constant, not an empty column"; + EXPECT_EQ(folded.column->getInt(0), -42); +} diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp index 854543454833..982150c48c34 100644 --- a/src/Interpreters/tests/gtest_filecache.cpp +++ b/src/Interpreters/tests/gtest_filecache.cpp @@ -8,6 +8,8 @@ #include +#include +#include #include #include @@ -23,6 +25,7 @@ #include #include #include +#include #if CLICKHOUSE_CLOUD #include #endif @@ -43,7 +46,10 @@ #include #include #include +#include +#include #include +#include #include #include @@ -2713,3 +2719,384 @@ TEST_F(FileCacheTest, SLRUDowngradeMetric) ASSERT_EQ(events[ProfileEvents::FilesystemCacheDowngradedFileSegments].load(), downgraded_before + 1); ASSERT_EQ(events[ProfileEvents::FilesystemCacheEvictedFileSegments].load(), evicted_before); } + +namespace +{ + +/// Behaves like a remote reader (e.g. `ReadBufferFromS3`): supports right-bounded reads (so +/// `getRemoteReadBuffer` uses it as is, without wrapping) and reports the remote object's metadata. +class FakeRemoteReadBuffer : public BoundedReadBuffer +{ +public: + explicit FakeRemoteReadBuffer(std::unique_ptr impl_) : BoundedReadBuffer(std::move(impl_)) {} + + std::optional getRemoteFileMetadata() const override + { + return RemoteFileMetadata{.size = static_cast(fs::file_size(getFileName())), .last_modification_time = 0}; + } +}; + +/// The query scope required for reading through the cache (a query id and a query context bound to +/// the current thread). +struct TestQueryScope +{ + explicit TestQueryScope(const std::string & query_id = "query_id") + { + ServerUUID::setRandomForUnitTests(); + + Poco::XML::DOMParser dom_parser; + std::string xml(R"CONFIG( +)CONFIG"); + Poco::AutoPtr document = dom_parser.parseString(xml); + Poco::AutoPtr config = new Poco::Util::XMLConfiguration(document); + getMutableContext().context->setConfig(config); + + query_context = DB::Context::createCopy(getContext().context); + query_context->makeQueryContext(); + query_context->setCurrentQueryId(query_id); + chassert(&DB::CurrentThread::get() == &thread_status); + query_scope = DB::QueryScope::create(query_context); + } + + DB::ThreadStatus thread_status; + ContextMutablePtr query_context; + DB::QueryScope query_scope; +}; + +void writeSourceFile(const std::string & path, const std::string & data) +{ + WriteBufferFromFile wb(path, DBMS_DEFAULT_BUFFER_SIZE); + wb.write(data.data(), data.size()); + wb.next(); + wb.finalize(); +} + +std::string makeSourceData(size_t size) +{ + std::string data(size, 0); + for (size_t i = 0; i < size; ++i) + data[i] = 'a' + i % 26; + return data; +} + +} + +/// Concurrent readBigAt calls on AsynchronousBoundedReadBuffer over a cached buffer, with a +/// prefetch in flight: the callers must not race on consuming it. +TEST_F(FileCacheTest, CachedReadBufferConcurrentReadBigAtWithPrefetch) +{ + TestQueryScope query_scope; + + ReadSettings read_settings; + read_settings.enable_filesystem_cache = true; + read_settings.local_fs_settings.method = LocalFSReadMethod::pread; + + const std::string data = makeSourceData(300); + std::string file_path = fs::current_path() / "test_concurrent_read_big_at"; + writeSourceFile(file_path, data); + + auto read_buffer_creator = [&]() -> std::unique_ptr + { + return std::make_unique(createReadBufferFromFileBase(file_path, read_settings, std::nullopt, std::nullopt)); + }; + + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_file_segment_size] = 16; + settings[FileCacheSetting::max_size] = 1000; + settings[FileCacheSetting::max_elements] = 100; + settings[FileCacheSetting::boundary_alignment] = 1; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("concurrent_read_big_at", settings); + cache->initialize(); + + auto key = DB::FileCacheKey::fromPath(file_path); + + ThreadPoolRemoteFSReader remote_fs_reader(4, 0); + + constexpr size_t num_threads = 4; + constexpr size_t num_iterations = 100; + /// Smaller than the file, so the prefetch is usually still in flight when the reads run. + constexpr size_t buffer_size = 64; + + for (size_t iteration = 0; iteration < num_iterations; ++iteration) + { + auto cached_buffer = std::make_unique( + file_path, key, cache, FileCache::getCommonOrigin(), read_buffer_creator, + read_settings.filesystem_cache_settings, buffer_size, buffer_size, + "test", data.size(), false, false, std::nullopt, nullptr); + + AsynchronousBoundedReadBuffer read_buffer( + std::move(cached_buffer), remote_fs_reader, buffer_size, + /* min_bytes_for_seek */ 0, Priority{0}, /* page_cache_block_size */ 0, /* enable_prefetches_log */ false); + + read_buffer.prefetch(Priority{0}); + + std::atomic ready{0}; + std::array errors; + std::vector threads; + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t] + { + /// Barrier, to maximize the chance that the readBigAt calls overlap. + ready.fetch_add(1); + while (ready.load() < num_threads) + ; + + try + { + /// Threads read different, partially overlapping ranges. + const size_t offset = (t < 2) ? 10 * (t + 1) : 50 * t; + const size_t count = 150; + std::string buf(count, 0); + size_t total = 0; + while (total < count) + { + size_t read = read_buffer.readBigAt(buf.data() + total, count - total, offset + total, nullptr); + if (read == 0) + break; + total += read; + } + if (total != count) + errors[t] = fmt::format("short read: {} instead of {}", total, count); + else if (memcmp(buf.data(), data.data() + offset, count) != 0) + errors[t] = "read data does not match file contents"; + } + catch (...) + { + errors[t] = getCurrentExceptionMessage(true); + } + }); + } + + for (auto & thread : threads) + thread.join(); + + for (size_t t = 0; t < num_threads; ++t) + ASSERT_EQ(errors[t], "") << "thread " << t << ", iteration " << iteration; + } +} + +/// Concurrent readBigAt calls on a cached buffer constructed with unknown file size: they race +/// on the lazy initialization of file_size (tryGetFileSize), which must be synchronized. +TEST_F(FileCacheTest, CachedReadBufferConcurrentReadBigAtUnknownFileSize) +{ + TestQueryScope query_scope; + + ReadSettings read_settings; + read_settings.enable_filesystem_cache = true; + read_settings.local_fs_settings.method = LocalFSReadMethod::pread; + + const std::string data = makeSourceData(300); + std::string file_path = fs::current_path() / "test_concurrent_read_big_at_unknown_size"; + writeSourceFile(file_path, data); + + auto read_buffer_creator = [&]() -> std::unique_ptr + { + return std::make_unique(createReadBufferFromFileBase(file_path, read_settings, std::nullopt, std::nullopt)); + }; + + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_file_segment_size] = 16; + settings[FileCacheSetting::max_size] = 1000; + settings[FileCacheSetting::max_elements] = 100; + settings[FileCacheSetting::boundary_alignment] = 1; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("concurrent_read_big_at_unknown_size", settings); + cache->initialize(); + + auto key = DB::FileCacheKey::fromPath(file_path); + + constexpr size_t num_threads = 4; + constexpr size_t num_iterations = 100; + + for (size_t iteration = 0; iteration < num_iterations; ++iteration) + { + /// Zero size is treated as unknown: the first readBigAt calls resolve it lazily. + auto cached_buffer = std::make_shared( + file_path, key, cache, FileCache::getCommonOrigin(), read_buffer_creator, + read_settings.filesystem_cache_settings, DBMS_DEFAULT_BUFFER_SIZE, DBMS_DEFAULT_BUFFER_SIZE, + "test", /* file_size */ 0, false, false, std::nullopt, nullptr); + + std::atomic ready{0}; + std::array errors; + std::vector threads; + + for (size_t t = 0; t < num_threads; ++t) + { + threads.emplace_back([&, t] + { + /// Barrier, to maximize the chance that the readBigAt calls overlap. + ready.fetch_add(1); + while (ready.load() < num_threads) + ; + + try + { + const size_t offset = 50 * t; + const size_t count = 150; + std::string buf(count, 0); + size_t total = 0; + while (total < count) + { + size_t read = cached_buffer->readBigAt(buf.data() + total, count - total, offset + total, nullptr); + if (read == 0) + break; + total += read; + } + if (total != count) + errors[t] = fmt::format("short read: {} instead of {}", total, count); + else if (memcmp(buf.data(), data.data() + offset, count) != 0) + errors[t] = "read data does not match file contents"; + } + catch (...) + { + errors[t] = getCurrentExceptionMessage(true); + } + }); + } + + for (auto & thread : threads) + thread.join(); + + for (size_t t = 0; t < num_threads; ++t) + ASSERT_EQ(errors[t], "") << "thread " << t << ", iteration " << iteration; + } +} + +TEST_F(FileCacheTest, QueryLimitContextRevivedDuringRelease) +{ + /// Regression for STID 4192-71db: a holder for some query_id decides it is the last one and + /// releases its query context, but a concurrent holder for the same query_id revives the + /// context first. The release must then be a no-op: the revived context must survive (so the + /// per-query download limit keeps being enforced for the rest of the query) and a later release + /// of the revived context must not fail with "Attempt to release query context that does not exist". + + CachePriorityGuard cache_guard; + CacheStateGuard state_guard; + FileCacheQueryLimit query_limit; + + const std::string query_id = "query_id_revive"; + FilesystemCacheSettings cache_settings; + cache_settings.max_download_size_per_query = 1024; + + /// holder1 takes the context; query_map and holder1 both reference it (use_count == 2). + auto context1 = query_limit.getOrSetQueryContext(query_id, cache_settings, cache_guard.writeLock()); + ASSERT_TRUE(context1 != nullptr); + ASSERT_EQ(context1.use_count(), 2); + + /// holder2 revives the same context before holder1 releases (getOrSetQueryContext returns the + /// existing entry). Now query_map, holder1 and holder2 all reference it (use_count == 3). + auto context2 = query_limit.getOrSetQueryContext(query_id, cache_settings, cache_guard.writeLock()); + ASSERT_EQ(context1.get(), context2.get()); + ASSERT_EQ(context1.use_count(), 3); + + /// holder1 releases. The map still maps query_id to the live context and another holder is + /// alive, so the entry must be kept (no erase, no throw) and nothing is handed back for + /// destruction. + FileCacheQueryLimit::QueryContextPtr doomed1; + ASSERT_NO_THROW(doomed1 = query_limit.removeQueryContext(query_id, context1, cache_guard.writeLock())); + ASSERT_EQ(doomed1, nullptr); + context1.reset(); + + /// Enforcement is preserved: the revived context is still discoverable. + { + DB::ThreadStatus thread_status; + auto query_context = DB::Context::createCopy(getContext().context); + query_context->makeQueryContext(); + query_context->setCurrentQueryId(query_id); + auto query_scope_holder = DB::QueryScope::create(query_context); + + auto found = query_limit.tryGetQueryContext(state_guard.lock()); + ASSERT_EQ(found.get(), context2.get()); + } + + /// holder2 is now the last holder; releasing it actually removes the entry, once, and hands the + /// orphaned context back so it is destroyed by the caller outside the cache lock. + const auto * context2_raw = context2.get(); + FileCacheQueryLimit::QueryContextPtr doomed2; + ASSERT_NO_THROW(doomed2 = query_limit.removeQueryContext(query_id, context2, cache_guard.writeLock())); + ASSERT_EQ(doomed2.get(), context2_raw); + ASSERT_EQ(doomed2.use_count(), 1); + context2.reset(); + + /// After full release the context is gone. + { + DB::ThreadStatus thread_status; + auto query_context = DB::Context::createCopy(getContext().context); + query_context->makeQueryContext(); + query_context->setCurrentQueryId(query_id); + auto query_scope_holder = DB::QueryScope::create(query_context); + + auto found = query_limit.tryGetQueryContext(state_guard.lock()); + ASSERT_EQ(found.get(), nullptr); + } +} + +TEST_F(FileCacheTest, QueryLimitConcurrentReleaseNoLeak) +{ + /// Regression for #109508: two holders for the same query_id release "at the same time". + /// A query with parallel read streams has several holders (each CachedOnDiskReadBufferFromFile + /// creates its own), so use_count is > 2. If the last-holder decision reads use_count before this + /// holder drops its own reference (or drops it outside the lock), both releasers observe the shared + /// count, both skip the erase, and after both drop their reference only the map entry remains and is + /// never removed. That orphans query_map[query_id] for the lifetime of the cache and lets a later + /// query reusing the same query_id pick up stale per-query limit state. The fix drops each holder's + /// reference under the lock and erases once the map entry is the sole owner. + + CachePriorityGuard cache_guard; + CacheStateGuard state_guard; + FileCacheQueryLimit query_limit; + + const std::string query_id = "query_id_concurrent_release"; + FilesystemCacheSettings cache_settings; + cache_settings.max_download_size_per_query = 1024; + + /// Two holders take the same context; query_map + both holders reference it (use_count == 3). + auto context1 = query_limit.getOrSetQueryContext(query_id, cache_settings, cache_guard.writeLock()); + auto context2 = query_limit.getOrSetQueryContext(query_id, cache_settings, cache_guard.writeLock()); + ASSERT_EQ(context1.get(), context2.get()); + ASSERT_EQ(context1.use_count(), 3); + + /// Keep a raw pointer to assert which release actually surrenders the context for destruction. + const auto * context_raw = context1.get(); + + /// Both holders decide to release while both are still alive (the interleaving that leaks): each + /// removeQueryContext drops that holder's reference under the lock. The first keeps the entry (one + /// holder still alive) and returns nullptr; the second erases it and returns the now-orphaned + /// context so the caller destroys it after the cache lock is released. Neither throws. + FileCacheQueryLimit::QueryContextPtr doomed1; + FileCacheQueryLimit::QueryContextPtr doomed2; + ASSERT_NO_THROW(doomed1 = query_limit.removeQueryContext(query_id, context1, cache_guard.writeLock())); + ASSERT_NO_THROW(doomed2 = query_limit.removeQueryContext(query_id, context2, cache_guard.writeLock())); + + /// removeQueryContext resets each passed reference, so both are already null here. + ASSERT_EQ(context1, nullptr); + ASSERT_EQ(context2, nullptr); + + /// Only the last release hands the context back for out-of-lock destruction; the earlier one + /// returns nullptr because another holder was still alive. + ASSERT_EQ(doomed1, nullptr); + ASSERT_EQ(doomed2.get(), context_raw); + ASSERT_EQ(doomed2.use_count(), 1); + + /// The entry must be gone: with the pre-fix logic both releases skipped the erase and the entry + /// leaked, so tryGetQueryContext would still find it. + { + DB::ThreadStatus thread_status; + auto query_context = DB::Context::createCopy(getContext().context); + query_context->makeQueryContext(); + query_context->setCurrentQueryId(query_id); + auto query_scope_holder = DB::QueryScope::create(query_context); + + auto found = query_limit.tryGetQueryContext(state_guard.lock()); + ASSERT_EQ(found.get(), nullptr); + } +} diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp index 066b74111891..13e8060c8f30 100644 --- a/src/Parsers/ASTAlterQuery.cpp +++ b/src/Parsers/ASTAlterQuery.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -553,7 +554,9 @@ void ASTAlterCommand::formatImpl(WriteBuffer & ostr, const FormatSettings & sett else if (type == ASTAlterCommand::MODIFY_DATABASE_SETTING) { ostr << "MODIFY SETTING "; - settings_changes->format(ostr, settings, state, frame); + auto modified_frame{frame}; + modified_frame.create_engine_name = DataLake::DATABASE_ENGINE_NAME; + settings_changes->format(ostr, settings, state, modified_frame); } else if (type == ASTAlterCommand::MODIFY_QUERY) { diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index bfa96fd9d8c8..e0dd429d1a8f 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1388,7 +1388,9 @@ inline static bool makeHexOrBinStringLiteral(IParser::Pos & pos, ASTPtr & node, binStringDecode(str_begin, str_end, res_pos, word_size); } - return makeStringLiteral(pos, node, String(reinterpret_cast(res.data()), res.size()), expected); + /// The buffer is sized for the worst case; a binary literal whose length is not a multiple of + /// eight can write fewer bytes than that, and the unwritten tail is uninitialized memory. + return makeStringLiteral(pos, node, String(res_begin, res_pos - res_begin), expected); } bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index dbc35334d0bd..9849e16ff8d8 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -200,7 +200,8 @@ String IAST::formatWithPossiblyHidingSensitiveData( bool show_secrets, bool print_pretty_type_names, IdentifierQuotingRule identifier_quoting_rule, - IdentifierQuotingStyle identifier_quoting_style) const + IdentifierQuotingStyle identifier_quoting_style, + bool ignore_redundant_parentheses) const { WriteBufferFromOwnString buf; FormatSettings settings(one_line); @@ -208,6 +209,7 @@ String IAST::formatWithPossiblyHidingSensitiveData( settings.print_pretty_type_names = print_pretty_type_names; settings.identifier_quoting_rule = identifier_quoting_rule; settings.identifier_quoting_style = identifier_quoting_style; + settings.ignore_redundant_parentheses = ignore_redundant_parentheses; format(buf, settings); return wipeSensitiveDataAndCutToLength(buf.str(), max_length, !show_secrets); } @@ -256,6 +258,18 @@ String IAST::formatWithSecretsMultiLine() const /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); } +String IAST::formatIgnoringRedundantParentheses() const +{ + return formatWithPossiblyHidingSensitiveData( + /*max_length=*/0, + /*one_line=*/true, + /*show_secrets=*/true, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks, + /*ignore_redundant_parentheses=*/true); +} + bool IAST::childrenHaveSecretParts() const { checkStackSize(); @@ -380,9 +394,9 @@ std::string IAST::dumpTree(size_t indent) const /// In operator-chain context (`frame.need_parens == true`) we keep the parens here so the output /// is `(expr AS alias)`, because the parser would not accept `(expr) AS alias OP rhs` at the top /// level of a SELECT element / WHERE clause (the alias terminates the SELECT element parser). -static bool decideParensEmission(const IAST & node, IAST::FormatStateStacked & frame) +static bool decideParensEmission(const IAST & node, const IAST::FormatSettings & settings, IAST::FormatStateStacked & frame) { - const bool parens = node.isParenthesized() && !frame.wrapped_in_parens; + const bool parens = node.isParenthesized() && !frame.wrapped_in_parens && !settings.ignore_redundant_parentheses; frame.wrapped_in_parens = false; if (!parens) return false; @@ -471,7 +485,7 @@ void IAST::format(WriteBuffer & ostr, const FormatSettings & settings) const { FormatState state; FormatStateStacked frame; - const bool parens = decideParensEmission(*this, frame); + const bool parens = decideParensEmission(*this, settings, frame); if (parens) ostr.write('('); formatImpl(ostr, settings, state, std::move(frame)); @@ -482,7 +496,7 @@ void IAST::format(WriteBuffer & ostr, const FormatSettings & settings) const void IAST::format(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { checkStackSize(); - const bool parens = decideParensEmission(*this, frame); + const bool parens = decideParensEmission(*this, settings, frame); if (parens) ostr.write('('); formatImpl(ostr, settings, state, std::move(frame)); @@ -493,7 +507,7 @@ void IAST::format(WriteBuffer & ostr, const FormatSettings & settings, FormatSta void IAST::format(FormattingBuffer out) const { checkStackSize(); - const bool parens = decideParensEmission(*this, out.frame); + const bool parens = decideParensEmission(*this, out.settings, out.frame); if (parens) out.ostr.write('('); formatImpl(out.ostr, out.settings, out.state, out.frame); diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index e4698ccd7213..fb51cefbca01 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -335,6 +335,10 @@ class IAST : public TypePromotion bool enforce_strict_identifier_format; /// This is needed for distributed queries with the old analyzer. Remove it after removing the old analyzer. bool collapse_identical_nodes_to_aliases; + /// Do not print the redundant parentheses that the user has written around an expression + /// (the `parenthesized` flag), so that `(a)` and `a` produce the same text. Used to store + /// and to compare table definition expressions - see `formatIgnoringRedundantParentheses`. + bool ignore_redundant_parentheses = false; explicit FormatSettings( bool one_line_, @@ -413,7 +417,8 @@ class IAST : public TypePromotion bool show_secrets, bool print_pretty_type_names, IdentifierQuotingRule identifier_quoting_rule, - IdentifierQuotingStyle identifier_quoting_style) const; + IdentifierQuotingStyle identifier_quoting_style, + bool ignore_redundant_parentheses = false) const; /** formatForLogging and formatForErrorMessage always hide secrets. This inconsistent * behaviour is due to the fact such functions are called from Client which knows nothing about @@ -425,6 +430,17 @@ class IAST : public TypePromotion String formatWithSecretsOneLine() const; String formatWithSecretsMultiLine() const; + /** Same as `formatWithSecretsOneLine`, but the redundant parentheses that the user has written + * around an expression are not printed, so `(a)` and `a` give the same text. + * + * Use it for the definition expressions of a table (keys, `TTL`, indices, projections, + * constraints, column defaults) that are stored in ZooKeeper or in a part, and for comparing + * two such definitions: whether the parentheses were written is not a property of the table, + * and the servers that did not remember them (before the parentheses became a part of the AST) + * stored the same text this method produces. + */ + String formatIgnoringRedundantParentheses() const; + virtual bool hasSecretParts() const { return childrenHaveSecretParts(); } void cloneChildren(); diff --git a/src/Parsers/ParserPreparedStatement.cpp b/src/Parsers/ParserPreparedStatement.cpp index 2f3a15aea63f..5ac769702913 100644 --- a/src/Parsers/ParserPreparedStatement.cpp +++ b/src/Parsers/ParserPreparedStatement.cpp @@ -90,7 +90,14 @@ bool ParserExecute::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) for (size_t i = 0; i < ast_args->children.size(); ++i) { - result->arguments.push_back(fieldToString(ast_args->children[i]->as()->value)); + /// The expression list parser accepts arbitrary expressions, but only literals are valid here. + const auto * literal = ast_args->children[i]->as(); + if (!literal) + { + expected.add(pos, "literal"); + return false; + } + result->arguments.push_back(fieldToString(literal->value)); } if (!close_bracket.ignore(pos, expected)) return false; diff --git a/src/Parsers/ParserStreamSettings.cpp b/src/Parsers/ParserStreamSettings.cpp index 2206f51a8e84..a5f829301e47 100644 --- a/src/Parsers/ParserStreamSettings.cpp +++ b/src/Parsers/ParserStreamSettings.cpp @@ -48,7 +48,12 @@ bool parseCursorObject(IParser::Pos & pos, Expected & expected, Map & flat, cons /// Peek at next token: `{` starts a nested object, integer is a leaf. if (pos->type == TokenType::OpeningCurlyBrace) { - if (!parseCursorObject(pos, expected, flat, new_path)) + /// This helper recurses directly instead of going through IParserBase::parse, + /// so the depth has to be accounted for here to keep `max_parser_depth` in effect. + pos.increaseDepth(); + const bool parsed = parseCursorObject(pos, expected, flat, new_path); + pos.decreaseDepth(); + if (!parsed) return false; } else diff --git a/src/Planner/ActionsChain.cpp b/src/Planner/ActionsChain.cpp index 1b4fc35663a8..7031ac3b792c 100644 --- a/src/Planner/ActionsChain.cpp +++ b/src/Planner/ActionsChain.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -12,6 +13,58 @@ namespace DB { +namespace +{ + +/// Compares two constant values exactly like `Field::operator==`, except the aggregate-state +/// leaves, which are compared by both the aggregate function name and the serialized state +/// instead of throwing when the names differ. The plain `Field` comparison of aggregate states +/// throws for different function names even when the types compare equal (they may be compatible +/// only by state representation, e.g. `quantileState` vs `quantilesState(0.9)`). Such states must +/// compare as different here: this comparison drives the replacement of a COLUMN output with the +/// same-name INPUT, and substituting a state of a different aggregate function would change the +/// downstream behavior (e.g. the result of `finalizeAggregation`), unlike in the block structure +/// checks where a relaxed comparison is a pure validation. +bool sameConstantValue(const Field & lhs, const Field & rhs); + +bool sameConstantValueVectors(const FieldVector & lhs, const FieldVector & rhs) +{ + if (lhs.size() != rhs.size()) + return false; + + for (size_t i = 0; i < lhs.size(); ++i) + if (!sameConstantValue(lhs[i], rhs[i])) + return false; + + return true; +} + +bool sameConstantValue(const Field & lhs, const Field & rhs) +{ + if (lhs.getType() != rhs.getType()) + return false; + + switch (lhs.getType()) + { + case Field::Types::AggregateFunctionState: + { + const auto & lhs_state = lhs.safeGet(); + const auto & rhs_state = rhs.safeGet(); + return lhs_state.name == rhs_state.name && lhs_state.data == rhs_state.data; + } + case Field::Types::Array: + return sameConstantValueVectors(lhs.safeGet(), rhs.safeGet()); + case Field::Types::Tuple: + return sameConstantValueVectors(lhs.safeGet(), rhs.safeGet()); + case Field::Types::Map: + return sameConstantValueVectors(lhs.safeGet(), rhs.safeGet()); + default: + return lhs == rhs; + } +} + +} + ActionsChainStep::ActionsChainStep(ActionsAndProjectInputsFlagPtr actions_, bool use_actions_nodes_as_output_columns_, ColumnsWithTypeAndName additional_output_columns_) @@ -86,8 +139,9 @@ void ActionsChainStep::finalizeInputAndOutputColumns(const NameSet & child_input && input_node->result_type && output_node->result_type && input_node->result_type->equals(*output_node->result_type) - && assert_cast(*input_node->column).getField() - == assert_cast(*output_node->column).getField(); + && sameConstantValue( + assert_cast(*input_node->column).getField(), + assert_cast(*output_node->column).getField()); if (same_const_value) output_node = input_node; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index dd4ef0a462a1..636360c00921 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -378,6 +378,12 @@ bool applyTrivialCountIfPossible( if (!count_func) return false; + /// `arrayJoin` in the argument multiplies rows above the source read, so the aggregate does not + /// observe `totalRows()` rows. Must precede `optimize_trivial_count`: storages that count in + /// read() act on that flag even when this function later declines. + if (hasFunctionNode(aggregates.front(), "arrayJoin")) + return false; + /// Some storages can optimize trivial count in read() method instead of totalRows() because it still can /// require reading some data (but much faster than reading columns). /// Set a special flag in query info so the storage will see it and optimize count in read() method. @@ -1300,7 +1306,17 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (query_plan.isInitialized() && !select_query_options.build_logical_plan && parallelReplicasEnabledForStorage(storage, query_context, settings)) { - if (query_context->canUseParallelReplicasCustomKey() && query_context->getClientInfo().distributed_depth == 0) + /// The custom-key read below replaces the plan with a remote read at the fixed stage + /// `WithMergeableStateAfterAggregationAndLimit`, so it is only allowed when the requested + /// stage is not below that: a plan built up to a partial stage - e.g. a `Merge` table plans + /// its children up to `WithMergeableState` when one of the underlying tables is read through + /// an interpreter - must not receive finalized (post-aggregation, post-LIMIT) data instead + /// of the partial aggregation states its consumer expects. + const bool to_stage_supports_custom_key = select_query_options.to_stage == QueryProcessingStage::Complete + || select_query_options.to_stage == QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit; + + if (query_context->canUseParallelReplicasCustomKey() && to_stage_supports_custom_key + && query_context->getClientInfo().distributed_depth == 0) { if (auto cluster = query_context->getClusterForParallelReplicas(); query_context->canUseParallelReplicasCustomKeyForCluster(*cluster)) diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 0cf3e2c5d161..9fd58d8d0bb5 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -1136,6 +1136,11 @@ static std::shared_ptr tryDirectJoin(const std::shared_ptrsecond)) + return {}; + auto right_table_expression_column_with_storage_column_name = right_table_expression_column; right_table_expression_column_with_storage_column_name.name = column_mapping_it->second; right_table_expression_header_with_storage_column_names.insert(right_table_expression_column_with_storage_column_name); diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index dda4e14ec2f4..eac9eb3e5e95 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -1412,7 +1412,14 @@ namespace DB || std::is_same_v>) { const auto & decimal_type = assert_cast(column_type.get()); - arrow_type = arrow::decimal(decimal_type->getPrecision(), decimal_type->getScale()); + const auto precision = decimal_type->getPrecision(); + /// Reproduces what the removed `arrow::decimal` did: `Decimal256` above precision 38, + /// `Decimal128` otherwise. `arrow::smallest_decimal` is deliberately not used here - + /// Arrow gained `Decimal32` and `Decimal64`, so it would narrow small decimals and + /// change the schema we write. + arrow_type = precision > 38 + ? arrow::decimal256(precision, decimal_type->getScale()) + : arrow::decimal128(precision, decimal_type->getScale()); return true; } diff --git a/src/Processors/Formats/Impl/Parquet/Prefetcher.cpp b/src/Processors/Formats/Impl/Parquet/Prefetcher.cpp index 1141cfe870a2..c673a467b727 100644 --- a/src/Processors/Formats/Impl/Parquet/Prefetcher.cpp +++ b/src/Processors/Formats/Impl/Parquet/Prefetcher.cpp @@ -36,9 +36,14 @@ void Prefetcher::init(ReadBuffer * reader_, const ReadOptions & options, FormatP range_sets.resize(1); } -Prefetcher::~Prefetcher() +void Prefetcher::shutdownTasks() { shutdown->shutdown(); +} + +Prefetcher::~Prefetcher() +{ + shutdownTasks(); /// Assert that all PrefetchHandle-s were destroyed. chassert(std::all_of(requests.begin(), requests.end(), [](const RequestState & req) @@ -212,7 +217,9 @@ std::vector Prefetcher::splitRange( size_t subrange_end = 0; for (const auto & [start, length] : subranges) { - if (start < range.start || length > range.end - start) + /// `start > range.end` has to be checked separately: `range.end - start` would + /// underflow and let a subrange past the end of the range through. + if (start < range.start || start > range.end || length > range.end - start) throw Exception(ErrorCodes::INCORRECT_DATA, "Subrange out of bounds: [{}, {}) not in [{}, {})", start, start + length, range.start, range.end); subrange_start = std::min(subrange_start, start); subrange_end = std::max(subrange_end, start + length); @@ -259,6 +266,15 @@ std::vector Prefetcher::splitRange( chassert(parent_req->state.load(std::memory_order_relaxed) == RequestState::State::HasTask); Task * task = parent_req->task; + + /// The parent range was already coalesced into a task, so the check above was skipped. Validate + /// against the task instead, before touching refcount or RequestState-s: `task_offset` below is + /// otherwise unchecked and getRangeData would hand out a span outside the task's buffer. + size_t task_end = task->offset + task->length; + for (const auto & [start, length] : subranges) + if (start < task->offset || start > task_end || length > task_end - start) + throw Exception(ErrorCodes::INCORRECT_DATA, "Subrange out of bounds: [{}, {}) not in read task [{}, {})", start, start + length, task->offset, task_end); + task->refcount.fetch_add(subranges.size()); for (size_t i = 0; i < subranges.size(); ++i) @@ -453,6 +469,12 @@ std::span Prefetcher::getRangeData(const PrefetchHandle & request) rethrowException(task); chassert(s == Task::State::Done); + /// Both `buf` and `cached_region` below cover at least [task->offset, task->offset + task->length). + /// Checked instead of asserted: handing out a span past the end would be a segfault in release builds. + if (req->task_offset > task->length || req->length > task->length - req->task_offset) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Prefetched range [{}, {}) is outside its read task [{}, {})", + task->offset + req->task_offset, task->offset + req->task_offset + req->length, task->offset, task->offset + task->length); + if (task->cached_region.has_value()) { /// Zero-copy path: serve data directly from cache cells. diff --git a/src/Processors/Formats/Impl/Parquet/Prefetcher.h b/src/Processors/Formats/Impl/Parquet/Prefetcher.h index 40796dd10342..135492aff741 100644 --- a/src/Processors/Formats/Impl/Parquet/Prefetcher.h +++ b/src/Processors/Formats/Impl/Parquet/Prefetcher.h @@ -29,6 +29,10 @@ class Prefetcher /// Waits for in-progress reads to complete, cancels queued reads that haven't started yet. ~Prefetcher(); + /// Same handshake as the destructor. After this returns, no background task reads through the + /// ReadBuffer passed to init() anymore, so that buffer may be destroyed. Idempotent. + void shutdownTasks(); + /// Not thread safe. /// All ranges must be registered before any reading happens (except direct readSync). /// Ranges are allowed to overlap a little, but this decreases the effectiveness of range diff --git a/src/Processors/Formats/Impl/Parquet/ReadManager.cpp b/src/Processors/Formats/Impl/Parquet/ReadManager.cpp index b6375ccb96c2..3040d1c1df67 100644 --- a/src/Processors/Formats/Impl/Parquet/ReadManager.cpp +++ b/src/Processors/Formats/Impl/Parquet/ReadManager.cpp @@ -127,9 +127,15 @@ void ReadManager::init(FormatParserSharedResourcesPtr parser_shared_resources_, flushMemoryUsageDiff(std::move(diff)); } -ReadManager::~ReadManager() +void ReadManager::shutdownTasks() { shutdown->shutdown(); + reader.prefetcher.shutdownTasks(); +} + +ReadManager::~ReadManager() +{ + shutdownTasks(); } void ReadManager::cancel() noexcept diff --git a/src/Processors/Formats/Impl/Parquet/ReadManager.h b/src/Processors/Formats/Impl/Parquet/ReadManager.h index 7073492d2174..3816d9bd348b 100644 --- a/src/Processors/Formats/Impl/Parquet/ReadManager.h +++ b/src/Processors/Formats/Impl/Parquet/ReadManager.h @@ -44,6 +44,11 @@ class ReadManager ~ReadManager(); + /// Same handshake as the destructor, but keeps `reader` and its metadata intact. After this + /// returns, no decode task runs anymore, so nothing can re-enter the prefetcher. Idempotent. + /// Drain this before the prefetcher: decode tasks read ranges through it. + void shutdownTasks(); + struct ReadResult { Chunk chunk; diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index a3c91158f695..64e25a301549 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -615,6 +615,7 @@ void Reader::initializePrefetches() { size_t len = size_t(column.meta->meta_data.bloom_filter_length); max_header_length = std::min(max_header_length, len); + column.bloom_filter_data_bytes = len; column.bloom_filter_data_prefetch = prefetcher.registerRange( size_t(column.meta->meta_data.bloom_filter_offset), len, /*likely_to_be_used=*/ false); @@ -700,6 +701,7 @@ void Reader::initializePrefetches() auto it = std::upper_bound(all_offsets.begin(), all_offsets.end(), offset); size_t end = it == all_offsets.end() ? prefetcher.getFileSize() : *it; + column.bloom_filter_data_bytes = end - offset; column.bloom_filter_data_prefetch = prefetcher.registerRange( offset, end - offset, /*likely_to_be_used=*/ false); } @@ -836,6 +838,13 @@ void Reader::processBloomFilterHeader(ColumnChunk & column, const PrimitiveColum const size_t bytes_per_block = 32; if (column.bloom_filter_header.numBytes <= 0 || column.bloom_filter_header.numBytes % bytes_per_block != 0) throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid bloom filter size."); + /// The bitset must fit in the bloom filter byte range the file declared, otherwise the block + /// subranges below would point outside the data we fetched. + if (header_size > column.bloom_filter_data_bytes || + size_t(column.bloom_filter_header.numBytes) > column.bloom_filter_data_bytes - header_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "Bloom filter bitset of {} bytes doesn't fit in {} bytes of bloom filter " + "data (including a {}-byte header) at offset {}. Use setting input_format_parquet_bloom_filter_push_down=0 to ignore.", + column.bloom_filter_header.numBytes, column.bloom_filter_data_bytes, header_size, column.meta->meta_data.bloom_filter_offset); size_t num_blocks = size_t(column.bloom_filter_header.numBytes) / bytes_per_block; const auto & hashes = column_info.bloom_filter_hashes; diff --git a/src/Processors/Formats/Impl/Parquet/Reader.h b/src/Processors/Formats/Impl/Parquet/Reader.h index 105cb07a3061..09d523fe327c 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.h +++ b/src/Processors/Formats/Impl/Parquet/Reader.h @@ -315,6 +315,10 @@ struct Reader /// TODO [parquet]: Check that all handles and tokens are reset after correct stages. PrefetchHandle bloom_filter_header_prefetch; PrefetchHandle bloom_filter_data_prefetch; + /// Length of bloom_filter_data_prefetch, i.e. how many bytes of bloom filter (header + + /// bitset) the file claims to have. Upper bound if the file didn't say (see + /// need_to_find_bloom_filter_lengths_the_hard_way). + size_t bloom_filter_data_bytes = 0; PrefetchHandle dictionary_page_prefetch; PrefetchHandle column_index_prefetch; PrefetchHandle offset_index_prefetch; diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 6a238834caec..8436e55647f9 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -235,6 +235,19 @@ void ParquetV3BlockInputFormat::onCancel() noexcept reader->cancel(); } +void ParquetV3BlockInputFormat::resetReadBuffer() +{ + { + /// Background tasks read through a non-owning pointer to the buffers the base class is + /// about to release, so they have to be stopped first. `reader` stays alive: + /// getMatchedBuckets() reads row group metadata after the source is exhausted. + std::lock_guard lock(reader_mutex); + if (reader) + reader->shutdownTasks(); + } + IInputFormat::resetReadBuffer(); +} + void ParquetV3BlockInputFormat::resetParser() { { diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h index 2c5baf00ea92..77f3eb2ecd70 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h @@ -50,6 +50,8 @@ class ParquetV3BlockInputFormat final : public IInputFormat void resetParser() override; + void resetReadBuffer() override; + String getName() const override { return "ParquetV3BlockInputFormat"; } const BlockMissingValues * getMissingValues() const override; diff --git a/src/Processors/Merges/Algorithms/MergeTreeReadInfo.cpp b/src/Processors/Merges/Algorithms/MergeTreeReadInfo.cpp index b9506fd82207..084b325a76a5 100644 --- a/src/Processors/Merges/Algorithms/MergeTreeReadInfo.cpp +++ b/src/Processors/Merges/Algorithms/MergeTreeReadInfo.cpp @@ -43,7 +43,7 @@ bool isVirtualRow(const Chunk & chunk) return false; } -void setVirtualRow(Chunk & chunk, const Block & header, bool apply_virtual_row_conversions) +Block setVirtualRow(Chunk & chunk, const Block & header, bool apply_virtual_row_conversions) { auto read_info = chunk.getChunkInfos().extract(); chassert(read_info); @@ -78,6 +78,8 @@ void setVirtualRow(Chunk & chunk, const Block & header, bool apply_virtual_row_c } chunk.setColumns(ordered_columns, 1); + + return pk_block; } } diff --git a/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h b/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h index 1b485e096d88..1a14f082ccd9 100644 --- a/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h +++ b/src/Processors/Merges/Algorithms/MergeTreeReadInfo.h @@ -29,6 +29,8 @@ size_t getPartLevelFromChunk(const Chunk & chunk); bool isVirtualRow(const Chunk & chunk); -void setVirtualRow(Chunk & chunk, const Block & header, bool apply_virtual_row_conversions); +/// Returns the block the virtual row was built from, i.e. the sort columns it can announce exactly. +/// Every other column of `header` is filled with a type default and bounds nothing. +Block setVirtualRow(Chunk & chunk, const Block & header, bool apply_virtual_row_conversions); } diff --git a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp index c0409b36d92d..9beabcb0e5f9 100644 --- a/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/MergingSortedAlgorithm.cpp @@ -6,6 +6,9 @@ #include #include #include +#include +#include + namespace DB { @@ -54,6 +57,30 @@ static void checkVirtualRowBoundary(const SortCursorImpl & cursor, Columns & vir virtual_row_boundary.clear(); } +/// A virtual row announces the boundary of its source's next output, +/// but only for the sort columns present in its pk block. +/// Comparing those would make the merge trust a fictional boundary, +/// so the first merge that sees a virtual row must +/// have a sort description the row covers completely. +static void checkVirtualRowCoversSortDescription(const Block & pk_block, const SortDescription & description) +{ + for (const auto & column_description : description) + { + if (!pk_block.has(column_description.column_name)) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Virtual row does not cover sort column '{}'." + "Virtual row columns: {}, sort description [{}]", + column_description.column_name, pk_block.dumpNames(), + fmt::join(description | std::views::transform([](const auto & d) { return d.column_name; }), ", ")); + } +} + +#ifndef NDEBUG +constexpr static bool do_debug_checks = true; +#else +constexpr static bool do_debug_checks = false; +#endif + MergingSortedAlgorithm::MergingSortedAlgorithm( SharedHeader header_, @@ -115,7 +142,9 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) if (!isVirtualRow(input.chunk)) continue; - setVirtualRow(input.chunk, *header, apply_virtual_row_conversions); + auto pk_block = setVirtualRow(input.chunk, *header, apply_virtual_row_conversions); + if constexpr (do_debug_checks) + checkVirtualRowCoversSortDescription(pk_block, description); input.skip_last_row = true; } @@ -134,13 +163,14 @@ void MergingSortedAlgorithm::initialize(Inputs inputs) cursors[source_num] = SortCursorImpl(*header, chunk.getColumns(), chunk.getNumRows(), description, source_num); } -#ifndef NDEBUG - for (size_t source_num = 0; source_num < current_inputs.size(); ++source_num) + if constexpr (do_debug_checks) { - if (current_inputs[source_num].skip_last_row && !has_collation) - rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); + for (size_t source_num = 0; source_num < current_inputs.size(); ++source_num) + { + if (current_inputs[source_num].skip_last_row && !has_collation) + rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); + } } -#endif if (sorting_queue_strategy == SortingQueueStrategy::Default) { @@ -165,7 +195,9 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) bool is_virtual_row = isVirtualRow(input.chunk); if (is_virtual_row) { - setVirtualRow(input.chunk, *header, apply_virtual_row_conversions); + auto pk_block = setVirtualRow(input.chunk, *header, apply_virtual_row_conversions); + if constexpr (do_debug_checks) + checkVirtualRowCoversSortDescription(pk_block, description); input.skip_last_row = true; } @@ -174,16 +206,14 @@ void MergingSortedAlgorithm::consume(Input & input, size_t source_num) current_inputs[source_num].swap(input); cursors[source_num].reset(current_inputs[source_num].chunk.getColumns(), *header, current_inputs[source_num].chunk.getNumRows()); -#ifndef NDEBUG - /// See `initialize` for why we gate on `apply_virtual_row_conversions`. - if (is_virtual_row && !has_collation) - rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); - else - checkVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num], description, source_num); -#else - UNUSED(rememberVirtualRowBoundary); - UNUSED(checkVirtualRowBoundary); -#endif + if constexpr (do_debug_checks) + { + /// See `initialize` for why we gate on `apply_virtual_row_conversions`. + if (is_virtual_row && !has_collation) + rememberVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num]); + else + checkVirtualRowBoundary(cursors[source_num], virtual_row_boundary[source_num], description, source_num); + } if (sorting_queue_strategy == SortingQueueStrategy::Default) { diff --git a/src/Processors/QueryPlan/IntersectOrExceptStep.cpp b/src/Processors/QueryPlan/IntersectOrExceptStep.cpp index 73dd4b64ff1d..c3bd96048624 100644 --- a/src/Processors/QueryPlan/IntersectOrExceptStep.cpp +++ b/src/Processors/QueryPlan/IntersectOrExceptStep.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -21,6 +22,16 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +static bool containsAggregateStateColumn(const IColumn & column) +{ + if (typeid_cast(&column)) + return true; + + bool found = false; + column.forEachSubcolumn([&](const auto & subcolumn) { found = found || containsAggregateStateColumn(*subcolumn); }); + return found; +} + static SharedHeader checkHeaders(const SharedHeaders & input_headers) { if (input_headers.empty()) @@ -59,6 +70,17 @@ static SharedHeader checkHeaders(const SharedHeaders & input_headers) if (!common[col].column || !isColumnConst(*common[col].column)) continue; + /// Aggregate-state values cannot be compared as `Field`: the comparison throws when the + /// aggregate function type names differ, and they may legitimately differ between branches + /// when the functions have the same state representation (e.g. `quantileState` and + /// `quantilesState(0.9)`). Don't keep constness for them, materialize instead. + if (containsAggregateStateColumn(assert_cast(*common[col].column).getDataColumn())) + { + common[col].column = common[col].column->convertToFullColumnIfConst(); + materialized = true; + continue; + } + const Field value = assert_cast(*common[col].column).getField(); bool keep_const = true; for (const auto & header : input_headers) diff --git a/src/Processors/QueryPlan/JoinStepLogical.cpp b/src/Processors/QueryPlan/JoinStepLogical.cpp index 60c1b8bd25b0..e067c87a1346 100644 --- a/src/Processors/QueryPlan/JoinStepLogical.cpp +++ b/src/Processors/QueryPlan/JoinStepLogical.cpp @@ -1471,19 +1471,152 @@ void JoinStepLogical::buildPhysicalJoin( node = std::move(new_node); } -std::optional JoinStepLogical::getFilterActions(JoinTableSide side, const SharedHeader & stream_header) +using NameToColumnMap = std::unordered_map; + +static const ColumnConst * findInlinableConstant(const ActionsDAG::Node * node, const NameToColumnMap & constants) +{ + if (node->type != ActionsDAG::ActionType::INPUT) + return nullptr; + + auto it = constants.find(node->result_name); + if (it == constants.end() || !it->second.type->equals(*node->result_type)) + return nullptr; + + return typeid_cast(it->second.column.get()); +} + +static void inlineConstantInputs(ActionsDAG & dag, const NameToColumnMap & constants) +{ + std::unordered_set bound_inputs(dag.getInputs().begin(), dag.getInputs().end()); + + for (const auto & node : dag.getNodes()) + { + if (node.type != ActionsDAG::ActionType::INPUT || bound_inputs.contains(&node)) + continue; + + const auto * constant = findInlinableConstant(&node, constants); + if (!constant) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot evaluate condition, column {} is neither available in the stream nor a known constant", + node.result_name); + + ActionsDAG::Node const_node; + const_node.type = ActionsDAG::ActionType::COLUMN; + const_node.result_name = node.result_name; + const_node.result_type = node.result_type; + const_node.column = ColumnConst::create(constant->getDataColumnPtr(), 0); + + const_cast(node) = std::move(const_node); + } +} + +static bool areOppositeJoinSides(const JoinActionRef & lhs, const JoinActionRef & rhs) +{ + return (lhs.fromLeft() && rhs.fromRight()) || (lhs.fromRight() && rhs.fromLeft()); +} + +static bool canBeEvaluatedOnSide( + const JoinActionRef & condition, JoinTableSide side, const NameToColumnMap & constants) +{ + if (side == JoinTableSide::Left && (condition.fromLeft() || condition.fromNone())) + return true; + if (side == JoinTableSide::Right && condition.fromRight()) + return true; + + if (constants.empty()) + return false; + + /// Skip evalutation of equiality conditions, since it's used for join key extraction + auto [op, lhs, rhs] = condition.asBinaryPredicate(); + bool is_equality = op == JoinConditionOperator::Equals || op == JoinConditionOperator::NullSafeEquals; + if (is_equality && areOppositeJoinSides(lhs, rhs)) + return false; + + bool reads_own_column = false; + std::stack stack; + stack.push(condition); + + std::unordered_set visited; + while (!stack.empty()) + { + auto action = stack.top(); + stack.pop(); + + const auto * raw_node = action.getNode(); + if (!visited.insert(raw_node).second) + continue; + + if (raw_node->type == ActionsDAG::ActionType::INPUT) + { + if (side == JoinTableSide::Left ? action.fromLeft() : action.fromRight()) + reads_own_column = true; + else if (!findInlinableConstant(raw_node, constants)) + return false; + } + else if (raw_node->type == ActionsDAG::ActionType::ALIAS) + { + for (const auto & argument : action.getArguments()) + stack.push(argument); + } + else if (raw_node->type == ActionsDAG::ActionType::FUNCTION + && raw_node->function_base + && raw_node->function_base->isDeterministic()) + { + for (const auto & argument : action.getArguments()) + stack.push(argument); + } + else if (raw_node->type == ActionsDAG::ActionType::COLUMN) + { + /// Column represent a constant value, so it can be evaluated on any side + continue; + } + else + { + return false; + } + } + + return reads_own_column; +} + +std::optional JoinStepLogical::getFilterActions( + JoinTableSide side, const SharedHeader & left_header, const SharedHeader & right_header) { if (!canPushDownFromOn(join_operator, side)) return {}; - /// Check if condition can be extracted completely - const bool allow_join_on_const = TableJoin::isEnabledAlgorithm(join_settings.join_algorithms, JoinAlgorithm::HASH); + const auto & stream_header = side == JoinTableSide::Left ? left_header : right_header; + const auto & opposite_header = side == JoinTableSide::Right ? left_header : right_header; - auto & join_expression = join_operator.expression; - if (auto filter_condition = concatConditions(join_expression, side, /*can_extract_everything=*/allow_join_on_const)) - return ActionsDAG::createActionsForConjunction({filter_condition.getNode()}, stream_header->getColumnsWithTypeAndName()); + NameToColumnMap header_constants; + { + for (const auto & column : opposite_header->getColumnsWithTypeAndName()) + if (column.column && isColumnConst(*column.column)) + header_constants.emplace(column.name, column); + } + + ActionsDAG::NodeRawConstPtrs extracted; + std::vector kept; + kept.reserve(join_operator.expression.size()); + for (const auto & condition : join_operator.expression) + { + if (canBeEvaluatedOnSide(condition, side, header_constants)) + extracted.push_back(toBoolIfNeeded(condition).getNode()); + else + kept.push_back(condition); + } + + if (extracted.empty()) + return {}; + + auto filter_actions = ActionsDAG::createActionsForConjunction(extracted, stream_header->getColumnsWithTypeAndName()); + if (!filter_actions) + return {}; + + inlineConstantInputs(filter_actions->dag, header_constants); - return {}; + join_operator.expression = std::move(kept); + return filter_actions; } static void remapNodes(ActionsDAG::NodeRawConstPtrs & keys, const ActionsDAG::NodeMapping & node_map) diff --git a/src/Processors/QueryPlan/JoinStepLogical.h b/src/Processors/QueryPlan/JoinStepLogical.h index e1cf6eefaea2..cd4b9e54441f 100644 --- a/src/Processors/QueryPlan/JoinStepLogical.h +++ b/src/Processors/QueryPlan/JoinStepLogical.h @@ -116,7 +116,11 @@ class JoinStepLogical final : public IQueryPlanStep } void addConditions(ActionsDAG actions_dag); - std::optional getFilterActions(JoinTableSide side, const SharedHeader & stream_header); + + /// Extract the part of the JOIN ON expression that can be evaluated on `side` alone, to be applied + /// as a filter on that input. + std::optional getFilterActions( + JoinTableSide side, const SharedHeader & left_header, const SharedHeader & right_header); struct ActionsDAGWithKeys { diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 98ba6cc62cad..d910ab68b296 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -369,26 +370,32 @@ struct JoinActionRefPairHash } }; -static std::vector getJoiningKeysForJoinStep(const JoinOperator & join_operator) +/// Invokes `callback(lhs, rhs)` per Equals / NullSafeEquals predicate, `lhs` normalised to the left side. +template +static void forEachEquiJoinKey(const JoinOperator & join_operator, Callback && callback) { - std::vector joining_keys; for (const auto & predicate : join_operator.expression) { auto [predicate_op, lhs, rhs] = predicate.asBinaryPredicate(); if (predicate_op != JoinConditionOperator::Equals && predicate_op != JoinConditionOperator::NullSafeEquals) continue; - if (lhs.fromRight() && rhs.fromLeft()) std::swap(lhs, rhs); else if (!lhs.fromLeft() || !rhs.fromRight()) continue; + callback(lhs, rhs); + } +} - auto left_column = lhs.getColumn(); - auto right_column = rhs.getColumn(); - if (!left_column.type->equals(*right_column.type)) - continue; +static std::vector getJoiningKeysForJoinStep(const JoinOperator & join_operator) +{ + std::vector joining_keys; + forEachEquiJoinKey(join_operator, [&](const JoinActionRef & lhs, const JoinActionRef & rhs) + { + if (!lhs.getColumn().type->equals(*rhs.getColumn().type)) + return; joining_keys.emplace_back(lhs, rhs); - } + }); return joining_keys; } @@ -681,6 +688,95 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: equivalent_right_stream_column_to_left_stream_column[rhs_original_name] = lhs_column; } + /// Register the cross-type equi-key pairs that `buildEquialentSetsForJoinStepLogical` skips: its + /// Union-Find needs the two input types to be equal, plain name substitution does not. + /// + /// Substitution needs two other things. The replacement must carry the type the replaced name has + /// in the JOIN output, because that is what the filter's nodes were typed against, and it must + /// evaluate to the value that output column holds, because the filter's own semantics are defined + /// on that value. + /// + /// A cross-type equi-key gives both once the replacement is cast the way the JOIN casts that key: + /// the two sides are compared in their least supertype, so `CAST(, supertype)` is + /// exactly what is behind the JOIN output column. Demanding that the JOIN output type is that + /// supertype keeps the cast widening - a narrowing one would change what the predicate returns - + /// and rejects a column the JOIN altered for an unrelated reason, such as `join_use_nulls` widening + /// it to `Nullable`, where the replacement no longer matches the output. + /// + /// The cast node itself is only added once we know a filter really reaches that side, so the two + /// lists below carry what is needed to build it. + struct CrossTypeReplacement + { + JoinActionRef source; + DataTypePtr target_type; + String name; + }; + std::vector cross_type_replacements_for_left_stream; + std::vector cross_type_replacements_for_right_stream; + + if (logical_join + && (!left_stream_filter_push_down_input_columns_available + || !right_stream_filter_push_down_input_columns_available)) + { + const auto & join_output_header = *join_header; + + auto create_cast_name = [&](const String & replaced_name) + { + String name = fmt::format("__filterpushdown_cast{}", replaced_name); + int counter = 0; + for (; left_stream_input_header->has(name) || right_stream_input_header->has(name); ++counter) + name = fmt::format("__filterpushdown_cast_{}{}", counter, replaced_name); + return name; + }; + + /// Makes `replaced_name` substitutable by the opposite side's key, cast to `supertype`. + auto add_replacement = [&]( + std::unordered_map & equivalent_columns, + std::vector & replacements, + const String & replaced_name, + const JoinActionRef & source, + const DataTypePtr & supertype) + { + if (equivalent_columns.contains(replaced_name)) + return; + + const auto * replaced = join_output_header.findByName(replaced_name); + if (!replaced || !replaced->type->equals(*supertype)) + return; + + /// The side that already has the supertype is not cast by the JOIN either. + if (source.getType()->equals(*supertype)) + { + equivalent_columns[replaced_name] = source.getColumn(); + return; + } + + auto name = create_cast_name(replaced_name); + equivalent_columns[replaced_name] = ColumnWithTypeAndName(nullptr, supertype, name); + replacements.push_back({source, supertype, std::move(name)}); + }; + + forEachEquiJoinKey(logical_join->getJoinOperator(), [&](const JoinActionRef & lhs, const JoinActionRef & rhs) + { + /// Equal types are already covered by the equivalent sets above. + if (lhs.getType()->equals(*rhs.getType())) + return; + + auto supertype = tryGetLeastSupertype(DataTypes{lhs.getType(), rhs.getType()}); + if (!supertype) + return; + + add_replacement( + equivalent_left_stream_column_to_right_stream_column, + cross_type_replacements_for_right_stream, + lhs.getColumnName(), rhs, supertype); + add_replacement( + equivalent_right_stream_column_to_left_stream_column, + cross_type_replacements_for_left_stream, + rhs.getColumnName(), lhs, supertype); + }); + } + Names left_stream_available_columns_to_push_down = get_available_columns_for_filter(true /*push_to_left_stream*/, left_stream_filter_push_down_input_columns_available); Names right_stream_available_columns_to_push_down = get_available_columns_for_filter(false /*push_to_left_stream*/, right_stream_filter_push_down_input_columns_available); @@ -807,6 +903,33 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: return required_actions; }; + /// Materializes the casts the cross-type equi-key substitutions above refer to by name, so that + /// `fix_predicate_for_join_logical_step` can compute them from the stream's own input columns. + auto add_cross_type_replacement_actions = [&]( + const std::vector & replacements, + const auto & filter_dag_inputs, + std::vector & required_actions) + { + for (const auto & replacement : replacements) + { + auto is_used = [&](const auto * input) { return input->result_name == replacement.name; }; + if (std::ranges::none_of(filter_dag_inputs, is_used)) + continue; + + /// The cast is built without a context, and that is not a shortcut. There is no cast of + /// this key to reuse: the JOIN's actions only keep the cast behind its output column, + /// which converts the opposite side, and the key matching casts are added later, by the + /// conversion to the physical join - also without a context. So the pushed-down predicate + /// is built the same way as the values the key matching compares. Nor can a context make + /// the conversion differ: for the type pairs that have a least supertype, none of the + /// context-dependent conversion settings apply - they concern parsing from `String` and + /// serialization to `String`, which never appear as a supertype cast - and the date-time + /// overflow behavior is pinned by `createInternalCast` whether or not a context is given. + required_actions.push_back(JoinActionRef::transform({replacement.source}, + [&](ActionsDAG & dag, auto && args) { return &dag.addCast(*args.at(0), replacement.target_type, replacement.name, nullptr); })); + } + }; + if (join_filter_push_down_actions.left_stream_filter_to_push_down) { if (logical_join) @@ -819,6 +942,7 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: lhs = JoinActionRef::transform({lhs}, [&](ActionsDAG & dag, auto && args) { return &dag.addAlias(*args.at(0), it->second); }); required_actions_from_join.push_back(lhs); } + add_cross_type_replacement_actions(cross_type_replacements_for_left_stream, filter_dag_inputs, required_actions_from_join); auto pre_filter_dag = JoinExpressionActions::getSubDAG(required_actions_from_join); *join_filter_push_down_actions.left_stream_filter_to_push_down = fix_predicate_for_join_logical_step( std::move(*join_filter_push_down_actions.left_stream_filter_to_push_down), std::move(pre_filter_dag)); @@ -853,6 +977,7 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: rhs = JoinActionRef::transform({rhs}, [&](ActionsDAG & dag, auto && args) { return &dag.addAlias(*args.at(0), it->second); }); required_actions_from_join.push_back(rhs); } + add_cross_type_replacement_actions(cross_type_replacements_for_right_stream, filter_dag_inputs, required_actions_from_join); auto pre_filter_dag = JoinExpressionActions::getSubDAG(required_actions_from_join); *join_filter_push_down_actions.right_stream_filter_to_push_down = fix_predicate_for_join_logical_step( std::move(*join_filter_push_down_actions.right_stream_filter_to_push_down), std::move(pre_filter_dag)); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 2c1412ec856e..21d2f1980074 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -667,8 +667,7 @@ SortingInputOrder buildInputOrderFromSortDescription( } /// If the prefix description is used, we can't restore the full description from PK value. - /// TODO: partial sort description can be used as well. Implement support later. - if (order_key_prefix_descr.size() < description.size() || pk_column_names.size() < next_sort_key) + if (pk_column_names.size() < next_sort_key) can_optimize_virtual_row = false; auto order_info = std::make_shared(order_key_prefix_descr, next_sort_key, read_direction, limit); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp index 79b7b02b49e4..fa3a12d2f15b 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTopK.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -104,7 +105,14 @@ size_t tryOptimizeTopK(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const auto & sort_column = sorting_step->getInputHeaders().front()->getByName(sort_column_name); - const bool where_clause = filter_step || read_from_mergetree_step->getPrewhereInfo(); + /// A row-level policy filter restricts the rows inside the reader just like a `WHERE` / `PREWHERE`, + /// so it must count as a `where_clause` as well. Otherwise a query filtered only by a row policy leaves + /// `where_clause == false`, `MergeTreeDataSelectExecutor` enables `perform_top_k_optimization` and narrows + /// the read to the top-K marks before the policy runs: the policy then discards the rows in those marks + /// and the query returns fewer rows than the `LIMIT` - or none at all - even though later marks hold rows + /// the policy keeps. + const bool where_clause + = filter_step || read_from_mergetree_step->getPrewhereInfo() || read_from_mergetree_step->getRowLevelFilter(); ///remove alias if (sort_column_name.contains('.')) @@ -158,7 +166,9 @@ size_t tryOptimizeTopK(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, /// Dynamic and Variant columns cannot be reliably filtered: their lessOrEquals /// returns Nullable(UInt8) rather than UInt8, causing an "Unexpected return type" - /// logical error when the prewhere filter is executed. Skip the optimization for them. + /// logical error when the prewhere filter is executed. Comparison functions also + /// reject zero-sized tuples even though ORDER BY supports them. Skip the optimization + /// for these types. /// /// For variable-length types (e.g. String, Array, Map, Tuple containing variable-length /// elements), the per-row threshold comparison cost can exceed its savings — most notably @@ -166,10 +176,12 @@ size_t tryOptimizeTopK(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, /// path behind an explicit opt-in. Nullable and Tuple of fixed-length types are still /// considered fixed-length (haveMaximumSizeOfValue forwards through them). const bool sort_column_is_variable_length = !sort_column.type->haveMaximumSizeOfValue(); + const auto * sort_column_tuple_type = typeid_cast(sort_column.type.get()); bool use_dynamic_filtering = settings.use_top_k_dynamic_filtering && !read_from_mergetree_step->getPrewhereInfo() && !isDynamic(sort_column.type) && !isVariant(sort_column.type) + && (!sort_column_tuple_type || !sort_column_tuple_type->getElements().empty()) && (!sort_column_is_variable_length || settings.use_top_k_dynamic_filtering_for_variable_length_types); /// When read-in-order optimization is enabled and the sort column is a prefix diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp index 7026cb4e05e8..44d9c801cf2b 100644 --- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp +++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp @@ -67,6 +67,14 @@ bool canUseProjectionForReadingStep(ReadFromMergeTree * reading) if (reading->getAnalyzedResult() && reading->getAnalyzedResult()->readFromProjection()) return false; + /// A distributed read (make_distributed_plan) was already turned into a sharded read by an + /// earlier optimization pass. A projection match would replace this single read with a Union of + /// the surviving-parts read and the projection read, and only one branch carries the sharded + /// flag -> the branches expose different shard lists and makeDistributedPlan asserts on the + /// mismatch. Keep the read whole; the projection optimization is a no-op for distributed reads. + if (reading->getDistributedReadBucketCount() > 0) + return false; + if (reading->isQueryWithFinal()) return false; diff --git a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp index e58775068e69..327eec3203c0 100644 --- a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp +++ b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp @@ -14,16 +14,19 @@ static size_t trySplitJoin(QueryPlan::Node * node, QueryPlan::Nodes & nodes) if (!join_step || node->children.size() != 2 || typeid_cast(node->children.back()->step.get())) return 0; + const auto & lhs_header = node->children.front()->step->getOutputHeader(); + const auto & rhs_header = node->children.back()->step->getOutputHeader(); + size_t num_new_nodes = 0; for (auto [idx, side]: {std::make_pair(0, JoinTableSide::Left), std::make_pair(1, JoinTableSide::Right)}) { auto & child_node = *node->children.at(idx); - const auto & header = child_node.step->getOutputHeader(); - auto fitler_dag = join_step->getFilterActions(side, header); - if (!fitler_dag) + const auto & header = side == JoinTableSide::Left ? lhs_header : rhs_header; + auto filter_dag = join_step->getFilterActions(side, lhs_header, rhs_header); + if (!filter_dag) continue; - const auto & filter_column_name = fitler_dag->dag.getOutputs()[fitler_dag->filter_pos]->result_name; - QueryPlanStepPtr step = std::make_unique(header, std::move(fitler_dag->dag), filter_column_name, fitler_dag->remove_filter); + const auto & filter_column_name = filter_dag->dag.getOutputs()[filter_dag->filter_pos]->result_name; + QueryPlanStepPtr step = std::make_unique(header, std::move(filter_dag->dag), filter_column_name, filter_dag->remove_filter); step->setStepDescription("Join filter"); auto * new_node = &nodes.emplace_back(std::move(child_node)); @@ -50,19 +53,7 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes, const Op if (expr.hasStatefulFunctions()) return 0; - bool filter_name_clashs_with_input = false; - if (filter_step->removesFilterColumn()) - { - for (const auto * input : expr.getInputs()) - { - if (input->result_name == filter_column_name) - { - filter_name_clashs_with_input = true; - break; - } - } - } - + const auto * filter_dag_node = expr.tryFindInOutputs(filter_column_name); auto split = expr.splitActionsForFilter(filter_column_name); if (split.second.trivial()) @@ -78,20 +69,8 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes, const Op node->children.swap(filter_node.children); node->children.push_back(&filter_node); - std::string split_filter_name = filter_column_name; - if (filter_name_clashs_with_input) - { - split_filter_name = "__split_filter"; - - for (auto & filter_output : split.first.getOutputs()) - { - if (filter_output->result_name == filter_column_name) - { - filter_output = &split.first.addAlias(*filter_output, split_filter_name); - break; - } - } - } + /// The filter node may have been renamed by the split to avoid clashing with an input of the same name. + std::string split_filter_name = split.split_nodes_mapping.at(filter_dag_node)->result_name; filter_node.step = std::make_unique( filter_node.children.at(0)->step->getOutputHeader(), diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 5478fca2b7d0..c401595f9d27 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -1225,7 +1225,9 @@ Pipes readByLayers( merging_pipes[i] = step_getter(layers[i]); auto & filter_function = filters[i]; - if (!filter_function) + /// An empty per-layer pipe has no header. It carries nothing to filter and is removed when the + /// layer pipes are united, so do not attempt to add a transform to it. + if (!filter_function || merging_pipes[i].empty()) continue; auto syntax_result = TreeRewriter(context).analyze(filter_function, primary_key.expression->getRequiredColumnsWithTypes()); diff --git a/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp b/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp index 08fcf3126283..a8c7d2d2f1a0 100644 --- a/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp +++ b/src/Processors/QueryPlan/QueryPlanStepRegistry.cpp @@ -64,6 +64,7 @@ void registerBroadcastSendStep(QueryPlanStepRegistry & registry); void registerBroadcastReceiveStep(QueryPlanStepRegistry & registry); void registerReadFromMergeTreeStep(QueryPlanStepRegistry & registry); +void registerReadNothingStep(QueryPlanStepRegistry & registry); void registerReadFromTableStep(QueryPlanStepRegistry & registry); void registerReadFromTableFunctionStep(QueryPlanStepRegistry & registry); void registerBuildRuntimeFilterStep(QueryPlanStepRegistry & registry); @@ -105,6 +106,7 @@ void QueryPlanStepRegistry::registerPlanSteps() registerBroadcastReceiveStep(registry); registerReadFromMergeTreeStep(registry); + registerReadNothingStep(registry); registerReadFromTableStep(registry); registerReadFromTableFunctionStep(registry); registerBuildRuntimeFilterStep(registry); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index d2b8853b2148..c8495e4e5edf 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1838,8 +1838,16 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( for (auto && non_intersecting_parts_range : split_ranges_result.non_intersecting_parts_ranges) non_intersecting_parts_by_primary_key.push_back(std::move(non_intersecting_parts_range)); + /// A layer may produce an empty pipe (the in-order getter creates one source per part, + /// and a layer may end up with no parts). An empty pipe has no header, so it must not + /// reach `createProjection` or `addMergingFinal` below. Dropping it is safe here: + /// unlike the join-by-shards path, the per-layer pipes are simply united, so their + /// positions carry no meaning. for (auto && merging_pipe : split_ranges_result.merging_pipes) - pipes.push_back(std::move(merging_pipe)); + { + if (!merging_pipe.empty()) + pipes.push_back(std::move(merging_pipe)); + } } else { @@ -2674,6 +2682,10 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( bool is_initial_query = context_->getClientInfo().query_kind == ClientInfo::QueryKind::INITIAL_QUERY; bool distributed_index_analysis_enabled = !final_second_pass + /// Projection parts are identified only by the projection name, which is identical in every + /// parent part, so per-part analysis results cannot be attributed back, and remote replicas + /// resolve part names against the parent table. Analyze projection parts locally. + && !projection_parts_exist && settings[Setting::distributed_index_analysis] && (settings[Setting::distributed_index_analysis_for_non_shared_merge_tree] || data.isSharedStorage()) && (total_parts >= distributed_index_analysis_min_parts_to_activate) diff --git a/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp b/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp index 5e7ef89e439d..5abf674215e9 100644 --- a/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp +++ b/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp @@ -189,7 +189,7 @@ bool ReadFromObjectStorageStep::requestReadingInOrder() const InputOrderInfoPtr ReadFromObjectStorageStep::getDataOrder() const { - return convertSortingKeyToInputOrder(getStorageMetadata()->getSortingKey()); + return convertSortingKeyToInputOrder(storage_snapshot->metadata->getSortingKey()); } } diff --git a/src/Processors/QueryPlan/ReadFromObjectStorageStep.h b/src/Processors/QueryPlan/ReadFromObjectStorageStep.h index 5e6062f97d32..bc34ecd360ee 100644 --- a/src/Processors/QueryPlan/ReadFromObjectStorageStep.h +++ b/src/Processors/QueryPlan/ReadFromObjectStorageStep.h @@ -33,7 +33,6 @@ class ReadFromObjectStorageStep : public SourceStepWithFilter StorageMetadataPtr getStorageMetadata() const { return storage_snapshot->metadata; } - void applyFilters(ActionDAGNodes added_filter_nodes) override; void updatePrewhereInfo(const PrewhereInfoPtr & prewhere_info_value) override; bool canUpdatePrewhereInfoMultipleTimes() const override { return false; } diff --git a/src/Processors/QueryPlan/ReadNothingStep.cpp b/src/Processors/QueryPlan/ReadNothingStep.cpp index e2eaf2a1b8ff..2cb204a664c3 100644 --- a/src/Processors/QueryPlan/ReadNothingStep.cpp +++ b/src/Processors/QueryPlan/ReadNothingStep.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -20,4 +22,21 @@ void ReadNothingStep::initializePipeline(QueryPipelineBuilder & pipeline, const pipeline.init(Pipe(std::make_shared(getOutputHeader()))); } +void ReadNothingStep::serialize(Serialization & ctx) const +{ + /// The output header is the whole state, and the plan writes it generically for every node. + (void)ctx; +} + +QueryPlanStepPtr ReadNothingStep::deserialize(Deserialization & ctx) +{ + return std::make_unique(ctx.output_header); +} + +void registerReadNothingStep(QueryPlanStepRegistry & registry); +void registerReadNothingStep(QueryPlanStepRegistry & registry) +{ + registry.registerStep("ReadNothing", &ReadNothingStep::deserialize); +} + } diff --git a/src/Processors/QueryPlan/ReadNothingStep.h b/src/Processors/QueryPlan/ReadNothingStep.h index 5cbeaaa4b303..825d9bb784a9 100644 --- a/src/Processors/QueryPlan/ReadNothingStep.h +++ b/src/Processors/QueryPlan/ReadNothingStep.h @@ -15,6 +15,11 @@ class ReadNothingStep : public ISourceStep QueryPlanStepPtr clone() const override; void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + + void serialize(Serialization & ctx) const override; + bool isSerializable() const override { return true; } + + static QueryPlanStepPtr deserialize(Deserialization & ctx); }; } diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index a7a1e1d85e31..443e3c3f4fa1 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -22,6 +23,16 @@ namespace ErrorCodes extern const int PARAMETER_OUT_OF_BOUND; } +static bool containsAggregateStateColumn(const IColumn & column) +{ + if (typeid_cast(&column)) + return true; + + bool found = false; + column.forEachSubcolumn([&](const auto & subcolumn) { found = found || containsAggregateStateColumn(*subcolumn); }); + return found; +} + static SharedHeader checkHeaders(const SharedHeaders & input_headers) { if (input_headers.empty()) @@ -60,6 +71,17 @@ static SharedHeader checkHeaders(const SharedHeaders & input_headers) if (!common[col].column || !isColumnConst(*common[col].column)) continue; + /// Aggregate-state values cannot be compared as `Field`: the comparison throws when the + /// aggregate function type names differ, and they may legitimately differ between branches + /// when the functions have the same state representation (e.g. `quantileState` and + /// `quantilesState(0.9)`). Don't keep constness for them, materialize instead. + if (containsAggregateStateColumn(assert_cast(*common[col].column).getDataColumn())) + { + common[col].column = common[col].column->convertToFullColumnIfConst(); + materialized = true; + continue; + } + const Field value = assert_cast(*common[col].column).getField(); bool keep_const = true; for (const auto & header : input_headers) diff --git a/src/Processors/Transforms/FilterTransform.cpp b/src/Processors/Transforms/FilterTransform.cpp index a54b9127698a..dbeb45229a5f 100644 --- a/src/Processors/Transforms/FilterTransform.cpp +++ b/src/Processors/Transforms/FilterTransform.cpp @@ -1,6 +1,8 @@ #include +#include #include +#include #include #include #include @@ -10,6 +12,8 @@ #include #include #include +#include +#include #include #include #include @@ -66,6 +70,43 @@ Block FilterTransform::transformHeader( return result; } +/// constant folding in prepare misses an empty set behind a Nullable argument - no constness at 0 rows +static bool isAlwaysFalseByEmptySet(const ActionsDAG::Node * node) +{ + while (node->type == ActionsDAG::ActionType::ALIAS) + node = node->children.at(0); + + if (node->type != ActionsDAG::ActionType::FUNCTION) + return false; + + const auto & function_name = node->function_base->getName(); + + if (function_name == "and") + return std::any_of(node->children.begin(), node->children.end(), isAlwaysFalseByEmptySet); + + /// notIn over an empty set is always true, and the -IgnoreSet variants must not fold + if (function_name != "in" && function_name != "globalIn") + return false; + + const IColumn * set_column = node->children[1]->column.get(); + if (!set_column) + return false; + + if (const auto * const_column = typeid_cast(set_column)) + set_column = &const_column->getDataColumn(); + + const auto * column_set = typeid_cast(set_column); + if (!column_set) + return false; + + auto future_set = column_set->getData(); + if (!future_set) + return false; + + auto set = future_set->get(); + return set && set->getTotalRowCount() == 0; +} + FilterTransform::FilterTransform( SharedHeader header_, ExpressionActionsPtr expression_, @@ -125,12 +166,15 @@ IProcessor::Status FilterTransform::prepare() if (!always_false && expression && !on_totals) { - auto header = expression->getActionsDAG().updateHeader(getInputPort().getHeader()); - auto & column = header.getByPosition(filter_column_position).column; - if (column) + const auto & actions_dag = expression->getActionsDAG(); + always_false = isAlwaysFalseByEmptySet(&actions_dag.findInOutputs(filter_column_name)); + + if (!always_false) { - ConstantFilterDescription constant_filter(*column); - always_false = constant_filter.always_false; + auto header = actions_dag.updateHeader(getInputPort().getHeader()); + auto & column = header.getByPosition(filter_column_position).column; + if (column) + always_false = ConstantFilterDescription(*column).always_false; } } } diff --git a/src/QueryPipeline/BlockIO.cpp b/src/QueryPipeline/BlockIO.cpp index 1e392858c902..ced516e02a5e 100644 --- a/src/QueryPipeline/BlockIO.cpp +++ b/src/QueryPipeline/BlockIO.cpp @@ -25,8 +25,11 @@ void BlockIO::reset() */ /// TODO simplify it all - releaseWorkloadResources(); + /// Reset the pipeline before releasing workload resources: pipeline threads hold raw pointers + /// to `MemoryReservation` (see `WorkloadResources` in `PipelineExecutor`), so the reservation + /// must outlive them. resetPipeline(/*cancel=*/false); + releaseWorkloadResources(); process_list_entries.clear(); /// TODO Do we need also reset callbacks? In which order? @@ -60,7 +63,14 @@ BlockIO::~BlockIO() void BlockIO::onFinish(std::chrono::system_clock::time_point finish_time) { - releaseWorkloadResources(); + /// Release the query slot as early as possible: until it is released the query keeps occupying a + /// concurrency slot even though the client already considers the query finished, which can needlessly + /// block the next query. This is safe while the pipeline is still running because pipeline threads do + /// not touch the query slot. + /// The memory reservation is different: pipeline threads hold raw pointers to it (see `WorkloadResources` + /// in `PipelineExecutor`) and read it until the pipeline is finalized below, so releasing it here would + /// be a data race. It is released a bit later instead — the extra hold is brief and harmless. + releaseQuerySlot(); if (finalize_query_pipeline) { /// Keep the same teardown order as in resetPipeline: @@ -71,6 +81,9 @@ void BlockIO::onFinish(std::chrono::system_clock::time_point finish_time) } else resetPipeline(/*cancel=*/false); + + /// Safe now: the pipeline (and its threads) have been finalized and joined. + releaseMemoryReservation(); } void BlockIO::onException(bool log_as_error) @@ -115,4 +128,22 @@ void BlockIO::releaseWorkloadResources() const } } +void BlockIO::releaseQuerySlot() const +{ + for (const auto & entry : process_list_entries) + { + if (entry) + entry->getQueryStatus()->releaseQuerySlot(); + } +} + +void BlockIO::releaseMemoryReservation() const +{ + for (const auto & entry : process_list_entries) + { + if (entry) + entry->getQueryStatus()->releaseMemoryReservation(); + } +} + } diff --git a/src/QueryPipeline/BlockIO.h b/src/QueryPipeline/BlockIO.h index 359bffb41379..239f73f2975f 100644 --- a/src/QueryPipeline/BlockIO.h +++ b/src/QueryPipeline/BlockIO.h @@ -88,9 +88,18 @@ struct BlockIO /// Set is_all_data_sent in system.processes for this query. void setAllDataSent() const; - /// Release query slot early to allow client to reuse it for his next query. + /// Release all acquired workload resources (query slot and memory reservation). + /// Only safe once the pipeline has been stopped (see `releaseMemoryReservation`). void releaseWorkloadResources() const; + /// Release the query slot early to allow the client to reuse it for its next query. + /// Safe while the pipeline is still running: pipeline threads do not access the query slot. + void releaseQuerySlot() const; + + /// Release the memory reservation. MUST be called only after the pipeline has been finalized, + /// because pipeline threads hold raw pointers to `MemoryReservation`. + void releaseMemoryReservation() const; + void resetPipeline(bool cancel); private: diff --git a/src/Server/ArrowFlight/ArrowFlightServer.cpp b/src/Server/ArrowFlight/ArrowFlightServer.cpp index d44d92769abd..87627bbce6af 100644 --- a/src/Server/ArrowFlight/ArrowFlightServer.cpp +++ b/src/Server/ArrowFlight/ArrowFlightServer.cpp @@ -1391,6 +1391,7 @@ arrow::Status ArrowFlightServer::DoAction( if (std::holds_alternative(value)) { /// std::monostate means "reset to default" (SET setting = DEFAULT). + query_context->checkSettingsConstraintsForSettingsReset({setting}, SettingSource::QUERY); session_context->resetSettingsToDefaultValue({setting}); } else diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 526ee3774a49..79b2fed844be 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -267,6 +267,25 @@ bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::st } +std::optional CertificateReloader::getCertificate(const std::string & prefix) const +{ + std::lock_guard lock{data_mutex}; + + auto it = data_index.find(prefix); + if (it == data_index.end()) + return {}; + + auto current = it->second->data.get(); + if (!current || current->certs_chain.empty()) + return {}; + + /// `X509` is reference counted and immutable, so the certificate can be shared with the caller. + X509 * leaf_certificate = static_cast(current->certs_chain.front()); + X509_up_ref(leaf_certificate); + return X509Certificate(leaf_certificate); +} + + CertificateReloader::Data::Data(std::string cert_path, std::string key_path, std::string pass_phrase) : certs_chain(X509Certificate::fromFile(cert_path)), key(KeyPair::fromFile(key_path, pass_phrase)) { diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 03a59b84db88..b06f6c7b7209 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -98,6 +99,11 @@ class CertificateReloader /// A callback for OpenSSL int setCertificate(SSL * ssl, const MultiData * pdata); + /// The leaf certificate that is currently served for `prefix` connections, if there is one. + /// It is not necessarily the certificate of the corresponding `SSL_CTX`: certificates are installed + /// per connection, and with `` the context itself never gets a certificate at all. + std::optional getCertificate(const std::string & prefix) const; + private: CertificateReloader() = default; diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 02a5aa79e154..9c70fa460dd1 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -65,6 +65,7 @@ namespace ErrorCodes extern const int SYNTAX_ERROR; extern const int OPENSSL_ERROR; extern const int UNEXPECTED_PACKET_FROM_CLIENT; + extern const int UNKNOWN_PACKET_FROM_CLIENT; } PostgreSQLHandler::PostgreSQLHandler( @@ -405,9 +406,18 @@ void PostgreSQLHandler::cancelRequest() inline std::unique_ptr PostgreSQLHandler::receiveStartupMessage(int payload_size) { + /// The declared size is read from the wire before any authentication, and the message is read + /// into memory in full, so it has to be bounded. PostgreSQL uses the same limit. + static constexpr Int32 max_startup_message_size = 10000; + std::unique_ptr message; try { + if (payload_size < 8 || payload_size > max_startup_message_size) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Startup message declares a size of {} bytes, while it must be between 8 and {} bytes", + payload_size, max_startup_message_size); + message = message_transport->receiveWithPayloadSize(payload_size - 8); } catch (const Exception &) diff --git a/src/Server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp index 4d04bef4696e..82a4b416d25c 100644 --- a/src/Server/PrometheusRequestHandler.cpp +++ b/src/Server/PrometheusRequestHandler.cpp @@ -49,6 +49,7 @@ namespace Setting namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int CANNOT_WRITE_TO_OSTREAM; extern const int SUPPORT_IS_DISABLED; extern const int NOT_IMPLEMENTED; } @@ -385,7 +386,8 @@ class PrometheusRequestHandler::ReadImpl : public ImplWithContext response.set("Content-Encoding", "snappy"); ProtobufZeroCopyOutputStreamFromWriteBuffer zero_copy_output_stream{std::make_unique(getOutputStream(response))}; - read_response.SerializeToZeroCopyStream(&zero_copy_output_stream); + if (!read_response.SerializeToZeroCopyStream(&zero_copy_output_stream)) + throw Exception(ErrorCodes::CANNOT_WRITE_TO_OSTREAM, "Failed to serialize the Prometheus ReadResponse"); zero_copy_output_stream.finalize(); #else diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 992a40c6a91b..43bb4fbd1fbe 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1253,7 +1253,7 @@ bool AlterCommand::isTTLAlter(const StorageInMemoryMetadata & metadata) const if (!metadata.table_ttl.definition_ast) return true; /// If TTL had not been changed, do not require mutations - return metadata.table_ttl.definition_ast->formatWithSecretsOneLine() != ttl->formatWithSecretsOneLine(); + return metadata.table_ttl.definition_ast->formatIgnoringRedundantParentheses() != ttl->formatIgnoringRedundantParentheses(); } if (!ttl || type != MODIFY_COLUMN) @@ -1262,7 +1262,7 @@ bool AlterCommand::isTTLAlter(const StorageInMemoryMetadata & metadata) const bool column_ttl_changed = true; for (const auto & [name, ttl_ast] : metadata.columns.getColumnTTLs()) { - if (name == column_name && ttl->formatWithSecretsOneLine() == ttl_ast->formatWithSecretsOneLine()) + if (name == column_name && ttl->formatIgnoringRedundantParentheses() == ttl_ast->formatIgnoringRedundantParentheses()) { column_ttl_changed = false; break; @@ -1479,11 +1479,13 @@ void AlterCommands::prepare(const StorageInMemoryMetadata & metadata, bool share auto columns = metadata.columns; std::unordered_set columns_with_full_type_modify; + /// Used to tell whether a command restates the definition the table already has, so it must not + /// depend on whether the redundant parentheses were written on one side and not on the other. auto ast_to_str = [](const ASTPtr & query) -> String { if (!query) return ""; - return query->formatWithSecretsOneLine(); + return query->formatIgnoringRedundantParentheses(); }; for (size_t i = 0; i < size(); ++i) diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 55a4b8c5979e..0e9e800bc622 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -145,10 +145,14 @@ bool ColumnDescription::operator==(const ColumnDescription & other) const && ast_to_str(ttl) == ast_to_str(other.ttl); } +/// This is how a column is serialized into ZooKeeper, and `ColumnsDescription::operator==` compares +/// two sets of columns through it, so the text must not depend on the redundant parentheses the user +/// has written around a `DEFAULT`, `CODEC` or `TTL` expression. static String formatASTStateAware(IAST & ast, IAST::FormatState & state) { WriteBufferFromOwnString buf; IAST::FormatSettings settings(true); + settings.ignore_redundant_parentheses = true; ast.format(buf, settings, state, IAST::FormatStateStacked()); return buf.str(); } diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index fd30ea9a4e91..e5f363785d89 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -37,7 +37,7 @@ String ConstraintsDescription::toString() const for (const auto & constraint : constraints) list.children.push_back(constraint); - return list.formatWithSecretsOneLine(); + return list.formatIgnoringRedundantParentheses(); } ConstraintsDescription ConstraintsDescription::parse(const String & str) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index a77f425b9f75..45f2b2210783 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -119,6 +119,12 @@ class IStorage : public std::enable_shared_from_this, public TypePromo /// Returns true if the storage receives data from a remote server or servers. virtual bool isRemote() const { return false; } + /// Returns true for storages that do not store data themselves but read it from other tables, + /// e.g. `Distributed`, `Merge`, `Buffer`, `Alias`. The `_table` and `_database` virtual columns + /// of the rows read from such a storage carry the name of the table that actually produced + /// each row, which is not necessarily the name of this storage. + virtual bool readsFromOtherTables() const { return false; } + /// Returns true if the storage is a view of a table or another view. virtual bool isView() const { return false; } diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 59a3337fd602..f8fbe44b755f 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -94,6 +94,11 @@ IndexDescription IndexDescription::getIndexFromAST( if (index_definition->name.empty()) throw Exception(ErrorCodes::INCORRECT_QUERY, "Skip index must have name in definition."); + /// Without escaping the name becomes a part of the index file name as is, see `getIndexFileName`. + if (!escape_filenames && index_definition->name.contains('/')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Skip index name ({}) cannot contain '/' with `escape_index_filenames` disabled", index_definition->name); + auto index_type = index_definition->getType(); if (!index_type) throw Exception(ErrorCodes::INCORRECT_QUERY, "TYPE is required for index"); @@ -228,7 +233,7 @@ String IndicesDescription::explicitToString() const list.children.push_back(index.definition_ast); } - return list.formatWithSecretsOneLine(); + return list.formatIgnoringRedundantParentheses(); } String IndicesDescription::allToString() const @@ -240,7 +245,7 @@ String IndicesDescription::allToString() const for (const auto & index : *this) list.children.push_back(index.definition_ast); - return list.formatWithSecretsOneLine(); + return list.formatIgnoringRedundantParentheses(); } diff --git a/src/Storages/Kafka/StorageKafkaUtils.cpp b/src/Storages/Kafka/StorageKafkaUtils.cpp index 71b447b170cd..e57359dfce1a 100644 --- a/src/Storages/Kafka/StorageKafkaUtils.cpp +++ b/src/Storages/Kafka/StorageKafkaUtils.cpp @@ -193,7 +193,13 @@ void registerStorageKafka(StorageFactory & factory) auto num_consumers = (*kafka_settings)[KafkaSetting::kafka_num_consumers].value; auto max_consumers = std::max(getNumberOfCPUCoresToUse(), 16); - if (!args.getLocalContext()->getSettingsRef()[Setting::kafka_disable_num_consumers_limit] && num_consumers > max_consumers) + /// The limit depends on the local CPU count, so a definition read back from metadata may have been + /// accepted on a bigger server. Validate only a freshly introduced one, so existing tables stay loadable. + const bool is_fresh_definition = args.mode <= LoadingStrictnessLevel::CREATE + || (args.mode == LoadingStrictnessLevel::ATTACH && !args.query.attach_short_syntax); + + if (is_fresh_definition + && !args.getLocalContext()->getSettingsRef()[Setting::kafka_disable_num_consumers_limit] && num_consumers > max_consumers) { throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -254,7 +260,8 @@ void registerStorageKafka(StorageFactory & factory) return std::make_shared( args.table_id, args.getContext(), args.columns, args.comment, std::move(kafka_settings), collection_name); - if (!args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_kafka_offsets_storage_in_keeper] && !args.query.attach) + if (args.mode <= LoadingStrictnessLevel::CREATE + && !args.getLocalContext()->getSettingsRef()[Setting::allow_experimental_kafka_offsets_storage_in_keeper]) throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, "Storing the Kafka offsets in Keeper is experimental. Set `allow_experimental_kafka_offsets_storage_in_keeper` setting " diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index c3ef5b93917e..35b112a0bd94 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -1053,6 +1053,20 @@ std::optional RefreshTask::executeRefreshUnlocked(int32_t root_znode_versi query_for_logging, normalized_query_hash, refresh_query.get(), refresh_context, Stopwatch{CLOCK_MONOTONIC}.getStart(), internal); refresh_context->setProcessListElement(process_list_entry->getQueryStatus()); + + /// Publish the query status before interpreting the query, not just around the pipeline executor + /// below: planning runs nested pipelines for `IN (subquery)` sets, and only the status cancels those. + { + std::unique_lock exec_lock(execution.executor_mutex); + if (execution.interrupt_execution.load()) + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Refresh for view {} cancelled", view_storage_id.getFullTableName()); + execution.executing_query_status = process_list_entry->getQueryStatus(); + } + SCOPE_EXIT({ + std::unique_lock exec_lock(execution.executor_mutex); + execution.executing_query_status = nullptr; + }); + refresh_context->setProgressCallback([this](const Progress & prog) { execution.progress.incrementPiecewiseAtomically(prog); @@ -1471,14 +1485,26 @@ bool RefreshTask::updateCoordinationState(CoordinationZnode root, bool running, void RefreshTask::interruptExecution() { chassert(!mutex.try_lock()); - std::unique_lock lock(execution.executor_mutex); - if (execution.interrupt_execution.exchange(true)) - return; - if (execution.executor) + std::shared_ptr query_status; { - execution.executor->cancel(); - LOG_DEBUG(getLogger(), "Cancelling refresh in {}", set_handle.getID().getFullNameNotQuoted()); + std::unique_lock lock(execution.executor_mutex); + if (execution.interrupt_execution.exchange(true)) + return; + query_status = execution.executing_query_status; + if (execution.executor) + { + execution.executor->cancel(); + LOG_DEBUG(getLogger(), "Cancelling refresh in {}", set_handle.getID().getFullNameNotQuoted()); + } } + + /// Also mark the refresh query killed, not just cancel the pipeline: a refresh blocked in I/O + /// (e.g. a filesystem-cache download wait) doesn't observe pipeline cancellation and would keep + /// running, so shutdown()'s deactivate() — and any DROP driving it, including SharedCatalog + /// state apply — would block until the I/O returned on its own. Done outside executor_mutex + /// because cancelQuery() cancels registered executors, which take their own locks. + if (query_status) + query_status->cancelQuery(CancelReason::CANCELLED_BY_USER); } std::tuple RefreshTask::getAndLockTargetTable(const StorageID & storage_id, const ContextPtr & context) diff --git a/src/Storages/MaterializedView/RefreshTask.h b/src/Storages/MaterializedView/RefreshTask.h index 45c4c27b608d..f2e363eacb9f 100644 --- a/src/Storages/MaterializedView/RefreshTask.h +++ b/src/Storages/MaterializedView/RefreshTask.h @@ -21,6 +21,7 @@ namespace DB { class PipelineExecutor; +class QueryStatus; class StorageMaterializedView; class ASTRefreshStrategy; @@ -273,7 +274,7 @@ class RefreshTask : public std::enable_shared_from_this Finished, }; - /// Protects interrupt_execution and executor. + /// Protects interrupt_execution, executor and executing_query_status. /// Can be locked while holding `mutex`. std::mutex executor_mutex; /// If there's a refresh in progress, it can be aborted by setting this flag and cancel()ling @@ -281,6 +282,9 @@ class RefreshTask : public std::enable_shared_from_this /// `out_of_schedule_refresh_requested`, etc. std::atomic_bool interrupt_execution {false}; PipelineExecutor * executor = nullptr; + /// Process-list entry of the in-flight refresh query, so interruptExecution() can mark it + /// killed. Set as soon as the query enters the process list, before it is interpreted. + std::shared_ptr executing_query_status; /// Interrupts internal CREATE/EXCHANGE/DROP queries that refresh does. Only used during shutdown. StopSource cancel_ddl_queries; Progress progress; diff --git a/src/Storages/MergeTree/ColumnsSubstreams.cpp b/src/Storages/MergeTree/ColumnsSubstreams.cpp index 98cb99db9421..1885e4312550 100644 --- a/src/Storages/MergeTree/ColumnsSubstreams.cpp +++ b/src/Storages/MergeTree/ColumnsSubstreams.cpp @@ -70,21 +70,33 @@ size_t ColumnsSubstreams::getSubstreamPosition( const NameAndTypePair & name_and_type, const ISerialization::SubstreamPath & substream_path, const MergeTreeSettingsPtr & storage_settings) const +{ + if (auto position = tryGetSubstreamPosition(column_position, name_and_type, substream_path, storage_settings)) + return *position; + + auto substream = ISerialization::getFileNameForStream(name_and_type, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get position for substream {}: column {} with position {} doesn't have such substream", substream, name_and_type.name, column_position); +} + +std::optional ColumnsSubstreams::tryGetSubstreamPosition( + size_t column_position, + const NameAndTypePair & name_and_type, + const ISerialization::SubstreamPath & substream_path, + const MergeTreeSettingsPtr & storage_settings) const { ISerialization::StreamFileNameSettings stream_file_name_settings(*storage_settings); auto substream = ISerialization::getFileNameForStream(name_and_type, substream_path, stream_file_name_settings); if (auto position = tryGetSubstreamPosition(column_position, substream)) - return *position; + return position; /// To be able to read old parts after changes in stream file name settings, try to change settings and try to find it again. if (ISerialization::tryToChangeStreamFileNameSettingsForNotFoundStream(substream_path, stream_file_name_settings)) { substream = ISerialization::getFileNameForStream(name_and_type, substream_path, stream_file_name_settings); - if (auto position = tryGetSubstreamPosition(column_position, substream)) - return *position; + return tryGetSubstreamPosition(column_position, substream); } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get position for substream {}: column {} with position {} doesn't have such substream", substream, name_and_type.name, column_position); + return std::nullopt; } diff --git a/src/Storages/MergeTree/ColumnsSubstreams.h b/src/Storages/MergeTree/ColumnsSubstreams.h index 26dcec9e270a..00aabac7c4f6 100644 --- a/src/Storages/MergeTree/ColumnsSubstreams.h +++ b/src/Storages/MergeTree/ColumnsSubstreams.h @@ -30,6 +30,7 @@ class ColumnsSubstreams size_t getSubstreamPosition(size_t column_position, const String & substream) const; std::optional tryGetSubstreamPosition(size_t column_position, const String & substream) const; size_t getSubstreamPosition(size_t column_position, const NameAndTypePair & name_and_type, const ISerialization::SubstreamPath & substream_path, const MergeTreeSettingsPtr & storage_settings) const; + std::optional tryGetSubstreamPosition(size_t column_position, const NameAndTypePair & name_and_type, const ISerialization::SubstreamPath & substream_path, const MergeTreeSettingsPtr & storage_settings) const; std::optional tryGetSubstreamPosition(const String & substream) const; size_t getFirstSubstreamPosition(size_t column_position) const; size_t getLastSubstreamPosition(size_t column_position) const; diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index adbcacddc499..d58dd37232b5 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1709,6 +1709,8 @@ bool KeyCondition::hasOnlyConjunctions() const } +DataTypePtr getArgumentTypeOfMonotonicFunction(const IFunctionBase & func); + static Field applyFunctionForField( const FunctionBasePtr & func, const DataTypePtr & arg_type, @@ -1752,29 +1754,24 @@ static FieldRef applyFunction(const FunctionBasePtr & func, const DataTypePtr & { /// When cache is missed, we calculate the whole column where the field comes from. This will avoid repeated calculation. ColumnsWithTypeAndName args{(*columns)[field.column_idx]}; - /// Strip outer `LowCardinality` from the argument column and type before executing, keeping the - /// cached result full too. A monotonic-function chain is built against the outer-LowCardinality - /// stripped key type (`applyFunctionChainToColumn` strips it the same way), so a specialized - /// wrapper such as the UInt8->Bool `CAST` does `checkAndGetColumn` on the raw - /// column and aborts with a bad cast on a `ColumnLowCardinality` (e.g. a `LowCardinality(Bool)` - /// key compared with a `LowCardinality` constant). `removeLowCardinality` / - /// `convertToFullColumnIfLowCardinality` are no-ops for non-LC inputs. - if (args[0].column && args[0].column->lowCardinality()) + /// Normalize the chain's input only: the incoming index column may still be `LowCardinality` + /// while the chain was built against a stripped key type. Interior links need nothing, because + /// each is built against the previous function's result type, which the cache below preserves. + if (args[0].column && args[0].column->lowCardinality() && !getArgumentTypeOfMonotonicFunction(*func)->lowCardinality()) { args[0].column = args[0].column->convertToFullColumnIfLowCardinality(); args[0].type = removeLowCardinality(args[0].type); } - field.columns->emplace_back(ColumnWithTypeAndName {nullptr, removeLowCardinality(func->getResultType()), result_name}); + /// Invariant: every function receives the argument type it was built for, so the cached result + /// keeps this function's own result type and representation. + field.columns->emplace_back(ColumnWithTypeAndName {nullptr, func->getResultType(), result_name}); (*columns)[result_idx].column - = func->execute(args, (*columns)[result_idx].type, args.front().column->size(), /* dry_run = */ false) - ->convertToFullColumnIfLowCardinality(); + = func->execute(args, (*columns)[result_idx].type, args.front().column->size(), /* dry_run = */ false); } return {field.columns, field.row_idx, result_idx}; } -DataTypePtr getArgumentTypeOfMonotonicFunction(const IFunctionBase & func); - /// Sequentially applies functions to the column, returns `true` /// if all function arguments are compatible with functions /// signatures, and none of the functions produce `NULL` output. @@ -3982,6 +3979,11 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu func_name = String(reversed); } + /// What the chain actually produces, which is what any cast appended below will be fed. This + /// stays unstripped: only the copy used to choose the comparison supertype is stripped. + DataTypePtr chain_result_type + = chain.empty() ? recursiveRemoveLowCardinality(key_expr_type) : chain.back()->getResultType(); + key_expr_type = recursiveRemoveLowCardinality(key_expr_type); DataTypePtr key_expr_type_not_null; bool key_expr_type_is_nullable = false; @@ -4069,7 +4071,9 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu ? DataTypePtr(std::make_shared(common_type)) : common_type; - auto func_cast = createInternalCast({key_expr_type, {}}, common_type_maybe_nullable, CastType::nonAccurate, {}, node.getTreeContext().getQueryContext()); + /// Declared against the type this cast is actually given, not the stripped + /// `key_expr_type` used to pick the supertype. + auto func_cast = createInternalCast({chain_result_type, {}}, common_type_maybe_nullable, CastType::nonAccurate, {}, node.getTreeContext().getQueryContext()); /// If we know the given range only contains one value, then we treat all functions as positive monotonic. if (!single_point && !func_cast->hasInformationAboutMonotonicity()) @@ -5054,6 +5058,10 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( DataTypePtr current_type, bool single_point) { + /// The chain was built against a recursively `LowCardinality`-stripped key type, so seed it with the + /// stripped type here rather than in each caller: several of them pass the key column's raw type. + current_type = recursiveRemoveLowCardinality(current_type); + for (const auto & func : functions) { /// We check the monotonicity of each function on a specific range. @@ -5558,12 +5566,10 @@ BoolMask KeyCondition::checkInHyperrectangle( if (!element.monotonic_functions_chain.empty()) { key_range_storage = hyperrectangle[key_column]; - /// The chain was built in `extractAtomFromTree` against an - /// `LowCardinality`-stripped key type; the runtime type must match. std::optional new_range = applyMonotonicFunctionsChainToRange( *key_range_storage, element.monotonic_functions_chain, - recursiveRemoveLowCardinality(data_types[key_column]), + data_types[key_column], single_point ); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a507f0695d4c..2ab0b088c20d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7884,6 +7884,12 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r /// Subdirectories in the part's directory. It's used to restore projections. std::unordered_set subdirs; + /// A restored part is committed data the moment RESTORE is acknowledged, so it must get the same + /// durability an inserted part gets: fsync the file contents when the table enables fsync_after_insert. + /// Only meaningful on a local disk - on object storage the object is durable once finalized. The part + /// directory itself is fsynced later by IMergeTreeDataPart::renameTo (gated on fsync_part_directory). + const bool fsync_files = (*getSettings())[MergeTreeSetting::fsync_after_insert] && !disk->isRemote(); + /// Copy files from the backup to the directory `tmp_part_dir`. disk->createDirectories(temp_part_dir); @@ -7925,7 +7931,7 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r } else { - size_t file_size = backup->copyFileToDisk(part_path_in_backup_fs / filename, disk, temp_part_dir / filename, WriteMode::Rewrite); + size_t file_size = backup->copyFileToDisk(part_path_in_backup_fs / filename, disk, temp_part_dir / filename, WriteMode::Rewrite, fsync_files); reservation->update(reservation->getSize() - file_size); } } @@ -9981,9 +9987,11 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour if (my_snapshot->getColumns().getAllPhysical().sizeOfDifference(src_snapshot->getColumns().getAllPhysical())) throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS, "Tables have different structure"); + /// The definitions are compared as text, so the text must not depend on whether the user has + /// written redundant parentheses: `PARTITION BY (a)` and `PARTITION BY a` are the same key. auto query_to_string = [] (const ASTPtr & ast) { - return ast ? ast->formatWithSecretsOneLine() : ""; + return ast ? ast->formatIgnoringRedundantParentheses() : ""; }; if (query_to_string(my_snapshot->getSortingKeyAST()) != query_to_string(src_snapshot->getSortingKeyAST())) @@ -10006,10 +10014,10 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour std::unordered_set my_query_strings; for (const auto & description : my_descriptions) - my_query_strings.insert(description.definition_ast->formatWithSecretsOneLine()); + my_query_strings.insert(description.definition_ast->formatIgnoringRedundantParentheses()); for (const auto & src_description : src_descriptions) - if (!my_query_strings.contains(src_description.definition_ast->formatWithSecretsOneLine())) + if (!my_query_strings.contains(src_description.definition_ast->formatIgnoringRedundantParentheses())) return false; return true; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 11e24e70d325..a63165727789 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -78,6 +78,7 @@ namespace DB namespace Setting { extern const SettingsBool per_part_index_stats; + extern const SettingsBool apply_deleted_mask; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; extern const SettingsString force_data_skipping_indices; extern const SettingsBool force_index_by_date; @@ -1377,6 +1378,9 @@ void MergeTreeDataSelectExecutor::filterPartsByQueryConditionCache( const auto & settings = context->getSettingsRef(); if (!settings[Setting::use_query_condition_cache] || !settings[Setting::allow_experimental_analyzer] + /// `apply_deleted_mask = 0` must return deleted rows, so it cannot reuse entries written + /// by normal reads: those may exclude a granule whose only matching rows are deleted. + || !settings[Setting::apply_deleted_mask] || (!select_query_info.prewhere_info && !select_query_info.filter_actions_dag) || (vector_search_parameters.has_value()) /// vector search has filter in the ORDER BY || select_query_info.isFinal() diff --git a/src/Storages/MergeTree/MergeTreeIOSettings.cpp b/src/Storages/MergeTree/MergeTreeIOSettings.cpp index e98a4eca280c..02faffb89769 100644 --- a/src/Storages/MergeTree/MergeTreeIOSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeIOSettings.cpp @@ -120,7 +120,12 @@ MergeTreeReaderSettings MergeTreeReaderSettings::createFromContext(const Context && (settings[Setting::max_streams_to_max_threads_ratio] > 1 || settings[Setting::max_streams_for_merge_tree_reading] > 1); result.enable_multiple_prewhere_read_steps = settings[Setting::enable_multiple_prewhere_read_steps]; result.force_short_circuit_execution = settings[Setting::query_plan_merge_filters]; - result.use_query_condition_cache = settings[Setting::use_query_condition_cache] && settings[Setting::allow_experimental_analyzer]; + /// `apply_deleted_mask = 0` reads deleted rows, so its entries and those of normal reads are not + /// interchangeable. The setting is a debugging aid, so such queries skip the cache instead of + /// getting a key space of their own. Mirrored on the read side in MergeTreeDataSelectExecutor. + result.use_query_condition_cache = settings[Setting::use_query_condition_cache] + && settings[Setting::allow_experimental_analyzer] + && settings[Setting::apply_deleted_mask]; result.use_deserialization_prefixes_cache = settings[Setting::merge_tree_use_deserialization_prefixes_cache]; result.use_prefixes_deserialization_thread_pool = settings[Setting::merge_tree_use_prefixes_deserialization_thread_pool]; result.secondary_indices_enable_bulk_filtering = settings[Setting::secondary_indices_enable_bulk_filtering]; diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 0fac0ac03fab..00f41b544889 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -545,6 +545,22 @@ bool MergeTreeIndexConditionBloomFilter::traverseFunction(const RPNBuilderTreeNo return false; } +/// True when converting the constant to the element type yields the exact bytes the index holds, so +/// hashing it is equivalent to the comparison. Floats are excluded: `-0.0` equals but hashes apart. +static bool bloomFilterHashDomainMatches(const DataTypePtr & value_type, const DataTypePtr & nested_type) +{ + if (!value_type) + return false; + + auto value = removeLowCardinalityAndNullable(value_type); + auto element = removeLowCardinalityAndNullable(nested_type); + + if (isFloat(value) || isFloat(element)) + return false; + + return (isInteger(value) && isInteger(element)) || value->equals(*element); +} + bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( const String & function_name, const RPNBuilderTreeNode & key_node, @@ -569,12 +585,21 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( if (function_name == "notIn" || function_name == "globalNotIn") out.function = RPNElement::FUNCTION_NOT_IN; + /// `nullIn` (transform_null_in=1) selects the same rows as `in` only for a NULL-free, + /// single-column, non-Array set whose type matches the index; otherwise no pruning. + if ((function_name == "nullIn" || function_name == "globalNullIn") && prepared_set + && prepared_set->getDataTypes().size() == 1 && !prepared_set->hasNull() + && prepared_set->areTypesEqual(0, index_type) + && !typeid_cast(index_type.get())) + out.function = RPNElement::FUNCTION_IN; + return true; } /// Try to match the column name to a JSONAllPaths index for JSON subcolumn IN filtering. /// tryMatchNodeToJSONIndex handles both plain subcolumns and CAST-wrapped expressions. /// NOT IN is not supported because after BoolMask inversion it never skips any granules. + /// nullIn/globalNullIn are deliberately not wired here: JSON paths need per-path NULL checks. if (auto json_info = tryMatchNodeToJSONIndex(key_node, header, "JSONAllPaths")) { if (function_name != "in" && function_name != "globalIn") @@ -676,6 +701,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( return false; } + /// nullIn/globalNullIn are deliberately not wired here, as in the JSON branch above. if (function_name == "in" || function_name == "globalIn") out.function = RPNElement::FUNCTION_IN; @@ -685,7 +711,35 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( return true; } - return false; + /// `arrayJoin(col) IN (set)` needs a set element in the granule, same as `hasAny(col, set)`. + /// `notIn` is not derivable: a granule holding a set element still yields rows outside the set. + if (function_name != "in" && function_name != "globalIn") + return false; + if (!column) + return false; + + auto array_join_argument = key_node.getArrayJoinArgument(); + if (!array_join_argument) + return false; + + auto array_column_name = array_join_argument->getColumnName(); + if (!header.has(array_column_name)) + return false; + + size_t position = header.getPositionByName(array_column_name); + const auto * array_type = typeid_cast(header.getByPosition(position).type.get()); + if (!array_type) + return false; + + const auto & array_nested_type = array_type->getNestedType(); + if (!bloomFilterHashDomainMatches(type, array_nested_type)) + return false; + + const auto & converted_column = castColumn(ColumnWithTypeAndName{column, type, ""}, array_nested_type); + out.predicate.emplace_back( + std::make_pair(position, BloomFilterHash::hashWithColumn(array_nested_type, converted_column, 0, column->size()))); + out.function = RPNElement::FUNCTION_HAS_ANY; + return true; } @@ -801,6 +855,32 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( { auto key_column_name = key_node.getColumnName(); + /// `arrayJoin(col) = const` needs an element equal to the constant, same as `has(col, const)`. + /// `notEquals` is not derivable: a granule holding the constant still yields differing rows. + if (function_name == "equals") + { + if (auto array_join_argument = key_node.getArrayJoinArgument()) + { + auto array_column_name = array_join_argument->getColumnName(); + if (header.has(array_column_name)) + { + size_t position = header.getPositionByName(array_column_name); + const auto * array_type = typeid_cast(header.getByPosition(position).type.get()); + if (array_type && bloomFilterHashDomainMatches(value_type, array_type->getNestedType())) + { + const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType()); + auto converted_field = convertFieldToType(value_field, *actual_type, value_type.get()); + if (converted_field.isNull()) + return false; + + out.function = RPNElement::FUNCTION_HAS; + out.predicate.emplace_back(std::make_pair(position, BloomFilterHash::hashWithField(actual_type.get(), converted_field))); + return true; + } + } + } + } + if (header.has(key_column_name)) { size_t position = header.getPositionByName(key_column_name); diff --git a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp index 797c9281d94e..09b1964bc711 100644 --- a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp @@ -2,12 +2,8 @@ #include #include -#include #include -#include -#include - namespace DB { @@ -40,34 +36,57 @@ std::optional tryMatchJSONSubcolumnToIndex( const Names & index_columns, const String & json_function_name) { - /// Try all possible dot splits of the column name. - /// For "t.json.some.path" this produces: - /// ("t", "json.some.path"), ("t.json", "some.path"), ("t.json.some", "path") - for (auto [candidate_col, subcolumn_part] : Nested::getAllColumnAndSubcolumnPairs(column_name)) + /// Scan the index columns, not the dot positions of the name: the name can embed a folded + /// constant, so its length is unbounded while `index_columns` is not. + const std::string_view name = column_name; + const size_t json_column_offset = json_function_name.size() + 1; + + std::string_view matched_json_column; + std::string_view matched_subcolumn; + size_t matched_position = 0; + bool matched = false; + + for (size_t position = 0; position < index_columns.size(); ++position) { - auto index_column_name = fmt::format("{}({})", json_function_name, candidate_col); - auto it = std::find(index_columns.begin(), index_columns.end(), index_column_name); - if (it == index_columns.end()) + const std::string_view entry = index_columns[position]; + + /// Entry must be `json_function_name(X)` with a non-empty X. + if (entry.size() < json_column_offset + 2 || entry.back() != ')' || !entry.starts_with(json_function_name) + || entry[json_function_name.size()] != '(') continue; - /// Sub-object access (^ prefix) is not supported for index filtering - if (subcolumn_part.starts_with("^")) - return std::nullopt; + const std::string_view json_column = entry.substr(json_column_offset, entry.size() - json_column_offset - 1); - String path = extractPathFromSubcolumn(subcolumn_part); - if (path.empty()) - return std::nullopt; + /// The name must be `X.`. + if (json_column.size() + 1 >= name.size() || !name.starts_with(json_column) || name[json_column.size()] != '.') + continue; - size_t position = static_cast(std::distance(index_columns.begin(), it)); + /// Shortest X wins, ties resolve to the first entry: several entries can match one name. + if (matched && json_column.size() >= matched_json_column.size()) + continue; - return JSONSubcolumnIndexInfo{ - .json_column_name = String(candidate_col), - .path = std::move(path), - .header_position = position, - }; + matched_json_column = json_column; + matched_subcolumn = name.substr(json_column.size() + 1); + matched_position = position; + matched = true; } - return std::nullopt; + if (!matched) + return std::nullopt; + + /// Sub-object access (^ prefix) is not supported for index filtering + if (matched_subcolumn.starts_with("^")) + return std::nullopt; + + String path = extractPathFromSubcolumn(matched_subcolumn); + if (path.empty()) + return std::nullopt; + + return JSONSubcolumnIndexInfo{ + .json_column_name = String(matched_json_column), + .path = std::move(path), + .header_position = matched_position, + }; } std::optional tryMatchNodeToJSONIndex( diff --git a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h index 3cb0778c9019..bdaa87495c24 100644 --- a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h +++ b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h @@ -19,8 +19,9 @@ struct JSONSubcolumnIndexInfo }; /// Try to match a column name from the filter DAG to a JSON index column in the header. -/// Iterates all dot positions in `column_name` to handle JSON columns whose names contain dots -/// (e.g., `my.json` JSON or `t Tuple(json JSON)` with index on `JSONAllPaths(t.json)`). +/// Scans the index columns, not the dot positions of `column_name`, so cost is independent of the +/// name's length. JSON columns whose own names contain dots are handled (e.g., `my.json` JSON or +/// `t Tuple(json JSON)` with index on `JSONAllPaths(t.json)`); the shortest matching one wins. /// /// The `json_function_name` parameter specifies which index function to look for (e.g. "JSONAllPaths", /// "JSONAllValues"). diff --git a/src/Storages/MergeTree/MergeTreeIndexText.cpp b/src/Storages/MergeTree/MergeTreeIndexText.cpp index b1e745250099..835fee7e1668 100644 --- a/src/Storages/MergeTree/MergeTreeIndexText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexText.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -64,6 +65,7 @@ namespace ErrorCodes extern const int INCORRECT_NUMBER_OF_COLUMNS; extern const int CORRUPTED_DATA; extern const int SUPPORT_IS_DISABLED; + extern const int TOO_LARGE_STRING_SIZE; } namespace MergeTreeSetting @@ -72,6 +74,7 @@ namespace MergeTreeSetting extern const MergeTreeSettingsBool text_index_dictionary_block_frontcoding_compression; extern const MergeTreeSettingsNonZeroUInt64 text_index_posting_list_block_size; extern const MergeTreeSettingsTextIndexPostingListCodec text_index_posting_list_codec; + extern const MergeTreeSettingsMergeTreeTextIndexSerializationVersion text_index_serialization_version; extern const MergeTreeSettingsBool allow_experimental_text_index_positions; } @@ -87,6 +90,11 @@ static constexpr UInt64 MAX_CARDINALITY_FOR_EMBEDDED_POSTINGS = 6; static_assert(MAX_CARDINALITY_FOR_EMBEDDED_POSTINGS <= MAX_CARDINALITY_FOR_RAW_POSTINGS, "MAX_CARDINALITY_FOR_EMBEDDED_POSTINGS must be less or equal to MAX_CARDINALITY_FOR_RAW_POSTINGS"); static_assert(PostingListBuilder::max_small_size <= MAX_CARDINALITY_FOR_RAW_POSTINGS, "max_small_size must be less than or equal to MAX_CARDINALITY_FOR_RAW_POSTINGS"); +/// The enum values are written verbatim into the text index header and must remain stable. +static_assert(static_cast(MergeTreeTextIndexSerializationVersion::V0_Initial) == 0); +static_assert(static_cast(MergeTreeTextIndexSerializationVersion::V1_WithCodec) == 1); +static_assert(static_cast(MergeTreeTextIndexSerializationVersion::V2_WithPositions) == 2); + /// Kept as a fixed default rather than a MergeTree setting: a mutable table-level default would let /// an index's positions value change after parts exist, mixing positional and non-positional parts /// within one index. @@ -136,7 +144,7 @@ DictionaryBlock::DictionaryBlock(ColumnPtr tokens_, std::vector(TextIndexHeader::Version::WithCodec); - - if (serialization_version < required_version) + if (serialization_version < MergeTreeTextIndexSerializationVersion::V1_WithCodec) { /// Pre-WithCodec parts don't persist the codec type, but Bitpacking was the only /// compression codec at the time, so an IsCompressed posting list must be Bitpacking. @@ -248,14 +254,14 @@ PostingListPtr PostingsSerialization::deserialize(ReadBuffer & istr, UInt64 head /// If the posting list is completely in the buffer, avoid copying. if (istr.position() && istr.position() + num_bytes <= istr.buffer().end()) { - auto result = std::make_shared(PostingList::read(istr.position())); + auto result = std::make_shared(PostingList::readSafe(istr.position(), num_bytes)); istr.position() += num_bytes; return result; } deserialization_buffer.resize(num_bytes); istr.readStrict(deserialization_buffer.data(), num_bytes); - return std::make_shared(PostingList::read(deserialization_buffer.data())); + return std::make_shared(PostingList::readSafe(deserialization_buffer.data(), num_bytes)); } } @@ -346,6 +352,9 @@ ColumnPtr deserializeTokensFrontCoding(ReadBuffer & istr, size_t num_tokens) { UInt64 first_token_size = 0; readVarUInt(first_token_size, istr); + /// Prevent a corrupt or malicious .dct file from allocating huge amounts of memory + if (first_token_size > SerializationString::MAX_STRING_SIZE) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Corrupted text index dictionary: first token size ({}) exceeds the maximum ({})", first_token_size, SerializationString::MAX_STRING_SIZE); offset += first_token_size; if (offset > data.size()) data.resize_exact(roundUpToPowerOfTwoOrZero(std::max(offset, data.size() * 2))); @@ -363,7 +372,26 @@ ColumnPtr deserializeTokensFrontCoding(ReadBuffer & istr, size_t num_tokens) UInt64 data_size = 0; readVarUInt(data_size, istr); - offset += lcp + data_size; + /// Reject a corrupted or malicious `.dct`: an out-of-range `lcp` or an overflowing `lcp + data_size` would wrap `offset`, skip the resize, and cause an out-of-bounds write below. + const UInt64 previous_token_size = data_offset - previous_token_offset; + if (lcp > previous_token_size) + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "Corrupted text index dictionary: front-coding longest common prefix ({}) exceeds the previous token size ({})", + lcp, previous_token_size); + + UInt64 token_size = 0; + UInt64 next_offset = 0; + if (common::addOverflow(lcp, data_size, token_size) || common::addOverflow(offset, token_size, next_offset)) + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "Corrupted text index dictionary: front-coding token size overflows (lcp = {}, data_size = {})", + lcp, data_size); + + if (token_size > SerializationString::MAX_STRING_SIZE) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Corrupted text index dictionary: front-coding token size ({}) exceeds the maximum ({})", token_size, SerializationString::MAX_STRING_SIZE); + + offset = next_offset; if (offset > data.size()) data.resize_exact(roundUpToPowerOfTwoOrZero(std::max(offset, data.size() * 2))); @@ -864,6 +892,7 @@ void serializeTokensRaw( for (size_t i = block_begin; i < block_end; ++i) { auto current_token = token_getter(i); + TextIndexSerialization::checkTokenSize(current_token.size()); writeVarUInt(current_token.size(), ostr); ostr.write(current_token.data(), current_token.size()); } @@ -882,6 +911,7 @@ void serializeTokensFrontCoding( size_t block_end) { const auto & first_token = token_getter(block_begin); + TextIndexSerialization::checkTokenSize(first_token.size()); writeVarUInt(first_token.size(), ostr); ostr.write(first_token.data(), first_token.size()); @@ -889,6 +919,7 @@ void serializeTokensFrontCoding( for (size_t i = block_begin + 1; i < block_end; ++i) { auto current_token = token_getter(i); + TextIndexSerialization::checkTokenSize(current_token.size()); auto lcp = computeCommonPrefixLength(previous_token, current_token); writeVarUInt(lcp, ostr); writeVarUInt(current_token.size() - lcp, ostr); @@ -1043,6 +1074,12 @@ TokenPostingsInfo TextIndexSerialization::serializePostings( return info; } +void TextIndexSerialization::checkTokenSize(size_t token_size) +{ + if (token_size > SerializationString::MAX_STRING_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_STRING_SIZE, "Too large string size: {}. The maximum is: {}.", token_size, SerializationString::MAX_STRING_SIZE); +} + void TextIndexSerialization::serializeTokens(const ColumnString & tokens, WriteBuffer & ostr, TokensFormat format) { serializeTokensImpl( @@ -1083,14 +1120,22 @@ void TextIndexSerialization::serializeTokenInfo(WriteBuffer & ostr, const TokenP } } -void TextIndexSerialization::serializeHeader(const DictionarySparseIndex & sparse_index, IPostingListCodec::Type posting_list_codec_type, MergeTreeIndexVersion version, bool has_positions, WriteBuffer & ostr) +void TextIndexSerialization::serializeHeader(MergeTreeTextIndexSerializationVersion version, const DictionarySparseIndex & sparse_index, IPostingListCodec::Type posting_list_codec_type, bool has_positions, WriteBuffer & ostr) { - UInt64 codec_type = static_cast(posting_list_codec_type); + /// `textIndexCreator` raises the version to one that can represent the codec + /// and positions, so a violation here is a logical error, not a user error. + if (posting_list_codec_type != IPostingListCodec::Type::None && version < MergeTreeTextIndexSerializationVersion::V1_WithCodec) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Text index version 'v0_initial' does not support a posting list codec"); + + if (has_positions && version < MergeTreeTextIndexSerializationVersion::V2_WithPositions) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Text index version {} does not support positions", static_cast(version)); writeVarUInt(static_cast(version), ostr); - writeVarUInt(codec_type, ostr); - if (version >= static_cast(TextIndexHeader::Version::WithPositions)) + if (version >= MergeTreeTextIndexSerializationVersion::V1_WithCodec) + writeVarUInt(static_cast(posting_list_codec_type), ostr); + + if (version >= MergeTreeTextIndexSerializationVersion::V2_WithPositions) writeVarUInt(static_cast(has_positions), ostr); chassert(sparse_index.tokens->size() == sparse_index.offsets_in_file->size()); @@ -1107,13 +1152,13 @@ TextIndexHeader TextIndexSerialization::deserializeHeaderPrefix(ReadBuffer & ist UInt64 version = 0; readVarUInt(version, istr); - if (version > static_cast(TextIndexHeader::Version::WithPositions)) + if (version > static_cast(MergeTreeTextIndexSerializationVersion::V2_WithPositions)) throw Exception(ErrorCodes::CORRUPTED_DATA, "Unsupported version of sparse index ({})", version); TextIndexHeader header; - header.version = static_cast(version); + header.version = static_cast(version); - if (version >= static_cast(TextIndexHeader::Version::WithCodec)) + if (header.version >= MergeTreeTextIndexSerializationVersion::V1_WithCodec) { UInt64 codec_type = 0; readVarUInt(codec_type, istr); @@ -1124,7 +1169,7 @@ TextIndexHeader TextIndexSerialization::deserializeHeaderPrefix(ReadBuffer & ist header.codec_type = static_cast(codec_type); } - if (version >= static_cast(TextIndexHeader::Version::WithPositions)) + if (header.version >= MergeTreeTextIndexSerializationVersion::V2_WithPositions) { UInt64 has_positions = 0; readVarUInt(has_positions, istr); @@ -1364,6 +1409,7 @@ DictionarySparseIndex serializeTokensAndPostings( chassert(dictionary_mark.offset_in_decompressed_block == 0); const auto & first_token = sorted_tokens[block_begin].token; + TextIndexSerialization::checkTokenSize(first_token.size()); sparse_index_offsets_data.emplace_back(dictionary_mark.offset_in_compressed_file); sparse_index_str.insertData(first_token.data(), first_token.size()); @@ -1427,12 +1473,8 @@ void MergeTreeIndexGranuleTextWritable::serializeBinaryWithMultipleStreams(Merge positions_stream = it->second; } - /// Positional parts need a WithPositions reader. - auto serialization_version = static_cast( - params.positions ? TextIndexHeader::Version::WithPositions : TextIndexHeader::Version::WithCodec); - auto postings_codec = PostingListCodecFactory::createPostingListCodec(posting_list_codec_type); - PostingsSerialization postings_serialization(std::move(postings_codec), serialization_version); + PostingsSerialization postings_serialization(std::move(postings_codec), params.serialization_version); auto sparse_index_block = serializeTokensAndPostings( sorted_tokens, @@ -1442,7 +1484,7 @@ void MergeTreeIndexGranuleTextWritable::serializeBinaryWithMultipleStreams(Merge postings_serialization, positions_stream); - TextIndexSerialization::serializeHeader(sparse_index_block, posting_list_codec_type, serialization_version, params.positions, index_stream->compressed_hashing); + TextIndexSerialization::serializeHeader(params.serialization_version, sparse_index_block, posting_list_codec_type, params.positions, index_stream->compressed_hashing); } void MergeTreeIndexGranuleTextWritable::deserializeBinary(ReadBuffer &, MergeTreeIndexVersion) @@ -1920,17 +1962,35 @@ MergeTreeIndexPtr textIndexCreator(StorageMetadataPtr metadata_snapshot, const I UInt64 positions = extractFieldOption(options, ARGUMENT_POSITIONS).value_or(DEFAULT_POSITIONS); + String posting_list_codec_name = extractFieldOption(options, ARGUMENT_POSTING_LIST_CODEC) + .value_or(settings[MergeTreeSetting::text_index_posting_list_codec].toString()); + + auto posting_list_codec = PostingListCodecFactory::createPostingListCodec(posting_list_codec_name, index.name); + bool has_codec = posting_list_codec && posting_list_codec->getType() != IPostingListCodec::Type::None; + + /// The setting is a preference to preserve compatibility, not a hard constraint. + /// If the setting contradicts the index features on the current version, the index features take precedence. + using enum MergeTreeTextIndexSerializationVersion; + MergeTreeTextIndexSerializationVersion min_version = V0_Initial; + MergeTreeTextIndexSerializationVersion max_version = V2_WithPositions; + + if (has_codec) + min_version = V1_WithCodec; + + if (positions) + min_version = V2_WithPositions; + + const MergeTreeTextIndexSerializationVersion version_setting = settings[MergeTreeSetting::text_index_serialization_version]; + MergeTreeTextIndexSerializationVersion serialization_version = std::clamp(version_setting, min_version, max_version); + MergeTreeIndexTextParams index_params{ dictionary_block_size, dictionary_block_frontcoding_compression, posting_list_block_size, positions, std::move(preprocessor_ast), - std::move(postprocessor_ast)}; - - String posting_list_codec_name = extractFieldOption(options, ARGUMENT_POSTING_LIST_CODEC) - .value_or(settings[MergeTreeSetting::text_index_posting_list_codec].toString()); - auto posting_list_codec = PostingListCodecFactory::createPostingListCodec(posting_list_codec_name, index.name); + std::move(postprocessor_ast), + serialization_version}; if (!options.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected text index arguments: {}", fmt::join(std::views::keys(options), ", ")); @@ -1974,6 +2034,8 @@ void textIndexValidator(const IndexDescription & index, bool /*attach*/, const M "Text index argument '{}' is experimental. Enable it with the MergeTree setting " "`allow_experimental_text_index_positions = 1`.", ARGUMENT_POSITIONS); + /// The `text_index_serialization_version` setting is not validated against the index features: + /// it is a preference, and `textIndexCreator` raises it to a version that can represent them. String posting_list_codec_name = extractFieldOption(options, ARGUMENT_POSTING_LIST_CODEC) .value_or(settings[MergeTreeSetting::text_index_posting_list_codec].toString()); PostingListCodecFactory::createPostingListCodec(posting_list_codec_name, index.name); diff --git a/src/Storages/MergeTree/MergeTreeIndexText.h b/src/Storages/MergeTree/MergeTreeIndexText.h index 654b7c2d4e6a..db4459c7a00b 100644 --- a/src/Storages/MergeTree/MergeTreeIndexText.h +++ b/src/Storages/MergeTree/MergeTreeIndexText.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -82,6 +83,7 @@ struct MergeTreeIndexTextParams size_t positions = 0; ASTPtr preprocessor; ASTPtr postprocessor; + MergeTreeTextIndexSerializationVersion serialization_version = MergeTreeTextIndexSerializationVersion::V0_Initial; }; using PostingList = roaring::Roaring; @@ -159,7 +161,7 @@ struct TokenPostingsInfo; struct PostingsSerialization { - PostingsSerialization(PostingListCodecPtr posting_list_codec_, MergeTreeIndexVersion serialization_version_); + PostingsSerialization(PostingListCodecPtr posting_list_codec_, MergeTreeTextIndexSerializationVersion serialization_version_); enum Flags : UInt64 { @@ -188,7 +190,7 @@ struct PostingsSerialization private: PostingListCodecPtr posting_list_codec; - MergeTreeIndexVersion serialization_version; + MergeTreeTextIndexSerializationVersion serialization_version; /// Reusable buffers to avoid repeated heap allocations during deserialization. std::vector raw_postings_buffer; @@ -270,16 +272,9 @@ using DictionarySparseIndexPtr = std::shared_ptr; struct TextIndexHeader { - enum class Version - { - Initial = 0, - WithCodec = 1, - WithPositions = 2, - }; - - MergeTreeIndexVersion version = static_cast(Version::Initial); + MergeTreeTextIndexSerializationVersion version = MergeTreeTextIndexSerializationVersion::V0_Initial; IPostingListCodec::Type codec_type = IPostingListCodec::Type::None; - /// Persisted for version >= WithPositions. + /// Persisted for version >= V2_WithPositions. bool has_positions = false; DictionarySparseIndex sparse_index; }; @@ -300,7 +295,9 @@ struct TextIndexSerialization static void serializeTokens(const ColumnString & tokens, WriteBuffer & ostr, TokensFormat format); static void serializeTokenInfo(WriteBuffer & ostr, const TokenPostingsInfo & token_info); - static void serializeHeader(const DictionarySparseIndex & sparse_index, IPostingListCodec::Type posting_list_codec_type, MergeTreeIndexVersion version, bool has_positions, WriteBuffer & ostr); + /// Reject a token the reader would refuse (throws `TOO_LARGE_STRING_SIZE`); call before copying a token elsewhere. + static void checkTokenSize(size_t token_size); + static void serializeHeader(MergeTreeTextIndexSerializationVersion version, const DictionarySparseIndex & sparse_index, IPostingListCodec::Type posting_list_codec_type, bool has_positions, WriteBuffer & ostr); static TextIndexHeader deserializeHeader(ReadBuffer & istr); /// Reads only the version and posting list codec from the start of the header, without the @@ -358,7 +355,7 @@ struct MergeTreeIndexGranuleText final : public IMergeTreeIndexGranule void setCurrentRange(RowsRange range) { current_range = std::move(range); } const String & getIndexIdForCaches() const { return index_id_for_caches; } IPostingListCodec::Type getPostingsCodecType() const { return postings_codec_type; } - MergeTreeIndexVersion getSerializationVersion() const { return serialization_version; } + MergeTreeTextIndexSerializationVersion getSerializationVersion() const { return serialization_version; } static PostingListPtr readPostingsBlock( MergeTreeIndexReaderStream & stream, @@ -395,7 +392,7 @@ struct MergeTreeIndexGranuleText final : public IMergeTreeIndexGranule /// Codec type used to serialize postings in this granule. IPostingListCodec::Type postings_codec_type = IPostingListCodec::Type::None; /// On-disk serialization version of the text index header. - MergeTreeIndexVersion serialization_version = static_cast(TextIndexHeader::Version::Initial); + MergeTreeTextIndexSerializationVersion serialization_version = MergeTreeTextIndexSerializationVersion::V0_Initial; }; /// Text index granule created on writing of the index. diff --git a/src/Storages/MergeTree/MergeTreeIndices.cpp b/src/Storages/MergeTree/MergeTreeIndices.cpp index 43c78d6c0f66..e477f371bf2a 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INCORRECT_QUERY; + extern const int BAD_ARGUMENTS; } bool indexFileExistsInChecksums( @@ -50,6 +51,14 @@ String getIndexFileName(const String & index_name, bool escape_filename) { if (escape_filename) return escapeForFileName(String(SKIP_INDEX_FILE_PREFIX) + index_name); + + /// Here the name becomes a part of the file name as is, so a '/' in it would turn into a path + /// separator. `getIndexFromAST` rejects such names, but `escape_index_filenames` can also be + /// switched off by `ALTER TABLE ... MODIFY SETTING` after the index was created. + if (index_name.contains('/')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Skip index name ({}) cannot contain '/' with `escape_index_filenames` disabled", index_name); + return String(SKIP_INDEX_FILE_PREFIX) + index_name; } diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp index 39dffd7d3700..b050d1e5a727 100644 --- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp @@ -230,6 +230,7 @@ MergeTreeReadPoolBase::buildReadTaskInfo(const RangesInDataPart & part_with_rang auto columns_list = storage_snapshot->getColumnsByNames(options, column_names); auto mutation_steps = read_task_info.alter_conversions->getMutationSteps(part_info, columns_list, storage_snapshot->metadata, getContext()); + read_task_info.has_on_fly_mutation_steps = !mutation_steps.empty(); std::move(mutation_steps.begin(), mutation_steps.end(), std::back_inserter(read_task_info.mutation_steps)); } diff --git a/src/Storages/MergeTree/MergeTreeReadTask.cpp b/src/Storages/MergeTree/MergeTreeReadTask.cpp index 542a3268144d..5359e19e3dd9 100644 --- a/src/Storages/MergeTree/MergeTreeReadTask.cpp +++ b/src/Storages/MergeTree/MergeTreeReadTask.cpp @@ -480,7 +480,22 @@ bool MergeTreeReadTask::appliesMutationsBeforePrewhere() const /// predicate but does not apply the mutations (apply_mutations_on_fly = 0) would wrongly skip /// them. The read path already bypasses the cache in this case; this keeps the write path /// symmetric. - return !info->mutation_steps.empty() || !info->patch_parts.empty(); + /// + /// Only filters that vary between queries count. A materialized lightweight delete does not: + /// `_row_exists` is committed part data, so every query reading the part sees the same rows. + /// Its step lands in `mutation_steps` too, hence the checks below instead of testing that list. + /// (`apply_deleted_mask = 0` is the exception and skips the cache entirely, see + /// MergeTreeReaderSettings::createFromContext.) + if (!info->patch_parts.empty()) + return true; + + /// Not `alter_conversions->hasMutations()`: a pending mutation that touches no column this + /// query reads produces no step and rewrites nothing the query observes. + if (info->has_on_fly_mutation_steps) + return true; + + /// An unmaterialized lightweight delete is applied from the mutations snapshot at read time. + return info->alter_conversions && info->alter_conversions->hasLightweightDelete(); } } diff --git a/src/Storages/MergeTree/MergeTreeReadTask.h b/src/Storages/MergeTree/MergeTreeReadTask.h index 42dfd5869e3d..ff8ea6c22193 100644 --- a/src/Storages/MergeTree/MergeTreeReadTask.h +++ b/src/Storages/MergeTree/MergeTreeReadTask.h @@ -110,6 +110,9 @@ struct MergeTreeReadTaskInfo MergedPartOffsetsPtr merged_part_offsets; /// Prewhere steps that should be applied to execute on-fly mutations for part. PrewhereExprSteps mutation_steps; + /// Whether `mutation_steps` holds steps for on-fly mutations, as opposed to only the step that + /// applies an already materialized lightweight-delete mask. + bool has_on_fly_mutation_steps = false; /// Patches that should be applied for part. PatchPartsForReader patch_parts; /// Column names to read during PREWHERE and WHERE diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 35f00883e668..c5f7384441c6 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -403,9 +403,7 @@ void MergeTreeReaderCompact::initSubcolumnsDeserializationOrder() /// that do not exist in this part (e.g. MapBucketIndexes in old bucketed Map parts). enumerate_settings.check_stream_exists_callback = [&, column_pos = *pos](const ISerialization::SubstreamPath & substream_path) -> bool { - auto substream = ISerialization::getFileNameForStream( - column, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); - return columns_substreams.tryGetSubstreamPosition(column_pos, substream).has_value(); + return columns_substreams.tryGetSubstreamPosition(column_pos, column_from_part, substream_path, storage_settings).has_value(); }; auto order = getSubcolumnsDeserializationOrder(column, subcolumns_data, columns_substreams.getColumnSubstreams(*pos), enumerate_settings, ISerialization::StreamFileNameSettings(*storage_settings)); @@ -456,9 +454,7 @@ void MergeTreeReaderCompact::readPrefix(size_t column_idx, size_t from_mark, Mer { check_stream_exists_callback = [&](const ISerialization::SubstreamPath & substream_path) -> bool { - auto substream = ISerialization::getFileNameForStream( - column, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); - return columns_substreams.tryGetSubstreamPosition(*column_positions[column_idx], substream).has_value(); + return columns_substreams.tryGetSubstreamPosition(*column_positions[column_idx], column, substream_path, storage_settings).has_value(); }; } diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 83cd40fd655a..2c315a3c2c4b 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -192,20 +192,35 @@ void MergeTreeReaderStream::seekToMark(const MarkInCompressedFile & mark) } } +namespace +{ + +/// Index of the first mark after `from` that points to a different position, or `marks_count` if +/// there is none. Marks are non-decreasing positions in the file, so equal marks form contiguous +/// runs and binary search is valid. +size_t findNextDifferentMark(const MergeTreeMarksGetter & marks, size_t from, size_t marks_count) +{ + auto indices = collections::range(from, marks_count); + auto less_mark = [&](size_t lhs, size_t rhs) + { + return marks.getMark(lhs, 0).asTuple() < marks.getMark(rhs, 0).asTuple(); + }; + + auto it = std::upper_bound(indices.begin(), indices.end(), from, std::move(less_mark)); + return it == indices.end() ? marks_count : *it; +} + +} + bool MergeTreeReaderStream::hasAtMostNDistinctMarks(size_t max_transitions) const { auto marks = marks_loader->loadMarks(); - size_t num_transitions = 0; - MarkInCompressedFile last_mark{std::numeric_limits::max(), std::numeric_limits::max()}; - for (size_t i = 0; i < marks_count; ++i) + + size_t num_distinct = 0; + for (size_t pos = 0; pos < marks_count; pos = findNextDifferentMark(*marks, pos, marks_count)) { - auto mark = marks->getMark(i, 0); - if (mark != last_mark) - { - last_mark = mark; - if (++num_transitions > max_transitions) - return false; - } + if (++num_distinct > max_transitions) + return false; } return true; } @@ -306,17 +321,11 @@ size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) /// Mark 8, points to [84995, 7738] /// Mark 9, points to [126531, 8637] <--- what we are looking for - auto indices = collections::range(right_mark, marks_count); - auto next_different_mark = [&](auto lhs, auto rhs) - { - return marks_getter->getMark(lhs, 0).asTuple() < marks_getter->getMark(rhs, 0).asTuple(); - }; - - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_mark)); - if (it == indices.end()) + size_t next_different_mark = findNextDifferentMark(*marks_getter, right_mark, marks_count); + if (next_different_mark == marks_count) return file_size; - right_mark = *it; + right_mark = next_different_mark; } /// Special case for streams with dynamic/object structure. diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index b6d3fe39d1e0..6da19a06333c 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -38,6 +38,7 @@ class MergeTreeReaderStream /// Returns true if the mark file has at most `max_transitions` distinct /// consecutive (offset_in_compressed_file, offset_in_decompressed_block) /// positions. Loads marks from cache if available. + /// Costs at most `max_transitions` binary searches over the marks. bool hasAtMostNDistinctMarks(size_t max_transitions) const; /// Seeks to start of @row_index mark. Column position is implementation defined. diff --git a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp index aabdc399a716..8702396e28ab 100644 --- a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp @@ -116,14 +116,13 @@ void MergeTreeReaderTextIndex::setIndexGranule(MergeTreeIndexGranulePtr index_gr phrase_search_doc_ids.clear(); auto postings_codec = PostingListCodecFactory::createPostingListCodec(granule->getPostingsCodecType()); - /// Lazy mode requires the per-segment block-index section (from `WithCodec` onward) and + /// Lazy mode requires the per-segment block-index section (from `V1_WithCodec` onward) and /// pure-token queries — pattern predicates take the eager materialize path. - auto required_version = static_cast(TextIndexHeader::Version::WithCodec); const auto & condition_text = assert_cast(*index.condition); use_lazy_mode = lazy_mode_requested && postings_codec->getType() != IPostingListCodec::Type::None - && granule->getSerializationVersion() >= required_version + && granule->getSerializationVersion() >= MergeTreeTextIndexSerializationVersion::V1_WithCodec && !condition_text.hasSearchPatterns(); postings_serialization = PostingsSerialization(std::move(postings_codec), granule->getSerializationVersion()); diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 7d9f3f5cef74..37e212fcae22 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -692,6 +692,22 @@ namespace ErrorCodes Allow creating text indexes with the experimental `positions` argument which stores token positions to support exact phrase matching. )", BETA) \ + DECLARE(MergeTreeTextIndexSerializationVersion, text_index_serialization_version, MergeTreeTextIndexSerializationVersion::V1_WithCodec, R"( +The preferred on-disk serialization format version for writing text indexes. + +The setting is a preference rather than a hard constraint: if the configured version cannot +represent an index, a newer version that can represent it is chosen automatically, and +writing a text index never fails because of this setting. + +During a rolling upgrade, pin the format with the profile-level `compatibility` setting on +the already upgraded servers, so that they keep writing the format that older servers can still read. + +Possible values: + +- `v0_initial` — The original format. Does not persist the posting list codec type. +- `v1_with_codec` — Persists the posting list codec type in the text index header. +- `v2_with_positions` — Persists token positions for indexes with `positions`. +)", 0) \ DECLARE(UInt64, merge_selecting_sleep_ms, 5000, R"( Minimum time to wait before trying to select parts to merge again after no parts were selected. A lower setting will trigger selecting tasks in diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 73cd1e51b2ec..9a9e2954029a 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -63,7 +63,8 @@ struct MutableColumnsAndConstraints; M(CLASS_NAME, MergeTreeMapSerializationVersion) \ M(CLASS_NAME, MergeTreePartMinMaxIndexColumns) \ M(CLASS_NAME, SearchOrphanedPartsDisks) \ - M(CLASS_NAME, TextIndexPostingListCodec) + M(CLASS_NAME, TextIndexPostingListCodec) \ + M(CLASS_NAME, MergeTreeTextIndexSerializationVersion) MERGETREE_SETTINGS_SUPPORTED_TYPES(MergeTreeSettings, DECLARE_SETTING_TRAIT) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index af8baabc3397..48255dcf4ecb 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -1242,31 +1242,63 @@ static NameToNameVector collectFilesForRenames( if (updated_columns_in_patches.contains(command.rename_to)) continue; - String escaped_name_from = escapeForFileName(command.column_name); - String escaped_name_to = escapeForFileName(command.rename_to); - - ISerialization::StreamCallback callback = [&](const ISerialization::SubstreamPath & substream_path) + const auto * substreams = source_part->getColumnsSubstreams().tryGetColumnSubstreams(command.column_name); + if (substreams) { + /// Use columns_substreams.txt as the source of truth for substream file names. + /// This way the file renames stay consistent with the new columns_substreams.txt + /// produced by addRenamedColumnToColumnsSubstreams (both use getFileNameForRenamedColumnStream + /// on the same source names). auto storage_settings = source_part->storage.getSettings(); + for (const auto & substream : *substreams) + { + auto stream_from = IMergeTreeDataPart::getStreamNameOrHash(substream, ".bin", source_part->checksums); + if (!stream_from) + continue; - String full_stream_from = ISerialization::getFileNameForStream(command.column_name, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); - String full_stream_to = boost::replace_first_copy(full_stream_from, escaped_name_from, escaped_name_to); - - auto stream_from = IMergeTreeDataPart::getStreamNameOrHash(full_stream_from, ".bin", source_part->checksums); - if (!stream_from) - return; + String renamed = ISerialization::getFileNameForRenamedColumnStream( + command.column_name, command.rename_to, substream); + String stream_to = replaceFileNameToHashIfNeeded( + renamed, *storage_settings, &new_part->getDataPartStorage()); - String stream_to = replaceFileNameToHashIfNeeded(full_stream_to, *storage_settings, &new_part->getDataPartStorage()); + if (*stream_from != stream_to) + { + add_rename(*stream_from + ".bin", stream_to + ".bin"); + add_rename(*stream_from + mrk_extension, stream_to + mrk_extension); + } + } + } + else + { + /// Fallback for parts without columns_substreams.txt (discarded due to corruption or old parts). + /// Use getStreamNameForColumn with bidirectional fallback to find the actual file + /// regardless of whether the part was written with a different escape_variant_subcolumn_filenames value. + String escaped_name_from = escapeForFileName(command.column_name); + String escaped_name_to = escapeForFileName(command.rename_to); - if (stream_from != stream_to) + ISerialization::StreamCallback callback = [&](const ISerialization::SubstreamPath & substream_path) { - add_rename(*stream_from + ".bin", stream_to + ".bin"); - add_rename(*stream_from + mrk_extension, stream_to + mrk_extension); - } - }; + auto storage_settings = source_part->storage.getSettings(); - if (auto serialization = source_part->tryGetSerialization(command.column_name)) - serialization->enumerateStreams(callback); + String full_stream_from = ISerialization::getFileNameForStream(command.column_name, substream_path, ISerialization::StreamFileNameSettings(*storage_settings)); + String full_stream_to = boost::replace_first_copy(full_stream_from, escaped_name_from, escaped_name_to); + + auto stream_from = IMergeTreeDataPart::getStreamNameForColumn(command.column_name, substream_path, ".bin", source_part->checksums, storage_settings); + if (!stream_from) + return; + + String stream_to = replaceFileNameToHashIfNeeded(full_stream_to, *storage_settings, &new_part->getDataPartStorage()); + + if (*stream_from != stream_to) + { + add_rename(*stream_from + ".bin", stream_to + ".bin"); + add_rename(*stream_from + mrk_extension, stream_to + mrk_extension); + } + }; + + if (auto serialization = source_part->tryGetSerialization(command.column_name)) + serialization->enumerateStreams(callback); + } } else if (command.type == MutationCommand::Type::UPDATE || command.type == MutationCommand::Type::READ_COLUMN || command.type == MutationCommand::Type::MATERIALIZE_COLUMN) { diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 7ef90c7e3659..47ed74cc8ff1 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -508,6 +508,24 @@ std::optional RPNBuilderTreeNode::toFunctionNodeOrNu return RPNBuilderFunctionTreeNode(getNodeWithoutAlias(dag_node), tree_context); } +std::optional RPNBuilderTreeNode::getArrayJoinArgument() const +{ + if (ast_node) + { + const auto * ast_function = typeid_cast(ast_node); + if (ast_function && ast_function->name == "arrayJoin" && ast_function->arguments + && ast_function->arguments->children.size() == 1) + return RPNBuilderTreeNode(ast_function->arguments->children[0].get(), tree_context); + return {}; + } + + const auto * node_without_alias = getNodeWithoutAlias(dag_node); + if (node_without_alias->type == ActionsDAG::ActionType::ARRAY_JOIN && node_without_alias->children.size() == 1) + return RPNBuilderTreeNode(node_without_alias->children[0], tree_context); + + return {}; +} + std::string RPNBuilderFunctionTreeNode::getFunctionName() const { if (ast_node) diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index ae872db77a95..a785294769b2 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -130,6 +130,11 @@ class RPNBuilderTreeNode /// Convert node to function node or null optional std::optional toFunctionNodeOrNull() const; + /** If this node is `arrayJoin(x)`, return its argument node `x`; otherwise std::nullopt. + * Handles both the DAG `ARRAY_JOIN` action node and the AST `ASTFunction` named `arrayJoin`. + */ + std::optional getArrayJoinArgument() const; + /// Get tree context const RPNBuilderTreeContext & getTreeContext() const { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index ef8844c8297c..fd69dce95b0e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -79,6 +79,7 @@ namespace FailPoints extern const char replicated_merge_tree_insert_retry_pause[]; extern const char replicated_merge_tree_restore_attach_retry[]; extern const char rmt_delay_commit_part[]; + extern const char rmt_pause_before_commit_local_part[]; extern const char rmt_dedup_conflict_part_name_missing[]; } @@ -903,6 +904,10 @@ std::vector ReplicatedMergeTreeSink::commitPart( auto sleep_before_commit_for_tests = [&] () { + /// The parts have been renamed but not committed yet, and the caller still holds the + /// table lock it took for the whole pipeline. + FailPointInjection::pauseFailPoint(FailPoints::rmt_pause_before_commit_local_part); + auto sleep_before_commit_local_part_in_replicated_table_ms = (*storage.getSettings())[MergeTreeSetting::sleep_before_commit_local_part_in_replicated_table_ms]; if (sleep_before_commit_local_part_in_replicated_table_ms.totalMilliseconds()) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 1069091962e1..d8360f74075e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -33,25 +32,11 @@ namespace ErrorCodes extern const int METADATA_MISMATCH; } -/// User-written parentheses around individual key elements (e.g. `PRIMARY KEY (col)`) are -/// syntactically meaningless in stored metadata. Strip them so the canonical form matches -/// what `KeyDescription::parse` produces when reading metadata back from ZooKeeper. -static void stripArtificialParens(IAST & ast) -{ - ast.setParenthesized(false); - if (auto * list = ast.as()) - for (auto & child : list->children) - if (child) - child->setParenthesized(false); -} - static String formattedAST(const ASTPtr & ast) { if (!ast) return ""; - auto cloned = ast->clone(); - stripArtificialParens(*cloned); - return cloned->formatWithSecretsOneLine(); + return ast->formatIgnoringRedundantParentheses(); } static String formattedASTNormalized(const ASTPtr & ast) @@ -60,8 +45,7 @@ static String formattedASTNormalized(const ASTPtr & ast) return ""; auto ast_normalized = ast->clone(); FunctionNameNormalizer::visit(ast_normalized.get()); - stripArtificialParens(*ast_normalized); - return ast_normalized->formatWithSecretsOneLine(); + return ast_normalized->formatIgnoringRedundantParentheses(); } ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTreeData & data, const StorageMetadataPtr & metadata_snapshot) diff --git a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp index 3b05c0db05e6..09786a9b73ba 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp +++ b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp @@ -1,15 +1,29 @@ #include #include +#include +#include #include +#include #include #include #include #include +#include +#include + +#include namespace DB { +namespace ErrorCodes +{ + extern const int ACCESS_DENIED; + extern const int UNKNOWN_IDENTIFIER; + extern const int NO_SUCH_COLUMN_IN_TABLE; +} + StorageFromMergeTreeProjection::StorageFromMergeTreeProjection( StorageID storage_id_, StoragePtr parent_storage_, StorageMetadataPtr parent_metadata_, ProjectionDescriptionRawPtr projection_) : IStorage(storage_id_) @@ -33,6 +47,78 @@ void StorageFromMergeTreeProjection::read( { context->checkAccess(AccessType::SELECT, parent_storage->getStorageID()); + const auto parent_storage_id = parent_storage->getStorageID(); + auto row_policy_filter = context->getRowPolicyFilter( + parent_storage_id.getDatabaseName(), parent_storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + + const bool has_row_policy = row_policy_filter && !row_policy_filter->isAlwaysTrue(); + + Names read_column_names = column_names; + if (has_row_policy) + { + /// aggregate projections fold many parent rows into one state, so a per-row policy can't be applied + if (projection->type != ProjectionDescription::Type::Normal) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from projection `{}` of table {} under a row policy: it is not a normal " + "projection, so the policy cannot be enforced before aggregation", + projection->name, parent_storage_id.getNameForLogs()); + + /// the policy is on the parent table; enforce it here or the projection leaks hidden rows + if (!query_info.planner_context || !query_info.table_expression) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot enforce the row policy of table {} on projection `{}` without the analyzer", + parent_storage_id.getNameForLogs(), projection->name); + + for (const auto & policy : row_policy_filter->policies) + if (context->hasQueryContext()) + context->getQueryContext()->addUsedRowPolicy(policy->getFullName().toString()); + + FilterDAGInfo filter_info; + try + { + /// resolve against the projection's own columns; anything it can't provide throws below + filter_info = buildFilterInfo( + row_policy_filter->expression->clone(), query_info.table_expression, query_info.planner_context); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::UNKNOWN_IDENTIFIER && e.code() != ErrorCodes::NO_SUCH_COLUMN_IN_TABLE) + throw; + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from projection `{}` of table {} because its row policy references a column " + "the projection does not store, so it cannot be enforced", + projection->name, parent_storage_id.getNameForLogs()); + } + + /// pull in the policy's columns; reject virtuals the projection reorders (position-relative) or + /// only exposes under a projection name absent on the parent (`_parent_part_offset`) + static const NameSet not_row_preserving{ + "_part_offset", "_part_index", "_part_granule_offset", "_block_offset", "_block_number", "_parent_part_offset"}; + for (const auto & name : filter_info.actions.getRequiredColumnsNames()) + { + if (not_row_preserving.contains(name)) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from projection `{}` of table {} because its row policy uses virtual column " + "`{}`, whose value the projection does not preserve, so it cannot be enforced", + projection->name, parent_storage_id.getNameForLogs(), name); + if (std::find(read_column_names.begin(), read_column_names.end(), name) == read_column_names.end()) + read_column_names.push_back(name); + } + + /// don't clobber a filter the planner already set for a policy on the table function (`_table_function.*`) + if (query_info.row_level_filter) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from projection `{}` of table {}: a row policy already applies to the table " + "function itself and cannot be combined with the parent table's row policy", + projection->name, parent_storage_id.getNameForLogs()); + + /// row-level filter runs before any user PREWHERE (a post-read filter would let PREWHERE see hidden rows) + query_info.row_level_filter = std::make_shared(std::move(filter_info)); + + /// planner left trivial-LIMIT on (it checks this tf's storage id, which has no policy); n rows then filter could yield < n + query_info.trivial_limit = 0; + } + /// A UNIQUE KEY parent rejects projection reads in the MergeTreeDataSelectExecutor /// constructor below (the universal projection-read chokepoint), since reading a /// projection part bypasses the parent's delete-bitmap filter. @@ -40,6 +126,15 @@ void StorageFromMergeTreeProjection::read( const auto & snapshot_data = assert_cast(*storage_snapshot->data); const auto & parts = snapshot_data.parts; + /// on-the-fly data mutations and patch parts are applied to the parent read but not to the projection + /// (mutations_snapshot is cleared below), so under a policy the projection could show stale hidden rows + if (has_row_policy + && (snapshot_data.mutations_snapshot->hasDataMutations() || snapshot_data.mutations_snapshot->hasPatchParts())) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from projection `{}` of table {} under a row policy while it has unmaterialized " + "mutations, since the projection may show stale values the policy would hide", + projection->name, parent_storage_id.getNameForLogs()); + RangesInDataParts projection_parts; for (const auto & part : *parts) { @@ -56,7 +151,7 @@ void StorageFromMergeTreeProjection::read( .readFromParts( std::make_shared(projection_parts), snapshot_data.mutations_snapshot->cloneEmpty(), - column_names, + read_column_names, storage_snapshot, query_info, context, diff --git a/src/Storages/MergeTree/TextIndexUtils.cpp b/src/Storages/MergeTree/TextIndexUtils.cpp index 35314ce920c9..0cf192d870ff 100644 --- a/src/Storages/MergeTree/TextIndexUtils.cpp +++ b/src/Storages/MergeTree/TextIndexUtils.cpp @@ -211,12 +211,12 @@ void BuildTextIndexTransform::writeTemporarySegment(size_t i) static PostingsSerialization createPostingsSerialization(const IMergeTreeIndex & index) { - const auto * codec = typeid_cast(index).getPostingListCodec(); + const auto & text_index = typeid_cast(index); + const auto * codec = text_index.getPostingListCodec(); auto codec_type = codec ? codec->getType() : IPostingListCodec::Type::None; auto codec_copy = PostingListCodecFactory::createPostingListCodec(codec_type); - /// The merged part is written in the current on-disk format. - return PostingsSerialization(std::move(codec_copy), static_cast(TextIndexHeader::Version::WithCodec)); + return PostingsSerialization(std::move(codec_copy), text_index.getParams().serialization_version); } static PostingsSerialization createSourcePostingsSerialization(MergeTreeIndexReaderStream & header_stream) @@ -249,7 +249,9 @@ MergeTextIndexesTask::MergeTextIndexesTask( input_streams.resize(segments.size()); output_tokens = ColumnString::create(); - params = typeid_cast(*index_ptr).getParams(); + + const auto & text_index = typeid_cast(*index_ptr); + params = text_index.getParams(); sparse_index_tokens = ColumnString::create(); sparse_index_offsets = ColumnUInt64::create(); @@ -428,6 +430,7 @@ void MergeTextIndexesTask::flushDictionaryBlock() chassert(current_mark.offset_in_decompressed_block == 0); auto first_token = output_tokens->getDataAt(0); + TextIndexSerialization::checkTokenSize(first_token.size()); assert_cast(*sparse_index_tokens).insertData(first_token.data(), first_token.size()); assert_cast(*sparse_index_offsets).insertValue(current_mark.offset_in_compressed_file); @@ -565,10 +568,7 @@ void MergeTextIndexesTask::finalize() auto * index_stream = output_streams.at(MergeTreeIndexSubstream::Type::Regular); DictionarySparseIndex sparse_index(std::move(sparse_index_tokens), std::move(sparse_index_offsets)); - - auto serialization_version = static_cast( - params.positions ? TextIndexHeader::Version::WithPositions : TextIndexHeader::Version::WithCodec); - TextIndexSerialization::serializeHeader(sparse_index, postings_serialization.getPostingListCodec()->getType(), serialization_version, params.positions, index_stream->compressed_hashing); + TextIndexSerialization::serializeHeader(params.serialization_version, sparse_index, postings_serialization.getPostingListCodec()->getType(), params.positions, index_stream->compressed_hashing); for (auto & stream : output_streams_holders) stream->finalize(); diff --git a/src/Storages/MergeTree/tests/gtest_posting_list_cursor.cpp b/src/Storages/MergeTree/tests/gtest_posting_list_cursor.cpp index bdf06f18d95c..6e39a88805b8 100644 --- a/src/Storages/MergeTree/tests/gtest_posting_list_cursor.cpp +++ b/src/Storages/MergeTree/tests/gtest_posting_list_cursor.cpp @@ -3622,12 +3622,14 @@ TEST(PostingListCursorTest, TextIndexHeaderPersistsCodecType) DictionarySparseIndex sparse_index(tokens->getPtr(), offsets->getPtr()); WriteBufferFromOwnString out; - TextIndexSerialization::serializeHeader(sparse_index, IPostingListCodec::Type::Bitpacking, static_cast(TextIndexHeader::Version::WithCodec), /*has_positions=*/ false, out); + TextIndexSerialization::serializeHeader( + MergeTreeTextIndexSerializationVersion::V1_WithCodec, + sparse_index, IPostingListCodec::Type::Bitpacking, /*has_positions=*/ false, out); ReadBufferFromString in(out.str()); auto sparse_index_data = TextIndexSerialization::deserializeHeader(in); - EXPECT_EQ(sparse_index_data.version, static_cast(TextIndexHeader::Version::WithCodec)); + EXPECT_EQ(sparse_index_data.version, MergeTreeTextIndexSerializationVersion::V1_WithCodec); EXPECT_EQ(sparse_index_data.codec_type, IPostingListCodec::Type::Bitpacking); EXPECT_EQ(sparse_index_data.sparse_index.size(), 1u); EXPECT_EQ(assert_cast(*sparse_index_data.sparse_index.tokens).getDataAt(0), "alpha"); @@ -3637,7 +3639,7 @@ TEST(PostingListCursorTest, TextIndexHeaderPersistsCodecType) TEST(PostingListCursorTest, TextIndexHeaderInitialVersionDefaultsToNoneCodec) { WriteBufferFromOwnString out; - writeVarUInt(static_cast(TextIndexHeader::Version::Initial), out); + writeVarUInt(static_cast(MergeTreeTextIndexSerializationVersion::V0_Initial), out); writeVarUInt(1u, out); auto tokens = ColumnString::create(); @@ -3651,13 +3653,60 @@ TEST(PostingListCursorTest, TextIndexHeaderInitialVersionDefaultsToNoneCodec) ReadBufferFromString in(out.str()); auto sparse_index_data = TextIndexSerialization::deserializeHeader(in); - EXPECT_EQ(sparse_index_data.version, static_cast(TextIndexHeader::Version::Initial)); + EXPECT_EQ(sparse_index_data.version, MergeTreeTextIndexSerializationVersion::V0_Initial); EXPECT_EQ(sparse_index_data.codec_type, IPostingListCodec::Type::None); EXPECT_EQ(sparse_index_data.sparse_index.size(), 1u); EXPECT_EQ(assert_cast(*sparse_index_data.sparse_index.tokens).getDataAt(0), "beta"); EXPECT_EQ(assert_cast(*sparse_index_data.sparse_index.offsets_in_file).getData()[0], 7u); } +TEST(PostingListCursorTest, TextIndexHeaderWriteInitialVersionOmitsCodec) +{ + auto tokens = ColumnString::create(); + tokens->insert("gamma"); + + auto offsets = ColumnUInt64::create(); + offsets->insertValue(13); + + DictionarySparseIndex sparse_index(tokens->getPtr(), offsets->getPtr()); + + WriteBufferFromOwnString out_initial; + TextIndexSerialization::serializeHeader( + MergeTreeTextIndexSerializationVersion::V0_Initial, + sparse_index, IPostingListCodec::Type::None, /*has_positions=*/ false, out_initial); + + WriteBufferFromOwnString out_with_codec; + TextIndexSerialization::serializeHeader( + MergeTreeTextIndexSerializationVersion::V1_WithCodec, + sparse_index, IPostingListCodec::Type::None, /*has_positions=*/ false, out_with_codec); + + /// The `Initial` header omits the single-byte codec type, so it is exactly one byte shorter. + EXPECT_EQ(out_initial.str().size() + 1, out_with_codec.str().size()); + + WriteBufferFromOwnString out_with_positions; + TextIndexSerialization::serializeHeader( + MergeTreeTextIndexSerializationVersion::V2_WithPositions, + sparse_index, IPostingListCodec::Type::None, /*has_positions=*/ true, out_with_positions); + + /// The `WithPositions` header adds the single-byte positions flag on top of the codec type. + EXPECT_EQ(out_with_codec.str().size() + 1, out_with_positions.str().size()); + + ReadBufferFromString in(out_initial.str()); + auto sparse_index_data = TextIndexSerialization::deserializeHeader(in); + + EXPECT_EQ(sparse_index_data.version, MergeTreeTextIndexSerializationVersion::V0_Initial); + EXPECT_EQ(sparse_index_data.codec_type, IPostingListCodec::Type::None); + EXPECT_EQ(sparse_index_data.sparse_index.size(), 1u); + EXPECT_EQ(assert_cast(*sparse_index_data.sparse_index.tokens).getDataAt(0), "gamma"); + EXPECT_EQ(sparse_index_data.sparse_index.getOffsetInFile(0), 13u); + + ReadBufferFromString in_with_positions(out_with_positions.str()); + auto with_positions_data = TextIndexSerialization::deserializeHeader(in_with_positions); + + EXPECT_EQ(with_positions_data.version, MergeTreeTextIndexSerializationVersion::V2_WithPositions); + EXPECT_TRUE(with_positions_data.has_positions); +} + // Section: row_offset beyond UInt32::max must throw — doc IDs are 32-bit, and // `values[i] - row_offset` would otherwise underflow `size_t` and write OOB. // Skipped under debug/sanitizers: `LOGICAL_ERROR` aborts there, so `EXPECT_THROW` diff --git a/src/Storages/MergeTree/tests/gtest_text_index_front_coding.cpp b/src/Storages/MergeTree/tests/gtest_text_index_front_coding.cpp new file mode 100644 index 000000000000..3a99606c124f --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_text_index_front_coding.cpp @@ -0,0 +1,177 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int CORRUPTED_DATA; +} + +namespace +{ + +using TokensFormat = TextIndexSerialization::TokensFormat; + +/// Appends a front-coded token: common-prefix length, suffix size, suffix bytes. +void writeFrontCodedToken(WriteBuffer & out, UInt64 lcp, UInt64 data_size, std::string_view suffix) +{ + writeVarUInt(lcp, out); + writeVarUInt(data_size, out); + out.write(suffix.data(), suffix.size()); +} + +/// Returns the error code thrown while deserializing the dictionary block, or 0 if none. +int deserializeTokensErrorCode(const std::string & buffer) +{ + ReadBufferFromMemory in(buffer.data(), buffer.size()); + try + { + TextIndexSerialization::deserializeTokens(in); + } + catch (const Exception & e) + { + return e.code(); + } + return 0; +} + +std::vector deserializeFrontCodedTokens(const std::string & buffer) +{ + ReadBufferFromMemory in(buffer.data(), buffer.size()); + auto [column, format] = TextIndexSerialization::deserializeTokens(in); + + EXPECT_EQ(format, static_cast(TokensFormat::FrontCodedStrings)); + + /// Cumulative offsets, no terminating zeros: token i is the range [offsets[i - 1], offsets[i]). + const auto & string_column = assert_cast(*column); + const auto & chars = string_column.getChars(); + const auto & offsets = string_column.getOffsets(); + + std::vector tokens; + size_t begin = 0; + for (size_t i = 0; i < string_column.size(); ++i) + { + size_t end = offsets[i]; + tokens.emplace_back(reinterpret_cast(chars.data()) + begin, end - begin); + begin = end; + } + return tokens; +} + +} + +/// A valid block must round-trip without being rejected by the hardening checks. +TEST(TextIndexFrontCoding, ValidBlockIsAccepted) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(3, out); /// num_tokens + + /// First token is stored verbatim; the rest are front-coded against the previous one. + writeVarUInt(3, out); + out.write("car", 3); + writeFrontCodedToken(out, /*lcp=*/ 3, /*data_size=*/ 1, "d"); + writeFrontCodedToken(out, /*lcp=*/ 3, /*data_size=*/ 1, "e"); + + const auto tokens = deserializeFrontCodedTokens(out.str()); + ASSERT_EQ(tokens.size(), 3u); + EXPECT_EQ(tokens[0], "car"); + EXPECT_EQ(tokens[1], "card"); + EXPECT_EQ(tokens[2], "care"); +} + +/// The exact scenario from the security report: 0xFFFFFFFFFFFFFF00 + 0x100 wraps to 0. +TEST(TextIndexFrontCoding, LcpPlusDataSizeOverflowIsRejected) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(2, out); /// num_tokens + + writeVarUInt(4, out); + out.write("test", 4); + writeFrontCodedToken(out, /*lcp=*/ 0xFFFFFFFFFFFFFF00ULL, /*data_size=*/ 0x100ULL, ""); + + EXPECT_EQ(deserializeTokensErrorCode(out.str()), ErrorCodes::CORRUPTED_DATA); +} + +/// `lcp` larger than the previous token would make the memcpy copy an attacker-controlled length. +TEST(TextIndexFrontCoding, LcpLargerThanPreviousTokenIsRejected) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(2, out); /// num_tokens + + writeVarUInt(4, out); + out.write("test", 4); + writeFrontCodedToken(out, /*lcp=*/ 100, /*data_size=*/ 0, ""); + + EXPECT_EQ(deserializeTokensErrorCode(out.str()), ErrorCodes::CORRUPTED_DATA); +} + +/// Valid `lcp`, but `data_size` so large that `lcp + data_size` overflows. +TEST(TextIndexFrontCoding, DataSizeOverflowIsRejected) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(2, out); /// num_tokens + + writeVarUInt(4, out); + out.write("test", 4); + writeFrontCodedToken(out, /*lcp=*/ 4, /*data_size=*/ 0xFFFFFFFFFFFFFFFFULL, ""); + + EXPECT_EQ(deserializeTokensErrorCode(out.str()), ErrorCodes::CORRUPTED_DATA); +} + +/// `lcp + data_size` does not overflow, but adding it to the running `offset` does. +TEST(TextIndexFrontCoding, RunningOffsetOverflowIsRejected) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(2, out); /// num_tokens + + writeVarUInt(4, out); + out.write("test", 4); + /// lcp (4) + data_size (0xFFFFFFFFFFFFFFFB) == 0xFFFFFFFFFFFFFFFF, but offset (4) + that overflows. + writeFrontCodedToken(out, /*lcp=*/ 4, /*data_size=*/ 0xFFFFFFFFFFFFFFFBULL, ""); + + EXPECT_EQ(deserializeTokensErrorCode(out.str()), ErrorCodes::CORRUPTED_DATA); +} + +/// A first token size that does not overflow but exceeds the size cap must be rejected before allocating. +TEST(TextIndexFrontCoding, FirstTokenTooLargeIsRejected) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(1, out); /// num_tokens + writeVarUInt(SerializationString::MAX_STRING_SIZE + 1, out); + + EXPECT_EQ(deserializeTokensErrorCode(out.str()), ErrorCodes::CORRUPTED_DATA); +} + +/// A reconstructed token size that does not overflow but exceeds the size cap must be rejected before allocating. +TEST(TextIndexFrontCoding, ReconstructedTokenTooLargeIsRejected) +{ + WriteBufferFromOwnString out; + writeVarUInt(static_cast(TokensFormat::FrontCodedStrings), out); + writeVarUInt(2, out); /// num_tokens + + writeVarUInt(4, out); + out.write("test", 4); + writeFrontCodedToken(out, /*lcp=*/ 4, /*data_size=*/ SerializationString::MAX_STRING_SIZE, ""); + + EXPECT_EQ(deserializeTokensErrorCode(out.str()), ErrorCodes::CORRUPTED_DATA); +} diff --git a/src/Storages/MergeTree/tests/gtest_text_index_postings_deserialization.cpp b/src/Storages/MergeTree/tests/gtest_text_index_postings_deserialization.cpp new file mode 100644 index 000000000000..922e8db7295a --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_text_index_postings_deserialization.cpp @@ -0,0 +1,112 @@ +#include + +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// Wraps `payload` the way `PostingsSerialization::serialize` writes an uncompressed posting list: +/// [VarUInt: number of bytes][portable serialization of a roaring bitmap]. +String encodePostingsPayload(const String & payload) +{ + WriteBufferFromOwnString out; + writeVarUInt(payload.size(), out); + out.write(payload.data(), payload.size()); + return out.str(); +} + +String serializePostings(const PostingList & postings) +{ + String payload(postings.getSizeInBytes(), '\0'); + postings.write(payload.data()); + return payload; +} + +PostingsSerialization makeSerialization() +{ + /// A posting list without the `IsCompressed` flag never consults the codec, but `PostingsSerialization` requires one. + auto codec = PostingListCodecFactory::createPostingListCodec(IPostingListCodec::Type::None); + return PostingsSerialization(std::move(codec), MergeTreeTextIndexSerializationVersion::V2_WithPositions); +} + +/// The whole payload is in the buffer, so `deserialize` deserializes it in place. +PostingList decodePostingsInPlace(const String & encoded) +{ + auto serialization = makeSerialization(); + ReadBufferFromString in(encoded); + return *serialization.deserialize(in, /*header=*/ 0, /*cardinality=*/ 0); +} + +/// The payload spans two buffers, so `deserialize` copies it out before deserializing. +PostingList decodePostingsAcrossBuffers(const String & encoded) +{ + auto serialization = makeSerialization(); + size_t split_at = encoded.size() / 2; + ReadBufferFromMemory head(encoded.data(), split_at); + ReadBufferFromMemory tail(encoded.data() + split_at, encoded.size() - split_at); + ConcatReadBuffer in(head, tail); + return *serialization.deserialize(in, /*header=*/ 0, /*cardinality=*/ 0); +} + +} + +/// Bounding the deserialization must not reject posting lists that the index writer produces. +TEST(TextIndexPostingsDeserializationTest, RoundTrip) +{ + PostingList empty; + + PostingList array_container; + for (uint32_t row_id = 0; row_id < 100; ++row_id) + array_container.add(row_id * 7); + + /// More than 4096 row ids per 65536-row range are stored as a bitset container. + PostingList bitset_container; + for (uint32_t row_id = 0; row_id < 30000; ++row_id) + bitset_container.add(row_id * 2); + + PostingList run_container; + run_container.addRangeClosed(1000, 500000); + run_container.runOptimize(); + + for (const auto & postings : {empty, array_container, bitset_container, run_container}) + { + const String encoded = encodePostingsPayload(serializePostings(postings)); + + auto decoded_in_place = decodePostingsInPlace(encoded); + EXPECT_EQ(decoded_in_place.cardinality(), postings.cardinality()); + EXPECT_TRUE(decoded_in_place == postings); + + auto decoded_across_buffers = decodePostingsAcrossBuffers(encoded); + EXPECT_EQ(decoded_across_buffers.cardinality(), postings.cardinality()); + EXPECT_TRUE(decoded_across_buffers == postings); + } +} + +/// A container that claims more values than the declared payload holds must not be deserialized: +/// the deserializer would read past the end of the buffer and return leaked heap bytes as row ids. +TEST(TextIndexPostingsDeserializationTest, TruncatedPayloadRejected) +{ + PostingList postings; + for (uint32_t row_id = 0; row_id < 3 * 4096; ++row_id) + postings.add(row_id * 16); + + const String payload = serializePostings(postings); + ASSERT_GT(payload.size(), 64u); + + /// Truncate the payload while the declared size stays consistent with what is actually passed in, + /// so only the container headers claim data that is not there. `Roaring::readSafe` reports this as + /// `std::runtime_error`, so assert on the common base rather than on the concrete exception type. + for (size_t size : {payload.size() / 2, payload.size() - 1}) + { + const String encoded = encodePostingsPayload(payload.substr(0, size)); + EXPECT_THROW(decodePostingsInPlace(encoded), std::exception) << "size = " << size; + EXPECT_THROW(decodePostingsAcrossBuffers(encoded), std::exception) << "size = " << size; + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 24930b88462a..f8f3e9c1c093 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -903,6 +903,7 @@ void IcebergMetadata::createInitial( if (local_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata].value) location_path = configuration_ptr->getTypeName() + "://" + configuration_ptr->getNamespace() + "/" + configuration_ptr->getRawPath().path; + auto [metadata_content_object, metadata_content] = createEmptyMetadataFile( location_path, *columns, partition_by, order_by, local_context, configuration_ptr->getDataLakeSettings()[DataLakeStorageSetting::iceberg_format_version]); auto compression_method_str = local_context->getSettingsRef()[Setting::iceberg_metadata_compression_method].value; @@ -914,6 +915,15 @@ void IcebergMetadata::createInitial( auto filename = fmt::format("{}metadata/v1{}.metadata.json", configuration_ptr->getRawPath().path, compression_suffix); + if (catalog) + { + /// Register the namespace before any files are written (but after all local + /// validation, so a rejected CREATE leaves no trace in the catalog): a catalog + /// that shares its storage view with the data (e.g. SeaweedFS) refuses to create + /// a namespace over the plain directory those files would leave behind. + catalog->createNamespaceIfNotExists(DataLake::parseTableName(table_id_.getTableName()).first, location_path); + } + try { writeMessageToFile(metadata_content, filename, object_storage, local_context, "*", "", compression_method); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index 0a02fd75c04f..80ac7b27d06d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -151,11 +152,15 @@ PruningReturnStatus ManifestFilesPruner::canBePruned( { const auto & partition_value = entry->parsed_entry->partition_key_value; std::vector index_value(partition_value.begin(), partition_value.end()); - for (auto & field : index_value) + for (size_t i = 0; i < index_value.size(); ++i) { + auto & field = index_value[i]; + const auto & type = partition_key->data_types.at(i); // NULL_LAST if (field.isNull()) field = POSITIVE_INFINITY; + else if (field.getType() == Field::Types::Int64 && WhichDataType(type).isDateTime64()) /// clickhouse used to write timestamp as simple long in avro + field = DecimalField(field.safeGet(), getDecimalScale(*type)); } bool can_be_true = partition_key_condition->mayBeTrueInRange( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 723ef8b2d00e..f6c989b13ad3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -482,6 +483,9 @@ IcebergSchemaProcessor::getComplexTypeFromObject( ContextPtr context_, bool is_subfield_of_root) { + /// The schema comes from the table metadata and can be nested arbitrarily deeply. + checkStackSize(); + String type_name = type->getValue(f_type); if (type_name == f_list) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 48f088cbe96e..613728590c33 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -9,9 +9,11 @@ #include #include #include -#include +#include #include #include +#include +#include #include #include #include @@ -650,7 +652,6 @@ Poco::Dynamic::Var getAvroType(DataTypePtr type) case TypeIndex::UInt64: case TypeIndex::Int64: case TypeIndex::DateTime: - case TypeIndex::DateTime64: case TypeIndex::Time: return "long"; case TypeIndex::Time64: @@ -661,6 +662,17 @@ Poco::Dynamic::Var getAvroType(DataTypePtr type) else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); } + case TypeIndex::DateTime64: + { + if (getDecimalScale(*type) != 6) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported type for iceberg {}", type->getName()); + + Poco::JSON::Object::Ptr timestamp_type = new Poco::JSON::Object; + timestamp_type->set("type", "long"); + timestamp_type->set("logicalType", "timestamp-micros"); + timestamp_type->set("adjust-to-utc", assert_cast(*type).hasExplicitTimeZone()); + return timestamp_type; + } case TypeIndex::Float32: return "float"; case TypeIndex::Float64: diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/Types.h b/src/Storages/ObjectStorage/DataLakes/Paimon/Types.h index f74d40c846b5..759730ee8593 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/Types.h +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/Types.h @@ -270,10 +270,6 @@ struct DataType type.root_type = RootDataType::ARRAY; auto nested_type = parse(inner_json_object, "element"); type.clickhouse_data_type = std::make_shared(nested_type.clickhouse_data_type); - if (nullable) - { - type.clickhouse_data_type = std::make_shared(type.clickhouse_data_type); - } } else if (real_type == "MAP") { @@ -281,15 +277,17 @@ struct DataType auto key_type = parse(inner_json_object, "key"); auto value_type = parse(inner_json_object, "value"); type.clickhouse_data_type = std::make_shared(key_type.clickhouse_data_type, value_type.clickhouse_data_type); - if (nullable) - { - type.clickhouse_data_type = std::make_shared(type.clickhouse_data_type); - } } else { throw Exception(); } + /// ClickHouse forbids Nullable(Array) and Nullable(Map), so a nullable composite is kept + /// unwrapped; the reader maps a NULL composite to an empty one. + if (nullable) + { + type.clickhouse_data_type = makeNullableSafe(type.clickhouse_data_type); + } return type; } } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index abd0cb896222..46b107d9f2d7 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,8 @@ #include #include #include +#include +#include #include #include #include @@ -51,6 +54,7 @@ namespace DB namespace Setting { extern const SettingsBool optimize_count_from_files; + extern const SettingsBool throw_on_hive_partitioning_resolution_failure; extern const SettingsBool use_hive_partitioning; extern const SettingsInt64 delta_lake_snapshot_start_version; extern const SettingsInt64 delta_lake_snapshot_end_version; @@ -66,6 +70,11 @@ namespace ErrorCodes extern const int FILE_ALREADY_EXISTS; } +namespace FailPoints +{ + extern const char datalake_simulate_missing_table_state[]; +} + String StorageObjectStorage::getPathSample(ContextPtr context) { const auto path = configuration->getRawPath(); @@ -220,9 +229,12 @@ StorageObjectStorage::StorageObjectStorage( configuration->check(context); - /// FIXME: We need to call getPathSample() lazily on select - /// in case it failed to be initialized in constructor. - if (updated_configuration && sample_path.empty() && need_resolve_sample_path && !configuration->getPartitionStrategy()) + /// Resolving the sample path requires listing the object storage. Defer it to the first use + /// of the table, so that CREATE, ATTACH and server startup do not depend on the endpoint. + hive_partitioning_sample_path_deferred = !is_table_function && need_resolve_sample_path && !need_resolve_columns_or_format; + + if (updated_configuration && sample_path.empty() && need_resolve_sample_path + && !hive_partitioning_sample_path_deferred && !configuration->getPartitionStrategy()) { try { @@ -315,14 +327,20 @@ StorageObjectStorage::StorageObjectStorage( metadata.partition_key = configuration->getPartitionStrategy()->getPartitionKeyDescription(); } - metadata.setVirtuals(VirtualColumnUtils::getVirtualsForFileLikeStorage( - metadata.columns, + metadata.setVirtuals(createVirtualColumns(metadata.columns, sample_path, context)); + + setInMemoryMetadata(metadata); +} + +VirtualColumnsDescription StorageObjectStorage::createVirtualColumns( + ColumnsDescription & columns, const std::string & sample_path, const ContextPtr & context) const +{ + return VirtualColumnUtils::getVirtualsForFileLikeStorage( + columns, context, format_settings, configuration->getPartitionStrategyType(), - sample_path)); - - setInMemoryMetadata(metadata); + sample_path); } String StorageObjectStorage::getName() const @@ -400,10 +418,82 @@ configuration->update(object_storage, query_context); return configuration->getExternalMetadata(); } +void StorageObjectStorage::resolveHivePartitioningSamplePathIfDeferred(const ContextPtr & query_context) +{ + if (!hive_partitioning_sample_path_deferred) + return; + + std::lock_guard lock(hive_partitioning_resolution_mutex); + if (hive_partitioning_sample_path_resolved) + return; + + /// Listing the storage happens on behalf of the triggering query, so it must use its context. + /// Rebuilding the client with any other one would ignore that session's credential restriction. + auto access_context = Context::createCopy(query_context); + access_context->setSetting("use_hive_partitioning", true); + + /// The resulting column types are shared by every session, so infer them from the server + /// settings instead of the ones of whichever query happens to resolve the table first. + auto inference_context = Context::createCopy(Context::getGlobalContextInstance()); + inference_context->setSetting("use_hive_partitioning", true); + + std::string sample_path; + try + { + configuration->update(object_storage, access_context); + sample_path = getPathSample(access_context); + } + catch (...) + { + /// A query running without hive partitioning may silently return different results. + if (query_context->getSettingsRef()[Setting::throw_on_hive_partitioning_resolution_failure]) + throw; + + /// An endpoint failure degrades only the triggering query and is retried by the next one. + LOG_WARNING( + log, + "Failed to list object storage, cannot use hive partitioning. " + "Error: {}", + getCurrentExceptionMessage(true)); + return; + } + + auto current_metadata = getInMemoryMetadataPtr(query_context, false); + auto new_metadata = *current_metadata; + + /// Errors thrown below stay unresolved on purpose, so they are reported until they go away. + auto [new_hive_partition_columns, new_file_columns] = HivePartitioningUtils::setupHivePartitioningForObjectStorage( + new_metadata.columns, + configuration, + sample_path, + /* inferred_schema */ false, + format_settings, + inference_context); + + if (!new_metadata.columns.empty() && new_file_columns.empty()) + { + throw Exception(ErrorCodes::INCORRECT_DATA, + "File without physical columns is not supported. Please try it with `use_hive_partitioning=0` and or `partition_strategy=wildcard`. File {}", + sample_path); + } + + hive_partition_columns_to_read_from_file_path = std::move(new_hive_partition_columns); + file_columns = std::move(new_file_columns); + + new_metadata.setVirtuals(createVirtualColumns(new_metadata.columns, sample_path, inference_context)); + setInMemoryMetadata(new_metadata); + + hive_partitioning_sample_path_resolved = true; +} + void StorageObjectStorage::updateExternalDynamicMetadataIfExists(ContextPtr query_context) { if (!configuration->isDataLakeConfiguration()) + { + /// Called before query analysis, so the hive virtual columns are visible to the triggering query. + resolveHivePartitioningSamplePathIfDeferred(query_context); return; + } /// Always force an update to pick up the latest snapshot version. /// Using if_not_updated_before=true would leave latest_snapshot_version @@ -467,13 +557,69 @@ std::optional StorageObjectStorage::totalBytes(ContextPtr query_context) void StorageObjectStorage::read( QueryPlan & query_plan, const Names & column_names, - const StorageSnapshotPtr & storage_snapshot, + const StorageSnapshotPtr & storage_snapshot_, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum /*processed_stage*/, size_t max_block_size, size_t num_streams) { + /// Some paths bypass updateExternalDynamicMetadataIfExists and reach read with the resolution pending. + resolveHivePartitioningSamplePathIfDeferred(local_context); + + auto storage_snapshot = storage_snapshot_; + + /// Test-only: emulate a snapshot that reached the read step without the pinned + /// datalake_table_state (the concurrent-commit race this PR fixes), so the + /// regression test reproduces deterministically. + fiu_do_on(FailPoints::datalake_simulate_missing_table_state, + { + if (configuration->isDataLakeConfiguration() && storage_snapshot->metadata + && storage_snapshot->metadata->datalake_table_state.has_value()) + { + auto stripped = std::make_shared(*storage_snapshot->metadata); + stripped->datalake_table_state.reset(); + storage_snapshot = std::make_shared(*this, std::move(stripped)); + } + }); + + /// The read pipeline needs a single, internally consistent metadata snapshot: the requested + /// columns (prepareReadingFromFormat below), the field-id mapping used to list/read files, and + /// the read-in-order sorting key must all come from the SAME data lake snapshot. Normally + /// updateExternalDynamicMetadataIfExists pins datalake_table_state during analysis, but a + /// concurrent commit (TOCTOU between setInMemoryMetadata and getInMemoryMetadataPtr) can leave + /// it unset here. Pin one coherent snapshot now, before prepareReadingFromFormat, so every + /// consumer observes the same state. Mirrors updateExternalDynamicMetadataIfExists. + if (configuration->isDataLakeConfiguration() && storage_snapshot->metadata + && !storage_snapshot->metadata->datalake_table_state.has_value()) + { + /// Initialize underlying datalake metadata if it has not been yet. + /// Cluster table function workers and other paths that bypass the analyzer + /// reach `read` without having had `update`/`updateExternalDynamicMetadataIfExists` + /// called, so `getTableStateSnapshot` would otherwise hit an "uninitialized" assertion. + if (is_table_function) + configuration->lazyInitializeIfNeeded(object_storage, local_context); + else + configuration->update(object_storage, local_context); + + if (auto state = configuration->getTableStateSnapshot(local_context)) + { + StorageInMemoryMetadata pinned = *storage_snapshot->metadata; + pinned.setDataLakeTableState(*state); + + /// Reload columns and sorting key from the same state so they cannot diverge. + if (configuration->shouldReloadSchemaForConsistency(local_context)) + { + if (auto rebuilt = configuration->buildStorageMetadataFromState(*state, local_context)) + pinned = rebuilt->withVirtuals(VirtualColumnUtils::getVirtualsForFileLikeStorage( + rebuilt->columns, local_context, format_settings, configuration->getPartitionStrategyType())); + } + + storage_snapshot = std::make_shared( + *this, std::make_shared(std::move(pinned))); + } + } + if (distributed_processing && local_context->getSettingsRef()[Setting::max_streams_for_files_processing_in_cluster_functions]) num_streams = clampClusterFunctionNumStreams( local_context->getSettingsRef()[Setting::max_streams_for_files_processing_in_cluster_functions]); @@ -993,6 +1139,9 @@ Pipe StorageObjectStorage::executeCommand(const String & command_name, const AST void StorageObjectStorage::alter(const AlterCommands & params, ContextPtr context, AlterLockHolder & /*alter_lock_holder*/) { + /// Do not interleave with the hive partitioning resolution, which also updates the metadata. + std::lock_guard lock(hive_partitioning_resolution_mutex); + auto metadata_snapshot = getInMemoryMetadataPtr(context, false); StorageInMemoryMetadata new_metadata = *metadata_snapshot; params.apply(new_metadata, context); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 5f1db8f2a527..122cd5ffa625 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -19,6 +19,7 @@ #include #include +#include #include namespace DB @@ -228,6 +229,11 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation /// Get path sample for hive partitioning implementation. String getPathSample(ContextPtr context); + /// Resolve the deferred hive partitioning sample path. Requires listing the object storage. + void resolveHivePartitioningSamplePathIfDeferred(const ContextPtr & query_context); + + VirtualColumnsDescription createVirtualColumns(ColumnsDescription & columns, const std::string & sample_path, const ContextPtr & context) const; + /// Creates ReadBufferIterator for schema inference implementation. static std::unique_ptr createReadBufferIterator( const ObjectStoragePtr & object_storage, @@ -253,6 +259,12 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation NamesAndTypesList hive_partition_columns_to_read_from_file_path; NamesAndTypesList file_columns; + /// Set only in the constructor when hive partitioning detection is deferred to the first use. + bool hive_partitioning_sample_path_deferred = false; + std::mutex hive_partitioning_resolution_mutex; + /// Stays false on a failed resolution, so the next query retries it. + bool hive_partitioning_sample_path_resolved TSA_GUARDED_BY(hive_partitioning_resolution_mutex) = false; + LoggerPtr log; std::shared_ptr catalog; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index c54be45de9a0..6936fa38b64f 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -371,6 +371,25 @@ std::shared_ptr StorageObjectStorageSource::createFileIterator( } else if (configuration->supportsFileIterator()) { + /// For datalake configurations, ensure datalake_table_state is present in the metadata + /// before calling iterate(). The state is normally set by + /// updateExternalDynamicMetadataIfExists() during query analysis, but it can be missing + /// due to race conditions between concurrent queries (TOCTOU between setInMemoryMetadata + /// and getInMemoryMetadataPtr), or when called from code paths that bypass the + /// analyzer/interpreter (e.g. schema inference, cluster functions with nullptr metadata). + if (configuration->isDataLakeConfiguration() + && (!storage_metadata || !storage_metadata->datalake_table_state.has_value())) + { + if (auto state = configuration->getTableStateSnapshot(local_context)) + { + auto fixed_metadata = storage_metadata + ? std::make_shared(*storage_metadata) + : std::make_shared(); + fixed_metadata->setDataLakeTableState(*state); + storage_metadata = std::move(fixed_metadata); + } + } + auto iter = configuration->iterate( filter_actions_dag, filter_actions_dag ? std::function{} : file_progress_callback, diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp index fa60b7b66327..0e764595e18c 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -346,7 +347,12 @@ void ObjectStorageQueueTableMetadata::checkImmutableFieldsEquals(const ObjectSto } } - if (columns != from_zk.columns) + /// Different versions serialize the same columns to a different text: the redundant parentheses + /// the user has written around a column `DEFAULT`, `CODEC` or `TTL` expression were kept by some + /// versions and are suppressed now (`IAST::FormatSettings::ignore_redundant_parentheses`), + /// so when the texts differ, compare the columns structurally before rejecting. + if (columns != from_zk.columns + && ColumnsDescription::parse(columns) != ColumnsDescription::parse(from_zk.columns)) throw Exception( ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in columns. " diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index e84ee800af46..a78358a745d9 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -152,6 +152,7 @@ namespace ErrorCodes extern const int KEEPER_EXCEPTION; extern const int QUERY_WAS_CANCELLED; extern const int TIMEOUT_EXCEEDED; + extern const int TABLE_IS_DROPPED; } namespace @@ -430,12 +431,18 @@ void StorageObjectStorageQueue::startup() /// Create metadata in keeper if it does not exits yet. /// Create a persistent node for the table under /registry node. bool created_new_metadata = false; - files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate( + /// Keep a local handle: the factory call and the startup below do keeper I/O, which must not + /// run under `mutex`. + auto metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate( zookeeper_name, zk_path, std::move(temp_metadata), getStorageID(), created_new_metadata); + { + std::lock_guard lock(mutex); + files_metadata = metadata; + } /// Register the metadata in startup(), unregister in shutdown. /// (If startup is never called, shutdown also won't be called.) @@ -452,6 +459,7 @@ void StorageObjectStorageQueue::startup() /* is_drop */created_new_metadata, /* keep_data_in_keeper */false); + std::lock_guard lock(mutex); files_metadata.reset(); } }); @@ -470,7 +478,7 @@ void StorageObjectStorageQueue::startup() }); /// Start background tasks. - files_metadata->startup(); + metadata->startup(); for (auto & task : streaming_tasks) task->activateAndSchedule(); @@ -513,11 +521,19 @@ void StorageObjectStorageQueue::shutdown(bool is_drop) tryLogCurrentException(log); } - if (files_metadata) + /// Drop the handle before the keeper cleanup below, so a concurrent reader sees the + /// shut-down state. The local copy keeps the metadata alive for that cleanup. + std::shared_ptr metadata; + { + std::lock_guard lock(mutex); + metadata = std::move(files_metadata); + } + + if (metadata) { try { - files_metadata->unregisterActive(getStorageID()); + metadata->unregisterActive(getStorageID()); } catch (...) { @@ -525,8 +541,6 @@ void StorageObjectStorageQueue::shutdown(bool is_drop) } ObjectStorageQueueMetadataFactory::instance().remove(zookeeper_name, zk_path, getStorageID(), is_drop, keep_data_in_keeper); - - files_metadata.reset(); } LOG_TRACE(log, "Shut down storage"); } @@ -652,7 +666,11 @@ void ReadFromObjectStorageQueue::initializePipeline(QueryPipelineBuilder & pipel { Pipes pipes; - size_t processing_threads_num = storage->getTableMetadata().processing_threads_num; + auto metadata = storage->tryGetFilesMetadata(); + if (!metadata) + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {} is dropped or detached", storage->getStorageID()); + + size_t processing_threads_num = metadata->getTableMetadata().processing_threads_num; createIterator(nullptr); @@ -666,6 +684,7 @@ void ReadFromObjectStorageQueue::initializePipeline(QueryPipelineBuilder & pipel parser_shared_resources, progress, iterator, + metadata, max_block_size, context, commit_once_processed)); @@ -686,6 +705,7 @@ std::shared_ptr StorageObjectStorageQueue::createSourc FormatParserSharedResourcesPtr parser_shared_resources, ProcessingProgressPtr progress_, std::shared_ptr file_iterator, + std::shared_ptr metadata, size_t max_block_size, ContextPtr local_context, bool commit_once_processed, @@ -717,7 +737,7 @@ std::shared_ptr StorageObjectStorageQueue::createSourc parser_shared_resources, commit_settings_copy, after_processing_settings_copy, - files_metadata, + std::move(metadata), local_context, max_block_size, shutdown_called, @@ -745,6 +765,12 @@ void StorageObjectStorageQueue::threadFunc(size_t streaming_tasks_index) auto component_guard = Coordination::setCurrentComponent("StorageObjectStorageQueue::threadFunc"); + /// One snapshot for the whole poll. Absent means the table is shutting down: nothing to poll, + /// and no rescheduling either, since shutdown() deactivates the tasks. + auto metadata = tryGetFilesMetadata(); + if (!metadata) + return; + const auto storage_id = getStorageID(); if (getContext()->getS3QueueDisableStreaming()) @@ -765,7 +791,7 @@ void StorageObjectStorageQueue::threadFunc(size_t streaming_tasks_index) { LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); - files_metadata->registerActive(storage_id); + metadata->registerActive(storage_id); if (streamToViews(streaming_tasks_index)) { @@ -810,7 +836,7 @@ void StorageObjectStorageQueue::threadFunc(size_t streaming_tasks_index) { try { - files_metadata->unregisterActive(storage_id); + metadata->unregisterActive(storage_id); } catch (...) { @@ -827,6 +853,15 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) Stopwatch watch; + /// A concurrent shutdown drops the metadata handle: nothing was consumed, and this is a + /// background poll, so returning is correct here rather than raising an error. + auto metadata = tryGetFilesMetadata(); + if (!metadata) + { + LOG_TEST(log, "Metadata is not available (table is shutting down)"); + return false; + } + auto table_id = getStorageID(); auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); if (!table) @@ -874,8 +909,9 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) } size_t total_rows = 0; - const size_t processing_threads_num = getTableMetadata().processing_threads_num; - const bool parallel_inserts = getTableMetadata().parallel_inserts; + const auto & table_metadata = metadata->getTableMetadata(); + const size_t processing_threads_num = table_metadata.processing_threads_num; + const bool parallel_inserts = table_metadata.parallel_inserts; const size_t threads = parallel_inserts ? 1 : processing_threads_num; LOG_TEST(log, "Using {} processing threads (processing_threads_num: {}, parallel_inserts: {}, async deduplicate: {})", @@ -928,6 +964,7 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) parser_shared_resources, processing_progress, file_iterator, + metadata, DBMS_DEFAULT_BUFFER_SIZE, queue_context, /*commit_once_processed=*/false, @@ -963,6 +1000,7 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) /*insert_succeeded=*/ false, rows, sources, + *metadata, transaction_start_time, getCurrentExceptionMessage(true), getCurrentExceptionCode()); @@ -998,7 +1036,7 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) throw Exception(ErrorCodes::FAULT_INJECTED, "Failed after insert"); }); - commit(/*insert_succeeded=*/ true, rows, sources, transaction_start_time); + commit(/*insert_succeeded=*/ true, rows, sources, *metadata, transaction_start_time); file_iterator->releaseFinishedBuckets(); file_iterator->refreshExpiringBucketLocks(); max_files_override = 0; @@ -1009,7 +1047,9 @@ bool StorageObjectStorageQueue::streamToViews(size_t streaming_tasks_index) return total_rows > 0; } -void StorageObjectStorageQueue::postProcess(const StoredObjects & successful_objects) const +void StorageObjectStorageQueue::postProcess( + const StoredObjects & successful_objects, + const ObjectStorageQueueMetadata & metadata) const { std::optional post_processor; @@ -1021,7 +1061,7 @@ void StorageObjectStorageQueue::postProcess(const StoredObjects & successful_obj type, object_storage, getName(), - files_metadata->getTableMetadata(), + metadata.getTableMetadata(), after_processing_settings); } @@ -1035,6 +1075,7 @@ void StorageObjectStorageQueue::commit( bool insert_succeeded, size_t inserted_rows, std::vector> & sources, + const ObjectStorageQueueMetadata & metadata, time_t transaction_start_time, const std::string & exception_message, int error_code) const @@ -1054,7 +1095,7 @@ void StorageObjectStorageQueue::commit( } // Use partition-based processing for both HIVE and REGEX modes - bool has_partitioning = files_metadata->getPartitioningMode() != ObjectStorageQueuePartitioningMode::NONE; + bool has_partitioning = metadata.getPartitioningMode() != ObjectStorageQueuePartitioningMode::NONE; if (has_partitioning) ObjectStorageQueueSource::preparePartitionProcessedRequests(requests, last_processed_file_per_partition); else @@ -1069,9 +1110,9 @@ void StorageObjectStorageQueue::commit( ProfileEvents::increment(ProfileEvents::ObjectStorageQueueCommitRequests, requests.size()); if (!successful_objects.empty() - && files_metadata->getTableMetadata().after_processing != ObjectStorageQueueAction::KEEP) + && metadata.getTableMetadata().after_processing != ObjectStorageQueueAction::KEEP) { - postProcess(successful_objects); + postProcess(successful_objects, metadata); } auto context = getContext(); @@ -1321,7 +1362,11 @@ void StorageObjectStorageQueue::checkAlterIsPossible(const AlterCommands & comma if (!new_metadata.hasSettingsChanges()) throw Exception(ErrorCodes::LOGICAL_ERROR, "No settings changes"); - const auto mode = getTableMetadata().getMode(); + auto metadata = tryGetFilesMetadata(); + if (!metadata) + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {} is dropped or detached", getStorageID()); + + const auto mode = metadata->getTableMetadata().getMode(); const auto & new_settings = new_metadata.settings_changes->as().changes; for (const auto & setting : new_settings) @@ -1431,7 +1476,11 @@ void StorageObjectStorageQueue::alter( SettingsChanges changed_settings; std::set new_settings_set; - const auto mode = getTableMetadata().getMode(); + auto metadata = tryGetFilesMetadata(); + if (!metadata) + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {} is dropped or detached", getStorageID()); + + const auto mode = metadata->getTableMetadata().getMode(); const size_t dependencies_count = getDependencies(); bool requires_detached_mv = false; @@ -1505,7 +1554,7 @@ void StorageObjectStorageQueue::alter( /// Alter settings which are stored in keeper. ObjectStorageQueueMetadata::getKeeperRetriesControl(log).retryLoop([&] { - files_metadata->alterSettings(changed_settings, local_context); + metadata->alterSettings(changed_settings, local_context); }); /// Alter settings which are not stored in keeper. @@ -1561,7 +1610,7 @@ void StorageObjectStorageQueue::alter( deduplication_v2 = change.value.safeGet(); } - files_metadata->updateSettings(changed_settings); + metadata->updateSettings(changed_settings); /// Reset streaming_iterator as it can hold state which we could have just altered. if (requires_detached_mv) streaming_file_iterator.reset(); @@ -1576,11 +1625,10 @@ zkutil::ZooKeeperPtr StorageObjectStorageQueue::getZooKeeper() const return getContext()->getDefaultOrAuxiliaryZooKeeper(zookeeper_name); } -const ObjectStorageQueueTableMetadata & StorageObjectStorageQueue::getTableMetadata() const +std::shared_ptr StorageObjectStorageQueue::tryGetFilesMetadata() const { - if (!files_metadata) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Files metadata is empty"); - return files_metadata->getTableMetadata(); + std::lock_guard lock(mutex); + return files_metadata; } std::shared_ptr @@ -1588,7 +1636,11 @@ StorageObjectStorageQueue::createFileIterator(ContextPtr local_context, const Ac { auto component_guard = Coordination::setCurrentComponent("StorageObjectStorageQueue::createFileIterator"); - const auto & table_metadata = getTableMetadata(); + auto metadata = tryGetFilesMetadata(); + if (!metadata) + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {} is dropped or detached", getStorageID()); + + const auto & table_metadata = metadata->getTableMetadata(); bool file_deletion_enabled = table_metadata.getMode() == ObjectStorageQueueMode::UNORDERED && (table_metadata.tracked_files_ttl_sec || table_metadata.tracked_files_limit); @@ -1602,7 +1654,7 @@ StorageObjectStorageQueue::createFileIterator(ContextPtr local_context, const Ac auto metadata_snapshot = getInMemoryMetadataPtr(local_context, false); return std::make_shared( - files_metadata, + metadata, object_storage, configuration, getStorageID(), @@ -1623,11 +1675,16 @@ ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const /// (because of the inconvenience of keeping them in sync with ObjectStorageQueueTableMetadata), /// so let's reconstruct. ObjectStorageQueueSettings settings; - /// If startup() for a table was not called, just use the default queue settings + /// If startup() for a table was not called, just use the default queue settings. + /// The same holds after shutdown(), which drops the metadata handle while `startup_finished` stays set. if (!startup_finished) return settings; - const auto & table_metadata = getTableMetadata(); + auto metadata = tryGetFilesMetadata(); + if (!metadata) + return settings; + + const auto & table_metadata = metadata->getTableMetadata(); settings[ObjectStorageQueueSetting::mode] = table_metadata.mode; settings[ObjectStorageQueueSetting::after_processing] = table_metadata.after_processing; if (zookeeper_name == zkutil::DEFAULT_ZOOKEEPER_NAME) @@ -1643,12 +1700,12 @@ ObjectStorageQueueSettings StorageObjectStorageQueue::getSettings() const settings[ObjectStorageQueueSetting::tracked_files_limit] = table_metadata.tracked_files_limit; settings[ObjectStorageQueueSetting::buckets] = table_metadata.buckets; - auto cleanup_interval_ms = files_metadata->getCleanupIntervalMS(); + auto cleanup_interval_ms = metadata->getCleanupIntervalMS(); settings[ObjectStorageQueueSetting::cleanup_interval_min_ms] = static_cast(cleanup_interval_ms.first); settings[ObjectStorageQueueSetting::cleanup_interval_max_ms] = static_cast(cleanup_interval_ms.second); - settings[ObjectStorageQueueSetting::persistent_processing_node_ttl_seconds] = static_cast(files_metadata->getPersistentProcessingNodeTTLSeconds()); - settings[ObjectStorageQueueSetting::use_persistent_processing_nodes] = files_metadata->usePersistentProcessingNode(); - const auto & file_statuses_cache = files_metadata->getFileStatusesCache(); + settings[ObjectStorageQueueSetting::persistent_processing_node_ttl_seconds] = static_cast(metadata->getPersistentProcessingNodeTTLSeconds()); + settings[ObjectStorageQueueSetting::use_persistent_processing_nodes] = metadata->usePersistentProcessingNode(); + const auto & file_statuses_cache = metadata->getFileStatusesCache(); settings[ObjectStorageQueueSetting::metadata_cache_size_bytes] = file_statuses_cache.maxSizeInBytes(); settings[ObjectStorageQueueSetting::metadata_cache_size_elements] = file_statuses_cache.maxCount(); @@ -1772,9 +1829,13 @@ void StorageObjectStorageQueue::waitForPathToBeProcessed( /// For unordered mode each file gets its own node under processed/ and failed/. /// For ordered mode the processed pointer is a shared node whose *data* is updated, /// while the failed node is still per-file. - const bool is_ordered = files_metadata->getTableMetadata().getMode() == ObjectStorageQueueMode::ORDERED; + auto metadata = tryGetFilesMetadata(); + if (!metadata) + throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {} is dropped or detached", getStorageID()); + + const bool is_ordered = metadata->getTableMetadata().getMode() == ObjectStorageQueueMode::ORDERED; - auto file_metadata = files_metadata->getFileMetadata(path); + auto file_metadata = metadata->getFileMetadata(path); const auto & processed_node_path = file_metadata->getProcessedNodePath(); const auto & failed_node_path = file_metadata->getFailedNodePath(); @@ -1834,7 +1895,7 @@ void StorageObjectStorageQueue::waitForPathToBeProcessed( /// that occurs between the state check and watch registration. ObjectStorageQueueMetadata::getKeeperRetriesControl(log).retryLoop([&] { - auto zk = files_metadata->getZooKeeper()->getKeeper(); + auto zk = metadata->getZooKeeper()->getKeeper(); if (is_ordered) { diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h index dd55c2763db3..a05f9d0132d3 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h @@ -148,7 +148,7 @@ class StorageObjectStorageQueue : public IStorage, WithContext size_t min_insert_block_size_bytes_for_materialized_views TSA_GUARDED_BY(mutex); std::unique_ptr temp_metadata; - std::shared_ptr files_metadata; + std::shared_ptr files_metadata TSA_GUARDED_BY(mutex); StorageObjectStorageConfigurationPtr configuration; ObjectStoragePtr object_storage; @@ -175,7 +175,10 @@ class StorageObjectStorageQueue : public IStorage, WithContext bool supportsOptimizationToSubcolumns() const override { return false; } bool supportsColumnsWithDynamicStructure() const override { return true; } - const ObjectStorageQueueTableMetadata & getTableMetadata() const; + /// Returns the metadata handle, or nullptr when the table has not started up or has + /// already been shut down. The returned handle must stay in scope for as long as anything + /// borrowed from it (e.g. its table metadata) is used. + std::shared_ptr tryGetFilesMetadata() const; std::shared_ptr createFileIterator(ContextPtr local_context, const ActionsDAG::Node * predicate); std::shared_ptr createSource( @@ -184,6 +187,7 @@ class StorageObjectStorageQueue : public IStorage, WithContext FormatParserSharedResourcesPtr parser_shared_resources, ProcessingProgressPtr progress_, std::shared_ptr file_iterator, + std::shared_ptr metadata, size_t max_block_size, ContextPtr local_context, bool commit_once_processed, @@ -198,12 +202,13 @@ class StorageObjectStorageQueue : public IStorage, WithContext /// A subset of logic executed by threadFunc. bool streamToViews(size_t streaming_tasks_index); /// Apply after_processing action to successfully processed files. - void postProcess(const StoredObjects & successful_objects) const; + void postProcess(const StoredObjects & successful_objects, const ObjectStorageQueueMetadata & metadata) const; /// Commit processed files to keeper as either successful or unsuccessful. void commit( bool insert_succeeded, size_t inserted_rows, std::vector> & sources, + const ObjectStorageQueueMetadata & metadata, time_t transaction_start_time, const std::string & exception_message = {}, int error_code = 0) const; diff --git a/src/Storages/ObjectStorageQueue/tests/gtest_object_storage_queue_metadata.cpp b/src/Storages/ObjectStorageQueue/tests/gtest_object_storage_queue_metadata.cpp new file mode 100644 index 000000000000..9c5436bf2b13 --- /dev/null +++ b/src/Storages/ObjectStorageQueue/tests/gtest_object_storage_queue_metadata.cpp @@ -0,0 +1,66 @@ +#include + +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int METADATA_MISMATCH; +} + +namespace +{ + +String makeMetadataJSON(const String & default_expression) +{ + /// The `columns` payload as `ColumnsDescription::toString` produces it. The versions that kept + /// the redundant parentheses of the user (before `IAST::FormatSettings::ignore_redundant_parentheses`) + /// stored `(y + 1)` where the current version stores `y + 1`, and the stored string is compared + /// with the local one on every server restart. + String columns = "columns format version: 1\n2 columns:\n`x` UInt64\tDEFAULT\t" + default_expression + "\n`y` UInt64\n"; + + String escaped_columns; + for (char c : columns) + { + if (c == '\n') + escaped_columns += "\\n"; + else if (c == '\t') + escaped_columns += "\\t"; + else if (c == '"' || c == '\\') + { + escaped_columns += '\\'; + escaped_columns += c; + } + else + escaped_columns += c; + } + + return R"({"format_name":"CSV","columns":")" + escaped_columns + R"(","mode":"unordered","after_processing":"keep"})"; +} + +} + +TEST(ObjectStorageQueueTableMetadata, ColumnsComparisonIgnoresRedundantParentheses) +{ + auto plain = ObjectStorageQueueTableMetadata::parse(makeMetadataJSON("y + 1")); + auto parenthesized = ObjectStorageQueueTableMetadata::parse(makeMetadataJSON("(y + 1)")); + auto different = ObjectStorageQueueTableMetadata::parse(makeMetadataJSON("y + 2")); + + /// A table created by a version that stored the redundant parentheses must be accepted + /// by a version that does not store them, and vice versa. + EXPECT_NO_THROW(plain.checkEquals(parenthesized)); + EXPECT_NO_THROW(parenthesized.checkEquals(plain)); + + /// A genuinely different default expression is still rejected. + try + { + plain.checkEquals(different); + FAIL() << "Expected METADATA_MISMATCH"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::METADATA_MISMATCH); + } +} diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index 594deabfd275..276a954bd89e 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -132,7 +132,8 @@ ProjectionsDescription ProjectionsDescription::clone() const bool ProjectionDescription::operator==(const ProjectionDescription & other) const { - return name == other.name && definition_ast->formatWithSecretsOneLine() == other.definition_ast->formatWithSecretsOneLine(); + return name == other.name + && definition_ast->formatIgnoringRedundantParentheses() == other.definition_ast->formatIgnoringRedundantParentheses(); } namespace @@ -256,6 +257,11 @@ ProjectionDescription ProjectionDescription::getProjectionFromAST( if (projection_definition->name.empty()) throw Exception(ErrorCodes::INCORRECT_QUERY, "Projection must have name in definition."); + /// The name is used unescaped as a directory name (`getDirectoryName`) inside a part directory, + /// so a '/' in it would address files outside of the part and outside of the data directory. + if (projection_definition->name.contains('/')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Projection name ({}) cannot contain '/'", projection_definition->name); + ProjectionDescription result; result.definition_ast = projection_definition->clone(); result.name = projection_definition->name; @@ -783,7 +789,7 @@ String ProjectionsDescription::toString() const for (const auto & projection : projections) list.children.push_back(projection.definition_ast); - return list.formatWithSecretsOneLine(); + return list.formatIgnoringRedundantParentheses(); } ProjectionsDescription ProjectionsDescription::parse( diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index e013a15de05f..35fd28dda220 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -155,7 +155,40 @@ StorageRabbitMQ::StorageRabbitMQ( String username; String password; - if ((*rabbitmq_settings)[RabbitMQSetting::rabbitmq_host_port].changed) + /// The connection is TLS when either the `rabbitmq_secure` setting is on or the + /// `rabbitmq_address` URI uses the `amqps` scheme; OpenSSL must be initialized in both cases. + bool secure_connection = (*rabbitmq_settings)[RabbitMQSetting::rabbitmq_secure].value; + + const auto address_string = getContext()->getMacros()->expand((*rabbitmq_settings)[RabbitMQSetting::rabbitmq_address]); + + if (!address_string.empty()) + { + std::optional address; + try + { + address.emplace(address_string); + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid `rabbitmq_address`: {}", e.what()); + } + + context_->getRemoteHostFilter().checkHostAndPort(address->hostname(), toString(address->port())); + + /// connectImpl takes the transport (amqp/amqps) from the URI scheme and ignores the + /// `rabbitmq_secure` setting for the address form, so reject a contradictory + /// `rabbitmq_secure = 1` rather than silently connecting in plaintext. Only for a fresh + /// CREATE though: an existing table must still attach on restart (it stays plaintext, as + /// it did before), so this validation does not brick upgrades. + if (mode <= LoadingStrictnessLevel::CREATE && secure_connection && !address->secure()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`rabbitmq_secure = 1` conflicts with the plaintext `amqp://` scheme in " + "`rabbitmq_address`; use an `amqps://` address for a secure connection"); + + secure_connection = address->secure(); + } + else if ((*rabbitmq_settings)[RabbitMQSetting::rabbitmq_host_port].changed) { username = setting_rabbitmq_username.empty() ? config.getString("rabbitmq.username", "") : setting_rabbitmq_username; password = setting_rabbitmq_password.empty() ? config.getString("rabbitmq.password", "") : setting_rabbitmq_password; @@ -172,7 +205,7 @@ StorageRabbitMQ::StorageRabbitMQ( context_->getRemoteHostFilter().checkHostAndPort(parsed_address.first, toString(parsed_address.second)); } - else if (!(*rabbitmq_settings)[RabbitMQSetting::rabbitmq_address].changed) + else throw Exception(ErrorCodes::BAD_ARGUMENTS, "RabbitMQ requires either `rabbitmq_host_port` or `rabbitmq_address` setting"); configuration = @@ -182,11 +215,11 @@ StorageRabbitMQ::StorageRabbitMQ( .username = username, .password = password, .vhost = config.getString("rabbitmq.vhost", getContext()->getMacros()->expand((*rabbitmq_settings)[RabbitMQSetting::rabbitmq_vhost])), - .secure = (*rabbitmq_settings)[RabbitMQSetting::rabbitmq_secure].value, - .connection_string = getContext()->getMacros()->expand((*rabbitmq_settings)[RabbitMQSetting::rabbitmq_address]) + .secure = secure_connection, + .connection_string = address_string }; - if (configuration.secure) + if (secure_connection) SSL_library_init(); if (!columns_.getMaterialized().empty() || !columns_.getAliases().empty() || !columns_.getDefaults().empty() || !columns_.getEphemeral().empty()) @@ -1420,7 +1453,7 @@ Optional parameters: - `rabbitmq_max_block_size` - Number of row collected before flushing data from RabbitMQ. Default: [max_insert_block_size](../../../operations/settings/settings.md#max_insert_block_size). - `rabbitmq_flush_interval_ms` - Timeout for flushing data from RabbitMQ. Default: [stream_flush_interval_ms](/operations/settings/settings#stream_flush_interval_ms). - `rabbitmq_queue_settings_list` - allows to set RabbitMQ settings when creating a queue. Available settings: `x-max-length`, `x-max-length-bytes`, `x-message-ttl`, `x-expires`, `x-priority`, `x-max-priority`, `x-overflow`, `x-dead-letter-exchange`, `x-queue-type`. The `durable` setting is enabled automatically for the queue. -- `rabbitmq_address` - Address for connection. Use ether this setting or `rabbitmq_host_port`. +- `rabbitmq_address` - Address for connection: `amqp(s)://user:password@host:port/vhost`. Use either this setting or `rabbitmq_host_port`; if both are set, `rabbitmq_address` is the one used. Its host and port are checked against [remote_url_allow_hosts](/reference/settings/server-settings/settings/remote#remote_url_allow_hosts). - `rabbitmq_vhost` - RabbitMQ vhost. Default: `'/'`. - `rabbitmq_queue_consume` - Use user-defined queues and do not make any RabbitMQ setup: declaring exchanges, queues, bindings. Default: `false`. - `rabbitmq_username` - RabbitMQ username. @@ -1435,7 +1468,8 @@ Optional parameters: ### SSL connection {#ssl-connection} -Use either `rabbitmq_secure = 1` or `amqps` in connection address: `rabbitmq_address = 'amqps://guest:guest@localhost/vhost'`. +With the `rabbitmq_host_port` form, set `rabbitmq_secure = 1` to use TLS. +With the `rabbitmq_address` form the transport comes from the URI scheme, so use `amqps`: `rabbitmq_address = 'amqps://guest:guest@localhost/vhost'`. `rabbitmq_secure` is ignored for the address form, and `rabbitmq_secure = 1` together with a plaintext `amqp://` address is rejected rather than silently connecting in cleartext. The default behaviour of the used library is not to check if the created TLS connection is sufficiently secure. Whether the certificate is expired, self-signed, missing or invalid: the connection is simply permitted. More strict checking of certificates can possibly be implemented in the future. Also format settings can be added along with rabbitmq-related settings. diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp index 065932a6eca5..4b3416ff5438 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp @@ -4,12 +4,15 @@ #include #include +#include #include #include #include #include #include #include +#include +#include #include #include #include @@ -20,9 +23,20 @@ #include +namespace ProfileEvents +{ + extern const Event SelectivityEstimatorInSetNotBuilt; + extern const Event SelectivityEstimatorInSetEstimatedFromSize; +} + namespace DB { +namespace Setting +{ + extern const SettingsUInt64 statistics_max_set_size_for_exact_selectivity_estimation; +} + RelationProfile ConditionSelectivityEstimator::estimateRelationProfile(const StorageMetadataPtr & metadata, const ActionsDAG::Node * filter, const ActionsDAG::Node * prewhere) const { if (filter == nullptr && prewhere == nullptr) @@ -298,14 +312,50 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr if (!future_set) return false; - auto prepared_set = future_set->buildOrderedSetInplace(rhs.getTreeContext().getQueryContext()); + /// Deliberately not `buildOrderedSetInplace`: this estimator is advisory - it only ranks + /// PREWHERE candidates - so it must not run a subquery to fill a set. A set that is not + /// built yet simply cannot be analysed, and the condition falls back to the default + /// selectivity, as it did for every subquery set before `ActionsDAG::Node::column` + /// became a `ColumnConst` and made these sets visible here. + auto prepared_set = future_set->getOrderedSetIfAlreadyBuilt(rhs.getTreeContext().getQueryContext()); if (!prepared_set || !prepared_set->hasExplicitSetElements()) + { + ProfileEvents::increment(ProfileEvents::SelectivityEstimatorInSetNotBuilt); return false; + } Columns columns = prepared_set->getSetElements(); if (columns.size() != 1) return false; + /// Turning the set into ranges below costs a `Field` per element, a sort, and one + /// statistics probe per element. Above the limit, estimate from the size of the set + /// and its bounds instead: still one pass over the set, for the bounds, but without the + /// sort or the per-element probes. The atom is finalized rather than turned into ranges: + /// a scalar selectivity cannot intersect with other predicates on the same column, only + /// multiply. + const auto max_set_size = node.getTreeContext().getQueryContext()->getSettingsRef() + [Setting::statistics_max_set_size_for_exact_selectivity_estimation]; + if (max_set_size && columns[0]->size() > max_set_size) + { + chassert(is_in_operator); + const bool negative = func_name != "in"; + const auto lhs_name = func.getArgumentAt(0).getColumnName(); + + /// An expression rather than a column (`lower(col) IN (...)`) has no statistics to + /// consult, and below the limit it is given a flat default by the "not a real column" + /// branch further down. Use that same default here, so that crossing the limit cannot + /// change the estimate for such an atom - and skip the size-based path, which would + /// otherwise read `set_size / ` as "matches everything". + if (metadata && !metadata->getColumns().tryGet(lhs_name)) + out.selectivity.true_sel = negative ? 1.0 - default_cond_equal_factor : default_cond_equal_factor; + else + out.selectivity = estimateSelectivityFromSetSize(metadata, lhs_name, *columns[0], negative); + + out.finalized = true; + return false; + } + Tuple tuple(columns[0]->size()); for (size_t i = 0; i < columns[0]->size(); ++i) tuple[i] = (*columns[0])[i]; @@ -573,6 +623,48 @@ UInt64 ConditionSelectivityEstimator::ColumnEstimator::estimateCardinality() con return stats->estimateCardinality(); } + +ConditionSelectivityEstimator::Selectivity ConditionSelectivityEstimator::estimateSelectivityFromSetSize( + const StorageMetadataPtr & metadata, const String & column_name, const IColumn & set_elements, bool negative) const +{ + ProfileEvents::increment(ProfileEvents::SelectivityEstimatorInSetEstimatedFromSize); + + /// `Set::appendSetElements` appends only the rows flagged by the deduplication filter, so this is the + /// set's exact number of distinct values, directly comparable with the column's estimated cardinality. + const size_t set_size = set_elements.size(); + + auto it = column_estimators.find(column_name); + if (it == column_estimators.end() || !isCompatibleStatistics(metadata, it->second.stats, column_name)) + { + /// No statistics: match what `finalize` assumes for a list of point ranges on an unknown column. + const Selectivity selectivity{std::min(static_cast(set_size) * default_cond_equal_factor, 1.0), 0}; + return negative ? selectivity.applyNot() : selectivity; + } + + /// First upper bound: a row outside the set's bounds cannot be in the set. This is exactly as + /// accurate as any other range atom on this column - `estimateRanges` degrades to the same + /// defaults whenever the statistics cannot answer for a range. + Selectivity selectivity{1.0, 0}; + Field min_value; + Field max_value; + set_elements.getExtremes(min_value, max_value, 0, set_size); + if (!min_value.isNull() && !max_value.isNull()) + selectivity = it->second.estimateRanges(PlainRanges(Range(min_value, true, max_value, true))); + + /// Second upper bound: at most `set_size` of the column's distinct values can match, and the + /// estimator assumes every distinct value carries the same share of rows, so at most + /// `set_size / cardinality` of them do. Only when the cardinality is measured - without a uniq + /// sketch `estimateCardinality` returns a fixed fraction of the row count, and dividing by that + /// guess would make the condition look arbitrarily selective and promote it into PREWHERE on no + /// evidence. + const UInt64 cardinality = it->second.estimateCardinality(); + if (cardinality && it->second.stats->hasCardinality()) + selectivity.true_sel + = std::min(selectivity.true_sel, static_cast(set_size) / static_cast(cardinality)); + + return negative ? selectivity.applyNot() : selectivity; +} + const ConditionSelectivityEstimator::AtomMap ConditionSelectivityEstimator::atom_map { { diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h index 12a9b31e103c..ae93efab1eae 100644 --- a/src/Storages/Statistics/ConditionSelectivityEstimator.h +++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h @@ -114,6 +114,13 @@ class ConditionSelectivityEstimator : public WithContext RelationProfile estimateRelationProfileImpl(std::vector & rpn, const StorageMetadataPtr & metadata) const; bool extractAtomFromTree(const StorageMetadataPtr & metadata, const RPNBuilderTreeNode & node, RPNElement & out) const; + + /// Selectivity of `column IN (set)` derived from the size of the set rather than from its contents: + /// the share of rows inside the set's bounding range, capped by the share of distinct values the set + /// can possibly cover. Costs one pass for the bounds and a single statistics probe, where turning the + /// set into ranges costs a `Field` per element, a sort and one probe per element. + Selectivity estimateSelectivityFromSetSize( + const StorageMetadataPtr & metadata, const String & column_name, const IColumn & set_elements, bool negative) const; UInt64 estimateSelectivity(const RPNBuilderTreeNode & node) const; /// Magic constants for estimating the selectivity of a condition no statistics exists. diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp index 87bcfaf9c1c0..6c1b6207e4f9 100644 --- a/src/Storages/Statistics/Statistics.cpp +++ b/src/Storages/Statistics/Statistics.cpp @@ -297,6 +297,11 @@ std::optional ColumnStatistics::estimateRange(const Range & range) cons return *right_count - *left_count; } +bool ColumnStatistics::hasCardinality() const +{ + return stats.contains(StatisticsType::Uniq); +} + UInt64 ColumnStatistics::estimateCardinality() const { if (stats.contains(StatisticsType::Uniq)) diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h index 9c9d49efdb9c..411cea3680aa 100644 --- a/src/Storages/Statistics/Statistics.h +++ b/src/Storages/Statistics/Statistics.h @@ -119,6 +119,10 @@ class ColumnStatistics UInt64 getNonNullRowCount() const; /// True iff null-count tracking is available for this column (e.g. via `Basic` on a Nullable column). bool hasNullCount() const; + /// True iff `estimateCardinality` is backed by a uniq sketch. When it is not, that method returns a + /// fixed fraction of the row count, which callers dividing by the cardinality must not mistake for + /// a measurement. + bool hasCardinality() const; UInt64 estimateCardinality() const; UInt64 estimateDefaults() const; diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index 969989d58bea..d6d356766599 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -301,7 +302,20 @@ void StorageAlias::mutate(const MutationCommands & commands, ContextPtr local_co QueryPipeline StorageAlias::updateLightweight(const MutationCommands & commands, ContextPtr local_context) { auto target_storage = getTargetTable(TargetAccess{local_context, AccessType::ALTER}); - return target_storage->updateLightweight(commands, local_context); + auto lock = target_storage->lockForShare( + local_context->getCurrentQueryId(), + local_context->getSettingsRef()[Setting::lock_acquire_timeout]); + + auto pipeline = target_storage->updateLightweight(commands, local_context); + + /// The caller locks the alias, not the target, so the target needs its own share lock held + /// until the pipeline has committed the patch part. + QueryPlanResourceHolder target_resources; + target_resources.storage_holders.emplace_back(target_storage); + target_resources.table_locks.emplace_back(std::move(lock)); + pipeline.addResources(std::move(target_resources)); + + return pipeline; } CancellationCode StorageAlias::killMutation(const String & mutation_id) diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index 0b161a2559f9..63ae3fc9f06a 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -27,6 +27,8 @@ class StorageAlias final : public IStorage, WithContext std::string getName() const override { return "Alias"; } + bool readsFromOtherTables() const override { return true; } + /// Get the target storage this alias points to StoragePtr getTargetTable(std::optional access_check = std::nullopt) const; StoragePtr tryGetTargetTable() const { return DatabaseCatalog::instance().tryGetTable(StorageID(target_database, target_table), getContext()); } diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 17c99a495b1b..7da071633206 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -175,6 +176,8 @@ StorageBuffer::StorageBuffer( , bg_pool(getContext()->getBufferFlushSchedulePool()) { StorageInMemoryMetadata storage_metadata; + /// Reached when loading already-validated metadata, which stores no column list for this engine. + /// A freshly created table infers its structure in `registerStorageBuffer` under the user's context. if (columns_.empty()) { auto dest_table = DatabaseCatalog::instance().getTable(destination_id, context_); @@ -1394,9 +1397,22 @@ void registerStorageBuffer(StorageFactory & factory) destination_id.table_name = destination_table; } + /// An omitted structure is inferred here, under the user's context: `StorageBuffer` holds only + /// a long-lived context and would read the destination's columns with no user at all. Loading + /// of already-validated metadata has no user either, so it keeps inferring in the constructor. + ColumnsDescription columns = args.columns; + if (columns.empty() && !destination_id.empty() + && !(isLoadingFromExistingMetadata(args.mode) || args.query.attach_short_syntax)) + { + args.getLocalContext()->checkAccess(AccessType::SHOW_COLUMNS, destination_id); + auto destination = DatabaseCatalog::instance().getTable(destination_id, args.getLocalContext()); + auto destination_metadata = destination->getInMemoryMetadataPtr(args.getLocalContext(), false); + columns = destination_metadata->getColumns(); + } + return std::make_shared( args.table_id, - args.columns, + columns, args.constraints, args.comment, args.getContext(), diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index aad37e3badfd..4685aa736cd5 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -88,6 +88,7 @@ friend class BufferSink; size_t max_block_size, size_t num_streams) override; bool isRemote() const override; + bool readsFromOtherTables() const override { return static_cast(destination_id); } bool supportsParallelInsert() const override { return true; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 6bba8e70fd4d..a51044cfe76d 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -450,6 +450,8 @@ StorageDistributed::StorageDistributed( throw Exception(ErrorCodes::BAD_ARGUMENTS, "Settings flush_on_detach=0 and background_insert_batch=1 are incompatible"); StorageInMemoryMetadata storage_metadata; + /// Only a definition loaded from validated metadata reaches here with no columns; the creators + /// infer an omitted structure themselves, under the user's context. if (columns_.empty()) { StorageID id = StorageID::createEmpty(); @@ -2798,9 +2800,25 @@ void registerStorageDistributed(StorageFactory & factory) distributed_settings[DistributedSetting::background_insert_max_sleep_time_ms] = context->getSettingsRef()[Setting::distributed_background_insert_max_sleep_time_ms]; + /// Infer an omitted structure under the user's context, so that the `SHOW_COLUMNS` check for a + /// local shard is not made against the global context the constructor holds. Skipped when the + /// definition comes from already-validated metadata, which has no user to check against. + ColumnsDescription columns = args.columns; + if (columns.empty() && !(isLoadingFromExistingMetadata(args.mode) || args.query.attach_short_syntax)) + { + /// Expanded first, so this resolves the same cluster the constructor will: a Replicated + /// database's implicit cluster is found by the expanded name only. + const String expanded_cluster_name = local_context->getMacros()->expand(cluster_name); + columns = getStructureOfRemoteTable( + *local_context->getCluster(expanded_cluster_name), + StorageID{remote_database, remote_table}, + local_context, + /* table_func_ptr = */ nullptr); + } + return std::make_shared( args.table_id, - args.columns, + columns, args.constraints, args.comment, remote_database, diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index db3f624718d7..d867e72ef907 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -116,6 +116,7 @@ class StorageDistributed final : public IStorage, WithContext bool canMoveConditionsToPrewhere() const override { return false; } bool isRemote() const override { return true; } + bool readsFromOtherTables() const override { return true; } StorageSnapshotPtr getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp index 1c4c0068a706..fb8fed317eaa 100644 --- a/src/Storages/StorageKeeperMap.cpp +++ b/src/Storages/StorageKeeperMap.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -25,9 +26,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -119,6 +123,58 @@ std::string formattedAST(const ASTPtr & ast) return ast->formatWithSecretsOneLine(); } +/// Some builds persisted a single primary-key expression as `(key)`, while others wrote `key`. +/// Normalize only outer parentheses that wrap one complete expression. This works with parsers +/// that either preserve or discard those parentheses and rejects comments or trailing syntax. +std::optional tryCanonicalPrimaryKey(const std::string & primary_key) +{ + if (!primary_key.ends_with('\n')) + return std::nullopt; + + std::string_view expression(primary_key); + expression.remove_suffix(1); + + auto try_parse = [](std::string_view text) + { + ParserExpression parser; + const char * pos = text.data(); + const char * end = pos + text.size(); + std::string error_message; + return tryParseQuery( + parser, + pos, + end, + error_message, + /*hilite=*/ false, + /*description=*/ "KeeperMap primary key", + /*allow_multi_statements=*/ false, + text.size(), + DBMS_DEFAULT_MAX_PARSER_DEPTH, + DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, + /*skip_insignificant=*/ true); + }; + + ASTPtr ast = try_parse(expression); + if (!ast) + return std::nullopt; + + while (expression.size() >= 2 && expression.front() == '(' && expression.back() == ')') + { + auto inner_expression = expression.substr(1, expression.size() - 2); + auto inner_ast = try_parse(inner_expression); + if (!inner_ast) + break; + + expression = inner_expression; + ast = std::move(inner_ast); + } + + if (expression != ast->formatWithSecretsOneLine()) + return std::nullopt; + + return std::string(expression); +} + void verifyTableId(const StorageID & table_id) { if (!table_id.hasUUID()) @@ -461,7 +517,7 @@ StorageKeeperMap::StorageKeeperMap( if (exists) { - isMetadataStringEqual(stored_metadata_string, metadata_string, /*throw_on_error=*/ true); + isMetadataStringCompatible(stored_metadata_string, metadata_string, /*throw_on_error=*/ true); auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent); @@ -658,7 +714,7 @@ VirtualColumnsDescription StorageKeeperMap::createVirtuals() return desc; } -bool StorageKeeperMap::isMetadataStringEqual( +bool StorageKeeperMap::isMetadataStringCompatible( const std::string & zk_metadata_string, const std::string & local_metadata_string, bool throw_on_error) const @@ -667,21 +723,24 @@ bool StorageKeeperMap::isMetadataStringEqual( return true; const std::string_view metadata_format_version_prefix = "KeeperMap metadata format version: 1\ncolumns: "; - const std::string_view primary_key_header = "primary key: "; + const std::string_view primary_key_header = "\nprimary key: "; - if (!local_metadata_string.starts_with(metadata_format_version_prefix)) + if (!zk_metadata_string.starts_with(metadata_format_version_prefix)) throw Exception(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "Invalid KeeperMap metadata format version or columns definition in ZK: {}", zk_metadata_string); - auto zk_pk_pos = zk_metadata_string.rfind(primary_key_header); + if (!local_metadata_string.starts_with(metadata_format_version_prefix)) + throw Exception(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "Invalid local KeeperMap metadata format version or columns definition: {}", local_metadata_string); + + auto zk_pk_pos = zk_metadata_string.find(primary_key_header, metadata_format_version_prefix.size()); if (zk_pk_pos == std::string::npos) throw Exception(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "Invalid KeeperMap metadata format version or primary key definition in ZK: {}", zk_metadata_string); - auto local_pk_pos = local_metadata_string.rfind(primary_key_header); + auto local_pk_pos = local_metadata_string.find(primary_key_header, metadata_format_version_prefix.size()); if (local_pk_pos == std::string::npos) throw Exception(ErrorCodes::CANNOT_PARSE_INPUT_ASSERTION_FAILED, "Invalid local KeeperMap metadata format version or primary key definition: {}", local_metadata_string); - auto local_columns = ColumnsDescription::parse(local_metadata_string.substr(metadata_format_version_prefix.size(), local_pk_pos - metadata_format_version_prefix.size())); - auto zk_columns = ColumnsDescription::parse(zk_metadata_string.substr(metadata_format_version_prefix.size(), zk_pk_pos - metadata_format_version_prefix.size())); + auto local_columns = ColumnsDescription::parse(local_metadata_string.substr(metadata_format_version_prefix.size(), local_pk_pos + 1 - metadata_format_version_prefix.size())); + auto zk_columns = ColumnsDescription::parse(zk_metadata_string.substr(metadata_format_version_prefix.size(), zk_pk_pos + 1 - metadata_format_version_prefix.size())); /// Comment may be added later with ALTER command, and since we don't update metadata during ALTER, we should not compare comments bool columns_equal = zk_columns.toString(/*include_comments=*/ false) == local_columns.toString(/*include_comments=*/ false); @@ -689,18 +748,26 @@ bool StorageKeeperMap::isMetadataStringEqual( auto zk_pk = zk_metadata_string.substr(zk_pk_pos + primary_key_header.size()); auto local_pk = local_metadata_string.substr(local_pk_pos + primary_key_header.size()); - bool pk_equal = zk_pk == local_pk; - - if (columns_equal && pk_equal) - return true; + if (zk_pk == local_pk) + { + if (columns_equal) + return true; + } + else if (columns_equal) + { + const auto canonical_zk_pk = tryCanonicalPrimaryKey(zk_pk); + const auto canonical_local_pk = tryCanonicalPrimaryKey(local_pk); + if (canonical_zk_pk && canonical_local_pk && *canonical_zk_pk == *canonical_local_pk) + return true; + } if (throw_on_error) { throw Exception( ErrorCodes::BAD_ARGUMENTS, "Path {} is already used but the stored {} definition doesn't match. Stored metadata: {}, local metadata: {}", - columns_equal ? "columns" : "primary key", zk_root_path, + columns_equal ? "primary key" : "columns", zk_metadata_string, local_metadata_string); } @@ -709,8 +776,8 @@ bool StorageKeeperMap::isMetadataStringEqual( log, "Path {} is already used but the stored {} definition doesn't match. Stored metadata: {}, local metadata: {}. " "Will use stored metadata", - columns_equal ? "columns" : "primary key", zk_root_path, + columns_equal ? "primary key" : "columns", zk_metadata_string, local_metadata_string); @@ -1405,7 +1472,7 @@ StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus(const ContextPtr return; } - if (!isMetadataStringEqual(stored_metadata_string, metadata_string, /*throw_on_error=*/ false)) + if (!isMetadataStringCompatible(stored_metadata_string, metadata_string, /*throw_on_error=*/ false)) { table_status = TableStatus::INVALID_METADATA; return; diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h index f11b165c52b5..4f16d4a5f4b8 100644 --- a/src/Storages/StorageKeeperMap.h +++ b/src/Storages/StorageKeeperMap.h @@ -138,7 +138,7 @@ class StorageKeeperMap final : public StorageWithCommonVirtualColumns, public IK TableStatus getTableStatus(const ContextPtr & context) const; - bool isMetadataStringEqual( + bool isMetadataStringCompatible( const std::string & zk_metadata_string, const std::string & local_metadata_string, bool throw_on_error) const; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 9f9af98e1ebe..2e5c0d2b36c5 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -1271,7 +1271,7 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p if (!backup->fileExists(file_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", file_path_in_backup); - backup->copyFileToDisk(file_path_in_backup, disk, data_file.path, WriteMode::Append); + backup->copyFileToDisk(file_path_in_backup, disk, data_file.path, WriteMode::Append, /* sync= */ false); } if (use_marks_file) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 7da726420895..1fbcff907866 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -31,11 +31,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -96,6 +98,7 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int SAMPLING_NOT_SUPPORTED; extern const int ALTER_OF_COLUMN_IS_FORBIDDEN; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; +extern const int DATABASE_ACCESS_DENIED; extern const int STORAGE_REQUIRES_PARAMETER; extern const int UNKNOWN_DATABASE; extern const int UNKNOWN_TABLE; @@ -485,7 +488,13 @@ StorageMetadataHandle StorageMerge::getInMemoryMetadataPtr(ContextPtr query_cont } catch (const Exception & e) { - if (e.code() != ErrorCodes::UNKNOWN_DATABASE) + /// The source database may have been dropped (`UNKNOWN_DATABASE`), or it may be the internal + /// database of temporary tables, which `getDatabaseIterator` refuses to enumerate + /// (`DATABASE_ACCESS_DENIED`). Neither should prevent resolving the table's own metadata: + /// the virtual columns of the source tables are a best-effort enrichment, and an actual read + /// still throws in `getDatabaseIterators`. In particular, loading a stored table definition + /// (`ATTACH`, backup `RESTORE`, replicated-database replay) validates the storage through here. + if (e.code() != ErrorCodes::UNKNOWN_DATABASE && e.code() != ErrorCodes::DATABASE_ACCESS_DENIED) throw; } @@ -1180,6 +1189,10 @@ SelectQueryInfo ReadFromMerge::getModifiedQueryInfo(const ContextMutablePtr & mo column_node = std::make_shared(*resolved_pair, modified_query_info.table_expression); } + /// The set registry of the freshly derived planner context is empty, and + /// `PlannerActionsVisitor` resolves `IN` through it. + collectSets(column_node, *modified_query_info.planner_context); + ColumnNodePtrWithHashSet empty_correlated_columns_set; PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, false /*use_column_identifier_as_action_node_name*/); actions_visitor.visit(*filter_actions_dag, column_node); @@ -1372,7 +1385,16 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter auto storage_columns = storage_metadata_snapshot->getColumns(); auto needed_columns = storage_columns.getAll(); - ASTPtr expr = row_policy_filter_ptr->expression; + /// `RowPolicyFilter::expression` is the parsed policy condition owned by `RowPolicyCache`. That AST is + /// shared: every query of every user reading this table gets the same nodes, and a policy defined on a + /// whole database is shared by all its tables. `TreeRewriter` and `ExpressionAnalyzer` rewrite the AST + /// they are given in place - they normalize identifiers, substitute the results of scalar subqueries for + /// the subqueries themselves, and record `ASTLiteral::unique_column_name` - so they must be handed a + /// private copy. Analyzing the shared AST is both a data race against concurrent readers of the same + /// policy and a correctness bug: a scalar subquery such as `USING x <= (SELECT max(v) FROM limits)` gets + /// replaced by its value in the cache and is then frozen for the rest of the server's lifetime. + /// `generateFilterActions` in `InterpreterSelectQuery` clones for the same reason. + ASTPtr expr = row_policy_filter_ptr->expression->clone(); auto syntax_result = TreeRewriter(local_context).analyze(expr, needed_columns); auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context}; @@ -1474,8 +1496,13 @@ StorageMerge::StorageListWithLocks ReadFromMerge::getSelectedTables( if (!storage) continue; + /// The `_table` and `_database` values of the rows are stamped by the table that + /// actually produces the rows. If the child table reads from other tables, its rows + /// carry those tables' names, not the child's own name, so pruning the child by its + /// name could incorrectly discard the rows the predicate selects. Such children are + /// always read, and the predicate is applied to the rows. if (storage.get() != storage_merge.get()) - if (!table_filter || table_filter(iterator->databaseName(), iterator->name())) + if (!table_filter || storage->readsFromOtherTables() || table_filter(iterator->databaseName(), iterator->name())) if (granted_show_on_all_tables || access->isGranted(AccessType::SHOW_TABLES, iterator->databaseName(), iterator->name())) { if (!granted_select_on_all_tables) @@ -1496,6 +1523,12 @@ StorageMerge::StorageListWithLocks ReadFromMerge::getSelectedTables( DatabaseTablesIteratorPtr StorageMerge::DatabaseNameOrRegexp::getDatabaseIterator(const String & database_name, ContextPtr local_context) const { + /// The internal database of temporary tables holds the temporary tables of all sessions and all users, + /// and it is not covered by access control, so direct access to it is denied, see `DatabaseCatalog::tryGetDatabaseAndTable`. + if (database_name == DatabaseCatalog::TEMPORARY_DATABASE) + throw Exception( + ErrorCodes::DATABASE_ACCESS_DENIED, "Direct access to `{}` database is not allowed", DatabaseCatalog::TEMPORARY_DATABASE); + auto database = DatabaseCatalog::instance().getDatabase(database_name); auto table_name_match = [this, database_name](const String & table_name_) -> bool @@ -1538,6 +1571,10 @@ StorageMerge::DatabaseTablesIterators StorageMerge::DatabaseNameOrRegexp::getDat for (const auto & db : databases) { + /// A regexp is not an explicit request for the internal database of temporary tables, so it is skipped silently. + if (db.first == DatabaseCatalog::TEMPORARY_DATABASE) + continue; + if (source_database_regexp->match(db.first)) database_table_iterators.emplace_back(getDatabaseIterator(db.first, local_context)); } @@ -1612,6 +1649,9 @@ void ReadFromMerge::convertAndFilterSourceStream( QueryAnalysisPass query_analysis_pass(modified_query_info.table_expression); query_analysis_pass.run(query_tree, local_context); + /// On the query info cache path nothing registered this expression's sets. + collectSets(query_tree, *modified_query_info.planner_context); + ColumnNodePtrWithHashSet empty_correlated_columns_set; PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, false /*use_column_identifier_as_action_node_name*/); const auto & [nodes, _] = actions_visitor.visit(actions_dag, query_tree); @@ -1823,7 +1863,12 @@ std::optional StorageMerge::tryGetColumnSizes() cons } catch (const Exception & e) { - if (e.code() == ErrorCodes::UNKNOWN_DATABASE) + /// The column sizes are a best-effort introspection (`system.columns`). The source database + /// may have been dropped (`UNKNOWN_DATABASE`), or it may be the internal database of temporary + /// tables, which `getDatabaseIterator` refuses to enumerate (`DATABASE_ACCESS_DENIED`) - such a + /// table can no longer be created, but a pre-existing definition still loads (`ATTACH`, backup + /// `RESTORE`, replicated-database replay) and must not break `system.columns`. + if (e.code() == ErrorCodes::UNKNOWN_DATABASE || e.code() == ErrorCodes::DATABASE_ACCESS_DENIED) return std::nullopt; throw; } @@ -1905,6 +1950,21 @@ void registerStorageMerge(StorageFactory & factory) String source_database_name_or_regexp = checkAndGetLiteralArgument(database_ast, "database_name"); + /// With an explicit column list, `CREATE` (or a full-definition `ATTACH`, which is CREATE-like user input) + /// does not need schema inference and would not read the source tables, so the unusable table definition + /// would be stored; deny it right away, the same way as reading does, see `DatabaseNameOrRegexp::getDatabaseIterator`. + /// Only fresh user-supplied definitions are denied. Loads of previously stored metadata (server startup, + /// short-syntax `ATTACH`) and replays of definitions that already exist elsewhere (`SECONDARY_CREATE`: + /// replicated-database DDL replay, backup `RESTORE`) stay loadable, so a table created before this check + /// existed can still be restored or materialized on a new replica. Reading from such a table is denied + /// anyway, so unlike `StorageDistributed` there is nothing a restoring user could reach through it. + bool fresh_user_definition = args.mode == LoadingStrictnessLevel::CREATE + || (args.mode == LoadingStrictnessLevel::ATTACH && !args.query.attach_short_syntax); + if (!is_regexp && source_database_name_or_regexp == DatabaseCatalog::TEMPORARY_DATABASE + && fresh_user_definition) + throw Exception( + ErrorCodes::DATABASE_ACCESS_DENIED, "Direct access to `{}` database is not allowed", DatabaseCatalog::TEMPORARY_DATABASE); + engine_args[1] = evaluateConstantExpressionAsLiteral(engine_args[1], args.getLocalContext()); String table_name_regexp = checkAndGetLiteralArgument(engine_args[1], "table_name_regexp"); @@ -2006,7 +2066,7 @@ SELECT * FROM WatchLog; - `_table` — The name of the table from which data was read. Type: [String](../../../sql-reference/data-types/string.md). - If you filter on `_table`, (for example `WHERE _table='xyz'`) only tables which satisfy the filter condition are read. + If you filter on `_table`, (for example `WHERE _table='xyz'`) only tables which satisfy the filter condition are read. A table that itself reads from other tables (`Distributed`, `Merge`, `Buffer`, `Alias`) returns rows carrying the name of the table that actually produced them, so such tables are always read and the filter is applied to their rows. - `_database` — Contains the name of the database from which data was read. Type: [String](../../../sql-reference/data-types/string.md). diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 096b9a98e98a..99d815791fea 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -46,6 +46,7 @@ class StorageMerge final : public IStorage, WithContext std::string getName() const override { return "Merge"; } bool isRemote() const override; + bool readsFromOtherTables() const override { return true; } /// The check is delayed to the read method. It checks the support of the tables used. bool supportsSampling() const override { return true; } diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp index abb925041ffd..c3ed12b9530d 100644 --- a/src/Storages/StorageMergeTreeIndex.cpp +++ b/src/Storages/StorageMergeTreeIndex.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int NOT_IMPLEMENTED; + extern const int ACCESS_DENIED; } class MergeTreeIndexSource final : public ISource, WithContext @@ -403,7 +405,18 @@ void StorageMergeTreeIndex::readImpl( } } - context->checkAccess(AccessType::SELECT, source_table->getStorageID(), columns_from_storage); + auto source_storage_id = source_table->getStorageID(); + context->checkAccess(AccessType::SELECT, source_storage_id, columns_from_storage); + + /// We cannot apply a row policy to granules, but the index leaks keys of the rows it hides + auto row_policy_filter = context->getRowPolicyFilter( + source_storage_id.getDatabaseName(), source_storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Cannot read from `mergeTreeIndex` because a row policy is applied on table {}. " + "Reading the index could violate the row policy", + source_storage_id.getNameForLogs()); auto sample_block = std::make_shared(storage_snapshot->getSampleBlockForColumns(column_names)); diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index e3ad0474a062..0386dea1aaf0 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -34,6 +34,10 @@ class StorageProxy : public IStorage /// storage that opts out of the rewrite (e.g. Distributed) does not re-advertise true. bool supportsOptimizationToSubcolumns() const override { return getNested()->supportsOptimizationToSubcolumns(); } bool supportsColumnsWithDynamicStructure() const override { return getNested()->supportsColumnsWithDynamicStructure(); } + /// `ReadFromMerge::getSelectedTables` prunes children by name based on this flag; a lazy + /// `StorageTableProxy` around a delegating storage (`Distributed`, `Merge`, `Buffer`, `Alias`) + /// answering false would let a `_table`/`_database` filter incorrectly prune the child. + bool readsFromOtherTables() const override { return getNested()->readsFromOtherTables(); } ColumnSizeByName getColumnSizes() const override { return getNested()->getColumnSizes(); } ColumnSizeByName getColumnSizes(const Names & columns) const override { return getNested()->getColumnSizes(columns); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 880dae5cd904..f3e8cfa5135b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1365,14 +1365,48 @@ void StorageReplicatedMergeTree::createReplicaAttempt(const StorageMetadataPtr & const auto & zk_mutation_pointer = response_exists[response_num++].data; const auto & zk_creator_info = response_exists[response_num++].data; + /// The `metadata` and `columns` nodes could have been written by a server version that + /// serialized the same table definition to a different text: the redundant parentheses + /// the user has written around key or column expressions were kept by some versions and + /// are suppressed now (`IAST::FormatSettings::ignore_redundant_parentheses`). When the + /// texts differ, compare structurally, so that a retry of a partially created replica + /// after an upgrade still recognizes its own nodes. These lambdas are only reached for + /// an empty, never-active replica at our own path; a structural mismatch or unparseable + /// node throws (e.g. `METADATA_MISMATCH`), which describes the problem better than the + /// `REPLICA_ALREADY_EXISTS` the fall-through create attempt would produce. + auto is_same_metadata = [&](const String & zk_metadata_str) + { + if (zk_metadata_str == local_metadata) + return true; + auto zk_metadata_parsed = ReplicatedMergeTreeTableMetadata::parseAndNormalize( + zk_metadata_str, + metadata_snapshot->getColumns(), + metadata_snapshot->add_minmax_index_for_numeric_columns, + metadata_snapshot->add_minmax_index_for_string_columns, + getContext()); + return ReplicatedMergeTreeTableMetadata(*this, metadata_snapshot).checkEquals( + zk_metadata_parsed, + metadata_snapshot->columns, + metadata_snapshot->virtuals, + getStorageID().getNameForLogs(), + getContext()); + }; + + auto is_same_columns = [&](const String & zk_columns_str) + { + if (zk_columns_str == local_columns) + return true; + return ColumnsDescription::parse(zk_columns_str) == metadata_snapshot->getColumns(); + }; + if (zk_host.empty() && zk_log_pointer.empty() && zk_queue.empty() && zk_parts.empty() && zk_flags.empty() && (zk_is_lost == "0" || zk_is_lost == "1") && - zk_metadata == local_metadata && - zk_columns == local_columns && + is_same_metadata(zk_metadata) && + is_same_columns(zk_columns) && zk_metadata_version == local_metadata_version && zk_min_unprocessed_insert_time.empty() && zk_max_processed_insert_time.empty() && @@ -7091,11 +7125,13 @@ void StorageReplicatedMergeTree::alter( applyMetadataChangesToCreateQuery(ast, future_metadata, query_context); } + /// The definitions below are written into Keeper and compared with what the other replicas have + /// written, so their text must not depend on the parentheses the user has written around them. auto ast_to_str = [](ASTPtr query) -> String { if (!query) return ""; - return query->formatWithSecretsOneLine(); + return query->formatIgnoringRedundantParentheses(); }; const auto zookeeper = getZooKeeperAndAssertNotReadonly(); @@ -7143,19 +7179,19 @@ void StorageReplicatedMergeTree::alter( /// list here and we cannot change this representation for compatibility. Also we have preparsed AST `sorting_key.expression_list_ast` /// in KeyDescription, but it contain version column for VersionedCollapsingMergeTree, which shouldn't be defined as a part of key definition AST. /// So the best compatible way is just to convert definition_ast to list and serialize it. In all other places key.expression_list_ast should be used. - future_metadata_in_zk.sorting_key = extractKeyExpressionList(future_metadata.sorting_key.definition_ast)->formatWithSecretsOneLine(); + future_metadata_in_zk.sorting_key = ast_to_str(extractKeyExpressionList(future_metadata.sorting_key.definition_ast)); } if (ast_to_str(future_metadata.sampling_key.definition_ast) != ast_to_str(current_metadata->sampling_key.definition_ast)) - future_metadata_in_zk.sampling_expression = extractKeyExpressionList(future_metadata.sampling_key.definition_ast)->formatWithSecretsOneLine(); + future_metadata_in_zk.sampling_expression = ast_to_str(extractKeyExpressionList(future_metadata.sampling_key.definition_ast)); if (ast_to_str(future_metadata.partition_key.definition_ast) != ast_to_str(current_metadata->partition_key.definition_ast)) - future_metadata_in_zk.partition_key = extractKeyExpressionList(future_metadata.partition_key.definition_ast)->formatWithSecretsOneLine(); + future_metadata_in_zk.partition_key = ast_to_str(extractKeyExpressionList(future_metadata.partition_key.definition_ast)); if (ast_to_str(future_metadata.table_ttl.definition_ast) != ast_to_str(current_metadata->table_ttl.definition_ast)) { if (future_metadata.table_ttl.definition_ast) - future_metadata_in_zk.ttl_table = future_metadata.table_ttl.definition_ast->formatWithSecretsOneLine(); + future_metadata_in_zk.ttl_table = ast_to_str(future_metadata.table_ttl.definition_ast); else /// TTL was removed future_metadata_in_zk.ttl_table = ""; } diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index c250499fc7be..c20ef4adb7fa 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -698,7 +698,7 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & if (!backup->fileExists(file_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", file_path_in_backup); - backup->copyFileToDisk(file_path_in_backup, disk, data_file_path, WriteMode::Append); + backup->copyFileToDisk(file_path_in_backup, disk, data_file_path, WriteMode::Append, /* sync= */ false); } /// Append the index. diff --git a/src/Storages/StorageTimeSeries.cpp b/src/Storages/StorageTimeSeries.cpp index 007009a9212d..7e715fe425ac 100644 --- a/src/Storages/StorageTimeSeries.cpp +++ b/src/Storages/StorageTimeSeries.cpp @@ -277,8 +277,8 @@ void StorageTimeSeries::dropInnerTableIfAny(bool sync, ContextPtr local_context) { if (auto inner_table_id = tryGetTargetTableID(target_kind, local_context)) { - /// Best-effort to make them work: the inner table name is almost always less than the TimeSeries name (so it's safe to lock DDLGuard). - /// (See the comment in StorageMaterializedView::dropInnerTableIfAny.) + /// DDLGuards must be locked in order of increasing table name, so the inner guard + /// may be requested only when this table's name sorts first. bool may_lock_ddl_guard = getStorageID().getQualifiedName() < inner_table_id.getQualifiedName(); InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, sync, /* ignore_sync_setting= */ true, may_lock_ddl_guard); diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 925911df1536..e0285baea377 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -1,4 +1,4 @@ -// autogenerated by tests/ci/version_helper.py +// autogenerated by ci/jobs/scripts/create_release.py const char * auto_contributors[] { "0x01f", "0xMihalich", diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index e0d766a266cb..acc5da0bf5de 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -1,10 +1,20 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include #include +#include #include #include #include @@ -21,9 +31,22 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include +#include +#include + +#include namespace DB @@ -33,12 +56,15 @@ namespace Setting extern const SettingsBool allow_experimental_codecs; extern const SettingsBool allow_suspicious_codecs; extern const SettingsBool allow_suspicious_ttl_expressions; + extern const SettingsBool variant_throw_on_type_mismatch; + extern const SettingsBool dynamic_throw_on_type_mismatch; } namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int BAD_TTL_EXPRESSION; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; } @@ -67,6 +93,979 @@ TTLAggregateDescription & TTLAggregateDescription::operator=(const TTLAggregateD namespace { +/// The product of alternative counts probed for one function node is bounded to keep CREATE TABLE cheap. +/// A TTL whose validation needs more joint probes than this is rejected as suspicious (fail closed) rather +/// than partially checked; `allow_suspicious_ttl_expressions` remains the escape hatch. +constexpr size_t max_probe_combinations = 256; + +[[noreturn]] void throwTooManyProbeCombinations(std::string_view expression_kind) +{ + throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, + "TTL {}expression uses a function over arguments with too many combinations " + "of AggregateFunction payloads to validate ({} probes at most are allowed). " + "Use typed subcolumns instead, or set `allow_suspicious_ttl_expressions` to allow it", + expression_kind, max_probe_combinations); +} + +/// Build the list of single-row "suspect" materializations of `type` for the DDL-time TTL probe. Each +/// returned column is one combination of payloads stored inside the suspect types found in `type` - a +/// direct `AggregateFunction` state, or a `Variant`/`Dynamic` carrier that may hold one; the consumer must +/// survive every one of them. An empty list means the type contains nothing in scope of the check, so the +/// default value is representative and no extra probes are needed. +/// +/// The suspect payload does not have to be the argument's top-level type: a consumer over +/// `Array(AggregateFunction(...))`, `Array(Dynamic)` or `Tuple(UInt32, Variant(...))` builds fine (the +/// container's default value is empty/NULL, so the nested consumer never runs) yet still fails on the +/// nested payloads during TTL execution. So the materialization recurses through +/// `Array`/`Tuple`/`Map`/`Nullable` (and `Variant` alternatives), wrapping each nested payload back into a +/// single-row container column. +std::vector collectSuspectMaterializations(const DataTypePtr & type, std::string_view expression_kind) +{ + WhichDataType which(type); + + if (which.isAggregateFunction()) + { + /// A direct AggregateFunction payload. For a top-level argument the default-value probe already + /// covers it, but when the state is nested inside a container whose default value is empty + /// (`Array`, `Map`) the element-level consumer never sees it: e.g. the `equals` built inside + /// `arrayRemove(arr, 0)` for `arr Array(AggregateFunction(max, UInt64))` only runs on the + /// elements of a non-empty row. Materialize one default state so the container branches below + /// wrap it into a single-row non-empty column. + return {type->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst()}; + } + + if (which.isDynamic()) + { + /// A `Dynamic` can store any type, so probing a single representative payload is not enough: + /// a consumer that happens to accept an AggregateFunction state can still throw on other + /// legal payloads (e.g. `finalizeAggregation(dyn)` accepts the state but rejects `UInt64` / + /// `String`). Probe a small representative set instead - the AggregateFunction state that + /// brings the column into scope, plus a numeric and a string payload - so only a genuinely + /// type-agnostic consumer survives every probe. The state goes first so that a consumer failing + /// on it gets the aggregate-specific error message. + static const std::vector representative_type_names = + {"AggregateFunction(max, UInt64)", "UInt64", "String"}; + + std::vector payloads; + payloads.reserve(representative_type_names.size()); + for (const auto & type_name : representative_type_names) + { + auto payload_type = DataTypeFactory::instance().get(type_name); + ColumnPtr payload = payload_type->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst(); + + auto dynamic_column = type->createColumn(); + auto & dynamic = assert_cast(*dynamic_column); + if (dynamic.addNewVariant(payload_type)) + { + auto discr = dynamic.getVariantInfo().variant_name_to_discriminator.at(payload_type->getName()); + dynamic.getVariantColumn().insertIntoVariantFrom(discr, *payload, 0); + } + else + { + /// The type cannot hold new variants (e.g. `Dynamic(max_types=0)`), so values are + /// stored in the shared variant - probe through it as well. + dynamic.insertValueIntoSharedVariant(*payload, payload_type, payload_type->getName(), 0); + } + payloads.push_back(std::move(dynamic_column)); + } + return payloads; + } + + if (which.isVariant()) + { + const auto & variant_type = assert_cast(*type); + const auto & variant_types = variant_type.getVariants(); + + /// Only a `Variant` that can actually carry an AggregateFunction state (directly, or through a + /// nested carrier inside an alternative) is in scope of this check. But once such a `Variant` is a + /// consumer's argument, *every* alternative must be probed, not only the aggregate-carrying ones: + /// a state-aware consumer can accept the AggregateFunction branch and still throw + /// `ILLEGAL_TYPE_OF_ARGUMENT` on a sibling alternative that a later row happens to store. For + /// example `finalizeAggregation(v)` with `v Variant(AggregateFunction(max, UInt32), UInt32)` + /// succeeds on the state branch but throws on a row storing the `UInt32` alternative during TTL + /// execution. Probing only the aggregate alternative would wrongly accept it. + std::vector> alternative_payloads(variant_types.size()); + bool has_suspect_alternative = false; + for (size_t discr = 0; discr < variant_types.size(); ++discr) + { + if (hasAggregateFunctionType(variant_types[discr])) + has_suspect_alternative = true; + alternative_payloads[discr] = collectSuspectMaterializations(variant_types[discr], expression_kind); + if (!alternative_payloads[discr].empty()) + has_suspect_alternative = true; + } + + if (!has_suspect_alternative) + return {}; + + std::vector result; + for (size_t discr = 0; discr < variant_types.size(); ++discr) + { + auto & payloads = alternative_payloads[discr]; + if (payloads.empty()) + payloads.push_back(variant_types[discr]->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst()); + + for (const auto & payload : payloads) + { + auto variant_column = variant_type.createColumn(); + assert_cast(*variant_column).insertIntoVariantFrom( + static_cast(discr), *payload, 0); + result.push_back(std::move(variant_column)); + } + if (result.size() > max_probe_combinations) + throwTooManyProbeCombinations(expression_kind); + } + return result; + } + + if (const auto * array_type = typeid_cast(type.get())) + { + auto nested = collectSuspectMaterializations(array_type->getNestedType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + { + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->getData().push_back(1); + result.push_back(ColumnArray::create(IColumn::mutate(payload), std::move(offsets))); + } + return result; + } + + if (const auto * tuple_type = typeid_cast(type.get())) + { + const auto & element_types = tuple_type->getElements(); + std::vector> element_payloads(element_types.size()); + bool has_suspect_element = false; + size_t total_combinations = 1; + for (size_t i = 0; i < element_types.size(); ++i) + { + element_payloads[i] = collectSuspectMaterializations(element_types[i], expression_kind); + if (element_payloads[i].empty()) + element_payloads[i].push_back(element_types[i]->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst()); + else + has_suspect_element = true; + + total_combinations *= element_payloads[i].size(); + if (total_combinations > max_probe_combinations) + throwTooManyProbeCombinations(expression_kind); + } + + if (!has_suspect_element) + return {}; + + /// The cartesian product of the element materializations, by a mixed-radix counter. + std::vector result; + result.reserve(total_combinations); + std::vector selection(element_types.size(), 0); + while (true) + { + Columns elements(element_types.size()); + for (size_t i = 0; i < element_types.size(); ++i) + elements[i] = element_payloads[i][selection[i]]; + result.push_back(ColumnTuple::create(std::move(elements))); + + size_t i = 0; + while (i < selection.size() && ++selection[i] == element_payloads[i].size()) + { + selection[i] = 0; + ++i; + } + if (i == selection.size()) + break; + } + return result; + } + + if (const auto * map_type = typeid_cast(type.get())) + { + /// A Map is stored as Array(Tuple(key, value)); reuse the Array/Tuple materializations and wrap + /// them back into a Map column. + auto nested = collectSuspectMaterializations(map_type->getNestedType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + result.push_back(ColumnMap::create(IColumn::mutate(payload))); + return result; + } + + if (const auto * nullable_type = typeid_cast(type.get())) + { + /// A carrier can hide under a Nullable wrapper too (e.g. `Nullable(Tuple(UInt32, Dynamic))` with + /// `enable_nullable_tuple_type`). The default Nullable row is NULL, so a consumer over it + /// short-circuits and never sees the nested payload; wrap each of them into a non-NULL row instead. + auto nested = collectSuspectMaterializations(nullable_type->getNestedType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + result.push_back(ColumnNullable::create(IColumn::mutate(payload), ColumnUInt8::create(1, UInt8(0)))); + return result; + } + + return {}; +} + +/// True if casting `from_type` to `to_type` can pick the stored `Variant`/`Dynamic` alternative by parsing +/// the *row contents* instead of deriving it from the source type. `FunctionCast::createColumnToVariantWrapper` +/// routes a string source to `createStringToVariantWrapper` under `cast_string_to_variant_use_inference` +/// (enabled by default), and `createColumnToDynamicWrapper` does the same under +/// `cast_string_to_dynamic_use_inference`. Container casts recurse into their elements, so this check +/// mirrors that recursion; a `Variant` source, in contrast, is never re-parsed (it goes to +/// `createVariantToDynamicWrapper`, which preserves the alternative each row already stores). +bool castMayInferPayloadFromString(const DataTypePtr & from_type, const DataTypePtr & to_type) +{ + /// The wrappers look through `Nullable`/`LowCardinality` on both sides. + auto from = removeNullable(removeLowCardinality(from_type)); + auto to = removeNullable(removeLowCardinality(to_type)); + + const WhichDataType which_to(*to); + if ((which_to.isVariant() || which_to.isDynamic()) && WhichDataType(*from).isStringOrFixedString()) + return true; + + if (const auto * from_array = typeid_cast(from.get())) + { + const auto * to_array = typeid_cast(to.get()); + return to_array && castMayInferPayloadFromString(from_array->getNestedType(), to_array->getNestedType()); + } + + if (const auto * from_tuple = typeid_cast(from.get())) + { + const auto * to_tuple = typeid_cast(to.get()); + if (!to_tuple || from_tuple->getElements().size() != to_tuple->getElements().size()) + return false; + for (size_t i = 0; i < from_tuple->getElements().size(); ++i) + if (castMayInferPayloadFromString(from_tuple->getElements()[i], to_tuple->getElements()[i])) + return true; + return false; + } + + if (const auto * from_map = typeid_cast(from.get())) + { + const auto * to_map = typeid_cast(to.get()); + return to_map + && (castMayInferPayloadFromString(from_map->getKeyType(), to_map->getKeyType()) + || castMayInferPayloadFromString(from_map->getValueType(), to_map->getValueType())); + } + + return false; +} + +/// Build the single-row representative values of a non-suspect type for executing a typed `CAST` during +/// the DDL-time probe. The plain default value is degenerate for wrappers that would hide the payload +/// structure from the consumers of the cast result: a `Nullable` default row is NULL (the +/// `Variant`/`Dynamic` adaptors short-circuit on it) and an `Array`/`Map` default is empty (element-level +/// consumers never run), so recurse through them, materializing a non-NULL row and one-element containers +/// instead. +/// +/// A `Variant` source needs *several* representatives, not one: a cast of a `Variant` to a carrier +/// preserves whichever alternative each row currently stores (`createVariantToDynamicWrapper`), so the +/// payload of the result is not fixed by a single representative row. The default `Variant` row is NULL, +/// and narrowing the consumer's domain to it would accept e.g. `length(CAST(v, 'Dynamic'))` for +/// `v Variant(String, UInt32)`, which throws `ILLEGAL_TYPE_OF_ARGUMENT` during TTL execution as soon as a +/// row stores the `UInt32` alternative. So every alternative is materialized and the cast is probed with +/// each of them; the union of the outputs is the domain the consumers are validated against. +std::vector makeRepresentativeColumns(const DataTypePtr & type, std::string_view expression_kind) +{ + if (const auto * nullable_type = typeid_cast(type.get())) + { + auto nested = makeRepresentativeColumns(nullable_type->getNestedType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + result.push_back(ColumnNullable::create(IColumn::mutate(payload), ColumnUInt8::create(1, UInt8(0)))); + return result; + } + + if (const auto * variant_type = typeid_cast(type.get())) + { + const auto & variant_types = variant_type->getVariants(); + std::vector result; + for (size_t discr = 0; discr < variant_types.size(); ++discr) + { + for (const auto & payload : makeRepresentativeColumns(variant_types[discr], expression_kind)) + { + auto variant_column = variant_type->createColumn(); + assert_cast(*variant_column).insertIntoVariantFrom( + static_cast(discr), *payload, 0); + result.push_back(std::move(variant_column)); + } + if (result.size() > max_probe_combinations) + throwTooManyProbeCombinations(expression_kind); + } + return result; + } + + if (const auto * array_type = typeid_cast(type.get())) + { + auto nested = makeRepresentativeColumns(array_type->getNestedType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + { + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->getData().push_back(1); + result.push_back(ColumnArray::create(IColumn::mutate(payload), std::move(offsets))); + } + return result; + } + + if (const auto * tuple_type = typeid_cast(type.get()); tuple_type && !tuple_type->getElements().empty()) + { + const auto & element_types = tuple_type->getElements(); + std::vector> element_payloads(element_types.size()); + size_t total_combinations = 1; + for (size_t i = 0; i < element_types.size(); ++i) + { + element_payloads[i] = makeRepresentativeColumns(element_types[i], expression_kind); + total_combinations *= element_payloads[i].size(); + if (total_combinations > max_probe_combinations) + throwTooManyProbeCombinations(expression_kind); + } + + /// The cartesian product of the element representatives, by a mixed-radix counter. + std::vector result; + result.reserve(total_combinations); + std::vector selection(element_types.size(), 0); + while (true) + { + Columns elements(element_types.size()); + for (size_t i = 0; i < element_types.size(); ++i) + elements[i] = element_payloads[i][selection[i]]; + result.push_back(ColumnTuple::create(std::move(elements))); + + size_t i = 0; + while (i < selection.size() && ++selection[i] == element_payloads[i].size()) + { + selection[i] = 0; + ++i; + } + if (i == selection.size()) + break; + } + return result; + } + + if (const auto * map_type = typeid_cast(type.get())) + { + auto nested = makeRepresentativeColumns(map_type->getNestedType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + result.push_back(ColumnMap::create(IColumn::mutate(payload))); + return result; + } + + if (const auto * low_cardinality_type = typeid_cast(type.get())) + { + auto nested = makeRepresentativeColumns(low_cardinality_type->getDictionaryType(), expression_kind); + std::vector result; + result.reserve(nested.size()); + for (const auto & payload : nested) + { + auto column = type->createColumn(); + column->insert((*payload)[0]); + result.push_back(std::move(column)); + } + return result; + } + + return {type->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst()}; +} + +/// A fingerprint of a single-row candidate materialization, used to deduplicate candidates when merging +/// the domains of several selector branches. `ColumnDynamic`/`ColumnVariant` hash the *type* of the +/// payload stored in the row together with its value, so equal fingerprints mean the two candidates +/// carry the same payload (and in particular the same payload type). +UInt64 candidateFingerprint(const ColumnPtr & column) +{ + SipHash hash; + hash.update(column->getDataType()); + column->updateHashWithValue(0, hash); + return hash.get64(); +} + +/// The positions of the *value* arguments of a selector function - one whose result is always one of its +/// arguments, chosen by the others. `{}` for anything else. +std::vector getSelectorValueArguments(const String & function_name, size_t arguments_count) +{ + std::vector value_arguments; + + if (function_name == "if" && arguments_count == 3) + { + /// if(cond, then, else) + value_arguments = {1, 2}; + } + else if (function_name == "multiIf" && arguments_count >= 3) + { + /// multiIf(cond_1, then_1, ..., cond_n, then_n[, else]) + for (size_t i = 1; i < arguments_count; i += 2) + value_arguments.push_back(i); + if (arguments_count % 2 == 1) + value_arguments.push_back(arguments_count - 1); + } + else if ((function_name == "coalesce" || function_name == "ifNull") && arguments_count >= 1) + { + for (size_t i = 0; i < arguments_count; ++i) + value_arguments.push_back(i); + } + + return value_arguments; +} + +/// Reject TTL expressions that feed an AggregateFunction state into a function which cannot consume it +/// (e.g. `toDateTime(state)`), while still accepting state-aware functions like `finalizeAggregation`. +/// +/// We only execute the individual functions that directly receive an argument whose type contains an +/// AggregateFunction state (including states nested inside Tuple/Array/Map/etc.). Executing the whole +/// expression instead would make DDL validity depend on synthetic default values: a data-dependent +/// error from an unrelated downstream function - e.g. division by zero in `intDiv(100, finalizeAggregation(state))` +/// when the default state finalizes to 0 - would turn a perfectly valid TTL into a CREATE TABLE failure. +/// Walking nodes individually also makes the check independent of short-circuit evaluation, so an +/// unsupported consumer hidden in a not-taken `if`/`multiIf` branch is still validated. +/// +/// Higher-order functions (e.g. `arrayMap`) keep their lambda body in a separate inner DAG owned by a +/// `FunctionCapture`. Executing the outer node on a synthetic empty array would reduce the lambda over +/// zero rows and never reach the body, so we recurse into the lambda DAG instead. Only the type error +/// is translated into a clear message; all other exceptions are rethrown. +/// +/// A synthetic default value catches a top-level AggregateFunction argument, but not one that is only an +/// alternative of a `Variant` column: the default `Variant` row is NULL, so the `Variant` function +/// adaptor short-circuits (returns NULL) and never runs the consumer on the AggregateFunction +/// alternative. To exercise it we additionally probe with a single-row `Variant` column whose only value +/// is that alternative (e.g. `toDateTime(v)` with `v Variant(AggregateFunction(max, DateTime64(3)), String)`). +/// +/// `Dynamic` erases its value types entirely: the static type never mentions AggregateFunction, yet any +/// row may carry a state (e.g. inserted via CAST to `Dynamic`), and a consumer like `toDateTime` would +/// only fail later, during TTL execution. Since the stored types cannot be enumerated at DDL time, we +/// probe every `Dynamic` argument with a *set* of representative single-row payloads - the +/// AggregateFunction state that brings the column into scope (`AggregateFunction(max, UInt64)`) plus a +/// numeric (`UInt64`) and a string (`String`) payload. Only a genuinely type-agnostic consumer +/// (`isNotNull`, `dynamicType`, `toString`, ...) survives every probe; a state-aware consumer such as +/// `finalizeAggregation(dyn)`, which accepts the state but throws `ILLEGAL_TYPE_OF_ARGUMENT` on other +/// legal payloads, is rejected here rather than at execution time. Such a TTL is one inserted row +/// away from breaking every merge of the table, so rejecting it at CREATE is the safer default; the +/// `allow_suspicious_ttl_expressions` setting and ATTACH remain available as escape hatches. +/// +/// A suspect payload can also sit *inside* a container argument - a direct state in +/// `Array(AggregateFunction(...))` or `Map(String, AggregateFunction(...))`, or a carrier in +/// `Array(Dynamic)`, `Tuple(Dynamic)`, `Map(String, Dynamic)`, `Nullable(Tuple(..., Dynamic))`, or a +/// `Variant` nested in any of them. The container's default value is empty (or NULL), so a consumer that +/// processes the elements (e.g. the `equals` built inside `arrayRemove`) never sees a payload during a +/// default-value probe, yet still fails on the stored payloads during TTL execution. The suspect +/// materializations therefore recurse through the container types and wrap each nested payload back into a +/// single-row container column. +/// +/// Enumerating payloads from a static type is only correct for *stored* columns, which can hold any value +/// of their type. A carrier *computed* inside the expression can have a much narrower runtime domain: +/// `CAST(state, 'Dynamic')` or `CAST(state, 'Variant(AggregateFunction(max, UInt32), UInt32)')` only ever +/// produces the aggregate-state payload, so probing its consumer with fabricated sibling payloads +/// (`UInt64`/`String`, or the `UInt32` alternative) would reject a valid TTL such as +/// `DELETE WHERE isNotNull(finalizeAggregation(CAST(state, 'Dynamic')))`. So each probe records the +/// function's *actual* output columns, and a parent consuming a computed carrier is validated against +/// those instead of the static enumeration. This propagation applies only when the probes cover the node's +/// whole runtime domain, which requires two conditions: +/// - at least one argument carries suspect payloads itself: a carrier computed purely from non-suspect +/// inputs (e.g. `JSONExtract(s, 'Dynamic')`) can produce payloads that depend on the data rather than +/// on the input types, which a single synthetic execution cannot reveal; +/// - every non-suspect argument is a constant, so its probe value is exactly its execution-time value. +/// A non-constant non-suspect argument can *select* which payload the result carries - e.g. in +/// `if(cond, CAST(state, 'Dynamic'), CAST(0, 'Dynamic'))` the probes run with the default `cond = 0` +/// and only ever record the second branch, hiding the aggregate-state payload of the first one. +/// When either condition fails, the node keeps the fail-closed static enumeration of its result type. +/// +/// A *selector* function - `if`, `multiIf`, `coalesce`, `ifNull` - is the exception to the second condition: +/// its result is always one of its value arguments, so a non-constant control argument can only choose +/// *which* of their domains the result comes from, never introduce a payload none of them can hold. The union +/// of the value arguments' domains is therefore propagated after all, and valid TTLs such as +/// `toDateTime(if(cond, CAST(n, 'Dynamic'), CAST(m, 'Dynamic')))` over `n`, `m UInt32` and +/// `toDateTime(if(cond, CAST(1, 'Dynamic'), CAST(2, 'Dynamic')))` are accepted. A selector converts every +/// value argument to its result type, so a branch whose own type differs from it - including a branch that is +/// no carrier at all, like `m` in `if(cond, CAST(n, 'Dynamic'), m)` with `m UInt32` - contributes the payloads +/// that conversion produces from its values, which is what the branch domains are converted to below. +/// +/// Higher-order functions cannot be executed here at all, so their result normally falls back to the static +/// enumeration too. `arrayMap` is the exception: its result is exactly the array of the values its lambda +/// body produces, and that body is walked as an inner DAG by the recursive call below - with every rule of +/// this check applied to it - so its candidate domain is wrapped into one-element arrays and propagated. +/// This accepts e.g. `toDateTime(arrayElement(arrayMap(x -> CAST(x, 'Dynamic'), arr), 1))` over +/// `arr Array(UInt32)`, whose elements can only ever hold the `UInt32` payload. +/// +/// A typed `CAST` is the exception to the first condition: its output payloads are fixed by the *source +/// type* alone (the cast wrapper fills the discriminators derived from it), independent of the values. So +/// `CAST(n, 'Dynamic')` with `n UInt32` can only ever store the `UInt32` payload, and probing its consumer +/// with the static enumeration would reject a valid TTL such as +/// `DELETE WHERE toDateTime(CAST(n, 'Dynamic')) < now()` over synthetic `AggregateFunction` payloads +/// the cast can never produce. An untainted `CAST` is instead executed on the representative values of its +/// source type (non-NULL, one-element containers - the plain default would hide nested payload structure - +/// and one value per `Variant` alternative, which the cast preserves row by row) and the union of its +/// actual outputs is propagated to the consumers. +/// +/// A cast of a *string* to a carrier is in turn the exception to that exception - it parses the stored +/// alternative out of the row contents, so it is value-dependent after all and stays fail-closed; see the +/// `source_payload_may_be_inferred` check below. +/// +/// `result_name`, when set, names an output of `actions_dag` whose candidate materializations are returned to +/// the caller. It is used for the inner DAG of a lambda: the domain of the lambda body is what a higher-order +/// function like `arrayMap` produces, so returning it lets the outer node propagate it too. +std::vector checkActionsDAGForAggregateFunctions( + const ActionsDAG & actions_dag, std::string_view expression_kind, const String * result_name = nullptr) +{ + /// Per-node "candidate" materializations: the single-row columns whose payloads the node can produce + /// at TTL execution time and that are in scope of this check. An empty list means the node's default + /// (or constant) value is representative and its consumers need no extra probes. + std::unordered_map> candidates_map; + + /// The candidate materializations of the *body* of each lambda argument, keyed by its capture node. + std::unordered_map> lambda_body_candidates; + + std::function & (const ActionsDAG::Node *)> candidates_of + = [&](const ActionsDAG::Node * node) -> const std::vector & + { + if (auto it = candidates_map.find(node); it != candidates_map.end()) + return it->second; + + std::vector candidates; + + if (node->column) + { + /// The node's value is a known constant, so it is exact: probe consumers with the actual + /// value instead of over-approximating it from the static type. Keep it a single-row clone + /// (possibly const) so functions requiring constant arguments still see one. + if (hasAggregateFunctionType(node->result_type) || hasDynamicType(node->result_type)) + candidates.push_back(node->column->cloneResized(1)); + } + else if (node->type == ActionsDAG::ActionType::ALIAS) + { + candidates = candidates_of(node->children.front()); + } + else if (node->type == ActionsDAG::ActionType::FUNCTION) + { + /// Descend into lambda bodies of higher-order functions to validate consumers hidden inside + /// them. The capture node itself produces a function value, nothing to materialize. + if (const auto * function_capture = dynamic_cast(node->function_base.get())) + { + const auto & lambda_result_name = function_capture->getCapture().return_name; + lambda_body_candidates[node] = checkActionsDAGForAggregateFunctions( + function_capture->getAcionsDAG(), expression_kind, &lambda_result_name); + return candidates_map.emplace(node, std::move(candidates)).first->second; + } + + const ActionsDAG::Node * lambda_argument = nullptr; + bool has_lambda_argument = false; + ColumnsWithTypeAndName arguments; + std::vector suspect_indexes; + std::vector *> suspect_columns; + bool has_dynamic_suspect = false; + bool non_suspect_args_are_constant = true; + arguments.reserve(node->children.size()); + for (size_t i = 0; i < node->children.size(); ++i) + { + const auto * child = node->children[i]; + + /// A lambda argument cannot be materialized into a column; the higher-order function + /// that receives it is validated through the captured lambda DAG above, so skip + /// executing it here. + if (WhichDataType(child->result_type).isFunction()) + { + has_lambda_argument = true; + lambda_argument = child->type == ActionsDAG::ActionType::ALIAS ? child->children.front() : child; + /// Make sure the lambda body has been walked (and its candidates recorded) before the + /// higher-order node below looks them up - the outer loop over the DAG nodes visits the + /// nodes in no particular order. + candidates_of(lambda_argument); + break; + } + + /// Preserve constant arguments as constants - some functions (e.g. `CAST`) require a + /// constant argument and otherwise throw an unrelated error during this synthetic execution. + ColumnPtr column = child->column + ? child->column->cloneResized(1) + : child->result_type->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst(); + arguments.emplace_back(std::move(column), child->result_type, child->result_name); + + const auto & child_candidates = candidates_of(child); + if (child_candidates.empty()) + { + if (!child->column) + non_suspect_args_are_constant = false; + continue; + } + suspect_indexes.push_back(i); + suspect_columns.push_back(&child_candidates); + if (hasDynamicType(child->result_type)) + has_dynamic_suspect = true; + } + + const bool result_in_scope = hasAggregateFunctionType(node->result_type) || hasDynamicType(node->result_type); + + if (has_lambda_argument) + { + /// The node was not executed (its output cannot be derived synthetically here), so if its + /// result can carry a suspect payload, fail closed with the static enumeration - except for + /// `arrayMap`, whose result is exactly an array of the lambda body's values, so the body's + /// candidate domain (computed in the inner DAG above, with every narrowing rule of this + /// check applied to it) describes the elements: wrap each of them into a one-element array. + if (result_in_scope) + { + const auto * result_array_type = typeid_cast(node->result_type.get()); + const auto * lambda_capture = lambda_argument && lambda_argument->type == ActionsDAG::ActionType::FUNCTION + ? dynamic_cast(lambda_argument->function_base.get()) + : nullptr; + const auto * lambda_candidates = lambda_capture && lambda_body_candidates.contains(lambda_argument) + ? &lambda_body_candidates.at(lambda_argument) + : nullptr; + + if (node->function_base->getName() == "arrayMap" && result_array_type && lambda_candidates + && !lambda_candidates->empty() + && result_array_type->getNestedType()->equals(*lambda_capture->getCapture().return_type)) + { + for (const auto & element : *lambda_candidates) + { + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->getData().push_back(1); + candidates.push_back(ColumnArray::create(element->cloneResized(1), std::move(offsets))); + } + } + else + candidates = collectSuspectMaterializations(node->result_type, expression_kind); + } + } + else if (suspect_indexes.empty()) + { + /// No argument carries a suspect payload, so there is nothing to probe this node with. + /// If its *result* is a carrier computed from non-suspect inputs, its runtime payloads + /// may be data-dependent (see above), so consumers get the fail-closed static enumeration. + /// The exception is a typed CAST, whose output payload type is fixed by the source type + /// alone: execute it once on a representative source value and propagate the actual + /// output domain instead (see above). + if (result_in_scope) + { + const auto & function_name = node->function_base->getName(); + + /// A cast of a *string* to a carrier is not source-type-determined: with + /// `cast_string_to_variant_use_inference` (on by default) and + /// `cast_string_to_dynamic_use_inference`, `createColumnToVariantWrapper` / + /// `createColumnToDynamicWrapper` parse the alternative out of the row contents, so + /// `CAST(s, 'Variant(String, UInt32, AggregateFunction(max, UInt32))')` stores the + /// `String` alternative for the representative `''` but the `UInt32` one for a row + /// `s = '42'`. A single representative value therefore says nothing about the runtime + /// domain. The settings are also per-query while the stored TTL expression is rebuilt + /// and executed under other contexts, so, like the strict probe above, the DDL-time + /// verdict must hold for either of them: keep such casts on the fail-closed path. + bool source_payload_may_be_inferred = false; + for (const auto & child : node->children) + if (!child->column && castMayInferPayloadFromString(child->result_type, node->result_type)) + source_payload_may_be_inferred = true; + + if ((function_name == "CAST" || function_name == "_CAST") && !source_payload_may_be_inferred) + { + /// A source type can need more than one representative value (a `Variant` needs + /// one per alternative), so run the cast over the cartesian product of them and + /// propagate the union of the outputs. + std::vector representative_indexes; + std::vector> representative_columns; + size_t total_combinations = 1; + for (size_t i = 0; i < node->children.size(); ++i) + { + if (node->children[i]->column) + continue; + representative_indexes.push_back(i); + representative_columns.push_back( + makeRepresentativeColumns(node->children[i]->result_type, expression_kind)); + total_combinations *= representative_columns.back().size(); + } + + /// Too many source combinations to enumerate (or none at all): the node itself is + /// valid, only the narrowing is given up, so fall back to the static enumeration + /// instead of failing the whole expression. + if (total_combinations == 0 || total_combinations > max_probe_combinations) + { + candidates = collectSuspectMaterializations(node->result_type, expression_kind); + } + else + { + std::vector selection(representative_indexes.size(), 0); + while (true) + { + ColumnsWithTypeAndName representative_arguments = arguments; + for (size_t r = 0; r < representative_indexes.size(); ++r) + representative_arguments[representative_indexes[r]].column + = representative_columns[r][selection[r]]; + + try + { + ColumnPtr cast_result = node->function_base->execute( + representative_arguments, node->result_type, /*input_rows_count=*/ 1, /*dry_run=*/ true); + candidates.push_back(cast_result->convertToFullColumnIfConst()); + } + catch (...) /// Ok: any failure here only means we cannot narrow the domain. + { + /// The cast failed on a synthetic representative value (a data-dependent + /// error, e.g. an unparseable default string). The node itself needs no + /// validation - fail closed to the static enumeration for its consumers. + candidates = collectSuspectMaterializations(node->result_type, expression_kind); + break; + } + + size_t r = 0; + while (r < selection.size() && ++selection[r] == representative_columns[r].size()) + { + selection[r] = 0; + ++r; + } + if (r == selection.size()) + break; + } + } + } + else + candidates = collectSuspectMaterializations(node->result_type, expression_kind); + } + } + else + { + /// Translate the "cannot consume an AggregateFunction state" type error into a clear TTL + /// message; rethrow anything else (e.g. a data-dependent error raised by a perfectly + /// valid consumer). + auto probe = [&](const ColumnsWithTypeAndName & probe_arguments, std::string_view hint) -> ColumnPtr + { + try + { + return node->function_base->execute(probe_arguments, node->result_type, /*input_rows_count=*/ 1, /*dry_run=*/ true); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT) + throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, + "TTL {}expression uses {}: {}", expression_kind, hint, e.message()); + throw; + } + }; + + constexpr std::string_view aggregate_state_hint = + "AggregateFunction column in a function that cannot handle it. " + "Use `finalizeAggregation` to extract the value first"; + + constexpr std::string_view dynamic_hint = + "a Dynamic column in a function that cannot handle all types a Dynamic column can store " + "(e.g. an AggregateFunction state), so TTL execution could fail depending on the inserted values. " + "Use a typed subcolumn instead, or set `allow_suspicious_ttl_expressions` to allow it"; + + /// All suspect arguments must be materialized in the same probe: substituting them one at + /// a time would leave the other carriers at their all-NULL defaults, letting the adaptor + /// short-circuit to NULL and hide a consumer that only fails when several carriers hold + /// states simultaneously (e.g. `d1 + d2` or `v1 = v2`). So the probes below run the + /// cartesian product of the candidate materializations across all suspect arguments. + size_t total_combinations = 1; + for (const auto * columns : suspect_columns) + { + total_combinations *= columns->size(); + if (total_combinations > max_probe_combinations) + throwTooManyProbeCombinations(expression_kind); + } + + std::vector selection(suspect_indexes.size(), 0); + while (true) + { + ColumnsWithTypeAndName probe_arguments = arguments; + for (size_t s = 0; s < suspect_indexes.size(); ++s) + probe_arguments[suspect_indexes[s]].column = (*suspect_columns[s])[selection[s]]; + ColumnPtr probe_result = probe(probe_arguments, has_dynamic_suspect ? dynamic_hint : aggregate_state_hint); + + /// When every non-suspect argument is a constant, the probe outputs are exactly the + /// payloads this node can produce from its suspect inputs - propagate them so a parent + /// consuming this computed carrier is validated against the real domain, not the + /// static enumeration of its result type. A non-constant non-suspect argument breaks + /// this: it is probed with a synthetic default value, but at execution time it can + /// select a payload the probes never produced (e.g. the condition of `if`), so such + /// nodes fall back to the static enumeration below instead. + if (result_in_scope && non_suspect_args_are_constant) + candidates.push_back(probe_result->convertToFullColumnIfConst()); + + /// Advance the mixed-radix counter over the candidates of each suspect argument. + size_t s = 0; + while (s < selection.size() && ++selection[s] == suspect_columns[s]->size()) + { + selection[s] = 0; + ++s; + } + if (s == selection.size()) + break; + } + + /// The probes above validated this node, but their outputs under-approximate its runtime + /// domain when a non-constant non-suspect argument can select the payload - fail closed + /// with the static enumeration for the parents in that case. + /// + /// A *selector* function is the exception: its result is always one of its value arguments, + /// converted to the result type, so whatever its non-constant control arguments choose at + /// execution time, the result stays inside the union of the converted value domains. That + /// union - deduplicated by fingerprint - is propagated instead of the static enumeration: + /// `if(cond, CAST(n, 'Dynamic'), CAST(m, 'Dynamic'))` with `n`, `m UInt32` can only ever + /// hold the `UInt32` payload, whichever branch `cond` takes, + /// `if(cond, CAST(1, 'Dynamic'), CAST(2, 'Dynamic'))` only the `UInt8` one, and + /// `if(cond, CAST(n, 'Dynamic'), m)` with `m UInt32` only numeric ones. A branch whose + /// domain does contain a state (e.g. `CAST(state, 'Dynamic')`) keeps its state candidate in + /// the union, so an unsupported parent consumer is still rejected by its probes. + if (result_in_scope && !non_suspect_args_are_constant) + { + bool selector_domain_is_proven = false; + const auto value_arguments = getSelectorValueArguments(node->function_base->getName(), node->children.size()); + if (!value_arguments.empty()) + { + /// The domain of one value branch, expressed in the result type of the selector. + /// The branch's own domain is its candidate list, or the representative values of its + /// static type when it carries no suspect payload itself; each value is then converted + /// to the result type exactly like the selector does at execution time. `{}` means the + /// domain could not be proven - e.g. a conversion that infers the payload out of a + /// string is value-dependent, so it says nothing about the runtime payloads. + auto branch_domain = [&](const ActionsDAG::Node * value_argument) -> std::vector + { + const auto & branch_candidates = candidates_of(value_argument); + if (value_argument->result_type->equals(*node->result_type)) + return branch_candidates; + + if (castMayInferPayloadFromString(value_argument->result_type, node->result_type)) + return {}; + + std::vector converted; + try + { + const auto & branch_values = branch_candidates.empty() + ? makeRepresentativeColumns(value_argument->result_type, expression_kind) + : branch_candidates; + for (const auto & value : branch_values) + converted.push_back( + castColumn({value, value_argument->result_type, value_argument->result_name}, + node->result_type)->convertToFullColumnIfConst()); + } + catch (...) /// Ok: any failure here only means we cannot narrow the domain. + { + return {}; + } + return converted; + }; + + selector_domain_is_proven = true; + std::vector union_candidates; + std::unordered_set union_fingerprints; + for (size_t index : value_arguments) + { + const auto value_candidates = branch_domain(node->children[index]); + if (value_candidates.empty()) + { + selector_domain_is_proven = false; + break; + } + for (const auto & candidate : value_candidates) + if (union_fingerprints.insert(candidateFingerprint(candidate)).second) + union_candidates.push_back(candidate); + } + + if (selector_domain_is_proven) + candidates.insert(candidates.end(), union_candidates.begin(), union_candidates.end()); + } + + if (!selector_domain_is_proven) + candidates = collectSuspectMaterializations(node->result_type, expression_kind); + } + } + } + else + { + /// An INPUT column (or any other node kind) can hold any value of its type - enumerate the + /// suspect payloads from the static type. + candidates = collectSuspectMaterializations(node->result_type, expression_kind); + } + + return candidates_map.emplace(node, std::move(candidates)).first->second; + }; + + for (const auto & node : actions_dag.getNodes()) + candidates_of(&node); + + if (!result_name) + return {}; + + /// The lambda body: its result is the single output of the inner DAG. If it cannot be found, give the + /// caller nothing and let it fall back to the static enumeration. + if (const auto * result_node = actions_dag.tryFindInOutputs(*result_name)) + return candidates_of(result_node); + return {}; +} + +/// RAII guard setting `variant_throw_on_type_mismatch` / `dynamic_throw_on_type_mismatch` on the query +/// context of the *current thread* - the only place the `Variant`/`Dynamic` function adaptors read them +/// from - and restoring the previous values on scope exit. Note the DDL `context` cannot be used for this: +/// on a server it has no query context, and the adaptors would not see settings changed on it. +class MismatchSettingsGuard +{ +public: + MismatchSettingsGuard(bool variant_throw, bool dynamic_throw) + { + if (CurrentThread::isInitialized()) + { + if (auto thread_query_context = CurrentThread::tryGetQueryContext()) + thread_context = std::const_pointer_cast(thread_query_context); + } + + if (!thread_context) + return; + + const auto & settings = thread_context->getSettingsRef(); + if (settings[Setting::variant_throw_on_type_mismatch] != variant_throw) + { + old_variant_throw = settings[Setting::variant_throw_on_type_mismatch]; + thread_context->setSetting("variant_throw_on_type_mismatch", Field(variant_throw)); + } + if (settings[Setting::dynamic_throw_on_type_mismatch] != dynamic_throw) + { + old_dynamic_throw = settings[Setting::dynamic_throw_on_type_mismatch]; + thread_context->setSetting("dynamic_throw_on_type_mismatch", Field(dynamic_throw)); + } + } + + ~MismatchSettingsGuard() + { + if (!thread_context) + return; + + if (old_variant_throw) + thread_context->setSetting("variant_throw_on_type_mismatch", Field(*old_variant_throw)); + if (old_dynamic_throw) + thread_context->setSetting("dynamic_throw_on_type_mismatch", Field(*old_dynamic_throw)); + } + +private: + ContextMutablePtr thread_context; + std::optional old_variant_throw; + std::optional old_dynamic_throw; +}; + +void checkTTLExpressionForAggregateFunctions(const ExpressionActionsPtr & expression, std::string_view expression_kind) +{ + /// The synthetic probe in `checkActionsDAGForAggregateFunctions` exercises consumers over `Variant`/`Dynamic` + /// columns carrying an AggregateFunction state. For consumers wrapped in the `Variant`/`Dynamic` function + /// adaptors, whether a type mismatch throws or is silently turned into NULL at *execution* is decided by + /// `variant_throw_on_type_mismatch` / `dynamic_throw_on_type_mismatch`, which the adaptors read from the + /// query context of the current thread. But a stored TTL expression is later rebuilt and executed under + /// several unrelated contexts: the *inserting* session in `MergeTreeDataWriter::updateTTL` (strict by + /// default), the background context during TTL merges (settings from the `background_profile` server + /// config, strict by default), and table loading on ATTACH/restart (no thread query context at all, so + /// the adaptors fall back to strict). The DDL-time verdict must therefore not depend on any one of them: + /// the probe always runs strict, which is the superset - an expression that survives the strict probe + /// only ever gets *more* lenient at execution (a mismatch turns into NULL instead of an exception), so it + /// is safe under every context, while anything rejected here would throw on the first + /// AggregateFunction-carrying row in at least the strict paths (e.g. a default-settings INSERT). + /// A server that deliberately runs everything lenient still has `allow_suspicious_ttl_expressions`. + /// (Conversion functions such as `toDateTime` handle `Variant`/`Dynamic` natively, ignore both settings + /// and always throw on a stored type they cannot convert, so for them the probe's verdict is the same + /// under any settings.) + MismatchSettingsGuard probe_guard(/*variant_throw=*/ true, /*dynamic_throw=*/ true); + + checkActionsDAGForAggregateFunctions(expression->getActionsDAG(), expression_kind); +} + void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const String & result_column_name, bool allow_suspicious) { /// Do not apply this check in ATTACH queries for compatibility reasons and if explicitly allowed. @@ -87,6 +1086,8 @@ void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const Strin func.getName()); } } + + checkTTLExpressionForAggregateFunctions(ttl_expression, /*expression_kind=*/ ""); } const auto & result_column = ttl_expression->getSampleBlock().getByName(result_column_name); @@ -196,6 +1197,38 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType return result; } +/// Collect the argument expressions of every aggregate function found in the AST. +static void collectAggregateFunctionArguments(const ASTPtr & ast, ASTs & arguments) +{ + if (const auto * function = ast->as(); function && AggregateUtils::isAggregateFunction(*function)) + { + if (function->arguments) + for (const auto & argument : function->arguments->children) + arguments.push_back(argument); + } + + for (const auto & child : ast->children) + collectAggregateFunctionArguments(child, arguments); +} + +/// Validate the aggregate-function arguments of a `GROUP BY ... SET` assignment. These argument +/// expressions (e.g. `toDateTime(ts)` in `SET out = max(toDateTime(ts))`) are evaluated later by +/// TTLAggregationAlgorithm and are not part of the main TTL expression, so an unsupported +/// AggregateFunction-state consumer there would otherwise pass CREATE TABLE and fail at merge time. +static void checkTTLGroupBySetForAggregateFunctions( + const ASTPtr & assignment_expression, const NamesAndTypesList & columns, const ContextPtr & context) +{ + ASTs aggregate_arguments; + collectAggregateFunctionArguments(assignment_expression, aggregate_arguments); + + for (const auto & argument : aggregate_arguments) + { + auto argument_ast = argument->clone(); + auto argument_expression = buildExpressionAndSets(argument_ast, columns, context).expression; + checkTTLExpressionForAggregateFunctions(argument_expression, /*expression_kind=*/ "GROUP BY SET "); + } +} + ExpressionAndSets TTLDescription::buildExpression(const ContextPtr & context) const { auto ast = expression_ast->clone(); @@ -231,6 +1264,20 @@ TTLDescription TTLDescription::getTTLFromAST( checkExpressionDoesntContainSubqueries(*result.expression_ast); + /// Building a TTL expression can itself consult `variant_throw_on_type_mismatch`: the `Variant` + /// function adaptor throws in its constructor when none of the alternatives is compatible with the + /// consumer, and under a lenient setting resolves the result to constant NULL instead. Such a lenient + /// build must not slip through DDL validation regardless of the session (or even the background + /// profile) settings, because it produces a table that is broken no matter how TTL runs later: the + /// constant fold prunes the referenced column from the stored TTL column list, so every subsequent + /// rebuild of the TTL expression fails with "Missing columns", and the table cannot even be re-attached + /// on server restart (loading has no query context, so the adaptor defaults to strict and throws). + /// Hence the validation build always runs strict. The escape hatches stay intact: on ATTACH or with + /// `allow_suspicious_ttl_expressions` the build behaves exactly as the session dictates. + std::optional build_guard; + if (!is_attach && !context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]) + build_guard.emplace(/*variant_throw=*/ true, /*dynamic_throw=*/ true); + auto ttl_ast = result.expression_ast->clone(); auto expression = buildExpressionAndSets(ttl_ast, columns.getAllPhysical(), context).expression; result.expression_columns = expression->getRequiredColumnsWithTypes(); @@ -291,6 +1338,9 @@ TTLDescription TTLDescription::getTTLFromAST( throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "Invalid expression for assignment of column {}. Should contain an aggregate function", assignment.column_name); + if (!is_attach && !context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]) + checkTTLGroupBySetForAggregateFunctions(ass_expression, columns.getAllPhysical(), context); + ass_expression = addTypeConversionToAST(std::move(ass_expression), columns.getPhysical(assignment.column_name).type->getName()); aggregations.emplace_back(assignment.column_name, std::move(ass_expression)); aggregation_columns_set.insert(assignment.column_name); @@ -311,6 +1361,13 @@ TTLDescription TTLDescription::getTTLFromAST( set_part.expression_result_column_name = value->getColumnName(); set_part.expression = expr_analyzer.getActions(false); + /// The post-aggregation expression (including the implicit cast to the target column type) + /// is executed later by TTLAggregationAlgorithm. When an aggregate returns an AggregateFunction + /// state itself (e.g. `any(ts)`), casting it to an incompatible target type (e.g. `DateTime`) + /// must be rejected here instead of failing during the TTL merge. + if (!is_attach && !context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]) + checkTTLExpressionForAggregateFunctions(set_part.expression, /*expression_kind=*/ "GROUP BY SET "); + result.set_parts.emplace_back(set_part); for (const auto & descr : expr_analyzer.getAnalyzedData().aggregate_descriptions) @@ -326,6 +1383,10 @@ TTLDescription TTLDescription::getTTLFromAST( } checkTTLExpression(expression, result.result_column, is_attach || context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]); + + if (where_expression && !is_attach && !context->getSettingsRef()[Setting::allow_suspicious_ttl_expressions]) + checkTTLExpressionForAggregateFunctions(where_expression, /*expression_kind=*/ "WHERE "); + return result; } diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 76c979bdae53..9669cd40ad7e 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -7,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,7 +17,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -37,6 +41,7 @@ #include #include #include +#include #include #include @@ -47,6 +52,7 @@ namespace DB namespace Setting { + extern const SettingsBool analyzer_compatibility_join_using_top_level_identifier; extern const SettingsDistributedProductMode distributed_product_mode; extern const SettingsUInt64 interactive_delay; extern const SettingsUInt64 max_bytes_to_transfer; @@ -66,6 +72,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int INCOMPATIBLE_TYPE_OF_JOIN; extern const int DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED; + extern const int UNSUPPORTED_METHOD; } namespace @@ -606,6 +613,129 @@ QueryTreeNodePtr getSubqueryFromTableExpression( return subquery_node; } +/// Does `query_node` expose `name` as a top-level projection column? +bool hasProjectionColumn(const QueryNode & query_node, const String & name) +{ + for (const auto & projection_column : query_node.getProjectionColumns()) + if (projection_column.name == name) + return true; + return false; +} + +/// Does the JOIN's left table expression expose `name` as a real column the shard can resolve? +bool leftTableHasColumn(const QueryTreeNodePtr & node, const String & name) +{ + /// Flat worklist over the left table expression (same node-kind coverage as the join tree). + QueryTreeNodes nodes_to_process{node}; + for (size_t i = 0; i < nodes_to_process.size(); ++i) + { + const auto current = nodes_to_process[i]; + if (!current) + continue; + + if (const auto * table_node = current->as()) + { + if (table_node->getStorageSnapshot()->tryGetColumn(GetColumnsOptions::All, name).has_value()) + return true; + } + else if (const auto * table_function_node = current->as()) + { + if (table_function_node->getStorageSnapshot()->tryGetColumn(GetColumnsOptions::All, name).has_value()) + return true; + } + else if (const auto * query_node = current->as()) + { + if (hasProjectionColumn(*query_node, name)) + return true; + } + else if (const auto * union_node = current->as()) + { + for (const auto & projection_column : union_node->computeProjectionColumns()) + if (projection_column.name == name) + return true; + } + else if (const auto * join_node = current->as()) + { + nodes_to_process.push_back(join_node->getLeftTableExpression()); + nodes_to_process.push_back(join_node->getRightTableExpression()); + } + else if (const auto * cross_join_node = current->as()) + { + for (const auto & table_expression : cross_join_node->getTableExpressions()) + nodes_to_process.push_back(table_expression); + } + else if (const auto * array_join_node = current->as()) + { + nodes_to_process.push_back(array_join_node->getTableExpression()); + } + } + return false; +} + +/// Throw only when nothing on the remote server can resolve the `JOIN USING` key. +void checkJoin(const JoinNode & join_node, const QueryNode & enclosing_query) +{ + const auto & using_list = join_node.getJoinExpression()->as(); + for (const auto & using_node : using_list.getNodes()) + { + /// USING key `N`: a `ColumnNode` whose expression is a `ListNode{left, right}` (see `QueryAnalyzer::resolveJoin`). + const auto * using_column = using_node->as(); + if (!using_column || !using_column->hasExpression()) + continue; + + const auto & using_elements = using_column->getExpression()->as().getNodes(); + if (using_elements.empty()) + continue; + + const auto & name = using_column->getColumnName(); + const auto & left_element = using_elements.front(); + + /// Marker: the left element carries a resolved alias body (a `ColumnNode` with an expression, or a non-`ColumnNode` with an alias); plain-column keys never reach the throw. + const auto * left_column = left_element->as(); + if (left_column) + { + if (!left_column->hasExpression()) + continue; + } + else if (!left_element->hasAlias()) + { + continue; + } + + /// Top-level alias re-emitted as a projection name in the shipped SQL, re-resolves on the shard. + if (hasProjectionColumn(enclosing_query, name)) + continue; + + /// A shadowed column resolves on the shard and may join differently than the initiator's alias; accepted. + if (leftTableHasColumn(join_node.getLeftTableExpression(), name)) + continue; + + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "JOIN {} using identifier '{}' is resolved from an alias nested in the SELECT list, which is not " + "supported for queries sent to remote servers. Move the alias to the top level of the SELECT list", + join_node.formatASTForErrorMessage(), name); + } +} + +/// Reject `JOIN USING` keys that no remote server can resolve; keys the shard can re-resolve are shipped. +void rejectUnshippableJoinUsingKeys(const QueryTreeNodePtr & root) +{ + /// The enclosing query travels with the node so each `JOIN USING` is checked against its own projection. + std::vector> nodes_to_process{{root.get(), nullptr}}; + while (!nodes_to_process.empty()) + { + auto [node, enclosing_query] = nodes_to_process.back(); + nodes_to_process.pop_back(); + if (const auto * query_node = node->as()) + enclosing_query = query_node; + else if (const auto * join_node = node->as(); join_node && join_node->isUsingJoinExpression() && enclosing_query) + checkJoin(*join_node, *enclosing_query); + for (const auto & child : node->getChildren()) + if (child) + nodes_to_process.emplace_back(child.get(), enclosing_query); + } +} + } QueryTreeNodePtr buildQueryTreeForShard( @@ -765,6 +895,11 @@ QueryTreeNodePtr buildQueryTreeForShard( createUniqueAliasesIfNecessary(query_tree_to_modify, planner_context->getQueryContext()); + /// Reject `JOIN USING` keys that no remote server can resolve; keys the shard can re-resolve are shipped. + /// Such keys can only be produced by the projection-alias resolution, so check only when it is enabled. + if (planner_context->getQueryContext()->getSettingsRef()[Setting::analyzer_compatibility_join_using_top_level_identifier]) + rejectUnshippableJoinUsingKeys(query_tree_to_modify); + // Get rid of the settings clause so we don't send them to remote. Thus newly non-important // settings won't break any remote parser. It's also more reasonable since the query settings // are written into the query context and will be sent by the query pipeline. diff --git a/tests/integration/compose/docker_compose_iceberg_seaweedfs_catalog.yml b/tests/integration/compose/docker_compose_iceberg_seaweedfs_catalog.yml new file mode 100644 index 000000000000..d0923879f178 --- /dev/null +++ b/tests/integration/compose/docker_compose_iceberg_seaweedfs_catalog.yml @@ -0,0 +1,29 @@ +services: + seaweedfs: + image: chrislusf/seaweedfs:4.41 + entrypoint: /bin/sh + command: + - -c + - | + mkdir -p /data /etc/seaweedfs + cat > /etc/seaweedfs/s3.json <<'EOF' + { + "identities": [ + { + "name": "clickhouse", + "credentials": [{"accessKey": "clickhouse", "secretKey": "clickhouse"}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"] + } + ] + } + EOF + exec weed mini -dir=/data -s3 -s3.config=/etc/seaweedfs/s3.json -tableBucket=analytics + ports: + - "${ICEBERG_REST_CATALOG_PORT}:8181" + healthcheck: + test: ["CMD-SHELL", "nc -w2 -z localhost 8333 && nc -w2 -z localhost 8181"] + interval: 1s + timeout: 5s + retries: 60 + start_period: 10s + cpus: 3 diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 308288194d87..2e81d9e52b21 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -41,7 +41,6 @@ import pymongo import pymysql import nats - from filelock import FileLock, Timeout from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient # Not an easy dep import cassandra.cluster @@ -52,6 +51,7 @@ import docker from dict2xml import dict2xml +from filelock import FileLock, Timeout from docker.models.containers import Container from kazoo.exceptions import KazooException from minio import Minio diff --git a/tests/integration/helpers/spark_tools.py b/tests/integration/helpers/spark_tools.py index d02704038cbe..72ee5c689a00 100644 --- a/tests/integration/helpers/spark_tools.py +++ b/tests/integration/helpers/spark_tools.py @@ -2,6 +2,8 @@ import os import pyspark +from pyspark.context import SparkContext +from pyspark.sql import SparkSession def write_spark_log_config(log_dir): @@ -38,6 +40,42 @@ def write_spark_log_config(log_dir): return props_path +def _gateway_is_live(): + """Round-trip one trivial call against the class-cached py4j gateway. + + Returns False when nothing is cached (nothing to reuse) or when the cached + handle no longer answers. + """ + gateway = SparkContext._gateway + if gateway is None: + return False + try: + gateway.jvm.System.currentTimeMillis() + return True + except Exception: + return False + + +def _reset_pyspark_class_state(): + """Drop the cached gateway so ``_ensure_initialized`` relaunches the JVM. + + ``_ensure_initialized`` skips the relaunch while ``SparkContext._gateway`` is + truthy, and neither ``SparkContext.stop()`` nor ``SparkSession.stop()`` clears + it. Each clear is tolerant so a pyspark upgrade cannot break the harness. + """ + for owner, attr in ( + (SparkContext, "_gateway"), + (SparkContext, "_jvm"), + (SparkContext, "_active_spark_context"), + (SparkSession, "_instantiatedSession"), + (SparkSession, "_activeSession"), + ): + try: + setattr(owner, attr, None) + except Exception: + pass + + class ResilientSparkSession: """Wrapper around SparkSession that automatically restarts on JVM/py4j failures. @@ -51,7 +89,21 @@ class ResilientSparkSession: def __init__(self, create_session_fn): self._create = create_session_fn - self._session = create_session_fn() + # Set before creating so a factory failure cannot leave the attribute + # missing, which would make __getattr__ recurse through _is_alive. + self._session = None + self._session = self._prepare_and_create() + + def _prepare_and_create(self): + """Create a session, first discarding a cached gateway that is dead. + + The reset is conditional: a live gateway must be reused, otherwise the + still-running JVM is orphaned and a needless one is launched. + """ + if SparkContext._gateway is not None and not _gateway_is_live(): + logging.warning("Cached py4j gateway is dead, discarding it") + _reset_pyspark_class_state() + return self._create() def _restart(self): logging.warning("Spark session is dead, restarting...") @@ -59,16 +111,16 @@ def _restart(self): self._session.stop() except Exception: pass - # Clear any cached singleton so getOrCreate builds a fresh one - pyspark.sql.SparkSession.builder._options = {} try: pyspark.sql.SparkSession._instantiatedSession = None except Exception: pass - self._session = self._create() + self._session = self._prepare_and_create() logging.warning("Spark session restarted successfully") def _is_alive(self): + if self.__dict__.get("_session") is None: + return False try: self._session.sparkContext._jsc.sc().defaultParallelism() return True diff --git a/tests/integration/test_acme_tls/configs/config_no_certificate.xml b/tests/integration/test_acme_tls/configs/config_no_certificate.xml new file mode 100644 index 000000000000..e6c7aa42844f --- /dev/null +++ b/tests/integration/test_acme_tls/configs/config_no_certificate.xml @@ -0,0 +1,11 @@ + + + test@clickhouse.com + true + + never-issued.integration-tests.clickhouse.com + + + https://127.0.0.1:14000/dir + + diff --git a/tests/integration/test_acme_tls/test_no_certificate.py b/tests/integration/test_acme_tls/test_no_certificate.py new file mode 100644 index 000000000000..7308ed944cdd --- /dev/null +++ b/tests/integration/test_acme_tls/test_no_certificate.py @@ -0,0 +1,32 @@ +import logging + +import pytest + +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + +no_certificate_cluster = ClickHouseCluster(__file__) +node = no_certificate_cluster.add_instance( + "node_no_certificate", + main_configs=["configs/config_no_certificate.xml"], + with_zookeeper=True, +) + + +@pytest.fixture(scope="module") +def started_no_certificate_cluster(): + try: + no_certificate_cluster.start() + yield no_certificate_cluster + finally: + no_certificate_cluster.shutdown() + + +def test_show_certificate_without_certificate(started_no_certificate_cluster): + # The certificate is provisioned by ACME, and the ACME server is unreachable, so the server + # runs with an SSL context that has no certificate at all. `showCertificate` used to + # dereference a null pointer in this case. + assert node.query("SELECT showCertificate()") == "{}\n" + assert node.query("SELECT 1") == "1\n" diff --git a/tests/integration/test_acme_tls/test_single_node.py b/tests/integration/test_acme_tls/test_single_node.py index b4af52255d7a..375289427763 100644 --- a/tests/integration/test_acme_tls/test_single_node.py +++ b/tests/integration/test_acme_tls/test_single_node.py @@ -66,3 +66,23 @@ def test_acme_authorization(started_single_replica_cluster): return raise Exception("Failed to get expected certificate issuer") + + +def test_show_certificate(started_single_replica_cluster): + # Let Pebble know where to find our server + requests.post( + 'http://10.5.11.3:8055/add-a', + json={'host': 'single.integration-tests.clickhouse.com', 'addresses': ['10.5.11.11']} + ) + + # With ACME the SSL context of the server never receives a certificate, so `showCertificate` + # has to report the certificate held by the certificate reloader. + certificate = "" + for _ in range(120): + certificate = node.query("SELECT showCertificate()") + if "CN=Pebble Intermediate CA" in certificate: + return + + time.sleep(1) + + raise Exception(f"showCertificate() did not report the ACME certificate: {certificate}") diff --git a/tests/integration/test_ai_functions/mock_ai_server.py b/tests/integration/test_ai_functions/mock_ai_server.py index 096e11e233ea..69e236101531 100644 --- a/tests/integration/test_ai_functions/mock_ai_server.py +++ b/tests/integration/test_ai_functions/mock_ai_server.py @@ -34,12 +34,15 @@ import http.server import json +import threading from urllib.parse import urlparse, parse_qs MOCK_PORT = 18123 DEFAULT_EMBED_DIM = 4 -# Single-threaded `HTTPServer` handles one request at a time, so a plain dict is safe. +# The server is threaded (see `ThreadingHTTPServer` below) so it can serve the concurrent AI calls a +# multi-threaded query issues. `_LOCK` guards the shared mutable state against those concurrent handlers. +_LOCK = threading.Lock() LAST_REQUEST = {"path": None, "body": None, "headers": {}} # Number of upcoming requests to the flaky endpoints (`/v1/chat/flaky`, `/v1/embeddings_flaky`) @@ -153,12 +156,15 @@ def do_GET(self): return if parsed.path == "/last-request": - self._send_json(200, LAST_REQUEST) + with _LOCK: + snapshot = dict(LAST_REQUEST) + self._send_json(200, snapshot) return if parsed.path == "/set-flaky": qs = parse_qs(parsed.query) - FLAKY["fails_remaining"] = int(qs.get("count", ["0"])[0]) + with _LOCK: + FLAKY["fails_remaining"] = int(qs.get("count", ["0"])[0]) self.send_response(200) self.send_header("Content-Type", "text/plain") self.end_headers() @@ -173,13 +179,17 @@ def do_POST(self): content_length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(content_length).decode("utf-8") if content_length else "" - LAST_REQUEST["path"] = parsed.path - LAST_REQUEST["body"] = body - LAST_REQUEST["headers"] = {k.lower(): v for k, v in self.headers.items()} + with _LOCK: + LAST_REQUEST["path"] = parsed.path + LAST_REQUEST["body"] = body + LAST_REQUEST["headers"] = {k.lower(): v for k, v in self.headers.items()} if parsed.path in ("/v1/chat/flaky", "/v1/embeddings_flaky"): - if FLAKY["fails_remaining"] > 0: - FLAKY["fails_remaining"] -= 1 + with _LOCK: + should_fail = FLAKY["fails_remaining"] > 0 + if should_fail: + FLAKY["fails_remaining"] -= 1 + if should_fail: # Simulate a transient network failure: close the connection without sending any # response, so the client sees EOF — a Poco network exception — rather than an HTTP # error status. This exercises the network-error retry path, distinct from the HTTP @@ -245,8 +255,17 @@ def log_message(self, format, *args): pass # suppress request logs +class MockServer(http.server.ThreadingHTTPServer): + daemon_threads = True + allow_reuse_address = True + # Absorb a burst of simultaneous connections from a multi-threaded query. The default backlog of + # 5 overflows when several pipeline threads each open a connection at once, dropping SYNs and + # making the client's connect time out. + request_queue_size = 128 + + if __name__ == "__main__": - server = http.server.HTTPServer(("0.0.0.0", MOCK_PORT), Handler) + server = MockServer(("0.0.0.0", MOCK_PORT), Handler) try: server.serve_forever() finally: diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 406407b8ab72..c7ca597bc011 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -70,7 +70,8 @@ def get_profile_events(query_id): ProfileEvents['AIInputTokens'] AS input_tokens, ProfileEvents['AIOutputTokens'] AS output_tokens, ProfileEvents['AIRowsProcessed'] AS rows_processed, - ProfileEvents['AIRowsSkipped'] AS rows_skipped + ProfileEvents['AIRowsSkipped'] AS rows_skipped, + peak_threads_usage AS peak_threads FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish' LIMIT 1 @@ -977,3 +978,350 @@ def test_embed_retry_respects_api_call_quota(started_cluster): assert int(events["api_calls"]) == 1 assert int(events["rows_processed"]) == 0 assert int(events["rows_skipped"]) == 1 + + +# --------------------------------------------------------------------------- +# How many API calls a query shape issues +# +# These assert `AIAPICalls`, not output: the count is a property of the planner and the +# row loop, and it is what an implementation that evaluated AI functions lazily would +# change. Exact integers, so they hold on any host given the pinned settings. +# --------------------------------------------------------------------------- + +LAZY_ROWS = 64 +LAZY_BLOCK = 16 +LAZY_DISTINCT = 4 +CHAT_CALL = "aiClassify(x, ['positive','negative','neutral'], map('credentials', 'ai_mock'))" +EMBED_CALL = "aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed'))" + + +@pytest.fixture(scope="module") +def call_count_tables(started_cluster): + """One-part tables for the call-count scenarios, plus a duplicate-heavy one.""" + instance.query("DROP TABLE IF EXISTS lazy_rows SYNC") + instance.query( + "CREATE TABLE lazy_rows (id UInt32, x String) ENGINE = MergeTree ORDER BY id" + ) + instance.query( + f"INSERT INTO lazy_rows SELECT number, concat('row ', toString(number)) " + f"FROM numbers({LAZY_ROWS})" + ) + instance.query("OPTIMIZE TABLE lazy_rows FINAL") + + instance.query("DROP TABLE IF EXISTS lazy_dup SYNC") + instance.query( + "CREATE TABLE lazy_dup (id UInt32, x String) ENGINE = MergeTree ORDER BY id" + ) + instance.query( + f"INSERT INTO lazy_dup SELECT number, concat('dup ', toString(number % " + f"{LAZY_DISTINCT})) FROM numbers({LAZY_ROWS})" + ) + instance.query("OPTIMIZE TABLE lazy_dup FINAL") + yield + instance.query("DROP TABLE IF EXISTS lazy_rows SYNC") + instance.query("DROP TABLE IF EXISTS lazy_dup SYNC") + + +def run_and_count_calls(sql, prefix, extra_settings=None): + """Run `sql` and return the number of provider requests it issued.""" + settings = dict(AI_SETTINGS) + settings["max_block_size"] = LAZY_BLOCK + settings["max_threads"] = 1 + # `preferred_block_size_bytes` can split a block below `max_block_size` on its own. + settings["preferred_block_size_bytes"] = 0 + if extra_settings: + settings.update(extra_settings) + qid = unique_query_id(prefix) + instance.query(sql, settings=settings, query_id=qid) + return int(get_profile_events(qid)["api_calls"]) + + +# `expected` is what the implementation does today; `ideal` is what a maximally lazy +# implementation would do. They differ only for dedup, which does not exist: identical +# inputs are embedded once per row (`aiEmbed.cpp`, the live-row collection loop). +@pytest.mark.parametrize( + "case, sql, expected, ideal, settings", + [ + ( + "filter", + f"SELECT {CHAT_CALL} FROM lazy_rows WHERE id % 8 = 0 FORMAT Null", + LAZY_ROWS // 8, + LAZY_ROWS // 8, + {}, + ), + ( + "limit", + f"SELECT {CHAT_CALL} FROM lazy_rows LIMIT 5 FORMAT Null", + 5, + 5, + {}, + ), + ( + "order_by_limit", + f"SELECT {CHAT_CALL} FROM lazy_rows ORDER BY id LIMIT 5 FORMAT Null", + 5, + 5, + {}, + ), + ( + "ai_predicate_last", + f"SELECT count() FROM lazy_rows WHERE id % 8 = 0 AND {CHAT_CALL} = 'positive' " + f"FORMAT Null", + LAZY_ROWS // 8, + LAZY_ROWS // 8, + {}, + ), + ( + "short_circuit_if", + f"SELECT if(id % 8 = 0, {CHAT_CALL}, '') FROM lazy_rows FORMAT Null", + LAZY_ROWS // 8, + LAZY_ROWS // 8, + {"short_circuit_function_evaluation": "force_enable"}, + ), + ( + "prewhere", + f"SELECT {CHAT_CALL} FROM lazy_rows PREWHERE id % 8 = 0 FORMAT Null", + LAZY_ROWS // 8, + LAZY_ROWS // 8, + {}, + ), + ( + # Batch size 1 makes one request per input, so the count can show dedup. + # It does not: every row is embedded even though there are four distinct values. + "no_dedup_of_identical_inputs", + f"SELECT {EMBED_CALL} FROM lazy_dup FORMAT Null", + LAZY_ROWS, + LAZY_DISTINCT, + {"ai_function_embedding_max_batch_size": 1}, + ), + ( + # The control for the case above: deduplicating in SQL costs four requests. + "distinct_subquery_control", + f"SELECT {EMBED_CALL} FROM (SELECT DISTINCT x FROM lazy_dup) FORMAT Null", + LAZY_DISTINCT, + LAZY_DISTINCT, + {"ai_function_embedding_max_batch_size": 1}, + ), + ( + # Common subexpression elimination: `aiEmbed` is deterministic, so evaluating + # it in both the filter and the projection must not double the requests. + "cse_filter_and_projection", + f"SELECT {EMBED_CALL} FROM lazy_rows WHERE length({EMBED_CALL}) > 0 FORMAT Null", + LAZY_ROWS, + LAZY_ROWS, + {"ai_function_embedding_max_batch_size": 1}, + ), + ], +) +def test_api_call_count_per_query_shape(call_count_tables, case, sql, expected, ideal, settings): + calls = run_and_count_calls(sql, f"calls_{case}", settings) + assert calls == expected, ( + f"{case}: {calls} API calls, expected {expected} (a maximally lazy implementation " + f"would issue {ideal})" + ) + + +def _create_quota_parts(name, parts=8, rows_per_part=8, index_granularity=None): + """Create a MergeTree table of `parts` unmerged parts (merges stopped) so a scan over it + produces several blocks - the shape needed to tell a per-query quota from a per-block one. + A single-part table cannot: one block is one allowance. `SYSTEM STOP MERGES` keeps a + background merge from collapsing the parts before the scan and masking the difference. + + `index_granularity` pins a small, fixed granule size (adaptive granularity disabled) so the + number of marks is deterministic - needed when a test relies on the read pool splitting the + scan across threads, which is driven by mark count.""" + instance.query(f"DROP TABLE IF EXISTS {name} SYNC") + create = f"CREATE TABLE {name} (id UInt32, x String) ENGINE = MergeTree ORDER BY id" + if index_granularity is not None: + create += f" SETTINGS index_granularity = {index_granularity}, index_granularity_bytes = 0" + instance.query(create) + instance.query(f"SYSTEM STOP MERGES {name}") + for part in range(parts): + base = part * rows_per_part + instance.query( + f"INSERT INTO {name} SELECT number + {base}, " + f"concat('row ', toString(number + {base})) FROM numbers({rows_per_part})" + ) + + +# `max_block_size` = 8 with 8-row parts gives one block per part, so a per-block tracker +# reaches at most 8 (< the caps below) and never fires, while a per-query tracker accumulates +# across all 64 rows. +_QUOTA_SCOPE_SETTINGS = { + "max_block_size": 8, + "max_threads": 1, + "preferred_block_size_bytes": 0, +} + + +def test_api_call_quota_is_per_query(started_cluster): + """`ai_function_max_api_calls_per_query` must bound the query, not each block of it. + + The tracker is shared per query (owned by the query `Context`), so every block and every + pipeline stream draws on one allowance. It used to be a stack local in `executeImpl` with + no shared state, so each block started with a fresh allowance and the effective ceiling + grew with the data. + """ + limit = 10 + _create_quota_parts("quota_parts") + try: + qid = unique_query_id("quota_scope") + instance.query( + f"SELECT {CHAT_CALL} FROM quota_parts FORMAT Null", + settings={ + **AI_SETTINGS, + **_QUOTA_SCOPE_SETTINGS, + "ai_function_max_api_calls_per_query": limit, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + calls = int(get_profile_events(qid)["api_calls"]) + finally: + instance.query("DROP TABLE IF EXISTS quota_parts SYNC") + + assert calls <= limit, ( + f"{calls} API calls with ai_function_max_api_calls_per_query = {limit}: the quota " + "is tracked per executeImpl call, so the query spent a multiple of its own cap" + ) + + +def test_api_call_quota_throws_per_query(started_cluster): + """With `ai_function_throw_on_quota_exceeded = 1` (the default) the query must raise once + the per-query call quota is reached. No single 8-row block reaches the cap of 10, so a + per-block tracker never throws and the query completes; the per-query tracker throws.""" + _create_quota_parts("quota_throw") + try: + error = instance.query_and_get_error( + f"SELECT {CHAT_CALL} FROM quota_throw FORMAT Null", + settings={ + **AI_SETTINGS, + **_QUOTA_SCOPE_SETTINGS, + "ai_function_max_api_calls_per_query": 10, + "ai_function_throw_on_quota_exceeded": 1, + }, + ) + finally: + instance.query("DROP TABLE IF EXISTS quota_throw SYNC") + + assert "AI API call limit reached" in error, error + + +def test_input_token_quota_is_per_query(started_cluster): + """`ai_function_max_input_tokens_per_query` must bound the query too. The mock reports + `prompt_tokens = 10` per chat call, so a per-block tracker tops out at 80 tokens per 8-row + block (< the 100-token cap) and never fires, while the per-query tracker stops the scan.""" + limit = 100 + _create_quota_parts("quota_tokens") + try: + qid = unique_query_id("quota_tokens") + instance.query( + f"SELECT {CHAT_CALL} FROM quota_tokens FORMAT Null", + settings={ + **AI_SETTINGS, + **_QUOTA_SCOPE_SETTINGS, + "ai_function_max_input_tokens_per_query": limit, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + input_tokens = int(get_profile_events(qid)["input_tokens"]) + finally: + instance.query("DROP TABLE IF EXISTS quota_tokens SYNC") + + assert input_tokens <= limit, ( + f"{input_tokens} input tokens with ai_function_max_input_tokens_per_query = {limit}: " + "the quota is tracked per executeImpl call, so the query spent a multiple of its cap" + ) + + +def test_api_call_quota_holds_under_concurrency(started_cluster): + """The API-call cap must hold when several pipeline threads reserve slots against the shared + tracker at once. The slot is claimed with an atomic bounded increment (`tryReserveApiCall`), so + two threads cannot both pass a stale check and overshoot. + + The table has enough marks (small pinned granularity over 16 parts) that the read pool hands the + scan - and thus the AI function - to several threads under `max_threads = 8`. `peak_threads_usage` + from `system.query_log` is the count of threads that ran simultaneously; asserting it is > 1 + means a green result proves the concurrent reservation path was exercised, not that the scan + happened to collapse to one stream. The query wants 2048 calls but must make at most `limit`.""" + limit = 10 + _create_quota_parts("quota_concurrent", parts=16, rows_per_part=128, index_granularity=8) + try: + qid = unique_query_id("quota_concurrent") + instance.query( + f"SELECT {CHAT_CALL} FROM quota_concurrent FORMAT Null", + settings={ + **AI_SETTINGS, + "max_block_size": 8, + "max_threads": 8, + "ai_function_max_api_calls_per_query": limit, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + events = get_profile_events(qid) + calls = int(events["api_calls"]) + peak_threads = int(events["peak_threads"]) + finally: + instance.query("DROP TABLE IF EXISTS quota_concurrent SYNC") + + assert peak_threads > 1, ( + f"query peaked at {peak_threads} simultaneous thread(s); the concurrent reservation path was " + "not exercised, so this test would not catch a check-then-act overshoot" + ) + assert calls <= limit, ( + f"{calls} API calls with ai_function_max_api_calls_per_query = {limit} and {peak_threads} peak " + "threads: concurrent streams overshot the per-query cap" + ) + + +def test_api_call_quota_ignores_subquery_settings(started_cluster): + """`ai_function_max_*_per_query` is read from the top-level query context, so a nested + subquery's own `SETTINGS` override of it does not apply: the whole query shares one budget + seeded from the outer settings. A subquery runs in a copied child context carrying its own + settings, but the quota tracker lives on the query context, so those overrides are ignored.""" + _create_quota_parts("quota_levels") # 8 parts x 8 rows = 64 rows, one API call per row + try: + # The subquery caps at 5, the outer query at 20. A result of 20 shows the outer + # (query-context) value governs; the subquery override (which would give 5, as would a + # min-of-both rule) is ignored. + qid = unique_query_id("quota_levels_outer_wins") + instance.query( + f"SELECT c FROM (SELECT {CHAT_CALL} AS c FROM quota_levels " + "SETTINGS ai_function_max_api_calls_per_query = 5) FORMAT Null", + settings={ + **AI_SETTINGS, + **_QUOTA_SCOPE_SETTINGS, + "ai_function_max_api_calls_per_query": 20, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + outer_wins = int(get_profile_events(qid)["api_calls"]) + + # The quota is set only in the subquery; the outer query leaves it at the default (far + # above 64). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - + # a quota set only in a subquery has no effect. + qid = unique_query_id("quota_levels_subquery_only") + instance.query( + f"SELECT c FROM (SELECT {CHAT_CALL} AS c FROM quota_levels " + "SETTINGS ai_function_max_api_calls_per_query = 5) FORMAT Null", + settings={ + **AI_SETTINGS, + **_QUOTA_SCOPE_SETTINGS, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + subquery_only = int(get_profile_events(qid)["api_calls"]) + finally: + instance.query("DROP TABLE IF EXISTS quota_levels SYNC") + + assert outer_wins == 20, ( + f"expected the top-level cap (20) to govern, got {outer_wins}: a subquery-scoped SETTINGS " + "override of ai_function_max_api_calls_per_query must not change the query budget" + ) + assert subquery_only == 64, ( + f"expected all 64 rows to run (a quota set only in the subquery is ignored), got {subquery_only}" + ) diff --git a/tests/integration/test_arrowflight_interface/test_sql_server.py b/tests/integration/test_arrowflight_interface/test_sql_server.py index 20bcfa2996b1..7f3306db4779 100644 --- a/tests/integration/test_arrowflight_interface/test_sql_server.py +++ b/tests/integration/test_arrowflight_interface/test_sql_server.py @@ -35,13 +35,13 @@ session_id = ''.join(random.choices(string.ascii_letters + string.digits, k=16)) -def get_client(): +def get_client(session_id_override=None): return FlightSQLClient( host=node.ip_address, port=8888, insecure=True, disable_server_verification=True, - metadata={'x-clickhouse-session-id': session_id}, + metadata={'x-clickhouse-session-id': session_id_override or session_id}, features={'metadata-reflection': 'true'}, # makes the client emit metadata retrieval commands upon connection ) @@ -402,6 +402,21 @@ def test_set_session_options_persistence(): assert _query_setting(client, "max_threads") == default_value +def test_reset_session_option_respects_settings_constraints(): + constraint_session_id = 'settings_constraints_' + ''.join( + random.choices(string.ascii_letters + string.digits, k=16) + ) + client = get_client(constraint_session_id) + + result = client.set_session_options({"readonly": "2"}) + assert len(result.errors) == 0 + + result = client.set_session_options({"readonly": None}) + assert "readonly" in result.errors + + assert _query_setting(client, "readonly") == "2" + + def test_cancel_flight_info(): client = get_client() diff --git a/tests/integration/test_database_iceberg/configs/display_secrets.xml b/tests/integration/test_database_iceberg/configs/display_secrets.xml new file mode 100644 index 000000000000..4f463fc8cbad --- /dev/null +++ b/tests/integration/test_database_iceberg/configs/display_secrets.xml @@ -0,0 +1,3 @@ + + 1 + diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 5ec77a1a8fbd..373abd5a2805 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -191,6 +191,7 @@ def started_cluster(): "configs/backups.xml", "configs/cluster.xml", "configs/text_log.xml", + "configs/display_secrets.xml", ], user_configs=[], stay_alive=True, @@ -1794,3 +1795,227 @@ def test_partitioning_by_string(started_cluster): create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) assert node.query(f"SELECT * FROM {CATALOG_NAME}.`{namespace}.{table_name}`") == "a:b,c[d=e/f%g?h\ttest\t12:00:00.000000\n" + + +def test_alter_database_settings_not_supported(started_cluster): + node = started_cluster.instances["node1"] + + db_name = f"iceberg_alter_settings_{uuid.uuid4().hex}" + create_clickhouse_iceberg_database(started_cluster, node, db_name) + + fake_token = f"fake_secret_token_{uuid.uuid4().hex}" + + qid_alter = uuid.uuid4().hex + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING warehouse = 'other_warehouse'" + ) + assert "BAD_ARGUMENTS" in error + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING onelake_bearer_token = '{fake_token}'", + query_id=qid_alter, + ) + assert "BAD_ARGUMENTS" in error + + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING no_such_setting = 1" + ) + assert "BAD_ARGUMENTS" in error or "UNKNOWN_SETTING" in error + + show_result = node.query(f"SHOW CREATE DATABASE {db_name}") + assert "other_warehouse" not in show_result + assert "onelake_bearer_token" not in show_result + node.query( + f"SELECT name FROM system.tables WHERE database = '{db_name}' SETTINGS show_data_lake_catalogs_in_system_tables = true" + ) + + node.query("SYSTEM FLUSH LOGS system.query_log") + logged_query = node.query( + f"SELECT arrayStringConcat(groupArray(query), '\\n') FROM system.query_log WHERE query_id = '{qid_alter}'" + ) + assert fake_token not in logged_query + assert "[HIDDEN]" in logged_query + + node.query(f"DROP DATABASE {db_name}") + + glue_db_name = f"glue_alter_settings_{uuid.uuid4().hex}" + node.query( + f""" + ATTACH DATABASE {glue_db_name} ENGINE = DataLakeCatalog('http://fake-glue:1') + SETTINGS catalog_type = 'glue', region = 'us-east-1', storage_endpoint = 'http://fake-glue:1/x' + """ + ) + error = node.query_and_get_error( + f"ALTER DATABASE {glue_db_name} MODIFY SETTING region = 'eu-west-1'" + ) + assert "NOT_IMPLEMENTED" in error + node.query(f"DROP DATABASE {glue_db_name}") + + +def test_alter_database_settings_rest_auth_header(started_cluster): + node = started_cluster.instances["node1"] + + db_name = f"rest_alter_auth_header_{uuid.uuid4().hex}" + old_header = f"Authorization: Bearer old_{uuid.uuid4().hex}" + new_header = f"Authorization: Bearer new_{uuid.uuid4().hex}" + + node.query( + f""" + ATTACH DATABASE {db_name} ENGINE = DataLakeCatalog('http://fake-rest:1/api') + SETTINGS catalog_type = 'rest', warehouse = 'wh', auth_header = '{old_header}' + """ + ) + + node.query( + f"ALTER DATABASE {db_name} MODIFY SETTING auth_header = '{new_header}'" + ) + + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING catalog_credential = 'id:secret'" + ) + assert "BAD_ARGUMENTS" in error + + show_result = node.query(f"SHOW CREATE DATABASE {db_name}") + assert new_header not in show_result + assert "[HIDDEN]" in show_result + + node.restart_clickhouse() + + engine_full_with_secrets = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name}'", + settings={"format_display_secrets_in_show_and_select": 1}, + ) + assert new_header in engine_full_with_secrets + assert old_header not in engine_full_with_secrets + + node.query(f"DROP DATABASE {db_name}") + + +def test_alter_database_settings_onelake_persistence(started_cluster): + node = started_cluster.instances["node1"] + + db_name = f"onelake_alter_persist_{uuid.uuid4().hex}" + old_token = f"secret_token_{uuid.uuid4().hex}" + new_token = f"secret_token_{uuid.uuid4().hex}" + + node.query( + f""" + ATTACH DATABASE {db_name} ENGINE = DataLakeCatalog('http://fake-onelake:1/api') + SETTINGS catalog_type = 'onelake', warehouse = 'wh', onelake_tenant_id = 'tenant-0', onelake_tenant_id = 'tenant-1', onelake_bearer_token = '{old_token}' + """ + ) + + node.query( + f"ALTER DATABASE {db_name} MODIFY SETTING onelake_tenant_id = 'tenant-2', onelake_bearer_token = '{new_token}'" + ) + + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING onelake_client_id = 'client-1'" + ) + assert "BAD_ARGUMENTS" in error + + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING warehouse = 'other_warehouse'" + ) + assert "BAD_ARGUMENTS" in error + + error = node.query_and_get_error( + f"ALTER DATABASE {db_name} MODIFY SETTING onelake_bearer_token = ''" + ) + assert "BAD_ARGUMENTS" in error + + show_result = node.query(f"SHOW CREATE DATABASE {db_name}") + assert "tenant-2" in show_result + assert new_token not in show_result + assert old_token not in show_result + assert "[HIDDEN]" in show_result + + engine_full_with_secrets = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name}'", + settings={"format_display_secrets_in_show_and_select": 1}, + ) + assert "tenant-2" in engine_full_with_secrets + assert new_token in engine_full_with_secrets + assert old_token not in engine_full_with_secrets + + node.restart_clickhouse() + + show_result = node.query(f"SHOW CREATE DATABASE {db_name}") + assert "tenant-2" in show_result + assert "tenant-0" not in show_result + assert "tenant-1" not in show_result + assert new_token not in show_result + assert "[HIDDEN]" in show_result + + engine_full = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name}'" + ) + assert "tenant-2" in engine_full + assert "tenant-0" not in engine_full + assert "tenant-1" not in engine_full + assert new_token not in engine_full + + engine_full_with_secrets = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name}'", + settings={"format_display_secrets_in_show_and_select": 1}, + ) + assert new_token in engine_full_with_secrets + assert old_token not in engine_full_with_secrets + + node.query(f"DROP DATABASE {db_name}") + + +def test_catalog_listing_error_surfaces_in_system_tables(started_cluster): + """ + Regression test: an error from the catalog while listing tables (e.g. expired + catalog credentials) must not be silently turned into an empty listing when the + user explicitly opted into showing datalake catalogs in system tables with + show_data_lake_catalogs_in_system_tables=1. Without the opt-in the old tolerant + behaviour is kept (system tables must not fail because of one broken catalog). + """ + node = started_cluster.instances["node1"] + + root_namespace = f"clickhouse_{uuid.uuid4()}" + namespace = f"{root_namespace}_test_listing_error" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(namespace) + create_table(catalog, namespace, "table_x") + + create_clickhouse_iceberg_database(started_cluster, node, CATALOG_NAME) + + node.query("SYSTEM ENABLE FAILPOINT datalake_get_tables_throw") + try: + assert ( + node.query( + f"SELECT count() FROM system.iceberg_files WHERE database = '{CATALOG_NAME}'" + ).strip() + == "0" + ) + + error = node.query_and_get_error( + f"SELECT count() FROM system.iceberg_files WHERE database = '{CATALOG_NAME}' " + "SETTINGS show_data_lake_catalogs_in_system_tables = 1" + ) + assert "Injected catalog listing failure" in error + + error = node.query_and_get_error( + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' " + "SETTINGS show_data_lake_catalogs_in_system_tables = 1" + ) + assert "Injected catalog listing failure" in error + + error = node.query_and_get_error( + f"SELECT name, engine FROM system.tables WHERE database = '{CATALOG_NAME}' " + "SETTINGS show_data_lake_catalogs_in_system_tables = 1" + ) + assert "Injected catalog listing failure" in error + finally: + node.query("SYSTEM DISABLE FAILPOINT datalake_get_tables_throw") + + result = node.query( + f"SELECT name FROM system.tables WHERE database = '{CATALOG_NAME}' " + "SETTINGS show_data_lake_catalogs_in_system_tables = 1" + ) + assert "table_x" in result + + node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") diff --git a/tests/integration/test_database_iceberg_seaweedfs_catalog/__init__.py b/tests/integration/test_database_iceberg_seaweedfs_catalog/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_database_iceberg_seaweedfs_catalog/test.py b/tests/integration/test_database_iceberg_seaweedfs_catalog/test.py new file mode 100644 index 000000000000..c36bef05d11a --- /dev/null +++ b/tests/integration/test_database_iceberg_seaweedfs_catalog/test.py @@ -0,0 +1,88 @@ +import time + +import pytest +import requests + +from helpers.cluster import ClickHouseCluster + +CATALOG_URL = "http://seaweedfs:8181/v1" +STORAGE_URL = "http://seaweedfs:8333/analytics" +ACCESS_KEY = "clickhouse" +SECRET_KEY = "clickhouse" + + +def wait_for_seaweedfs(cluster, timeout=120): + """Wait until both the Iceberg REST catalog and the pre-created bucket are ready.""" + catalog_url = f"http://localhost:{cluster.iceberg_rest_catalog_port}/v1/config" + deadline = time.monotonic() + timeout + while True: + try: + if requests.get(catalog_url, timeout=2).status_code == 200: + buckets = cluster.exec_in_container( + cluster.get_container_id("seaweedfs"), + ["sh", "-c", "echo s3.bucket.list | weed shell 2>/dev/null"], + ) + if "analytics" in buckets: + return + except Exception: + if time.monotonic() > deadline: + raise + if time.monotonic() > deadline: + raise TimeoutError("SeaweedFS did not become ready") + time.sleep(0.5) + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_seaweedfs_catalog.yml" + }, + ) + cluster.start() + wait_for_seaweedfs(cluster) + yield cluster + finally: + cluster.shutdown() + + +def test_create_insert_select(started_cluster): + node = started_cluster.instances["node1"] + + node.query( + f""" + CREATE DATABASE lake + ENGINE = DataLakeCatalog('{CATALOG_URL}', '{ACCESS_KEY}', '{SECRET_KEY}') + SETTINGS catalog_type = 'rest', + warehouse = 's3://analytics', + storage_endpoint = '{STORAGE_URL}', + catalog_credential = '{ACCESS_KEY}:{SECRET_KEY}', + oauth_server_uri = '{CATALOG_URL}/oauth/tokens' + """, + settings={"allow_experimental_database_iceberg": 1}, + ) + + node.query( + f""" + CREATE TABLE lake.`sales.returns` (id Int64, reason String) + ENGINE = IcebergS3('{STORAGE_URL}/sales/returns/', '{ACCESS_KEY}', '{SECRET_KEY}') + """, + settings={ + "allow_experimental_database_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + }, + ) + + node.query( + "INSERT INTO lake.`sales.returns` VALUES (1, 'damaged'), (2, 'wrong size')", + settings={"allow_experimental_insert_into_iceberg": 1}, + ) + + assert ( + node.query("SELECT * FROM lake.`sales.returns` ORDER BY id") + == "1\tdamaged\n2\twrong size\n" + ) diff --git a/tests/integration/test_rabbitmq_malicious_broker/__init__.py b/tests/integration/test_rabbitmq_malicious_broker/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_rabbitmq_malicious_broker/configs/allowed_hosts.xml b/tests/integration/test_rabbitmq_malicious_broker/configs/allowed_hosts.xml new file mode 100644 index 000000000000..ed9aeab0e30e --- /dev/null +++ b/tests/integration/test_rabbitmq_malicious_broker/configs/allowed_hosts.xml @@ -0,0 +1,7 @@ + + + localhost:19672 + localhost:19673 + localhost:19674 + + diff --git a/tests/integration/test_rabbitmq_malicious_broker/malicious_broker.py b/tests/integration/test_rabbitmq_malicious_broker/malicious_broker.py new file mode 100644 index 000000000000..439d33bfc11a --- /dev/null +++ b/tests/integration/test_rabbitmq_malicious_broker/malicious_broker.py @@ -0,0 +1,173 @@ +""" +A minimal, deliberately hostile AMQP 0-9-1 broker. + +It walks a connecting client through the handshake up to `Connection.Tune`, proposes a +maximum frame size of zero, and then sends a frame header that declares a payload of +almost 4 GiB followed by a stream of filler bytes. + +A client that takes the proposed maximum frame size at face value ends up reading that +stream into its fixed-size receive buffer, which is a heap out-of-bounds write. + +Given a certificate and key it speaks amqps instead, so the same attack can be driven through +the client's TLS receive path (which used to be entirely unbounded). +""" + +import socket +import struct +import sys +import threading + +FRAME_METHOD = 1 +FRAME_END = 0xCE + +CLASS_CONNECTION = 10 +METHOD_START = 10 +METHOD_TUNE = 30 +METHOD_TUNE_OK = 31 + +# Almost 4 GiB - the point is that it can not possibly fit in the client's receive buffer. +HUGE_PAYLOAD_SIZE = 0xFFFFFF00 + +# How much filler to push at the client after the oversized frame header. +FILLER_TOTAL = 8 * 1024 * 1024 +FILLER_CHUNK = b"A" * 65536 + + +def frame(frame_type, channel, payload): + return ( + struct.pack(">BHI", frame_type, channel, len(payload)) + + payload + + bytes([FRAME_END]) + ) + + +def long_string(value): + return struct.pack(">I", len(value)) + value + + +def connection_start(): + payload = struct.pack(">HHBB", CLASS_CONNECTION, METHOD_START, 0, 9) + payload += struct.pack(">I", 0) # empty server-properties field table + payload += long_string(b"PLAIN") + payload += long_string(b"en_US") + return frame(FRAME_METHOD, 0, payload) + + +def connection_tune(frame_max): + payload = struct.pack( + ">HHHIH", CLASS_CONNECTION, METHOD_TUNE, 2047, frame_max, 0 + ) + return frame(FRAME_METHOD, 0, payload) + + +def recv_exactly(conn, size): + data = b"" + while len(data) < size: + chunk = conn.recv(size - len(data)) + if not chunk: + return None + data += chunk + return data + + +def read_frame(conn): + """Read one whole AMQP frame: 7-byte header, payload, and the frame-end octet. + + recv is not frame-aware - on TCP/TLS it may return any positive prefix - so both the header + and the payload have to be read with recv_exactly, otherwise the next read starts in the + middle of this frame and the stream desyncs. Returns (frame_type, channel, payload) without + the end octet, or None if the peer went away. + """ + header = recv_exactly(conn, 7) + if header is None: + return None + frame_type, channel, size = struct.unpack(">BHI", header) + body = recv_exactly(conn, size + 1) # payload plus the frame-end octet + if body is None: + return None + return frame_type, channel, body[:size] + + +def handle(conn, frame_max): + try: + conn.settimeout(30) + + # The client opens with the 8-byte protocol header. + if recv_exactly(conn, 8) is None: + return + + conn.sendall(connection_start()) + + # Connection.StartOk - consume the whole frame so the next read stays frame-aligned. + if read_frame(conn) is None: + return + + conn.sendall(connection_tune(frame_max)) + + # Connection.TuneOk (possibly pipelined with Connection.Open, so read exactly one frame). + # Log the frame_max the client settled on: the test asserts it to prove the client clamps + # hostile proposals to [4096, 128 MiB] instead of echoing them back. + tune_ok = read_frame(conn) + if tune_ok is None: + return + frame_type, _channel, payload = tune_ok + if frame_type == FRAME_METHOD and len(payload) >= 10: + klass, method, _channel_max, client_frame_max = struct.unpack( + ">HHHI", payload[:10] + ) + if klass == CLASS_CONNECTION and method == METHOD_TUNE_OK: + print(f"client TuneOk frame_max={client_frame_max}", flush=True) + + # A frame header claiming a payload that is orders of magnitude larger than the + # client's receive buffer, immediately followed by data to fill it with. + conn.sendall(struct.pack(">BHI", FRAME_METHOD, 0, HUGE_PAYLOAD_SIZE)) + + sent = 0 + while sent < FILLER_TOTAL: + conn.sendall(FILLER_CHUNK) + sent += len(FILLER_CHUNK) + except OSError: + pass + finally: + conn.close() + + +def main(): + port = int(sys.argv[1]) + frame_max = int(sys.argv[2]) if len(sys.argv) > 2 else 0 + # Optional TLS: pass a cert and key to make the broker speak amqps, so the same attack can + # be driven through the client's TLS receive path. + certfile = sys.argv[3] if len(sys.argv) > 3 else None + keyfile = sys.argv[4] if len(sys.argv) > 4 else None + + tls_context = None + if certfile: + import ssl + + tls_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls_context.load_cert_chain(certfile, keyfile) + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", port)) + server.listen(16) + print( + f"listening on {port} ({'amqps' if tls_context else 'amqp'})," + f" proposing frame_max={frame_max}", + flush=True, + ) + + while True: + conn, _ = server.accept() + if tls_context is not None: + try: + conn = tls_context.wrap_socket(conn, server_side=True) + except OSError: + # e.g. a plain-TCP liveness probe that never completes the TLS handshake + conn.close() + continue + threading.Thread(target=handle, args=(conn, frame_max), daemon=True).start() + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_rabbitmq_malicious_broker/test.py b/tests/integration/test_rabbitmq_malicious_broker/test.py new file mode 100644 index 000000000000..352d50fe13cb --- /dev/null +++ b/tests/integration/test_rabbitmq_malicious_broker/test.py @@ -0,0 +1,290 @@ +""" +Tests that `ENGINE = RabbitMQ` survives a hostile broker. + +`CREATE TABLE ... ENGINE = RabbitMQ` connects to the broker synchronously, so everything the +broker says during the handshake is parsed before the statement returns. A broker that +proposes a maximum frame size of zero used to make the client accept a frame of arbitrary +size while its receive buffer stayed at 4096 bytes, which is a heap out-of-bounds write. +""" + +import os + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import wait_condition + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + +# Must match the ports allowed in configs/allowed_hosts.xml. +BROKER_PORT = 19672 +TLS_BROKER_PORT = 19673 +HUGE_FRAME_MAX_BROKER_PORT = 19674 + +# The client must clamp the broker's frame_max proposal to this range and reply with the +# clamped value in `Connection.TuneOk` (MIN_FRAME_SIZE / MAX_FRAME_SIZE in AMQP-CPP). +MIN_FRAME_SIZE = 4096 +MAX_FRAME_SIZE = 128 * 1024 * 1024 + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/allowed_hosts.xml"], + stay_alive=True, +) + + +def _wait_for_port(port): + wait_condition( + lambda: node.exec_in_container( + ["bash", "-c", f"exec 3<>/dev/tcp/127.0.0.1/{port} && echo OK"], + nothrow=True, + ), + lambda r: "OK" in r, + max_attempts=40, + delay=0.5, + ) + + +def start_malicious_broker(): + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "malicious_broker.py"), + "/malicious_broker.py", + ) + + # Plain amqp broker. + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /malicious_broker.py {BROKER_PORT}" + " > /var/log/clickhouse-server/malicious_broker.log 2>&1", + ], + detach=True, + user="root", + ) + _wait_for_port(BROKER_PORT) + + # amqps broker with a throwaway self-signed cert (the client does not verify it), so the same + # overflow can be driven through the TLS receive path. + node.exec_in_container( + [ + "bash", + "-c", + "openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=localhost" + " -keyout /broker_key.pem -out /broker_cert.pem 2>/dev/null", + ], + user="root", + ) + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /malicious_broker.py {TLS_BROKER_PORT} 0 /broker_cert.pem /broker_key.pem" + " > /var/log/clickhouse-server/malicious_broker_tls.log 2>&1", + ], + detach=True, + user="root", + ) + _wait_for_port(TLS_BROKER_PORT) + + # Plain amqp broker proposing an absurdly large frame_max, to cover the upper clamp. + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /malicious_broker.py {HUGE_FRAME_MAX_BROKER_PORT} {2**32 - 1}" + " > /var/log/clickhouse-server/malicious_broker_huge.log 2>&1", + ], + detach=True, + user="root", + ) + _wait_for_port(HUGE_FRAME_MAX_BROKER_PORT) + + +def _assert_negotiated_frame_max(broker_log, expected): + """The broker logs the frame_max from the client's `Connection.TuneOk` reply.""" + wait_condition( + lambda: node.exec_in_container( + [ + "bash", + "-c", + f"grep -a 'client TuneOk frame_max=' /var/log/clickhouse-server/{broker_log}" + " | tail -1", + ], + nothrow=True, + ), + lambda reply: f"frame_max={expected}" in reply, + max_attempts=20, + delay=0.5, + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + start_malicious_broker() + yield cluster + finally: + cluster.shutdown() + + +def test_broker_proposing_zero_frame_max(started_cluster): + """A broker proposing `frame_max = 0` must not be able to overflow the receive buffer.""" + error = node.query_and_get_error( + f""" + CREATE TABLE malicious (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_address = 'amqp://guest:guest@localhost:{BROKER_PORT}/', + rabbitmq_exchange_name = 'ex', + rabbitmq_format = 'JSONEachRow' + """ + ) + # Reaching the connection at all means an allowed address is not rejected by the + # `remote_url_allow_hosts` check that the test below exercises. + assert "CANNOT_CONNECT_RABBITMQ" in error, error + + # The server has to be alive and healthy - the point of the test is that the oversized + # frame was rejected rather than read into a buffer that is too small for it. Under a + # sanitizer build the out-of-bounds write takes the server down and this query fails. + assert node.query("SELECT 1") == "1\n" + + # Deterministic proof the frame-size guard actually fired (rather than the server merely + # surviving the overflow by luck on a non-sanitizer build): the library rejects the + # oversized frame with a `frame size exceeded` protocol error, which the handler logs. + wait_condition( + lambda: node.contains_in_log("frame size exceeded"), + lambda fired: fired, + max_attempts=20, + delay=0.5, + ) + + # The client must not take the proposed zero at face value: its TuneOk reply has to + # carry the lower clamp, which is what bounds the receive buffer. + _assert_negotiated_frame_max("malicious_broker.log", MIN_FRAME_SIZE) + + +def test_broker_proposing_zero_frame_max_over_tls(started_cluster): + """The same overflow driven through the TLS (amqps) receive path, which used to be unbounded. + + This is primarily a guard for the sanitizer lane: without the fix the oversized frame is read + past the end of the TLS receive buffer and a sanitizer build aborts here, so the server must + stay alive and answer afterwards. + """ + # The plaintext test already puts `frame size exceeded` into the shared server log, so + # require the count to grow rather than the substring to appear. + rejections_before = int(node.count_in_log("frame size exceeded")) + + error = node.query_and_get_error( + f""" + CREATE TABLE malicious_tls (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_address = 'amqps://guest:guest@localhost:{TLS_BROKER_PORT}/', + rabbitmq_exchange_name = 'ex', + rabbitmq_format = 'JSONEachRow' + """ + ) + assert "CANNOT_CONNECT_RABBITMQ" in error, error + assert node.query("SELECT 1") == "1\n" + + # Deterministic proof the guard fired on the TLS receive path as well, not just that the + # server survived (on a non-sanitizer build the overflow could go unnoticed otherwise). + wait_condition( + lambda: int(node.count_in_log("frame size exceeded")), + lambda count: count > rejections_before, + max_attempts=20, + delay=0.5, + ) + + _assert_negotiated_frame_max("malicious_broker_tls.log", MIN_FRAME_SIZE) + + +def test_broker_proposing_huge_frame_max(started_cluster): + """A frame_max proposal close to 4 GiB must be clamped to 128 MiB, not echoed back. + + The TuneOk reply is what bounds the receive buffer, so accepting the proposal verbatim + would let the broker legally announce frames of arbitrary size. + """ + rejections_before = int(node.count_in_log("frame size exceeded")) + + error = node.query_and_get_error( + f""" + CREATE TABLE malicious_huge (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_address = 'amqp://guest:guest@localhost:{HUGE_FRAME_MAX_BROKER_PORT}/', + rabbitmq_exchange_name = 'ex', + rabbitmq_format = 'JSONEachRow' + """ + ) + assert "CANNOT_CONNECT_RABBITMQ" in error, error + assert node.query("SELECT 1") == "1\n" + + # The ~4 GiB frame the broker sends next still exceeds the clamped 128 MiB. + wait_condition( + lambda: int(node.count_in_log("frame size exceeded")), + lambda count: count > rejections_before, + max_attempts=20, + delay=0.5, + ) + + _assert_negotiated_frame_max("malicious_broker_huge.log", MAX_FRAME_SIZE) + + +def test_remote_host_filter_applies_to_rabbitmq_address(started_cluster): + """`rabbitmq_address` must be checked against `remote_url_allow_hosts` too.""" + error = node.query_and_get_error( + """ + CREATE TABLE filtered (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_address = 'amqp://guest:guest@not-allowed-host:5672/', + rabbitmq_exchange_name = 'ex', + rabbitmq_format = 'JSONEachRow' + """ + ) + assert "UNACCEPTABLE_URL" in error, error + assert "not-allowed-host:5672" in error, error + + +def test_remote_host_filter_not_bypassed_by_host_port(started_cluster): + """An allowed `rabbitmq_host_port` must not smuggle a disallowed `rabbitmq_address` past the filter. + + Both settings can be given together, and `RabbitMQConnection::connectImpl` connects to + `rabbitmq_address` (the URI) in preference to `rabbitmq_host_port`. So validating only the + host-port form let an allowed `rabbitmq_host_port` (`localhost:19672` is in the allowlist) pair + with a disallowed `rabbitmq_address` and still reach the unvalidated host. The address must be + checked whenever it is set. + """ + error = node.query_and_get_error( + """ + CREATE TABLE both_settings (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'localhost:19672', + rabbitmq_address = 'amqp://guest:guest@not-allowed-host:5672/', + rabbitmq_exchange_name = 'ex', + rabbitmq_format = 'JSONEachRow' + """ + ) + assert "UNACCEPTABLE_URL" in error, error + assert "not-allowed-host:5672" in error, error + + +def test_rabbitmq_secure_conflicts_with_plaintext_address(started_cluster): + """`rabbitmq_secure = 1` with a plaintext `amqp://` address must be rejected. + + `RabbitMQConnection::connectImpl` takes the transport from the URI scheme and ignores the + `rabbitmq_secure` setting for the address form, so this combination used to connect in + cleartext despite the user having asked for TLS - a silent downgrade rather than an error. + """ + error = node.query_and_get_error( + f""" + CREATE TABLE secure_conflict (key UInt64, value UInt64) + ENGINE = RabbitMQ + SETTINGS rabbitmq_address = 'amqp://guest:guest@localhost:{BROKER_PORT}/', + rabbitmq_secure = 1, + rabbitmq_exchange_name = 'ex', + rabbitmq_format = 'JSONEachRow' + """ + ) + assert "rabbitmq_secure" in error, error + assert "amqps" in error, error diff --git a/tests/integration/test_scheduler_memory/test.py b/tests/integration/test_scheduler_memory/test.py index 1a5b97800aa9..ba6e9bc52b18 100644 --- a/tests/integration/test_scheduler_memory/test.py +++ b/tests/integration/test_scheduler_memory/test.py @@ -124,12 +124,14 @@ def test_reserve_memory(): node.query("SYSTEM FLUSH LOGS") + # NOTE: MemoryReservationDecreases is intentionally not asserted per-query here. + # The reservation is released at query teardown (BlockIO::onFinish), after the query's + # ProfileEvents have already been snapshotted into query_log, so the decrease is not + # reliably attributed to the query. Asserting it per-query would be race-prone. assert_profile_event(node, "test_production", "MemoryReservationIncreases", lambda x: x == 1) - assert_profile_event(node, "test_production", "MemoryReservationDecreases", lambda x: x == 1) assert_profile_event(node, "test_production", "MemoryReservationKilled", lambda x: x == 0) assert_profile_event(node, "test_production", "MemoryReservationFailed", lambda x: x == 0) assert_profile_event(node, "test_development", "MemoryReservationIncreases", lambda x: x == 1) - assert_profile_event(node, "test_development", "MemoryReservationDecreases", lambda x: x == 1) assert_profile_event(node, "test_development", "MemoryReservationKilled", lambda x: x == 0) assert_profile_event(node, "test_development", "MemoryReservationFailed", lambda x: x == 0) diff --git a/tests/integration/test_spark_session_recovery/__init__.py b/tests/integration/test_spark_session_recovery/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_spark_session_recovery/test.py b/tests/integration/test_spark_session_recovery/test.py new file mode 100644 index 000000000000..517fd04c22ce --- /dev/null +++ b/tests/integration/test_spark_session_recovery/test.py @@ -0,0 +1,102 @@ +import os +import signal + +import pyspark +from pyspark.context import SparkContext +from pyspark.sql import SparkSession + +from helpers import spark_tools +from helpers.spark_tools import ResilientSparkSession + + +def _jvm_proc(): + return getattr(SparkContext._gateway, "proc", None) + + +def _jvm_pid(): + return getattr(_jvm_proc(), "pid", None) + + +def _alive(pid): + try: + os.kill(pid, 0) + return True + except Exception: + return False + + +def _kill_jvm_and_reap(proc): + """Kill the gateway JVM and wait until reaped, not merely signalled: nothing + in pyspark waits on ``gateway.proc``, so a killed JVM stays a zombie whose + ``os.kill(pid, 0)`` keeps succeeding until ``wait()`` reaps it.""" + os.kill(proc.pid, signal.SIGKILL) + proc.wait(timeout=30) + assert not _alive(proc.pid) + + +def _create(): + return ( + pyspark.sql.SparkSession.builder.appName("spark_session_recovery") + .master("local[1]") + .getOrCreate() + ) + + +def test_recovers_dead_gateway_and_reuses_live_one(): + """pyspark caches the py4j gateway in class state and no stop() clears it, so + a session built after the JVM died must discard it, while one built while it + still answers must reuse it.""" + try: + session = ResilientSparkSession(_create) + first_proc = _jvm_proc() + first_pid = _jvm_pid() + assert session.range(3).count() == 3 + assert first_pid is not None + + # A stopped session leaves the gateway cached and the JVM running. + session.stop() + assert SparkContext._gateway is not None + assert _alive(first_pid) + + # No leak: the next session must reuse that live JVM, not launch another. + reused = ResilientSparkSession(_create) + assert _jvm_pid() == first_pid + assert reused.range(2).count() == 2 + + # _restart() on a live gateway must also reuse it. + reused._restart() + assert _jvm_pid() == first_pid + assert reused.range(4).count() == 4 + + # Recovery: with the JVM dead, the next session must relaunch it. Before + # the fix __init__ raised "... does not exist in the JVM" -- naming + # SparkSession$, not CI's SparkConf, because _restart left + # _instantiatedSession set and getOrCreate asks for that first. + _kill_jvm_and_reap(first_proc) + + recovered = ResilientSparkSession(_create) + second_proc = _jvm_proc() + second_pid = _jvm_pid() + assert second_pid != first_pid + assert recovered.range(5).count() == 5 + + # Attribute access on a wrapper whose JVM died goes through __getattr__ + # -> _restart, which must recover the same way __init__ does. + _kill_jvm_and_reap(second_proc) + + assert recovered.range(6).count() == 6 + assert _jvm_pid() not in (first_pid, second_pid) + recovered.stop() + finally: + # A failure leaves a live session or a dead gateway; both poison the next + # module here. getOrCreate reuses a live session via + # applyModifiableSettings, which cannot apply a static conf: stop it. + active = SparkSession._instantiatedSession + if active is not None: + try: + active.stop() # raises once the gateway is dead; tolerated + except Exception: + pass + # Conditional: dropping a live gateway would orphan its JVM. + if SparkContext._gateway is not None and not spark_tools._gateway_is_live(): + spark_tools._reset_pyspark_class_state() diff --git a/tests/integration/test_storage_kafka/test_batch_slow_0.py b/tests/integration/test_storage_kafka/test_batch_slow_0.py index c3fdcaaa1be5..5056b1c6a7ab 100644 --- a/tests/integration/test_storage_kafka/test_batch_slow_0.py +++ b/tests/integration/test_storage_kafka/test_batch_slow_0.py @@ -311,7 +311,6 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator data_prefix = data_prefix + [""] if format_opts.get("printable", False) == False: raw_message = "hex(_raw_message)" - k.kafka_produce(kafka_cluster, topic_name, data_prefix + data_sample) create_query = create_query_generator( f"kafka_{format_name}", "id Int64, blockNo UInt16, val1 String, val2 Float32, val3 UInt8", @@ -323,6 +322,10 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator "kafka_flush_interval_ms": 1000, }, ) + # Create both materialized views, then detach/re-attach the Kafka table, + # before producing any message. Creating the first view starts the + # streaming loop, so producing earlier lets the loop consume and commit + # the broken message before the errors view is attached, leaving it empty. instance.query( f""" DROP TABLE IF EXISTS test.kafka_{format_name}; @@ -338,8 +341,12 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator CREATE MATERIALIZED VIEW test.kafka_errors_{format_name}_mv ENGINE=MergeTree ORDER BY tuple() AS SELECT {raw_message} as raw_message, _error as error, _topic as topic, _partition as partition, _offset as offset FROM test.kafka_{format_name} WHERE length(_error) > 0; + + DETACH TABLE test.kafka_{format_name}; + ATTACH TABLE test.kafka_{format_name}; """ ) + k.kafka_produce(kafka_cluster, topic_name, data_prefix + data_sample) raw_expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} diff --git a/tests/integration/test_storage_s3_queue/configs/plain_rewritable_disk.xml b/tests/integration/test_storage_s3_queue/configs/plain_rewritable_disk.xml new file mode 100644 index 000000000000..ee32b5fd72a1 --- /dev/null +++ b/tests/integration/test_storage_s3_queue/configs/plain_rewritable_disk.xml @@ -0,0 +1,12 @@ + + + + + object_storage + local_blob_storage + plain_rewritable + /var/lib/clickhouse/disks/plain_rw/ + + + + diff --git a/tests/integration/test_storage_s3_queue/test_5.py b/tests/integration/test_storage_s3_queue/test_5.py index 37133502e6be..f62ce5696107 100644 --- a/tests/integration/test_storage_s3_queue/test_5.py +++ b/tests/integration/test_storage_s3_queue/test_5.py @@ -65,6 +65,7 @@ def started_cluster(): "configs/s3queue_log.xml", "configs/remote_servers.xml", "configs/disable_streaming.xml", + "configs/plain_rewritable_disk.xml", ], user_configs=[ "configs/users.xml", @@ -1294,6 +1295,137 @@ def test_failed_startup(started_cluster): assert len(zk.get(f"{keeper_path}")) > 0 +LOGICAL_ERROR_MARKER = "Logical error: 'Files metadata is empty'" + + +def assert_reports_table_is_dropped(node, table_name, database_name="default"): + """Every user-facing entry point that needs the queue metadata must report + TABLE_IS_DROPPED, and none of them may abort the server.""" + qualified = f"{database_name}.{table_name}" + for query in ( + f"SELECT count() FROM {qualified} SETTINGS stream_like_engine_allow_direct_select=1", + f"SYSTEM FLUSH OBJECT STORAGE QUEUE {qualified} PATH 'x'", + f"ALTER TABLE {qualified} MODIFY SETTING polling_min_timeout_ms=555", + ): + error = node.query_and_get_error(query) + assert "TABLE_IS_DROPPED" in error, f"{query} -> {error}" + + assert node.query("SELECT 1") == "1\n" + # A query-level error alone is satisfied by many unrelated failures, so pin the + # absence of the abort itself. The table name is unique per invocation, and the + # log is shared, so match the marker together with it. + assert not node.contains_in_log(LOGICAL_ERROR_MARKER) + assert not node.contains_in_log("Received signal Segmentation fault") + + +def test_select_after_failed_startup(started_cluster): + node = started_cluster.instances["instance"] + table_name = f"test_select_after_failed_startup_{generate_random_string()}" + + node.query("SYSTEM ENABLE FAILPOINT object_storage_queue_fail_startup") + try: + assert "Failed to startup" in create_table( + started_cluster, + node, + table_name, + "unordered", + f"{table_name}_data", + format="column1 UInt32, column2 String", + expect_error=True, + additional_settings={"keeper_path": f"/clickhouse/test_{table_name}"}, + ) + finally: + node.query("SYSTEM DISABLE FAILPOINT object_storage_queue_fail_startup") + + # startup() reset the metadata handle, but nothing rolled the catalog entry back. + assert ( + node.query(f"SELECT count() FROM system.tables WHERE name = '{table_name}'") + == "1\n" + ) + + assert_reports_table_is_dropped(node, table_name) + + +def test_select_after_failed_drop(started_cluster): + node = started_cluster.instances["instance"] + suffix = generate_random_string() + table_name = f"test_select_after_failed_drop_{suffix}" + database_name = f"db_{table_name}" + + node.query( + f"CREATE DATABASE {database_name} ENGINE = Atomic SETTINGS disk = 'plain_rw'" + ) + create_table( + started_cluster, + node, + table_name, + "ordered", + f"{table_name}_data", + format="column1 UInt32, column2 String", + database_name=database_name, + additional_settings={"keeper_path": f"/clickhouse/test_{table_name}"}, + ) + + # DROP shuts the table down before the database detaches it, so a failure in + # between leaves the table attached with its metadata handle already gone. + node.query( + "SYSTEM ENABLE FAILPOINT plain_object_storage_write_fail_on_directory_create" + ) + try: + assert "FAULT_INJECTED" in node.query_and_get_error( + f"DROP TABLE {database_name}.{table_name}" + ) + finally: + node.query( + "SYSTEM DISABLE FAILPOINT plain_object_storage_write_fail_on_directory_create" + ) + + assert ( + node.query( + f"SELECT count() FROM system.tables " + f"WHERE database = '{database_name}' AND name = '{table_name}'" + ) + == "1\n" + ) + + assert_reports_table_is_dropped(node, table_name, database_name) + + +def test_select_racing_drop(started_cluster): + node = started_cluster.instances["instance"] + + for i in range(10): + table_name = f"test_select_racing_drop_{generate_random_string()}" + create_table( + started_cluster, + node, + table_name, + "unordered", + f"{table_name}_data", + format="column1 UInt32, column2 String", + additional_settings={"keeper_path": f"/clickhouse/test_{table_name}"}, + ) + + # DROP shuts the storage down without waiting for readers, so it can drop the + # metadata handle while this SELECT is still building its query plan. The + # subquery sleeps for 3 seconds, so the DROP lands while the plan is still open. + select = node.get_query_request( + f"SELECT count() FROM {table_name} " + f"WHERE column1 > (SELECT sum(sleepEachRow(0.2)) FROM numbers(15)) " + f"SETTINGS stream_like_engine_allow_direct_select=1" + ) + time.sleep(1.2) + node.query(f"DROP TABLE {table_name} SYNC") + + # Also accepting a clean result would make this arm pass against the unfixed + # server: a SELECT that finished first never reads the dropped metadata. + _, error = select.get_answer_and_error() + assert "TABLE_IS_DROPPED" in error, f"iteration {i}: {error}" + + assert node.query("SELECT 1") == "1\n" + assert not node.contains_in_log(LOGICAL_ERROR_MARKER) + + def test_create_or_replace_table(started_cluster): node1 = started_cluster.instances["instance"] node2 = started_cluster.instances["instance2"] diff --git a/tests/integration/test_text_index_upgrade/configs/compatibility.xml b/tests/integration/test_text_index_upgrade/configs/compatibility.xml new file mode 100644 index 000000000000..993525074c35 --- /dev/null +++ b/tests/integration/test_text_index_upgrade/configs/compatibility.xml @@ -0,0 +1,12 @@ + + + + + 26.5 + + + diff --git a/tests/integration/test_text_index_upgrade/test.py b/tests/integration/test_text_index_upgrade/test.py index 4093cef3a120..353dd4980d08 100644 --- a/tests/integration/test_text_index_upgrade/test.py +++ b/tests/integration/test_text_index_upgrade/test.py @@ -2,7 +2,7 @@ from helpers.cluster import ClickHouseCluster -# 26.4 writes the pre-WithCodec header format; the new reader recovers the +# 26.4 writes the pre-V1_WithCodec header format; the new reader recovers the # posting list codec from the index DDL. We test both the default codec and # 'bitpacking', where DDL recovery is the only way to decode old segments. OLD_VERSION_TAG = "26.4" @@ -19,6 +19,18 @@ def started_cluster(): with_installed_binary=True, stay_alive=True, ) + # Same as `node`, but its default profile pins `compatibility` to a pre-26.6 + # version. After the upgrade this makes the new binary resolve + # `text_index_serialization_version` to `v0_initial` on its own, without persisting any setting + # into the table metadata, which is the realistic rolling-upgrade knob. + cluster.add_instance( + "node_compat", + image="clickhouse/clickhouse-server", + tag=OLD_VERSION_TAG, + with_installed_binary=True, + stay_alive=True, + user_configs=["configs/compatibility.xml"], + ) cluster.start() yield cluster finally: @@ -104,7 +116,7 @@ def create_and_populate(node, table, posting_list_codec): # Exercise the lazy posting list apply mode against the upgraded binary: -# pre-WithCodec granules silently fall back to eager mode, while the new-format +# pre-V1_WithCodec granules silently fall back to eager mode, while the new-format # part inserted after the upgrade actually uses the cursor-based reader. LAZY_APPLY_SETTINGS = { "allow_experimental_text_index_lazy_apply": 1, @@ -136,7 +148,7 @@ def test_text_index_upgrade(started_cluster, posting_list_codec): create_and_populate(node, table, posting_list_codec) - # Ground truth from the old server: pre-WithCodec layout on disk, old reader. + # Ground truth from the old server: pre-V1_WithCodec layout on disk, old reader. assert run_search_queries(node, table) == expected_results() # Swap the binary but keep the data dir; the new reader must load the @@ -144,7 +156,7 @@ def test_text_index_upgrade(started_cluster, posting_list_codec): node.restart_with_latest_version() # Same data, same queries, same answers under the upgraded binary. - # Lazy mode falls back to materialize for these pre-WithCodec granules. + # Lazy mode falls back to materialize for these pre-V1_WithCodec granules. assert run_search_queries(node, table, settings=LAZY_APPLY_SETTINGS) == expected_results() # Confirm the text index is engaged after upgrade; without this check a @@ -198,7 +210,7 @@ def test_text_index_upgrade(started_cluster, posting_list_codec): == "1" ) - # Merge across mixed-format parts: posting lists from the old pre-WithCodec + # Merge across mixed-format parts: posting lists from the old pre-V1_WithCodec # layout and the new layout are read back and re-emitted as one new-format # part. Fails if the new reader cannot decode old segments end-to-end. node.query(f"OPTIMIZE TABLE {table} FINAL") @@ -227,3 +239,202 @@ def test_text_index_upgrade(started_cluster, posting_list_codec): node.query(f"DROP TABLE {table} SYNC") node.restart_with_original_version() + + +# -------------------------------------------------------------------------------- +# The tests below focus on the `text_index_serialization_version` MergeTree setting that controls +# the on-disk text index format, and on its interaction with the posting list codec +# across an upgrade and a downgrade. +# -------------------------------------------------------------------------------- + + +# A part written by whatever binary is currently running: even k -> 'common', +# odd k -> 'rare', every row carries 'shared', and row 5042 carries the new token +# 'unique5042'. Mirrors the third part inserted by `test_text_index_upgrade`. +def insert_new_part(node, table): + node.query( + f""" + INSERT INTO {table} + SELECT + number + 5000, + concat( + 'shared ', + if(number % 2 = 0, 'common', 'rare'), + if(number = 42, ' unique5042', '') + ) + FROM numbers(2500) + """ + ) + + +# Expected `SEARCH_QUERIES` answers after `create_and_populate` followed by +# `insert_new_part`: 'common' and 'rare' each gain 1250 rows, 'unique42' stays on +# row 42, and the new token 'unique5042' lives once on row 5042. +MIXED_EXPECTED = [ + "3750", # hasToken 'common' + "3750", # hasToken 'rare' + "1", # hasToken 'unique42' + "0", # hasToken 'absent' + "3750", # hasAllTokens ['common', 'shared'] + "3751", # hasAnyTokens ['rare', 'unique42'] + "[42]", # arraySort(groupArray(k)) for 'unique42' +] + +NEW_TOKEN_QUERY = "SELECT count() FROM {table} WHERE hasToken(s, 'unique5042')" + + +def assert_single_active_part(node, table): + active_parts = node.query( + f"SELECT count() FROM system.parts WHERE table = '{table}' AND active" + ).strip() + assert active_parts == "1", ( + f"expected a single active part after OPTIMIZE FINAL, got {active_parts}" + ) + + +def assert_index_used(node, table): + # A full scan would answer the search queries correctly too, so confirm the text + # index is actually engaged via the query plan. The query must read a column: a + # bare `count()` is answered from the index cardinality by `ReadFromTextIndexCount`, + # whose plan has no `ReadFromMergeTree` step listing the used indexes. + explain = node.query( + f"EXPLAIN indexes = 1 SELECT k FROM {table} WHERE hasToken(s, 'unique42')" + ) + assert "Name: idx" in explain, f"text index `idx` not used:\n{explain}" + + +def test_change_codec_after_upgrade(started_cluster): + """Create the index on the old version with the default codec, upgrade, switch + the posting list codec to 'bitpacking', and verify the index keeps working + across parts written in two different on-disk codecs (and through a merge).""" + node = started_cluster.instances["node"] + table = "text_index_change_codec" + + # Old binary, default codec -> pre-V1_WithCodec ('v0_initial') header on disk. + create_and_populate(node, table, posting_list_codec=None) + assert run_search_queries(node, table) == expected_results() + + node.restart_with_latest_version() + try: + # New binary reads the old-format parts unchanged. + assert run_search_queries(node, table) == expected_results() + + # Switch the default codec: new parts must persist the codec type in the header. + node.query( + f"ALTER TABLE {table} MODIFY SETTING text_index_posting_list_codec = 'bitpacking'" + ) + + # The version setting is only a preference: 'v0_initial' cannot persist the codec + # type, so the write path silently bumps such an index to 'v1_with_codec'. + node.query( + f"ALTER TABLE {table} MODIFY SETTING text_index_serialization_version = 'v0_initial'" + ) + + # The new part is written with 'bitpacking' + the 'v1_with_codec' header, so the + # table now mixes 'none'/'v0_initial' and 'bitpacking'/'v1_with_codec' segments. + insert_new_part(node, table) + assert run_search_queries(node, table) == MIXED_EXPECTED + assert node.query(NEW_TOKEN_QUERY.format(table=table)).strip() == "1" + assert_index_used(node, table) + + # Merge across the two codecs: the reader must decode both layouts and the + # writer re-emits a single 'bitpacking'/'v1_with_codec' part. + node.query(f"OPTIMIZE TABLE {table} FINAL") + assert_single_active_part(node, table) + assert run_search_queries(node, table) == MIXED_EXPECTED + assert node.query(NEW_TOKEN_QUERY.format(table=table)).strip() == "1" + + node.query(f"DROP TABLE {table} SYNC") + finally: + node.restart_with_original_version() + + +def test_downgrade_after_writing_on_new_version(started_cluster): + """The point of `text_index_serialization_version`: a new server can keep writing the old + on-disk format so the data survives a rollback. Write 'v0_initial'-format parts + with the *new* binary, reset the setting so the metadata stays loadable by the + old binary, downgrade, and verify the old binary reads everything back.""" + node = started_cluster.instances["node"] + table = "text_index_downgrade_setting" + + # Old binary, default codec -> 'v0_initial' format on disk. + create_and_populate(node, table, posting_list_codec=None) + assert run_search_queries(node, table) == expected_results() + + node.restart_with_latest_version() + new_version_active = True + try: + # New binary reads the old-format parts unchanged. + assert run_search_queries(node, table) == expected_results() + + # Force the new binary to keep writing the old on-disk format. + node.query( + f"ALTER TABLE {table} MODIFY SETTING text_index_serialization_version = 'v0_initial'" + ) + + # This part and the merged part below are written by the *new* binary, but in + # the 'v0_initial' format because of the setting above. + insert_new_part(node, table) + assert run_search_queries(node, table) == MIXED_EXPECTED + node.query(f"OPTIMIZE TABLE {table} FINAL") + assert_single_active_part(node, table) + assert run_search_queries(node, table) == MIXED_EXPECTED + + # An explicit `text_index_serialization_version` in the metadata is an unknown setting for + # the old binary and would block ATTACH after the downgrade. Reset it; the + # parts already on disk keep their 'v0_initial' format. + node.query(f"ALTER TABLE {table} RESET SETTING text_index_serialization_version") + + node.restart_with_original_version() + new_version_active = False + + # The old binary reads the parts the new binary wrote in 'v0_initial' format, + # including the merged one. This is the downgrade guarantee. A 'v1_with_codec' + # part here would instead fail to load on the old server. + assert run_search_queries(node, table) == MIXED_EXPECTED + assert node.query(NEW_TOKEN_QUERY.format(table=table)).strip() == "1" + assert_index_used(node, table) + + node.query(f"DROP TABLE {table} SYNC") + finally: + if new_version_active: + node.restart_with_original_version() + + +def test_downgrade_with_compatibility_setting(started_cluster): + """The realistic rolling-upgrade knob: with `compatibility` pinned to a pre-26.6 + version in the default profile, the new server resolves `text_index_serialization_version` to + 'v0_initial' on its own, without persisting any setting into the table metadata, so + the data stays readable after a rollback - no ALTER and no RESET required.""" + node = started_cluster.instances["node_compat"] + table = "text_index_downgrade_compat" + + create_and_populate(node, table, posting_list_codec=None) + assert run_search_queries(node, table) == expected_results() + + node.restart_with_latest_version() + new_version_active = True + try: + assert run_search_queries(node, table) == expected_results() + + # No ALTER: `compatibility = '26.5'` from the default profile makes the new + # binary write the 'v0_initial' format, and nothing is persisted in metadata. + insert_new_part(node, table) + assert run_search_queries(node, table) == MIXED_EXPECTED + node.query(f"OPTIMIZE TABLE {table} FINAL") + assert_single_active_part(node, table) + assert run_search_queries(node, table) == MIXED_EXPECTED + + node.restart_with_original_version() + new_version_active = False + + # The metadata never mentioned `text_index_serialization_version`, so the old binary loads + # the table and reads the 'v0_initial'-format parts the new binary produced. + assert run_search_queries(node, table) == MIXED_EXPECTED + assert node.query(NEW_TOKEN_QUERY.format(table=table)).strip() == "1" + assert_index_used(node, table) + + node.query(f"DROP TABLE {table} SYNC") + finally: + if new_version_active: + node.restart_with_original_version() diff --git a/tests/performance/formatdatetime_nonconst_time_zone.xml b/tests/performance/formatdatetime_nonconst_time_zone.xml new file mode 100644 index 000000000000..2ec584fdee4e --- /dev/null +++ b/tests/performance/formatdatetime_nonconst_time_zone.xml @@ -0,0 +1,4 @@ + + SELECT sum(length(formatDateTime(toDateTime(1700000000) + number, '%F %T', if(number % 2 = 0, 'Europe/London', 'Europe/Berlin')))) FROM numbers_mt(20000000) SETTINGS max_threads = 1 + SELECT sum(length(formatDateTime(toDateTime(1700000000) + number, '%F %T', if(number % 2 = 0, 'Europe/London', 'Europe/Berlin')))) FROM numbers_mt(20000000) SETTINGS max_threads = 8 + diff --git a/tests/performance/group_by_dynamic_keys.xml b/tests/performance/group_by_dynamic_keys.xml new file mode 100644 index 000000000000..f69cf91e7280 --- /dev/null +++ b/tests/performance/group_by_dynamic_keys.xml @@ -0,0 +1,57 @@ + + + + 1 + 8 + 1 + + + CREATE TABLE t_json_group_by (j JSON) ENGINE = MergeTree ORDER BY tuple() + + INSERT INTO t_json_group_by + SELECT toJSONString(map('a', toString(number % 1000), 'b', toString(number % 7), + 'c', toString(number % 13), 'd', toString(number % 97), + 'e', toString(number % 31), 'f', toString(number % 53), + 'g', toString(number % 11)))::JSON + FROM numbers(1000000) + + + + SELECT (number % 1000)::Dynamic AS a, + (number % 7)::Dynamic AS b, + (number % 13)::Dynamic AS c, + (number % 97)::Dynamic AS d, + (number % 31)::Dynamic AS e, + (number % 53)::Dynamic AS f, + (number % 11)::Dynamic AS g + FROM numbers(2000000) + GROUP BY 1, 2, 3, 4, 5, 6, 7 + FORMAT Null + + + + + SELECT toString(number % 1000) AS a, + toString(number % 7) AS b, + toString(number % 13) AS c, + toString(number % 97) AS d, + toString(number % 31) AS e, + toString(number % 53) AS f, + toString(number % 11) AS g + FROM numbers(2000000) + GROUP BY 1, 2, 3, 4, 5, 6, 7 + FORMAT Null + + + SELECT j.a, j.b, j.c, j.d, j.e, j.f, j.g FROM t_json_group_by GROUP BY 1, 2, 3, 4, 5, 6, 7 FORMAT Null + + DROP TABLE IF EXISTS t_json_group_by + diff --git a/tests/performance/low_cardinality_many_marks_point_lookups.xml b/tests/performance/low_cardinality_many_marks_point_lookups.xml new file mode 100644 index 000000000000..1ac7788f14d6 --- /dev/null +++ b/tests/performance/low_cardinality_many_marks_point_lookups.xml @@ -0,0 +1,100 @@ + + + + CREATE TABLE lc_many_marks_point_lookups + ( + k String, + ts UInt32, + lc1 LowCardinality(String), + lc2 LowCardinality(String), + lc3 LowCardinality(String), + lc4 LowCardinality(String), + lc5 LowCardinality(String), + lc6 LowCardinality(String), + v UInt64 + ) + ENGINE = MergeTree + ORDER BY (k, ts) + SETTINGS index_granularity = 8, min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0 + + + + INSERT INTO lc_many_marks_point_lookups + SELECT + 'K' || toString(number % 200), + number % 5000, + ['alpha', 'beta', 'gamma'][(number % 3) + 1], + 'constant', + 'c3', 'c4', 'c5', 'c6', + number + FROM numbers(2000000) + SETTINGS max_insert_threads = 1, max_insert_block_size = 2000000 + + + + + SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K0' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K1' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K2' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K3' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K4' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K5' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K6' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K7' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K8' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K9' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K10' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K11' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K12' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K13' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K14' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K15' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K16' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K17' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K18' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K19' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K20' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K21' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K22' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K23' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K24' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K25' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K26' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K27' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K28' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K29' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K30' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K31' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K32' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K33' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K34' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K35' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K36' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K37' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K38' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K39' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K40' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K41' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K42' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K43' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K44' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K45' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K46' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K47' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K48' GROUP BY k + UNION ALL SELECT k, argMax((lc1, lc2, lc3, lc4, lc5, lc6, v), ts) FROM lc_many_marks_point_lookups WHERE k = 'K49' GROUP BY k + FORMAT Null + SETTINGS max_threads = 1 + + + DROP TABLE IF EXISTS lc_many_marks_point_lookups + diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.sh b/tests/queries/0_stateless/00900_long_parquet_load_2.sh index 3fb1dc1874b5..5841286887a2 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.sh @@ -48,6 +48,8 @@ EXCLUDE=( 04065_optional_map_wrapper_required_value.parquet 04065_optional_struct_under_list.parquet 04065_optional_struct_nullable_leaf_under_list.parquet + # Hand-crafted file with an inconsistent bloom filter size for the 04654 out-of-bounds test. + 04654_bloom_filter_bitset_out_of_bounds.parquet ) for NAME in $(find "$DATA_DIR" -type f \( -iname '*.parquet' -o -iname '*.parquet.gz' \) -print0 | xargs -0 -n 1 basename | LC_ALL=C sort | grep -vFf <(printf '%s\n' "${EXCLUDE[@]}")); do diff --git a/tests/queries/0_stateless/01429_join_on_error_messages.sql b/tests/queries/0_stateless/01429_join_on_error_messages.sql index 66bcc7111765..595eb916a045 100644 --- a/tests/queries/0_stateless/01429_join_on_error_messages.sql +++ b/tests/queries/0_stateless/01429_join_on_error_messages.sql @@ -2,9 +2,9 @@ SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON equals(a); -- { serverError SELECT 1 FROM (select 1 a) A JOIN (select 1 b) B ON less(a); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH, 62 } SET join_algorithm = 'partial_merge'; -SELECT 1 FROM (select 1 a) A JOIN (select 1 b, 1 c) B ON a = b OR a = c; -- { serverError NOT_IMPLEMENTED } +SELECT 1 FROM (select materialize(1) a) A JOIN (select materialize(1) b, materialize(1) c) B ON a = b OR a = c; -- { serverError NOT_IMPLEMENTED } -- works for a = b OR a = b because of equivalent disjunct optimization SET join_algorithm = 'grace_hash'; -SELECT 1 FROM (select 1 a) A JOIN (select 1 b, 1 c) B ON a = b OR a = c; -- { serverError NOT_IMPLEMENTED } +SELECT 1 FROM (select materialize(1) a) A JOIN (select materialize(1) b, materialize(1) c) B ON a = b OR a = c; -- { serverError NOT_IMPLEMENTED } -- works for a = b OR a = b because of equivalent disjunct optimization diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference index 2e2efc9df62d..d63416d61dd6 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference @@ -14,7 +14,10 @@ ExpressionTransform (Expression) ExpressionTransform × 2 (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) × 2 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 2020-10-01 9 2020-10-01 9 2020-10-01 9 @@ -32,9 +35,11 @@ ExpressionTransform ExpressionTransform × 2 (ReadFromMergeTree) ReverseTransform - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder) 0 → 1 - ReverseTransform - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder) 0 → 1 + ReverseTransform + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder) 0 → 1 2020-10-01 9 2020-10-01 9 2020-10-01 9 @@ -51,7 +56,10 @@ ExpressionTransform (Expression) ExpressionTransform × 2 (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) × 2 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + VirtualRowTransform + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 2020-10-11 0 2020-10-11 0 2020-10-11 0 diff --git a/tests/queries/0_stateless/03151_unload_index_race.sh b/tests/queries/0_stateless/03151_unload_index_race.sh index 4cdf06abae2f..c4a7f3c58d2e 100755 --- a/tests/queries/0_stateless/03151_unload_index_race.sh +++ b/tests/queries/0_stateless/03151_unload_index_race.sh @@ -51,12 +51,23 @@ function thread_alter_settings() function thread_query_table() { local TIMELIMIT=$((SECONDS+$1)) + local ATTEMPTS=0 + local SUCCESSES=0 while [ $SECONDS -lt "$TIMELIMIT" ]; do + ATTEMPTS=$((ATTEMPTS+1)) COUNT=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM t where not ignore(*);") - if [ "$COUNT" -ne "2000" ]; then - echo "$COUNT" + # Empty $COUNT is a transiently interrupted read; skip it so the compare emits no bash error. + if [ -n "$COUNT" ]; then + SUCCESSES=$((SUCCESSES+1)) + if [ "$COUNT" != "2000" ]; then + echo "wrong count: $COUNT" + fi fi done + # A reader that issued reads but never once got a valid count is a real failure, not bash noise. + if [ "$ATTEMPTS" -gt 0 ] && [ "$SUCCESSES" -eq 0 ]; then + echo "reader never got a successful count" + fi } export -f thread_alter_settings diff --git a/tests/queries/0_stateless/03212_variant_dynamic_cast_or_default.sql b/tests/queries/0_stateless/03212_variant_dynamic_cast_or_default.sql index 8cf9c1469efd..28befbdab284 100644 --- a/tests/queries/0_stateless/03212_variant_dynamic_cast_or_default.sql +++ b/tests/queries/0_stateless/03212_variant_dynamic_cast_or_default.sql @@ -3,6 +3,7 @@ set use_variant_as_common_type = 1; set allow_experimental_dynamic_type = 1; set allow_suspicious_low_cardinality_types = 1; set session_timezone = 'UTC'; +set cast_string_to_date_time_mode = 'basic'; select accurateCastOrDefault(variant, 'UInt32'), multiIf(number % 4 == 0, NULL, number % 4 == 1, number, number % 4 == 2, 'str_' || toString(number), range(number)) as variant from numbers(8); select accurateCastOrNull(variant, 'UInt32'), multiIf(number % 4 == 0, NULL, number % 4 == 1, number, number % 4 == 2, 'str_' || toString(number), range(number)) as variant from numbers(8); diff --git a/tests/queries/0_stateless/03257_reverse_sorting_key.reference b/tests/queries/0_stateless/03257_reverse_sorting_key.reference index b1e7f0cd00a0..ae9cccdebd87 100644 --- a/tests/queries/0_stateless/03257_reverse_sorting_key.reference +++ b/tests/queries/0_stateless/03257_reverse_sorting_key.reference @@ -67,10 +67,11 @@ ExpressionTransform (Sorting) FinishSortingTransform PartialSortingTransform - (Expression) - ExpressionTransform - (ReadFromMergeTree) - MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 + RemoveVirtualRowTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1 0 1000 0 1010 0 1020 diff --git a/tests/queries/0_stateless/03262_common_expression_optimization.sql b/tests/queries/0_stateless/03262_common_expression_optimization.sql index 7d609f780295..07f641b0e234 100644 --- a/tests/queries/0_stateless/03262_common_expression_optimization.sql +++ b/tests/queries/0_stateless/03262_common_expression_optimization.sql @@ -1,5 +1,6 @@ SET enable_analyzer = 1; SET optimize_extract_common_expressions = 1; +SET query_plan_read_in_order_through_join = 0; DROP TABLE IF EXISTS x; CREATE TABLE x (x Int64, A UInt8, B UInt8, C UInt8, D UInt8, E UInt8, F UInt8) ENGINE = MergeTree ORDER BY x; diff --git a/tests/queries/0_stateless/03363_hive_style_partition.sql b/tests/queries/0_stateless/03363_hive_style_partition.sql index d4438a887053..6e9f9fb5068e 100644 --- a/tests/queries/0_stateless/03363_hive_style_partition.sql +++ b/tests/queries/0_stateless/03363_hive_style_partition.sql @@ -76,8 +76,11 @@ INSERT INTO FUNCTION s3(s3_conn, filename='t_03363_parquet', format=Parquet, par -- Schema specified, but the hive partition column is missing in the schema (present in the data tho) INSERT INTO FUNCTION s3(s3_conn, filename='half_baked', format=Parquet, partition_strategy='hive') PARTITION BY year SELECT 1 AS key, 2020 AS year; --- Should fail because contains only partition columns in schema and `use_hive_partitioning=1` -CREATE TABLE s3_table_half_schema_with_format (year UInt64) engine=S3(s3_conn, filename='half_baked/**.parquet', format=Parquet) SETTINGS use_hive_partitioning=1; -- {serverError INCORRECT_DATA} +-- Contains only partition columns in schema and `use_hive_partitioning=1`. The sample path for +-- hive partitioning detection is resolved lazily, so CREATE succeeds and the first use fails. +CREATE TABLE s3_table_half_schema_with_format (year UInt64) engine=S3(s3_conn, filename='half_baked/**.parquet', format=Parquet) SETTINGS use_hive_partitioning=1; +SELECT DISTINCT * FROM s3_table_half_schema_with_format; -- {serverError INCORRECT_DATA} +DROP TABLE s3_table_half_schema_with_format; -- Should succeed because hive is off CREATE TABLE s3_table_half_schema_with_format (year UInt64) engine=S3(s3_conn, filename='half_baked/**.parquet', format=Parquet) SETTINGS use_hive_partitioning=0; diff --git a/tests/queries/0_stateless/03631_buffer_access_check.sh b/tests/queries/0_stateless/03631_buffer_access_check.sh index ffd0032c39ad..46464165a7b7 100755 --- a/tests/queries/0_stateless/03631_buffer_access_check.sh +++ b/tests/queries/0_stateless/03631_buffer_access_check.sh @@ -17,6 +17,9 @@ DROP USER IF EXISTS $user; CREATE USER $user; GRANT SELECT, CREATE, INSERT ON $db.test_buffer TO $user; GRANT TABLE ENGINE ON Buffer TO $user; +-- Creating the buffer without a column list infers the structure from the destination, which needs +-- SHOW COLUMNS on it. The read and write checks asserted below require SELECT and INSERT instead. +GRANT SHOW COLUMNS ON $db.test_table TO $user; EOF ${CLICKHOUSE_CLIENT} --user $user --query "CREATE TABLE $db.test_buffer ENGINE = Buffer($db, test_table, 1, 10, 100, 10000, 1000000, 10000000, 100000000)" diff --git a/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference b/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference index 183b6800e157..2f00d70af84b 100644 --- a/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference +++ b/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference @@ -18,8 +18,8 @@ 03711_type_fixedstr all_2_2_0 Compact ['all_16360560524467875160_4720191420039909589'] 03711_type_float all_1_1_0 Compact ['all_13346149972136598791_11741866962490268826'] 03711_type_float all_2_2_0 Compact ['all_13611278761629314394_2981841964782876041'] -03711_type_json all_1_1_0 Compact ['all_1019996165332126500_10512220182055512995'] -03711_type_json all_2_2_0 Compact ['all_3077914509782472429_9735021766957434210'] +03711_type_json all_1_1_0 Compact ['all_13130503899522743408_9022237074411647791'] +03711_type_json all_2_2_0 Compact ['all_7473068319567880740_1495509417517398706'] 03711_type_json_mdp0 all_1_1_0 Compact ['all_9636225467500754362_16767708078063468458'] 03711_type_json_mdp0 all_2_2_0 Compact ['all_12384973329383563319_5350484746181772478'] 03711_type_map all_1_1_0 Compact ['all_15715181759477494293_9711484488284658522'] diff --git a/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference.orig b/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference.orig new file mode 100644 index 000000000000..8255c0595999 --- /dev/null +++ b/tests/queries/0_stateless/03711_deduplication_blocks_part_log.reference.orig @@ -0,0 +1,112 @@ +<<<<<<< HEAD +03711_async_array all_1_1_0 Compact ['all_10571676162184711478_162537819996838366'] +03711_async_mixed all_1_1_0 Compact ['all_1230248051038658715_18131952419049248935'] +03711_async_string all_1_1_0 Compact ['all_17388740250088195039_14005593440692414930'] +03711_async_uint all_1_1_0 Compact ['all_16994774723362420241_11434403476823141314'] +03711_join_with all_1_1_0 Compact ['all_9563449748241099126_16240541781756294264'] +03711_join_with all_2_2_0 Compact ['all_16444784039195326156_10685096260700552877'] +03711_mv_table_1 all_1_1_0 Compact ['all_9139126859104319288_12565776327365534163'] +03711_mv_table_2 all_1_1_0 Compact ['all_17054059182758846721_9547493448332016625'] +03711_table all_1_1_0 Compact ['all_659810233567002352_16818222267659499314'] +03711_table all_2_2_0 Compact ['all_10695495171951104297_11784282495965162872'] +03711_type_array all_1_1_0 Compact ['all_8714782328307936551_5667215566625646845'] +03711_type_array all_2_2_0 Compact ['all_16147962101698235904_1457969930443025035'] +03711_type_decimal all_1_1_0 Compact ['all_13242298624230215860_14719097669563059311'] +03711_type_decimal all_2_2_0 Compact ['all_5209730246278546871_3546417517182497241'] +03711_type_dynamic all_1_1_0 Compact ['all_4355947499223422669_319868960027583601'] +03711_type_dynamic all_2_2_0 Compact ['all_13821245313499792332_16722045970011847000'] +03711_type_fixedstr all_1_1_0 Compact ['all_15043183714943801330_10001046138349709664'] +03711_type_fixedstr all_2_2_0 Compact ['all_16360560524467875160_4720191420039909589'] +03711_type_float all_1_1_0 Compact ['all_13346149972136598791_11741866962490268826'] +03711_type_float all_2_2_0 Compact ['all_3906978350429926008_5556109652846278465'] +03711_type_json all_1_1_0 Compact ['all_13130503899522743408_9022237074411647791'] +03711_type_json all_2_2_0 Compact ['all_7473068319567880740_1495509417517398706'] +03711_type_json_mdp0 all_1_1_0 Compact ['all_9636225467500754362_16767708078063468458'] +03711_type_json_mdp0 all_2_2_0 Compact ['all_12384973329383563319_5350484746181772478'] +03711_type_map all_1_1_0 Compact ['all_15715181759477494293_9711484488284658522'] +03711_type_map all_2_2_0 Compact ['all_16684526797205426442_11364293216111778856'] +03711_type_mixed all_1_1_0 Compact ['all_12986313597556146191_6773938392127322341'] +03711_type_mixed all_2_2_0 Compact ['all_10629709287848428236_5002643046732336804'] +03711_type_nullable all_1_1_0 Compact ['all_6450400810974626920_13339409127224583224'] +03711_type_nullable all_2_2_0 Compact ['all_4420175743708305773_8529015166913398714'] +03711_type_string all_1_1_0 Compact ['all_11302719525430258405_10531094102249515008'] +03711_type_string all_2_2_0 Compact ['all_9267555012450399540_4231923082176187394'] +03711_type_tuple all_1_1_0 Compact ['all_12313389176203636080_11707488636192631513'] +03711_type_tuple all_2_2_0 Compact ['all_12572740846975368560_14638259508703709964'] +03711_type_variant all_1_1_0 Compact ['all_5728325312273395383_4364882181208797392'] +03711_type_variant all_2_2_0 Compact ['all_13821245313499792332_16722045970011847000'] +||||||| 78da318f308 +03711_async_array all_1_1_0 Compact ['all_10571676162184711478_162537819996838366'] +03711_async_mixed all_1_1_0 Compact ['all_1230248051038658715_18131952419049248935'] +03711_async_string all_1_1_0 Compact ['all_17388740250088195039_14005593440692414930'] +03711_async_uint all_1_1_0 Compact ['all_16994774723362420241_11434403476823141314'] +03711_join_with all_1_1_0 Compact ['all_9563449748241099126_16240541781756294264'] +03711_join_with all_2_2_0 Compact ['all_16444784039195326156_10685096260700552877'] +03711_mv_table_1 all_1_1_0 Compact ['all_9139126859104319288_12565776327365534163'] +03711_mv_table_2 all_1_1_0 Compact ['all_17054059182758846721_9547493448332016625'] +03711_table all_1_1_0 Compact ['all_659810233567002352_16818222267659499314'] +03711_table all_2_2_0 Compact ['all_10695495171951104297_11784282495965162872'] +03711_type_array all_1_1_0 Compact ['all_8714782328307936551_5667215566625646845'] +03711_type_array all_2_2_0 Compact ['all_16147962101698235904_1457969930443025035'] +03711_type_decimal all_1_1_0 Compact ['all_13242298624230215860_14719097669563059311'] +03711_type_decimal all_2_2_0 Compact ['all_5209730246278546871_3546417517182497241'] +03711_type_dynamic all_1_1_0 Compact ['all_4355947499223422669_319868960027583601'] +03711_type_dynamic all_2_2_0 Compact ['all_13821245313499792332_16722045970011847000'] +03711_type_fixedstr all_1_1_0 Compact ['all_15043183714943801330_10001046138349709664'] +03711_type_fixedstr all_2_2_0 Compact ['all_16360560524467875160_4720191420039909589'] +03711_type_float all_1_1_0 Compact ['all_13346149972136598791_11741866962490268826'] +03711_type_float all_2_2_0 Compact ['all_3906978350429926008_5556109652846278465'] +03711_type_json all_1_1_0 Compact ['all_1019996165332126500_10512220182055512995'] +03711_type_json all_2_2_0 Compact ['all_3077914509782472429_9735021766957434210'] +03711_type_json_mdp0 all_1_1_0 Compact ['all_9636225467500754362_16767708078063468458'] +03711_type_json_mdp0 all_2_2_0 Compact ['all_12384973329383563319_5350484746181772478'] +03711_type_map all_1_1_0 Compact ['all_15715181759477494293_9711484488284658522'] +03711_type_map all_2_2_0 Compact ['all_16684526797205426442_11364293216111778856'] +03711_type_mixed all_1_1_0 Compact ['all_12986313597556146191_6773938392127322341'] +03711_type_mixed all_2_2_0 Compact ['all_10629709287848428236_5002643046732336804'] +03711_type_nullable all_1_1_0 Compact ['all_6450400810974626920_13339409127224583224'] +03711_type_nullable all_2_2_0 Compact ['all_4420175743708305773_8529015166913398714'] +03711_type_string all_1_1_0 Compact ['all_11302719525430258405_10531094102249515008'] +03711_type_string all_2_2_0 Compact ['all_9267555012450399540_4231923082176187394'] +03711_type_tuple all_1_1_0 Compact ['all_12313389176203636080_11707488636192631513'] +03711_type_tuple all_2_2_0 Compact ['all_12572740846975368560_14638259508703709964'] +03711_type_variant all_1_1_0 Compact ['all_5728325312273395383_4364882181208797392'] +03711_type_variant all_2_2_0 Compact ['all_13821245313499792332_16722045970011847000'] +======= +03711_async_array all_1_1_0 Compact ['all_12163406207865539514_8152447970315193352','all_10571676162184711478_162537819996838366'] +03711_async_mixed all_1_1_0 Compact ['all_3439323524453290355_11489010909856266077','all_7902204961601613769_15054220737605758260'] +03711_async_string all_1_1_0 Compact ['all_18366378000025117484_14590573803496157491','all_17388740250088195039_14005593440692414930'] +03711_async_uint all_1_1_0 Compact ['all_5082745088469868580_7252256098787103274','all_16994774723362420241_11434403476823141314'] +03711_join_with all_1_1_0 Compact ['all_2580935373579939333_3495131879591922993','all_9563449748241099126_16240541781756294264'] +03711_join_with all_2_2_0 Compact ['all_10894757277114140813_5551987281932530566','all_16444784039195326156_10685096260700552877'] +03711_mv_table_1 all_1_1_0 Compact ['all_14742009508090625091_12156664216397061683','all_9139126859104319288_12565776327365534163'] +03711_mv_table_2 all_1_1_0 Compact ['all_10596915804981540433_5083872654531293333','all_17054059182758846721_9547493448332016625'] +03711_table all_1_1_0 Compact ['all_6308706741995381342_2495791770474910886','all_659810233567002352_16818222267659499314'] +03711_table all_2_2_0 Compact ['all_8062692066948936483_8663586595462158314','all_10695495171951104297_11784282495965162872'] +03711_type_array all_1_1_0 Compact ['all_7554219048159323672_1834570215616001735','all_8714782328307936551_5667215566625646845'] +03711_type_array all_2_2_0 Compact ['all_7434001727450480760_6690212448691732934','all_16147962101698235904_1457969930443025035'] +03711_type_decimal all_1_1_0 Compact ['all_7684184047826418593_15812757330475102584','all_13242298624230215860_14719097669563059311'] +03711_type_decimal all_2_2_0 Compact ['all_634532208345180154_9629772920462485019','all_5209730246278546871_3546417517182497241'] +03711_type_dynamic all_1_1_0 Compact ['all_1540730310432290816_16789751043680302604','all_4355947499223422669_319868960027583601'] +03711_type_dynamic all_2_2_0 Compact ['all_14851697563340100724_16005079741405713107','all_13821245313499792332_16722045970011847000'] +03711_type_fixedstr all_1_1_0 Compact ['all_17985314417356456969_2895210787138854751','all_15043183714943801330_10001046138349709664'] +03711_type_fixedstr all_2_2_0 Compact ['all_13000366732311404918_11355265191565411399','all_16360560524467875160_4720191420039909589'] +03711_type_float all_1_1_0 Compact ['all_10624737782828499782_17001038845662030522','all_13346149972136598791_11741866962490268826'] +03711_type_float all_2_2_0 Compact ['all_11406211213571585261_6049832058489322699','all_13611278761629314394_2981841964782876041'] +03711_type_json all_1_1_0 Compact ['all_17819340285194221855_17940280122011694097','all_1019996165332126500_10512220182055512995'] +03711_type_json all_2_2_0 Compact ['all_7066743280452309921_3827619985751953528','all_3077914509782472429_9735021766957434210'] +03711_type_json_mdp0 all_1_1_0 Compact ['all_11631160247462064027_8543123880164603979','all_9636225467500754362_16767708078063468458'] +03711_type_json_mdp0 all_2_2_0 Compact ['all_18319892067131980069_13672152316001981107','all_12384973329383563319_5350484746181772478'] +03711_type_map all_1_1_0 Compact ['all_10366755564185433312_15561349019090402858','all_15715181759477494293_9711484488284658522'] +03711_type_map all_2_2_0 Compact ['all_13858502907989251383_4216642888537477170','all_16684526797205426442_11364293216111778856'] +03711_type_mixed all_1_1_0 Compact ['all_4012615538593842576_1586555934316423149','all_12986313597556146191_6773938392127322341'] +03711_type_mixed all_2_2_0 Compact ['all_4426146008548491652_6557819384918199127','all_12862193591957907725_4178690807237888508'] +03711_type_nullable all_1_1_0 Compact ['all_4108578075138708649_16834152406607355842','all_6450400810974626920_13339409127224583224'] +03711_type_nullable all_2_2_0 Compact ['all_6524205473696451243_18266494491118398121','all_4420175743708305773_8529015166913398714'] +03711_type_string all_1_1_0 Compact ['all_17262683803697682868_6814827972164202340','all_11302719525430258405_10531094102249515008'] +03711_type_string all_2_2_0 Compact ['all_5587100151848398865_11651291973255600594','all_9267555012450399540_4231923082176187394'] +03711_type_tuple all_1_1_0 Compact ['all_654707135789651453_282421467643318343','all_12313389176203636080_11707488636192631513'] +03711_type_tuple all_2_2_0 Compact ['all_8828604914513999383_1424881637542711460','all_12572740846975368560_14638259508703709964'] +03711_type_variant all_1_1_0 Compact ['all_15804267090154954306_10048205686306098427','all_5728325312273395383_4364882181208797392'] +03711_type_variant all_2_2_0 Compact ['all_225539553731134642_1163512679381952006','all_13821245313499792332_16722045970011847000'] +>>>>>>> origin/backport/26.3/115866 diff --git a/tests/queries/0_stateless/03741_s3_glob_table_path_pushdown.reference b/tests/queries/0_stateless/03741_s3_glob_table_path_pushdown.reference index e18b9666292c..b181e521017b 100644 --- a/tests/queries/0_stateless/03741_s3_glob_table_path_pushdown.reference +++ b/tests/queries/0_stateless/03741_s3_glob_table_path_pushdown.reference @@ -26,7 +26,7 @@ test/03741_data/nested/file4.parquet 10 20 20 -1 4 +2 4 0 1 1 3 0 0 diff --git a/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.reference b/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.reference index 21f7ae26c79c..51e52f4d05a1 100644 --- a/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.reference +++ b/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.reference @@ -93,6 +93,15 @@ Tuple(JSON) subcolumn Skip Parts: 1/2 Granules: 1/4 +ambiguous JSON column: path present in the shorter column +Parts: 1/1 +Granules: 1/1 +ambiguous JSON column: path present only in the longer column +Parts: 0/1 +Granules: 0/1 +result: ambiguous JSON column +1 +0 result: equals 1 result: typed subcolumn diff --git a/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.sql b/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.sql index fca3861d1721..586b64f5b3f7 100644 --- a/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.sql +++ b/tests/queries/0_stateless/04024_json_skip_index_bloom_filter.sql @@ -316,6 +316,38 @@ WHERE explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' OR explain LIKE '%Sk DROP TABLE t_json_tuple; +-- One index whose columns are JSONAllPaths(a) and JSONAllPaths(`a.b`): the name `a.b.` +-- matches both, so the choice decides which bloom filter is probed. The shorter JSON column wins, +-- so `a.b.y` resolves to column `a` path `b.y`, which is absent from `a` and prunes. +DROP TABLE IF EXISTS t_json_ambiguous; +SET allow_suspicious_indices = 1; +CREATE TABLE t_json_ambiguous +( + a JSON, + `a.b` JSON, + INDEX idx (JSONAllPaths(a), JSONAllPaths(`a.b`)) TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS index_granularity = 1; + +INSERT INTO t_json_ambiguous VALUES ('{"b": {"x": 1}}', '{"y": 2}'); + +SELECT 'ambiguous JSON column: path present in the shorter column'; +SELECT trimLeft(explain) +FROM (EXPLAIN indexes = 1 SELECT count() FROM t_json_ambiguous WHERE a.b.x = 1) +WHERE explain LIKE '%Parts:%' OR explain LIKE '%Granules:%'; + +SELECT 'ambiguous JSON column: path present only in the longer column'; +SELECT trimLeft(explain) +FROM (EXPLAIN indexes = 1 SELECT count() FROM t_json_ambiguous WHERE a.b.y = 2) +WHERE explain LIKE '%Parts:%' OR explain LIKE '%Granules:%'; + +SELECT 'result: ambiguous JSON column'; +SELECT count() FROM t_json_ambiguous WHERE a.b.x = 1; +SELECT count() FROM t_json_ambiguous WHERE a.b.y = 2; + +DROP TABLE t_json_ambiguous; + -- ============================================================================= -- Section 9: Correctness -- ============================================================================= diff --git a/tests/queries/0_stateless/04029_join_convert_to_fixed_hash_table.sql b/tests/queries/0_stateless/04029_join_convert_to_fixed_hash_table.sql index ebae0f9c6d4f..1d1c4cff1392 100644 --- a/tests/queries/0_stateless/04029_join_convert_to_fixed_hash_table.sql +++ b/tests/queries/0_stateless/04029_join_convert_to_fixed_hash_table.sql @@ -18,6 +18,7 @@ INSERT INTO t_right_neg VALUES (-2, 'r-2'), (0, 'r0'), (2, 'r2'); SET join_algorithm = 'hash'; SET enable_join_fixed_hash_table_conversion = 1; SET max_bytes_before_external_join = 0, max_bytes_ratio_before_external_join = 0; -- Disable automatic spilling for this test +SET query_plan_read_in_order_through_join = 0; -- Verify conversion for Int32 SELECT '-- trigger check Int32'; diff --git a/tests/queries/0_stateless/04058_explain_pretty_aggregation_sorting.sql b/tests/queries/0_stateless/04058_explain_pretty_aggregation_sorting.sql index 8c0e864b15dc..7b62368e5973 100644 --- a/tests/queries/0_stateless/04058_explain_pretty_aggregation_sorting.sql +++ b/tests/queries/0_stateless/04058_explain_pretty_aggregation_sorting.sql @@ -10,6 +10,7 @@ SET optimize_read_in_order = 1; SET optimize_distinct_in_order = 1; SET optimize_sorting_by_input_stream_properties = 1; SET allow_reorder_prewhere_conditions = 0; +SET read_in_order_use_virtual_row = 0; DROP TABLE IF EXISTS t1; diff --git a/tests/queries/0_stateless/04104_ast_fuzzer_preserves_caller_transaction.reference b/tests/queries/0_stateless/04104_ast_fuzzer_preserves_caller_transaction.reference new file mode 100644 index 000000000000..d2a726ed8bc4 --- /dev/null +++ b/tests/queries/0_stateless/04104_ast_fuzzer_preserves_caller_transaction.reference @@ -0,0 +1,6 @@ +1 +committed [1,2] +1 +2 +3 +rolled-back [1,2] diff --git a/tests/queries/0_stateless/04104_ast_fuzzer_preserves_caller_transaction.sql b/tests/queries/0_stateless/04104_ast_fuzzer_preserves_caller_transaction.sql new file mode 100644 index 000000000000..ef20421cfd96 --- /dev/null +++ b/tests/queries/0_stateless/04104_ast_fuzzer_preserves_caller_transaction.sql @@ -0,0 +1,60 @@ +-- Tags: no-ordinary-database, no-encrypted-storage +-- Regression test for a TSAN data race in `executeASTFuzzerQueries` (STID: 2604-385d). +-- +-- `executeASTFuzzerQueries` used to reset the transaction on the CALLER's query and +-- session context before creating fuzz copies: +-- +-- context->getQueryContext()->getSessionContext()->setCurrentTransaction(...) +-- context->setCurrentTransaction(...) +-- +-- That mutation was unsynchronized, racing with concurrent readers of the same context +-- (for example, `RESTORE ASYNC` background workers calling `Context::createCopy` under +-- the shared `Context::mutex`). It also had the surprising side effect of silently +-- clearing the user's active transaction. +-- +-- The reset now happens on the fuzz session context COPY (after `makeSessionContext`), +-- which is not visible to any other thread, so the caller's transaction state is +-- preserved. This test locks that behavior in. + +-- Make sure the test itself controls where the fuzzer runs (stress test profile sets +-- `ast_fuzzer_runs=5`; we pin our baseline so only queries that explicitly opt in fire +-- the finish callback). +SET ast_fuzzer_runs = 0; +SET ast_fuzzer_any_query = 0; + +-- Async inserts are not supported inside transactions; disable so the test does not +-- depend on `disable_async_inserts.xml` being applied to the server config. +SET async_insert = 0; + +-- Suppress error-level log messages from fuzzed queries that fail expectedly. +SET send_logs_level = 'fatal'; + +DROP TABLE IF EXISTS mt_txn_fuzz; +CREATE TABLE mt_txn_fuzz (n Int64) ENGINE = MergeTree ORDER BY n; + +-- Start a transaction, run a SELECT with `ast_fuzzer_runs > 0` (which fires +-- `executeASTFuzzerQueries` in the finish callback), then verify the transaction +-- is still alive by doing an INSERT + COMMIT. With the buggy behavior, the fuzzer +-- would clear the session's `merge_tree_transaction` and the subsequent INSERT +-- would run outside the transaction (or COMMIT would fail with INVALID_TRANSACTION). +BEGIN TRANSACTION; +INSERT INTO mt_txn_fuzz VALUES (1); +SELECT n FROM mt_txn_fuzz ORDER BY n SETTINGS ast_fuzzer_runs = 3; +INSERT INTO mt_txn_fuzz VALUES (2); +COMMIT; + +-- Both inserts should be visible (the transaction survived the fuzzer). +SELECT 'committed', arraySort(groupArray(n)) FROM mt_txn_fuzz; + +-- A subsequent transaction must also work (the session isn't stuck in some +-- weird state left over from the fuzzer's mutation of the parent context). +BEGIN TRANSACTION; +INSERT INTO mt_txn_fuzz VALUES (3); +SELECT n FROM mt_txn_fuzz ORDER BY n SETTINGS ast_fuzzer_runs = 3; +ROLLBACK; + +-- Rollback must have dropped the value 3 (the transaction was real, not a +-- no-op because the fuzzer ate it). +SELECT 'rolled-back', arraySort(groupArray(n)) FROM mt_txn_fuzz; + +DROP TABLE mt_txn_fuzz; diff --git a/tests/queries/0_stateless/04117_join_equi_key_filter_pushdown_right_full.reference b/tests/queries/0_stateless/04117_join_equi_key_filter_pushdown_right_full.reference new file mode 100644 index 000000000000..74fe9e3064da --- /dev/null +++ b/tests/queries/0_stateless/04117_join_equi_key_filter_pushdown_right_full.reference @@ -0,0 +1,50 @@ +RIGHT JOIN USING, equi-key WHERE, matched types: left MergeTree prunes granules +1 +RIGHT JOIN USING, equi-key WHERE, UInt8/UInt64 mismatch: left MergeTree prunes granules +1 +RIGHT JOIN USING, equi-key WHERE: result +1 +RIGHT ALL JOIN ON, equi-key WHERE: left MergeTree prunes granules +1 +RIGHT ANTI JOIN USING, equi-key WHERE: correctness preserved +1000000000 +RIGHT JOIN USING, equi-key WHERE, join_use_nulls: left MergeTree prunes granules +1 +RIGHT JOIN USING, equi-key WHERE, join_use_nulls: result +1 +RIGHT JOIN ON, equi-key WHERE, join_use_nulls: unmatched right row keeps its NULL left side +\N 1000000000 +FULL JOIN USING, equi-key WHERE: result (matched row) +1 +FULL JOIN USING, equi-key WHERE: result (right-only contributing row preserved) +1000000000 +FULL JOIN USING, equi-key WHERE: left-only rows still reachable via filter +1 +FULL JOIN, side-qualified equi-key predicate: unmatched rows preserved +0 3 +5 0 +FULL JOIN, side-qualified equi-key predicates on both sides: only the matched row survives +2 Value_2 2 Value_2 +RIGHT JOIN USING, Int32 / UInt32 keys widened to Int64: left MergeTree prunes granules +1 +RIGHT JOIN USING, Int32 / UInt32 keys widened to Int64: result +1 +RIGHT JOIN USING, Int32 / Int64 keys widened to Int64: left MergeTree prunes granules +1 +RIGHT JOIN USING, Int32 / Int64 keys widened to Int64: result +1 +RIGHT JOIN ON, cross-type equi-key: left MergeTree prunes granules +1 +RIGHT JOIN USING, cross-type: unmatched right row preserved +5 +1000000000 +RIGHT JOIN ON, predicate on the wider side: no narrowing substitution +1 +RIGHT JOIN USING, cross-type, join_use_nulls: left MergeTree still prunes granules +1 +RIGHT JOIN USING, cross-type, join_use_nulls: result +1 +FULL JOIN USING, cross-type: neither MergeTree prunes granules +1 +FULL JOIN USING, cross-type, join_use_nulls: neither MergeTree prunes granules +1 diff --git a/tests/queries/0_stateless/04117_join_equi_key_filter_pushdown_right_full.sql b/tests/queries/0_stateless/04117_join_equi_key_filter_pushdown_right_full.sql new file mode 100644 index 000000000000..7b7b582db145 --- /dev/null +++ b/tests/queries/0_stateless/04117_join_equi_key_filter_pushdown_right_full.sql @@ -0,0 +1,192 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: the granule assertions describe the local `MergeTree` read, which parallel +-- replicas replace, and the `RIGHT JOIN` shapes below hit the unrelated logical error of +-- https://github.com/ClickHouse/ClickHouse/issues/113292 there. + +-- Equi-key `WHERE` predicates must reach the left `MergeTree` input of a `RIGHT JOIN` as an index +-- condition, including when the two `USING` keys differ in type. +-- +-- Analyzer only. Under `enable_analyzer = 0` a `USING` key is renamed to `
.` in the right +-- input header while the `JOIN` output keeps the bare name, so the equivalence maps are keyed by a name +-- the filter never references and nothing is pushed. That path is left as is. +-- +-- Nothing may be pushed through a `FULL JOIN`: a dropped row only becomes a defaulted unmatched row, so +-- a predicate on that default both admits and discards rows wrongly. Those cases assert results only. + +SET enable_analyzer = 1; +SET query_plan_filter_push_down = 1; +SET query_plan_join_swap_table = 'false'; +SET enable_join_runtime_filters = 0; + +DROP TABLE IF EXISTS mt; +CREATE TABLE mt (k UInt64) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; +INSERT INTO mt SELECT number FROM numbers(1000000); + +SELECT 'RIGHT JOIN USING, equi-key WHERE, matched types: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt AS l RIGHT JOIN (SELECT toUInt64(1) AS k) AS r USING (k) WHERE k = 1 +) WHERE explain ILIKE '%Condition: (k in [1, 1])%'; + +SELECT 'RIGHT JOIN USING, equi-key WHERE, UInt8/UInt64 mismatch: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt AS l RIGHT JOIN (SELECT 1 AS k) AS r USING (k) WHERE k = 1 +) WHERE explain ILIKE '%Condition: (k in [1, 1])%'; + +SELECT 'RIGHT JOIN USING, equi-key WHERE: result'; +SELECT k FROM mt AS l RIGHT JOIN (SELECT 1 AS k) AS r USING (k) WHERE k = 1 ORDER BY k; + +SELECT 'RIGHT ALL JOIN ON, equi-key WHERE: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT l.k FROM mt AS l RIGHT JOIN (SELECT toUInt64(1) AS k) AS r ON l.k = r.k WHERE r.k = 1 +) WHERE explain ILIKE '%Condition: (k in [1, 1])%'; + +SELECT 'RIGHT ANTI JOIN USING, equi-key WHERE: correctness preserved'; +SELECT k FROM mt AS l RIGHT ANTI JOIN (SELECT toUInt64(0) AS k UNION ALL SELECT toUInt64(1000000000) AS k) AS r USING (k) WHERE k = 1000000000 ORDER BY k; + +-- A substitution has to carry the type the replaced name has in the `JOIN` output, and the only type +-- reachable by casting the opposite key is the least supertype of the two. `join_use_nulls` widens the +-- output past that supertype, so nothing is substituted and the predicate stays above the `JOIN`. +SET join_use_nulls = 1; + +SELECT 'RIGHT JOIN USING, equi-key WHERE, join_use_nulls: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt AS l RIGHT JOIN (SELECT 1 AS k) AS r USING (k) WHERE k = 1 +) WHERE explain ILIKE '%Condition: (k in [1, 1])%'; + +SELECT 'RIGHT JOIN USING, equi-key WHERE, join_use_nulls: result'; +SELECT k FROM mt AS l RIGHT JOIN (SELECT 1 AS k) AS r USING (k) WHERE k = 1 ORDER BY k; + +SELECT 'RIGHT JOIN ON, equi-key WHERE, join_use_nulls: unmatched right row keeps its NULL left side'; +SELECT l.k, r.k FROM mt AS l RIGHT JOIN (SELECT toUInt64(1000000000) AS k) AS r ON l.k = r.k WHERE r.k = 1000000000 ORDER BY 1, 2; + +SET join_use_nulls = 0; + +SELECT 'FULL JOIN USING, equi-key WHERE: result (matched row)'; +SELECT k FROM mt AS l FULL JOIN (SELECT toUInt64(1) AS k) AS r USING (k) WHERE k = 1 ORDER BY k; + +SELECT 'FULL JOIN USING, equi-key WHERE: result (right-only contributing row preserved)'; +SELECT k FROM mt AS l FULL JOIN (SELECT toUInt64(1000000000) AS k) AS r USING (k) WHERE k = 1000000000 ORDER BY k; + +SELECT 'FULL JOIN USING, equi-key WHERE: left-only rows still reachable via filter'; +SELECT count() FROM ( + SELECT k FROM mt AS l FULL JOIN (SELECT toUInt64(1000000000) AS k) AS r USING (k) WHERE k = 7 +); + +DROP TABLE mt; + +DROP TABLE IF EXISTS s1; +DROP TABLE IF EXISTS s2; +CREATE TABLE s1 (id UInt64) ENGINE = MergeTree ORDER BY id; +CREATE TABLE s2 (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO s1 VALUES (5); +INSERT INTO s2 VALUES (3); + +-- Pushing this would drop the only left row and lose the `(5, 0)` unmatched row it produces. +SELECT 'FULL JOIN, side-qualified equi-key predicate: unmatched rows preserved'; +SELECT lhs.id, rhs.id FROM s1 AS lhs FULL JOIN s2 AS rhs ON lhs.id = rhs.id WHERE rhs.id != 5 ORDER BY 1, 2; + +DROP TABLE s1; +DROP TABLE s2; + +DROP TABLE IF EXISTS u1; +DROP TABLE IF EXISTS u2; +CREATE TABLE u1 (id UInt64, value String) ENGINE = MergeTree ORDER BY id; +CREATE TABLE u2 (id UInt64, value String) ENGINE = MergeTree ORDER BY id; +INSERT INTO u1 VALUES (1, 'Value_1'), (2, 'Value_2'); +INSERT INTO u2 VALUES (2, 'Value_2'), (3, 'Value_3'); + +-- Opposite direction: pushing these would let the two defaulted unmatched rows escape. +SELECT 'FULL JOIN, side-qualified equi-key predicates on both sides: only the matched row survives'; +SELECT * FROM u1 AS lhs FULL JOIN u2 AS rhs ON lhs.id = rhs.id WHERE lhs.id != 0 AND rhs.id != 0 ORDER BY 1, 3; + +DROP TABLE u1; +DROP TABLE u2; + +-- A `USING` supertype that is wider than one or both inputs. The replacement is then the same `CAST` +-- the `JOIN` applies to its key, so the left input is still read through the primary key. + +DROP TABLE IF EXISTS mt_i32; +CREATE TABLE mt_i32 (k Int32) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; +INSERT INTO mt_i32 SELECT number FROM numbers(1000000); + +SELECT 'RIGHT JOIN USING, Int32 / UInt32 keys widened to Int64: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::UInt32 AS k) AS r USING (k) WHERE k = 1 +) WHERE explain LIKE '%Granules: 1/%'; + +SELECT 'RIGHT JOIN USING, Int32 / UInt32 keys widened to Int64: result'; +SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::UInt32 AS k) AS r USING (k) WHERE k = 1 ORDER BY k; + +SELECT 'RIGHT JOIN USING, Int32 / Int64 keys widened to Int64: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::Int64 AS k) AS r USING (k) WHERE k = 1 +) WHERE explain LIKE '%Granules: 1/%'; + +SELECT 'RIGHT JOIN USING, Int32 / Int64 keys widened to Int64: result'; +SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::Int64 AS k) AS r USING (k) WHERE k = 1 ORDER BY k; + +SELECT 'RIGHT JOIN ON, cross-type equi-key: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT l.k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::Int64 AS k) AS r ON l.k = r.k WHERE r.k = 1 +) WHERE explain LIKE '%Granules: 1/%'; + +SELECT 'RIGHT JOIN USING, cross-type: unmatched right row preserved'; +SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 5::UInt32 AS k UNION ALL SELECT 1000000000::UInt32 AS k) AS r USING (k) WHERE k >= 5 ORDER BY k; + +-- The opposite direction would need a narrowing `CAST`, which is not a substitution, so the predicate +-- stays above the `JOIN` and the wider key value survives it. +SELECT 'RIGHT JOIN ON, predicate on the wider side: no narrowing substitution'; +SELECT count() FROM ( + SELECT l.k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::Int64 AS k) AS r ON l.k = r.k WHERE l.k = 1 +); + +SET join_use_nulls = 1; + +-- A `RIGHT JOIN` takes its `USING` key from the right input, which `join_use_nulls` does not widen, so +-- the output type is still the plain supertype and the substitution stays exact: the left input keeps +-- pruning. The `EXPLAIN` assertion pins that, because the result alone would also hold if the pushdown +-- silently stopped happening. +SELECT 'RIGHT JOIN USING, cross-type, join_use_nulls: left MergeTree still prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::UInt32 AS k) AS r USING (k) WHERE k = 1 +) WHERE explain LIKE '%Granules: 1/%'; + +SELECT 'RIGHT JOIN USING, cross-type, join_use_nulls: result'; +SELECT k FROM mt_i32 AS l RIGHT JOIN (SELECT 1::UInt32 AS k) AS r USING (k) WHERE k = 1 ORDER BY k; + +SET join_use_nulls = 0; + +-- The safety boundary itself: a `FULL JOIN` turns a dropped row into a defaulted unmatched one, so +-- neither side may be filtered, whatever the substitution would allow. Asserted on the plan, not only +-- on the rows, so that a later change re-enabling a side is caught even when the value survives it. +DROP TABLE IF EXISTS mt_u32; +CREATE TABLE mt_u32 (k UInt32) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'; +INSERT INTO mt_u32 SELECT number FROM numbers(1000000); + +SELECT 'FULL JOIN USING, cross-type: neither MergeTree prunes granules'; +SELECT count() = 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt_i32 AS l FULL JOIN mt_u32 AS r USING (k) WHERE k = 1 +) WHERE explain LIKE '%Granules: 1/%'; + +SET join_use_nulls = 1; +SELECT 'FULL JOIN USING, cross-type, join_use_nulls: neither MergeTree prunes granules'; +SELECT count() = 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt_i32 AS l FULL JOIN mt_u32 AS r USING (k) WHERE k = 1 +) WHERE explain LIKE '%Granules: 1/%'; +SET join_use_nulls = 0; + +DROP TABLE mt_u32; +DROP TABLE mt_i32; diff --git a/tests/queries/0_stateless/04141_iceberg_concurrent_no_logical_error.reference b/tests/queries/0_stateless/04141_iceberg_concurrent_no_logical_error.reference new file mode 100644 index 000000000000..48d40a08a68d --- /dev/null +++ b/tests/queries/0_stateless/04141_iceberg_concurrent_no_logical_error.reference @@ -0,0 +1,2 @@ +1 +OK diff --git a/tests/queries/0_stateless/04141_iceberg_concurrent_no_logical_error.sh b/tests/queries/0_stateless/04141_iceberg_concurrent_no_logical_error.sh new file mode 100755 index 000000000000..3f8942b121c2 --- /dev/null +++ b/tests/queries/0_stateless/04141_iceberg_concurrent_no_logical_error.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, long +# Tag no-fasttest: Iceberg pulls in extra dependencies. +# Tag no-parallel: deliberately runs concurrent clients to provoke a TOCTOU race. +# Tag long: the concurrent read/write loop can run past the 180s flaky-check cap under msan; +# shrinking it would weaken the concurrency that provokes the race, so exempt it instead. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +LOG_FILE=$(mktemp -t iceberg_concurrent_XXXXXX.log) +trap "rm -f \"${LOG_FILE}\"; rm -rf \"${TABLE_PATH}\" 2>/dev/null" EXIT + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" + +# ORDER BY makes the planner exercise the read-in-order path, so both iterate and +# isDataSortedBySortingKey are reachable from the same workload. +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (c0 Int32) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') + ORDER BY c0 +" + +# >"${LOG_FILE}" 2>&1 >"${LOG_FILE}" 2>&1 /dev/null 2>&1; then + sleep 1 + if ! ${CLICKHOUSE_CLIENT} --query "SELECT 1" >/dev/null 2>&1; then + echo "FAIL: server not responding after concurrent Iceberg access (possible LOGICAL_ERROR abort)" + status=1 + fi +fi + +# Release: the same LOGICAL_ERROR is a catchable exception carrying the exact message to the +# client. Deterministic counterpart: 04305_iceberg_missing_table_state.sh. +if grep -qF "Can't extract iceberg table state" "${LOG_FILE}"; then + echo "FAIL: observed the 'Can't extract iceberg table state' LOGICAL_ERROR" + status=1 +fi + +if [ "$status" -ne 0 ]; then + cat "${LOG_FILE}" + exit 1 +fi + +${CLICKHOUSE_CLIENT} --query "SELECT count() >= 3 FROM ${TABLE}" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" +echo "OK" diff --git a/tests/queries/0_stateless/04305_ast_fuzzer_skips_backup_restore.reference b/tests/queries/0_stateless/04305_ast_fuzzer_skips_backup_restore.reference new file mode 100644 index 000000000000..a73a10164686 --- /dev/null +++ b/tests/queries/0_stateless/04305_ast_fuzzer_skips_backup_restore.reference @@ -0,0 +1,5 @@ +backup_restore_skipped 1 +backup_restore_not_executed 1 +plain_query_executed 1 +plain_query_not_skipped 1 +alive [1,2,3] diff --git a/tests/queries/0_stateless/04305_ast_fuzzer_skips_backup_restore.sql b/tests/queries/0_stateless/04305_ast_fuzzer_skips_backup_restore.sql new file mode 100644 index 000000000000..afee2315eadb --- /dev/null +++ b/tests/queries/0_stateless/04305_ast_fuzzer_skips_backup_restore.sql @@ -0,0 +1,87 @@ +-- Tags: no-ordinary-database, no-parallel +-- no-parallel: reads the server-global ProfileEvents counters ASTFuzzerSkippedBackupRestore +-- and ASTFuzzerQueries, so no other test may run fuzzed queries against the same server while +-- this one measures the deltas. + +-- The serverfuzz/stress profile sets ast_fuzzer_runs server-wide, which would make every +-- statement here fire the fuzzer and pollute the counters we measure. Pin the baseline to 0 +-- so only statements with an explicit SETTINGS ast_fuzzer_runs > 0 fire it. +SET ast_fuzzer_runs = 0; +SET ast_fuzzer_any_query = 0; +SET send_logs_level = 'fatal'; + +DROP TABLE IF EXISTS mt_fuzz_backup; +DROP TABLE IF EXISTS mt_fuzz_backup_restored; +DROP TABLE IF EXISTS fuzz_events; + +CREATE TABLE mt_fuzz_backup (n Int64) ENGINE = MergeTree ORDER BY n; +INSERT INTO mt_fuzz_backup VALUES (1), (2), (3); + +-- Snapshot both global counters in a single scan so they are consistent. sumIf over the +-- (possibly absent) rows yields 0 before an event has ever fired, so the delta arithmetic +-- below is well defined on a fresh server. +CREATE TABLE fuzz_events (label String, skipped Int64, executed Int64) ENGINE = Memory; +INSERT INTO fuzz_events +SELECT 'before', + toInt64(sumIf(value, event = 'ASTFuzzerSkippedBackupRestore')), + toInt64(sumIf(value, event = 'ASTFuzzerQueries')) +FROM system.events; + +-- executeASTFuzzerQueries runs in the query finish callback and re-executes mutated copies of +-- the just-finished query. A fuzzed BACKUP/RESTORE can start async work (BackupsWorker keeps the +-- query context alive for its background workers), and the fuzzer's per-iteration cleanup would +-- then mutate that escaped context without holding Context::mutex, racing with Context::createCopy +-- in the restore workers. The fuzzer skips BACKUP/RESTORE via an early continue that runs before +-- any other guard (depth/format/length) and bumps ASTFuzzerSkippedBackupRestore instead of +-- executing the query and bumping ASTFuzzerQueries. +BACKUP TABLE mt_fuzz_backup TO Memory('04305_backup') SETTINGS ast_fuzzer_runs = 3, ast_fuzzer_any_query = 1 FORMAT Null; +RESTORE TABLE mt_fuzz_backup AS mt_fuzz_backup_restored FROM Memory('04305_backup') SETTINGS ast_fuzzer_runs = 3, ast_fuzzer_any_query = 1 FORMAT Null; +-- The async variant escapes the context to background workers; the fuzzer must skip it too. +BACKUP TABLE mt_fuzz_backup TO Memory('04305_backup_async') SETTINGS async = 1, ast_fuzzer_runs = 3, ast_fuzzer_any_query = 1 FORMAT Null; + +INSERT INTO fuzz_events +SELECT 'after_backup', + toInt64(sumIf(value, event = 'ASTFuzzerSkippedBackupRestore')), + toInt64(sumIf(value, event = 'ASTFuzzerQueries')) +FROM system.events; + +-- Deterministic proof of the skip contract: the fuzzed BACKUP/RESTORE queries reached the +-- ASTBackupQuery guard and were counted as skipped. The guard is checked before the +-- depth/format/length early-continue paths, so this positive count is specific to the query +-- type and cannot be produced by those unrelated skips. If the guard in executeQuery.cpp is +-- removed, these fuzzed queries are executed instead (or the server crashes/hangs on the +-- reintroduced race), so this counter stays 0 and the assertion flips to 0. +SELECT 'backup_restore_skipped', + (SELECT skipped FROM fuzz_events WHERE label = 'after_backup') + - (SELECT skipped FROM fuzz_events WHERE label = 'before') > 0; + +-- The skipped queries were not executed, so the executed-query counter must not advance for them. +SELECT 'backup_restore_not_executed', + (SELECT executed FROM fuzz_events WHERE label = 'after_backup') + - (SELECT executed FROM fuzz_events WHERE label = 'before') = 0; + +-- Positive control: a plain fuzzable query is NOT a BACKUP/RESTORE, so it is executed (not +-- skipped). The executed counter must advance and the skip counter must not. This proves both +-- counters are live under these settings, so the values above are real and not dead counters. +SELECT 1 SETTINGS ast_fuzzer_runs = 3, ast_fuzzer_any_query = 1 FORMAT Null; + +INSERT INTO fuzz_events +SELECT 'after_select', + toInt64(sumIf(value, event = 'ASTFuzzerSkippedBackupRestore')), + toInt64(sumIf(value, event = 'ASTFuzzerQueries')) +FROM system.events; + +SELECT 'plain_query_executed', + (SELECT executed FROM fuzz_events WHERE label = 'after_select') + - (SELECT executed FROM fuzz_events WHERE label = 'after_backup') > 0; + +SELECT 'plain_query_not_skipped', + (SELECT skipped FROM fuzz_events WHERE label = 'after_select') + - (SELECT skipped FROM fuzz_events WHERE label = 'after_backup') = 0; + +-- Server is alive and the restored table holds the original rows. +SELECT 'alive', arraySort(groupArray(n)) FROM mt_fuzz_backup_restored; + +DROP TABLE mt_fuzz_backup_restored; +DROP TABLE mt_fuzz_backup; +DROP TABLE fuzz_events; diff --git a/tests/queries/0_stateless/04305_iceberg_missing_table_state.reference b/tests/queries/0_stateless/04305_iceberg_missing_table_state.reference new file mode 100644 index 000000000000..1ef92499f7c6 --- /dev/null +++ b/tests/queries/0_stateless/04305_iceberg_missing_table_state.reference @@ -0,0 +1,4 @@ +6 +1 +2 +3 diff --git a/tests/queries/0_stateless/04305_iceberg_missing_table_state.sh b/tests/queries/0_stateless/04305_iceberg_missing_table_state.sh new file mode 100755 index 000000000000..9dbcceb28256 --- /dev/null +++ b/tests/queries/0_stateless/04305_iceberg_missing_table_state.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +# Tag no-fasttest: Iceberg pulls in extra dependencies. +# Tag no-parallel: toggles a process-global failpoint. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap "rm -rf \"${TABLE_PATH}\" 2>/dev/null; ${CLICKHOUSE_CLIENT} --query \"SYSTEM DISABLE FAILPOINT datalake_simulate_missing_table_state\" 2>/dev/null" EXIT + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" + +# ORDER BY makes the planner reach isDataSortedBySortingKey. +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (c0 Int32) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') + ORDER BY c0 +" + +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 \ + --query "INSERT INTO ${TABLE} VALUES (1), (2), (3)" + +# The failpoint strips datalake_table_state from the snapshot reaching the read step, +# deterministically reproducing the concurrent-commit race. Without the fix the read-in-order +# probe throws "Can't extract iceberg table state"; with the fix read pins one consistent +# snapshot before reading, so both queries succeed. +${CLICKHOUSE_CLIENT} --query "SYSTEM ENABLE FAILPOINT datalake_simulate_missing_table_state" + +# Non-trivial read so the read pipeline actually runs (count is answered from metadata by +# trivial-count and never builds a read step). +${CLICKHOUSE_CLIENT} --query "SELECT sum(c0) FROM ${TABLE}" +# ORDER BY reaches isDataSortedBySortingKey, which reads the stripped snapshot directly and +# is the deterministic LOGICAL_ERROR site under the failpoint. +${CLICKHOUSE_CLIENT} --query "SELECT c0 FROM ${TABLE} ORDER BY c0" + +${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT datalake_simulate_missing_table_state" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" diff --git a/tests/queries/0_stateless/04305_variant_corrupted_discriminators_native.reference b/tests/queries/0_stateless/04305_variant_corrupted_discriminators_native.reference new file mode 100644 index 000000000000..b6300d051086 --- /dev/null +++ b/tests/queries/0_stateless/04305_variant_corrupted_discriminators_native.reference @@ -0,0 +1 @@ +Invalid discriminator value 2 (num_variants = 2) diff --git a/tests/queries/0_stateless/04305_variant_corrupted_discriminators_native.sh b/tests/queries/0_stateless/04305_variant_corrupted_discriminators_native.sh new file mode 100755 index 000000000000..007444a068b9 --- /dev/null +++ b/tests/queries/0_stateless/04305_variant_corrupted_discriminators_native.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Test that reading a Native file with an out-of-bounds Variant discriminator +# produces an informative INCORRECT_DATA error instead of a crash (heap OOB write). +# +# The file data_native/variant_corrupted_discriminators.native is a pregenerated +# Native file containing a single column `v` of type `Variant(String, UInt64)` with +# 3 rows. Valid discriminator values are 0 (String), 1 (UInt64), and 255 (NULL). +# The second row's discriminator was set to 2, which is out of bounds (>= num_variants). + +$CLICKHOUSE_LOCAL --table test --input-format Native \ + -q "SELECT * FROM test" \ + < "${CUR_DIR}/data_native/variant_corrupted_discriminators.native" \ + 2>&1 | grep -o 'Invalid discriminator value [0-9]* (num_variants = [0-9]*)' diff --git a/tests/queries/0_stateless/04306_delta_lake_merge_missing_table_state.reference b/tests/queries/0_stateless/04306_delta_lake_merge_missing_table_state.reference new file mode 100644 index 000000000000..1191247b6d9a --- /dev/null +++ b/tests/queries/0_stateless/04306_delta_lake_merge_missing_table_state.reference @@ -0,0 +1,2 @@ +1 +2 diff --git a/tests/queries/0_stateless/04306_delta_lake_merge_missing_table_state.sh b/tests/queries/0_stateless/04306_delta_lake_merge_missing_table_state.sh new file mode 100755 index 000000000000..37ddfff49c97 --- /dev/null +++ b/tests/queries/0_stateless/04306_delta_lake_merge_missing_table_state.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-msan +# Tag no-fasttest: delta-kernel pulls in extra dependencies. +# Tag no-parallel: toggles a process-global failpoint. +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so DeltaLakeLocal is absent. + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/107334 +# DeltaLakeMetadataDeltaKernel::iterate() throws LOGICAL_ERROR +# 'No version found in table state snapshot' when the storage snapshot reaching the read +# step has no pinned datalake_table_state (e.g. a first access through merge() or a cluster +# function, which builds the child snapshot without updateExternalDynamicMetadataIfExists()). +# The DeltaLake counterpart of 04305_iceberg_missing_table_state. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap "rm -rf \"${TABLE_PATH}\" 2>/dev/null; ${CLICKHOUSE_CLIENT} --query \"SYSTEM DISABLE FAILPOINT datalake_simulate_missing_table_state\" 2>/dev/null" EXIT + +mkdir -p "${TABLE_PATH}" +cp -r "${CUR_DIR}"/data_delta_lake/struct_column_mapping/* "${TABLE_PATH}" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} --allow_experimental_delta_kernel_rs=1 --query " + CREATE TABLE ${TABLE} ENGINE = DeltaLakeLocal('${TABLE_PATH}') +" + +# The failpoint strips datalake_table_state from the snapshot reaching the read step, +# deterministically reproducing the missing-state path that merge()/cluster reads hit. +# Without the fix the read throws "No version found in table state snapshot" in +# DeltaLakeMetadataDeltaKernel::iterate(); with the fix read() pins one consistent snapshot +# before reading, so the query succeeds. The failpoint exists only in this build, so on the +# unfixed master binary SYSTEM ENABLE FAILPOINT fails cleanly and the read is unaffected. +${CLICKHOUSE_CLIENT} --query "SYSTEM ENABLE FAILPOINT datalake_simulate_missing_table_state" + +# Non-trivial read (subcolumn, not count) so a real read pipeline runs into iterate(). +${CLICKHOUSE_CLIENT} --allow_experimental_delta_kernel_rs=1 \ + --query "SELECT c1.id FROM ${TABLE} ORDER BY c1.id" + +${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT datalake_simulate_missing_table_state" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" diff --git a/tests/queries/0_stateless/04306_variant_corrupted_compact_discriminator_native.reference b/tests/queries/0_stateless/04306_variant_corrupted_compact_discriminator_native.reference new file mode 100644 index 000000000000..923a6e889e2e --- /dev/null +++ b/tests/queries/0_stateless/04306_variant_corrupted_compact_discriminator_native.reference @@ -0,0 +1 @@ +Invalid discriminator value 5 (num_variants = 2) diff --git a/tests/queries/0_stateless/04306_variant_corrupted_compact_discriminator_native.sh b/tests/queries/0_stateless/04306_variant_corrupted_compact_discriminator_native.sh new file mode 100755 index 000000000000..8f1ea65110f6 --- /dev/null +++ b/tests/queries/0_stateless/04306_variant_corrupted_compact_discriminator_native.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Test that reading a Native file with an out-of-bounds discriminator in a +# COMPACT-mode Variant granule produces an INCORRECT_DATA error instead of a +# crash (heap OOB write). +# +# The file data_native/variant_corrupted_compact_discriminator.native contains +# a single column `v` of type `Variant(String, UInt64)` serialized in COMPACT +# mode with `compact_discr = 5`. Valid discriminator values are 0 (String), +# 1 (UInt64), and 255 (NULL); 5 is out of bounds (>= num_variants). + +$CLICKHOUSE_LOCAL --table test --input-format Native \ + -q "SELECT * FROM test" \ + < "${CUR_DIR}/data_native/variant_corrupted_compact_discriminator.native" \ + 2>&1 | grep -o 'Invalid discriminator value [0-9]* (num_variants = [0-9]*)' diff --git a/tests/queries/0_stateless/04308_arrow_oob_reads_value.sh b/tests/queries/0_stateless/04308_arrow_oob_reads_value.sh index a633412acff6..91ae90d85bcc 100755 --- a/tests/queries/0_stateless/04308_arrow_oob_reads_value.sh +++ b/tests/queries/0_stateless/04308_arrow_oob_reads_value.sh @@ -49,9 +49,15 @@ def inflate_row_count(data, orig_n, large_n=LARGE_N): pos = idx + 1 return data -# 8. Boolean: 1 row → 1-byte bit buffer, inflated to 16384 rows -d = write_arrow(pa.array([True], type=pa.bool_())) -open(f'{out}/bool.arrow', 'wb').write(inflate_row_count(d, 1)) +# 8. Boolean: 2 rows → still a 1-byte bit-packed buffer, so the inflated row count is the only +# corruption and the buffer length stays honest. A 1-row array puts the literal 1 in the +# buffer-length field too, and inflating it declares a buffer longer than the whole file, +# which is rejected before the per-element read. 4 and 8 collide with other metadata fields. +d = write_arrow(pa.array([True, False], type=pa.bool_())) +# Pin that: the needle must match the two row counts and nothing else. +pos = [i for i in range(0, len(d) - 7, 8) if struct.unpack_from('= yesterday() + AND event_time >= now() - 600 + AND query_id = '${QUERY_ID}' + AND type = 'QueryFinish' + AND current_database = currentDatabase() +") + +# MarkCacheMisses should be at most EXPECTED_MARKS. +# It can be lower if some marks were already cached, but must not be higher — +# that would mean unnecessary per-bucket data streams were loaded. +if [ "${MARK_CACHE_MISSES}" -le "${EXPECTED_MARKS}" ]; then + echo "OK" +else + echo "FAIL: MarkCacheMisses (${MARK_CACHE_MISSES}) > expected (${EXPECTED_MARKS})" +fi + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE_NAME}" diff --git a/tests/queries/0_stateless/04331_type_json_allow_duplicated_key_with_literal_and_nested_object_typed_paths.reference b/tests/queries/0_stateless/04331_type_json_allow_duplicated_key_with_literal_and_nested_object_typed_paths.reference new file mode 100644 index 000000000000..4d6bf159c24f --- /dev/null +++ b/tests/queries/0_stateless/04331_type_json_allow_duplicated_key_with_literal_and_nested_object_typed_paths.reference @@ -0,0 +1,5 @@ +{"a":42,"a":{"b":42}} 42 42 {"b":42} 42 +{"a":42,"a":{"b":42}} 42 42 {"b":42} 42 +{"a":{"b":42,"b":{"c":42}}} 42 42 42 +{"a":42,"a":{"b":42},"c":"hello","c":{"d":1}} 42 hello 42 1 42 hello +{"a":42,"a":{"b":42}} 42 42 42 diff --git a/tests/queries/0_stateless/04331_type_json_allow_duplicated_key_with_literal_and_nested_object_typed_paths.sql b/tests/queries/0_stateless/04331_type_json_allow_duplicated_key_with_literal_and_nested_object_typed_paths.sql new file mode 100644 index 000000000000..6897589f6765 --- /dev/null +++ b/tests/queries/0_stateless/04331_type_json_allow_duplicated_key_with_literal_and_nested_object_typed_paths.sql @@ -0,0 +1,28 @@ +set enable_analyzer=1; +set type_json_allow_duplicated_key_with_literal_and_nested_object=1; + +-- Basic: typed path with literal first, object second +select '{"a" : 42, "a" : {"b" : 42}}'::JSON(a Int32) as json, json.a, json.a.b, json.^a, json.@a; + +-- Basic: typed path with object first, literal second +select '{"a" : {"b" : 42}, "a" : 42}'::JSON(a Int32) as json, json.a, json.a.b, json.^a, json.@a; + +-- Typed path for nested key +select '{"a" : {"b" : 42, "b" : {"c" : 42}}}'::JSON(a.b Int32) as json, json.a.b, json.a.b.c, json.@a.b; + +-- Multiple typed paths +select '{"a" : 42, "a" : {"b" : 42}, "c" : "hello", "c" : {"d" : 1}}'::JSON(a Int32, c String) as json, json.a, json.c, json.a.b, json.c.d, json.@a, json.@c; + +-- Typed path with Nullable type +select '{"a" : 42, "a" : {"b" : 42}}'::JSON(a Nullable(Int32)) as json, json.a, json.a.b, json.@a; + +-- Same-type duplicates should still fail +select '{"a" : 42, "a" : 43}'::JSON(a Int32); -- {serverError INCORRECT_DATA} +select '{"a" : {"b" : 1}, "a" : {"c" : 2}}'::JSON(a Int32); -- {serverError INCORRECT_DATA} + +-- Three duplicates (literal + object + literal) should still fail +select '{"a" : 42, "a" : {"b" : 42}, "a" : 43}'::JSON(a Int32); -- {serverError INCORRECT_DATA} + +-- Without the setting, typed paths should still fail on duplicate keys +set type_json_allow_duplicated_key_with_literal_and_nested_object=0; +select '{"a" : 42, "a" : {"b" : 42}}'::JSON(a Int32); -- {serverError INCORRECT_DATA} diff --git a/tests/queries/0_stateless/04337_ttl_aggregate_function_column_reject.reference b/tests/queries/0_stateless/04337_ttl_aggregate_function_column_reject.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04337_ttl_aggregate_function_column_reject.sql b/tests/queries/0_stateless/04337_ttl_aggregate_function_column_reject.sql new file mode 100644 index 000000000000..1f268f56f840 --- /dev/null +++ b/tests/queries/0_stateless/04337_ttl_aggregate_function_column_reject.sql @@ -0,0 +1,1439 @@ +-- Verify that CREATE TABLE rejects TTL expressions referencing AggregateFunction columns at DDL time. + +-- Table-level TTL: toDateTime cannot accept AggregateFunction state +CREATE TABLE test_ttl_agg +( + key1 String, + key2 String, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY (key1, key2) +TTL toDateTime(ts) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Column-level TTL: same issue +CREATE TABLE test_ttl_agg_col +( + key1 String, + key2 String, + ts AggregateFunction(max, DateTime64(3)) TTL toDateTime(ts) + INTERVAL 1 DAY +) +ENGINE = MergeTree() +ORDER BY (key1, key2); -- { serverError BAD_TTL_EXPRESSION } + +-- TTL DELETE WHERE: toDateTime on AggregateFunction in WHERE clause +CREATE TABLE test_ttl_agg_where +( + key1 String, + key2 String, + d DateTime, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY (key1, key2) +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(ts) > toDateTime(0); -- { serverError BAD_TTL_EXPRESSION } + +-- AggregateFunction passed directly to arithmetic (plus) +CREATE TABLE test_ttl_agg_plus +( + key1 String, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL ts + INTERVAL 1 DAY; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- Nullable-wrapped conversion: CAST to Nullable(DateTime) should still be caught +CREATE TABLE test_ttl_agg_nullable +( + key1 String, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL assumeNotNull(CAST(ts, 'Nullable(DateTime)')) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Non-date intermediate conversion: toUInt32(aggfunc) fails at execution time too +CREATE TABLE test_ttl_agg_touint +( + ts AggregateFunction(max, UInt32) +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(toUInt32(ts)) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Nested state inside a Tuple: the AggregateFunction is not the top-level type, so it must be +-- found via the recursive type walk. +CREATE TABLE test_ttl_agg_tuple +( + key1 String, + ts Tuple(a UInt32, b AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL toDateTime(ts.2) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Nested state inside an Array. +CREATE TABLE test_ttl_agg_array +( + key1 String, + ts Array(AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL toDateTime(ts[1]) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Nested state inside a Map. +CREATE TABLE test_ttl_agg_map +( + key1 String, + ts Map(String, AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL toDateTime(ts['a']) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Nested state used only in DELETE WHERE: must be caught on the WHERE path too. +CREATE TABLE test_ttl_agg_tuple_where +( + key1 String, + d DateTime, + ts Tuple(a UInt32, b AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(ts.2) > toDateTime(0); -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a nested AggregateFunction state that is not referenced by the TTL must be accepted. +CREATE TABLE test_ttl_agg_tuple_not_referenced +( + key1 String, + d DateTime, + ts Tuple(a UInt32, b AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL d + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_tuple_not_referenced; + +-- Valid: a data-dependent error of a function that does not itself consume the AggregateFunction state +-- must NOT fail validation. +CREATE TABLE test_ttl_agg_divzero +( + ts AggregateFunction(sum, UInt32) +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(intDiv(toUInt32(100), finalizeAggregation(ts))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_divzero; + +-- Short-circuit branch: an unsupported AggregateFunction consumer hidden in a not-taken if/multiIf +-- branch must still be rejected. +CREATE TABLE test_ttl_agg_if_branch +( + cond UInt8, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL if(cond, toDateTime(ts), toDateTime(finalizeAggregation(ts))) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Lambda body: an unsupported AggregateFunction consumer inside a higher-order function's lambda must +-- be rejected. Validation recurses into the lambda DAG instead of executing the outer arrayMap over the +-- empty default array (which would never reach the lambda body). +CREATE TABLE test_ttl_agg_lambda +( + ts Array(AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL arrayMap(x -> toDateTime(x), ts)[1] + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a state-aware consumer inside a lambda body must be accepted. +CREATE TABLE test_ttl_agg_lambda_finalize +( + ts Array(AggregateFunction(max, DateTime64(3))) +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL arrayMap(x -> toDateTime(finalizeAggregation(x)), ts)[1] + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_lambda_finalize; + +-- Valid: finalizeAggregation can operate on AggregateFunction states +CREATE TABLE test_ttl_agg_finalize +( + key1 String, + key2 String, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY (key1, key2) +TTL toDateTime(finalizeAggregation(ts)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_finalize; + +-- Valid: state-aware functions like bitmapCardinality properly accept AggregateFunction +CREATE TABLE test_ttl_agg_bitmap +( + k UInt64, + bm AggregateFunction(groupBitmap, UInt64) +) +ENGINE = MergeTree() +ORDER BY k +TTL toDateTime(bitmapCardinality(bm)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_bitmap; + +-- Valid: expressions with potential division by zero should NOT be rejected at DDL time +CREATE TABLE test_ttl_intdiv +( + ts UInt32, + denom UInt32 DEFAULT 1 +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(intDiv(ts, denom)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_intdiv; + +-- GROUP BY SET: an unsupported AggregateFunction consumer inside a SET aggregate argument must be +-- rejected. +CREATE TABLE test_ttl_agg_group_by_set +( + key UInt64, + d DateTime, + ts AggregateFunction(max, DateTime64(3)), + out DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY GROUP BY key SET out = max(toDateTime(ts)); -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a state-aware consumer inside a SET aggregate argument must be accepted. +CREATE TABLE test_ttl_agg_group_by_set_finalize +( + key UInt64, + d DateTime, + ts AggregateFunction(max, DateTime64(3)), + out DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY GROUP BY key SET out = max(toDateTime(finalizeAggregation(ts))); + +DROP TABLE test_ttl_agg_group_by_set_finalize; + +-- GROUP BY SET: an aggregate that returns the AggregateFunction state itself (e.g. `any(ts)`) and is then +-- implicitly cast to an incompatible target column type must be rejected. The aggregate argument is just +-- `ts`, so this is caught by validating the post-aggregation (casted) SET expression, not the argument. +CREATE TABLE test_ttl_agg_group_by_set_cast +( + key UInt64, + d DateTime, + ts AggregateFunction(max, DateTime64(3)), + out DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY GROUP BY key SET out = any(ts); -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: AggregateFunction column exists but is not referenced in TTL +CREATE TABLE test_ttl_agg_not_referenced +( + key1 String, + d DateTime, + ts AggregateFunction(max, DateTime64(3)) +) +ENGINE = MergeTree() +ORDER BY key1 +TTL d + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_not_referenced; + +-- Valid: normal DateTime column in TTL (sanity check) +CREATE TABLE test_ttl_normal +( + key1 String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key1 +TTL d + INTERVAL 1 DAY; + +DROP TABLE test_ttl_normal; + +-- Variant column carrying an AggregateFunction alternative: the all-NULL default probe column would let +-- the Variant function adaptor short-circuit, so this used to pass CREATE TABLE and only fail at insert. +-- Table-level TTL: toDateTime cannot consume the AggregateFunction alternative of the Variant. +CREATE TABLE test_ttl_agg_variant +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3)), String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(v) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Column-level TTL on the Variant: same issue. +CREATE TABLE test_ttl_agg_variant_col +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3)), String) TTL toDateTime(v) + INTERVAL 1 DAY +) +ENGINE = MergeTree() +ORDER BY key; -- { serverError BAD_TTL_EXPRESSION } + +-- TTL DELETE WHERE using the Variant alternative. +CREATE TABLE test_ttl_agg_variant_where +( + key UInt64, + d DateTime, + v Variant(AggregateFunction(max, DateTime64(3)), String) +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(v) > toDateTime(0); -- { serverError BAD_TTL_EXPRESSION } + +-- Every AggregateFunction alternative is probed, so a Variant of two different states is also rejected. +CREATE TABLE test_ttl_agg_variant_two +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3)), AggregateFunction(sum, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(v) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a state-aware consumer (`finalizeAggregation`) reaching the AggregateFunction alternative of a +-- Variant must still be accepted - only the aggregate-carrying alternative is probed, not the consumer. +CREATE TABLE test_ttl_agg_variant_finalize +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3))), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(assumeNotNull(finalizeAggregation(v))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_variant_finalize; + +-- Valid: a Variant with an AggregateFunction alternative that is not referenced in the TTL is accepted. +CREATE TABLE test_ttl_agg_variant_not_referenced +( + key UInt64, + d DateTime, + v Variant(AggregateFunction(max, DateTime64(3)), String) +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_variant_not_referenced; + +-- Valid: the escape hatch `allow_suspicious_ttl_expressions` still lets the rejected expression through. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_variant_suspicious +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3)), String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(v) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_variant_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- Dynamic column: the static type never mentions AggregateFunction, but any row may carry a state +-- (e.g. inserted via CAST to Dynamic), so this used to pass CREATE TABLE and only fail during TTL +-- execution. The validator probes Dynamic arguments with a synthetic state. +-- Table-level TTL: toDateTime cannot consume an AggregateFunction state stored in the Dynamic. +CREATE TABLE test_ttl_agg_dynamic +( + key UInt64, + dyn Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(dyn) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Column-level TTL on the Dynamic: same issue. +CREATE TABLE test_ttl_agg_dynamic_col +( + key UInt64, + dyn Dynamic TTL toDateTime(dyn) + INTERVAL 1 DAY +) +ENGINE = MergeTree() +ORDER BY key; -- { serverError BAD_TTL_EXPRESSION } + +-- TTL DELETE WHERE using the Dynamic column. +CREATE TABLE test_ttl_agg_dynamic_where +( + key UInt64, + d DateTime, + dyn Dynamic +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(dyn) > toDateTime(0); -- { serverError BAD_TTL_EXPRESSION } + +-- Dynamic with no room for new variants stores every value in the shared variant; the probe goes +-- through the shared variant and the expression is still rejected. +CREATE TABLE test_ttl_agg_dynamic_shared +( + key UInt64, + dyn Dynamic(max_types=0), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(dyn) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a type-agnostic consumer (`isNotNull`) can handle any stored value, including a state. +CREATE TABLE test_ttl_agg_dynamic_agnostic +( + key UInt64, + d DateTime, + dyn Dynamic +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(dyn); + +DROP TABLE test_ttl_agg_dynamic_agnostic; + +-- Valid: a Dynamic column that is not referenced in the TTL is accepted. +CREATE TABLE test_ttl_agg_dynamic_not_referenced +( + key UInt64, + d DateTime, + dyn Dynamic +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_dynamic_not_referenced; + +-- Valid: the escape hatch `allow_suspicious_ttl_expressions` still lets the rejected expression through. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_dynamic_suspicious +( + key UInt64, + dyn Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(dyn) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_dynamic_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- Lowering `variant_throw_on_type_mismatch` to 0 makes the Variant function adaptor return NULL instead of +-- throwing on a type mismatch. The validation probe must still reject a suspicious TTL, because TTL merges +-- rebuild the expression under the background context (strict by default) and would otherwise break every merge. +SET variant_throw_on_type_mismatch = 0; + +CREATE TABLE test_ttl_agg_variant_lenient +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3)), String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(v) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +SET variant_throw_on_type_mismatch = 1; + +-- Same, for `dynamic_throw_on_type_mismatch`: a lenient session must not let a suspicious Dynamic TTL through. +SET dynamic_throw_on_type_mismatch = 0; + +CREATE TABLE test_ttl_agg_dynamic_lenient +( + key UInt64, + dyn Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(dyn) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +-- A type-agnostic Dynamic consumer is still accepted even under the lenient setting. +CREATE TABLE test_ttl_agg_dynamic_lenient_agnostic +( + key UInt64, + d DateTime, + dyn Dynamic +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(dyn); + +DROP TABLE test_ttl_agg_dynamic_lenient_agnostic; + +SET dynamic_throw_on_type_mismatch = 1; + +-- The conversion functions above (`toDateTime`) ignore the mismatch settings, so they alone cannot tell +-- which settings the probe runs under. Consumers that go through the `Variant`/`Dynamic` function adaptors +-- (e.g. `length`) do honor the settings: under a lenient session the adaptor would silently return NULL in +-- the probe, while the strict TTL execution paths - a default-settings INSERT computing TTLs in +-- `MergeTreeDataWriter::updateTTL`, background merges under the default `background_profile`, and table +-- loading on restart (no query context, adaptors fall back to strict) - would throw on the first row +-- carrying an AggregateFunction state. The probe therefore always runs strict, regardless of the session, +-- so a lenient session must still get such a TTL rejected. +SET dynamic_throw_on_type_mismatch = 0; + +CREATE TABLE test_ttl_agg_dynamic_lenient_adaptor +( + key UInt64, + dyn Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE length(dyn) > 3; -- { serverError BAD_TTL_EXPRESSION } + +SET dynamic_throw_on_type_mismatch = 1; + +SET variant_throw_on_type_mismatch = 0; + +CREATE TABLE test_ttl_agg_variant_lenient_adaptor +( + key UInt64, + v Variant(AggregateFunction(max, DateTime64(3)), String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE length(v) > 3; -- { serverError BAD_TTL_EXPRESSION } + +SET variant_throw_on_type_mismatch = 1; + +-- The Variant function adaptor also consults `variant_throw_on_type_mismatch` while *building* the +-- expression: when none of the alternatives is compatible with the consumer, the build itself either +-- throws (strict) or resolves the result to constant NULL (lenient). The lenient build must not slip +-- through DDL validation: the constant fold prunes the referenced column from the stored TTL column list, +-- so every later rebuild of the TTL expression fails with "Missing columns" (broken INSERTs and merges), +-- and the table cannot be re-attached on restart (loading has no query context, so the adaptor is strict +-- and throws). The validation build therefore always runs strict, and a lenient session gets the same +-- rejection a strict one does. +SET variant_throw_on_type_mismatch = 0; + +CREATE TABLE test_ttl_agg_variant_lenient_build +( + key UInt64, + v Variant(AggregateFunction(max, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNull(length(v)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +-- The escape hatch keeps the plain session behavior: with `allow_suspicious_ttl_expressions` the lenient +-- session resolves the all-incompatible consumer to NULL at build time and the CREATE is accepted. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_variant_lenient_build_suspicious +( + key UInt64, + v Variant(AggregateFunction(max, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNull(length(v)); + +DROP TABLE test_ttl_agg_variant_lenient_build_suspicious; + +SET allow_suspicious_ttl_expressions = 0; +SET variant_throw_on_type_mismatch = 1; + +-- A consumer over *several* Variant/Dynamic carriers must be probed with all of them materialized +-- simultaneously: substituting one at a time leaves the other side at its all-NULL default, the adaptor +-- short-circuits to NULL, and the bad joint combination (e.g. state + state) is never built or executed - +-- so the CREATE passed while a row with states on both sides still threw during TTL execution. +CREATE TABLE test_ttl_agg_two_dynamic +( + key UInt64, + d1 Dynamic, + d2 Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE d1 = d2; -- { serverError BAD_TTL_EXPRESSION } + +CREATE TABLE test_ttl_agg_two_variant +( + key UInt64, + v1 Variant(AggregateFunction(max, UInt64), UInt64), + v2 Variant(AggregateFunction(max, UInt64), UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE v1 = v2; -- { serverError BAD_TTL_EXPRESSION } + +-- A joint consumer that handles every alternative combination is still accepted. +CREATE TABLE test_ttl_agg_two_carriers_agnostic +( + key UInt64, + d1 Dynamic, + v1 Variant(AggregateFunction(max, UInt64), UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE concat(toString(isNotNull(d1)), toString(isNotNull(v1))) = '11'; + +DROP TABLE test_ttl_agg_two_carriers_agnostic; + +-- The escape hatch also covers the joint case. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_two_dynamic_suspicious +( + key UInt64, + d1 Dynamic, + d2 Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE d1 = d2; + +DROP TABLE test_ttl_agg_two_dynamic_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A state-aware consumer over a mixed Variant is rejected even when it can consume the +-- AggregateFunction alternative, because it still throws on a sibling alternative a later row may store: +-- `finalizeAggregation` accepts the AggregateFunction branch but throws ILLEGAL_TYPE_OF_ARGUMENT on the UInt32 branch. +CREATE TABLE test_ttl_agg_mixed_variant_finalize +( + key UInt64, + v Variant(AggregateFunction(max, UInt32), UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(v)); -- { serverError BAD_TTL_EXPRESSION } + +-- A Variant every alternative of which the consumer can handle is still accepted. +CREATE TABLE test_ttl_agg_all_state_alternatives +( + key UInt64, + v Variant(AggregateFunction(max, UInt32), AggregateFunction(min, UInt32)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(v)); + +DROP TABLE test_ttl_agg_all_state_alternatives; + +-- A type-agnostic consumer over the same mixed Variant is still accepted. +CREATE TABLE test_ttl_agg_mixed_variant_agnostic +( + key UInt64, + v Variant(AggregateFunction(max, UInt32), UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(v); + +DROP TABLE test_ttl_agg_mixed_variant_agnostic; + +-- The escape hatch also covers the mixed-Variant case. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_mixed_variant_suspicious +( + key UInt64, + v Variant(AggregateFunction(max, UInt32), UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(v)); + +DROP TABLE test_ttl_agg_mixed_variant_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A state-aware consumer over a Dynamic is rejected even when it can consume an AggregateFunction state, +-- because a Dynamic can store any type and the consumer still throws on other legal payloads: +-- `finalizeAggregation` accepts the synthetic state but throws ILLEGAL_TYPE_OF_ARGUMENT on a UInt64 / String row. +CREATE TABLE test_ttl_agg_dynamic_finalize +( + key UInt64, + dyn Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(dyn)); -- { serverError BAD_TTL_EXPRESSION } + +-- A type-agnostic Dynamic consumer that handles every representative payload is still accepted. +CREATE TABLE test_ttl_agg_dynamic_finalize_agnostic +( + key UInt64, + d DateTime, + dyn Dynamic +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(dyn) AND dynamicType(dyn) != 'UInt64'; + +DROP TABLE test_ttl_agg_dynamic_finalize_agnostic; + +-- The escape hatch also covers the state-aware Dynamic case. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_dynamic_finalize_suspicious +( + key UInt64, + dyn Dynamic, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(dyn)); + +DROP TABLE test_ttl_agg_dynamic_finalize_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A Variant/Dynamic carrier nested inside a container argument (Array/Tuple/Map) must be probed too: +-- the container's default value is empty, so a consumer that processes the elements (e.g. the `equals` +-- built inside `arrayRemove`) never sees a payload during a default-value probe, yet still rebuilds per +-- stored payload during TTL execution and throws on the first element carrying an AggregateFunction state. +CREATE TABLE test_ttl_agg_array_dynamic +( + key UInt64, + arr Array(Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(arr, 0)); -- { serverError BAD_TTL_EXPRESSION } + +-- The same through a Map value. +CREATE TABLE test_ttl_agg_map_dynamic +( + key UInt64, + m Map(String, Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(mapValues(m), 0)); -- { serverError BAD_TTL_EXPRESSION } + +-- The same through a Tuple element. +CREATE TABLE test_ttl_agg_tuple_dynamic +( + key UInt64, + tup Tuple(UInt32, Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove([tup], (0, 0)::Tuple(UInt32, Dynamic))); -- { serverError BAD_TTL_EXPRESSION } + +-- A Variant with an AggregateFunction alternative nested inside an Array is probed per alternative too. +CREATE TABLE test_ttl_agg_array_variant +( + key UInt64, + arr Array(Variant(AggregateFunction(max, UInt32), UInt32)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(arr, 0)); -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a consumer that does not touch the elements is accepted. +CREATE TABLE test_ttl_agg_array_dynamic_agnostic +( + key UInt64, + arr Array(Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE length(arr) > 3; + +DROP TABLE test_ttl_agg_array_dynamic_agnostic; + +-- Valid: a nested carrier that is not referenced in the TTL is accepted. +CREATE TABLE test_ttl_agg_array_dynamic_not_referenced +( + key UInt64, + arr Array(Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY; + +DROP TABLE test_ttl_agg_array_dynamic_not_referenced; + +-- The escape hatch also covers the nested-carrier case. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_array_dynamic_suspicious +( + key UInt64, + arr Array(Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(arr, 0)); + +DROP TABLE test_ttl_agg_array_dynamic_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A carrier hidden under a Nullable wrapper (Nullable(Tuple(..., Dynamic))) must be probed with a +-- non-NULL row too: the default Nullable row is NULL, so the consumer would otherwise never see the +-- nested payload at DDL time, yet still throw during TTL execution once a non-NULL row stores a state. +SET enable_nullable_tuple_type = 1; + +CREATE TABLE test_ttl_agg_nullable_tuple_dynamic +( + key UInt64, + tup Nullable(Tuple(UInt32, Dynamic)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE tup = tup; -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: a type-agnostic consumer over the Nullable wrapper is accepted. +CREATE TABLE test_ttl_agg_nullable_tuple_agnostic +( + key UInt64, + tup Nullable(Tuple(UInt32, Dynamic)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(tup); + +DROP TABLE test_ttl_agg_nullable_tuple_agnostic; + +-- The escape hatch also covers the Nullable-wrapped carrier. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_nullable_tuple_suspicious +( + key UInt64, + tup Nullable(Tuple(UInt32, Dynamic)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE tup = tup; + +DROP TABLE test_ttl_agg_nullable_tuple_suspicious; + +SET allow_suspicious_ttl_expressions = 0; +SET enable_nullable_tuple_type = 0; + +-- A direct AggregateFunction state nested in a container (not through a Variant/Dynamic carrier) must be +-- probed with a non-empty row too: the default Array/Map value is empty, so an element-level consumer +-- (e.g. the `equals` built inside `arrayRemove`) would otherwise never see the state at DDL time, yet +-- still throw during TTL execution once a row stores a state. +CREATE TABLE test_ttl_agg_array_state +( + key UInt64, + arr Array(AggregateFunction(max, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(arr, 0)); -- { serverError BAD_TTL_EXPRESSION } + +CREATE TABLE test_ttl_agg_map_state +( + key UInt64, + m Map(String, AggregateFunction(max, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(mapValues(m), mapValues(m)[1])); -- { serverError BAD_TTL_EXPRESSION } + +-- Valid: consumers that do not look into the state elements are accepted. +CREATE TABLE test_ttl_agg_array_state_agnostic +( + key UInt64, + arr Array(AggregateFunction(max, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE length(arr) > 3; + +DROP TABLE test_ttl_agg_array_state_agnostic; + +-- The escape hatch also covers the container-nested state. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_array_state_suspicious +( + key UInt64, + arr Array(AggregateFunction(max, UInt64)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE notEmpty(arrayRemove(arr, 0)); + +DROP TABLE test_ttl_agg_array_state_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A carrier *computed* from an AggregateFunction state has a narrower runtime domain than its static +-- type: `CAST(state, 'Dynamic')` or `CAST(state, 'Variant(AggregateFunction(max, UInt32), UInt32)')` +-- only ever produces the aggregate-state payload, so a state-aware consumer over it is valid. The probe +-- must validate it against the child's actual output, not fabricate the impossible sibling payloads. +CREATE TABLE test_ttl_agg_computed_dynamic_accept +( + key UInt64, + state AggregateFunction(max, UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(CAST(state, 'Dynamic'))); + +DROP TABLE test_ttl_agg_computed_dynamic_accept; + +CREATE TABLE test_ttl_agg_computed_variant_accept +( + key UInt64, + state AggregateFunction(max, UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(CAST(state, 'Variant(AggregateFunction(max, UInt32), UInt32)'))); + +DROP TABLE test_ttl_agg_computed_variant_accept; + +-- The computed carrier still holds the state: a consumer that cannot handle it stays rejected. +CREATE TABLE test_ttl_agg_computed_carrier_reject +( + key UInt64, + state AggregateFunction(max, UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(CAST(state, 'Dynamic')); -- { serverError BAD_TTL_EXPRESSION } + +-- A typed CAST of a non-suspect column propagates its actual output domain: here the runtime payload +-- is always `UInt32` (the cast picks the matching alternative), which `finalizeAggregation` cannot +-- consume, so the TTL is rejected against the real domain. +CREATE TABLE test_ttl_agg_untainted_carrier_reject +( + key UInt64, + num UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(CAST(num, 'Variant(AggregateFunction(max, UInt32), UInt32)'))); -- { serverError BAD_TTL_EXPRESSION } + +-- The escape hatch also covers consumers of computed carriers. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_computed_carrier_suspicious +( + key UInt64, + state AggregateFunction(max, UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY key +TTL toDateTime(CAST(state, 'Dynamic')); + +DROP TABLE test_ttl_agg_computed_carrier_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A non-constant non-suspect argument can select which payload a computed carrier holds, so the probe +-- outputs (taken with synthetic defaults, here `cond = 0`) do not cover its runtime domain: with +-- `cond = 1` the `if` returns the aggregate-state branch and `toDateTime` would throw during TTL +-- execution. Such nodes fall back to the fail-closed static enumeration and the consumer is rejected. +CREATE TABLE test_ttl_agg_selected_carrier_reject +( + cond UInt8, + state AggregateFunction(max, UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(state, 'Dynamic'), CAST(0, 'Dynamic'))); -- { serverError BAD_TTL_EXPRESSION } + +-- A type-agnostic consumer survives the static enumeration, so the same selected carrier is accepted. +CREATE TABLE test_ttl_agg_selected_carrier_accept +( + cond UInt8, + state AggregateFunction(max, UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(if(cond, CAST(state, 'Dynamic'), CAST(0, 'Dynamic'))); + +DROP TABLE test_ttl_agg_selected_carrier_accept; + +-- The escape hatch also covers payload-selecting expressions. +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_agg_selected_carrier_suspicious +( + cond UInt8, + state AggregateFunction(max, UInt64), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(state, 'Dynamic'), CAST(0, 'Dynamic'))); + +DROP TABLE test_ttl_agg_selected_carrier_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A typed CAST of an ordinary column to Dynamic can only ever store the payload type derived from the +-- source type (here `UInt32`), so its consumer must not be probed with synthetic AggregateFunction +-- payloads the cast can never produce. +CREATE TABLE test_ttl_cast_plain_number_accept +( + n UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(CAST(n, 'Dynamic')) < d; + +DROP TABLE test_ttl_cast_plain_number_accept; + +-- The representative source value for the cast probe is non-NULL, so a Nullable source still exercises +-- the consumer on the actual payload type instead of a NULL row that would short-circuit it. +CREATE TABLE test_ttl_cast_nullable_number_accept +( + nn Nullable(UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(CAST(nn, 'Dynamic')) < d; + +DROP TABLE test_ttl_cast_nullable_number_accept; + +-- Containers are materialized with one element for the cast probe, so element-level consumers of the +-- cast result are validated against the actual element payload type (accepted: `toDateTime` of a +-- `UInt32` element works) instead of an empty default that would hide them. Note this holds for direct +-- consumers only: a lambda body (e.g. inside `arrayExists`) is validated through the captured DAG, +-- where the element is a plain `Dynamic` input, so it keeps the fail-closed static enumeration. +CREATE TABLE test_ttl_cast_array_accept +( + arr Array(UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE toDateTime(arrayElement(CAST(arr, 'Array(Dynamic)'), 1)) < d; + +DROP TABLE test_ttl_cast_array_accept; + +-- A cast of a *string* to a carrier is not source-type-determined: `cast_string_to_variant_use_inference` +-- (on by default) and `cast_string_to_dynamic_use_inference` make the stored alternative depend on the row +-- contents, so the representative empty string says nothing about the runtime domain. Here the empty +-- string is stored as the `String` alternative, but a row `s = '42'` is stored as `UInt32`, and `length` +-- then throws `ILLEGAL_TYPE_OF_ARGUMENT` during TTL execution - so such casts keep the fail-closed path. +CREATE TABLE test_ttl_cast_string_to_variant_reject +( + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(s, 'Variant(String, UInt32, AggregateFunction(max, UInt32))')) > 3; -- { serverError BAD_TTL_EXPRESSION } + +-- The same holds for a cast of a string to `Dynamic`, and through `Nullable`/`LowCardinality` wrappers and +-- container elements, which the cast recurses into. +CREATE TABLE test_ttl_cast_string_to_dynamic_reject +( + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(s, 'Dynamic')) > 3; -- { serverError BAD_TTL_EXPRESSION } + +CREATE TABLE test_ttl_cast_nullable_string_to_dynamic_reject +( + ns Nullable(String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(ns, 'Dynamic')) > 3; -- { serverError BAD_TTL_EXPRESSION } + +CREATE TABLE test_ttl_cast_lc_string_to_dynamic_reject +( + ls LowCardinality(String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(ls, 'Dynamic')) > 3; -- { serverError BAD_TTL_EXPRESSION } + +CREATE TABLE test_ttl_cast_string_array_to_dynamic_reject +( + arr Array(String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(arrayElement(CAST(arr, 'Array(Dynamic)'), 1)) > 3; -- { serverError BAD_TTL_EXPRESSION } + +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_cast_string_to_variant_suspicious +( + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(s, 'Variant(String, UInt32, AggregateFunction(max, UInt32))')) > 3; + +DROP TABLE test_ttl_cast_string_to_variant_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +-- A consumer that cannot handle the cast's actual payload type is still rejected: the runtime payload +-- of `CAST(n, 'Dynamic')` is `UInt32`, which `finalizeAggregation` cannot consume. +CREATE TABLE test_ttl_cast_plain_number_reject +( + n UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE isNotNull(finalizeAggregation(CAST(n, 'Dynamic'))); -- { serverError BAD_TTL_EXPRESSION } + +DROP TABLE IF EXISTS test_ttl_cast_variant_source_reject; + +-- A `Variant` source is the exception to the "one representative value" rule: the cast preserves +-- whichever alternative each row stores, so the payload of the result is not fixed by a single +-- representative. Probing only the default (NULL) row would accept this expression, but a row storing +-- the `UInt32` alternative makes `length` throw `ILLEGAL_TYPE_OF_ARGUMENT` during TTL execution. +CREATE TABLE test_ttl_cast_variant_source_reject +( + v Variant(String, UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(v, 'Dynamic')) > 3; -- { serverError BAD_TTL_EXPRESSION } + +-- Narrowing is still exact for a `Variant` source: every alternative is probed, and a consumer that +-- handles all of them is accepted (`length` works on both `String` and `Array(UInt32)`), without being +-- confronted with synthetic payloads the cast can never produce. +CREATE TABLE test_ttl_cast_variant_source_accept +( + v Variant(String, Array(UInt32)), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(v, 'Dynamic')) > 3; + +DROP TABLE test_ttl_cast_variant_source_accept; + +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_cast_variant_source_suspicious +( + v Variant(String, UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL d + INTERVAL 1 DAY DELETE WHERE length(CAST(v, 'Dynamic')) > 3; + +DROP TABLE test_ttl_cast_variant_source_suspicious; + +SET allow_suspicious_ttl_expressions = 0; + +DROP TABLE IF EXISTS test_ttl_selector_same_domain_accept; + +-- A selector function (`if`, `multiIf`, `coalesce`, `ifNull`) returns one of its value arguments, so a +-- non-constant condition can only choose *which* of their domains the result comes from. Both branches here +-- can only ever hold the `UInt32` payload, so the narrowed domain survives the selector and the expression +-- is accepted instead of being confronted with the synthetic payloads of a plain `Dynamic` column. +CREATE TABLE test_ttl_selector_same_domain_accept +( + cond UInt8, + n UInt32, + m UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), CAST(m, 'Dynamic'))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_same_domain_accept; + +DROP TABLE IF EXISTS test_ttl_selector_multi_if_same_domain_accept; + +CREATE TABLE test_ttl_selector_multi_if_same_domain_accept +( + cond UInt8, + n UInt32, + m UInt32, + k UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(multiIf(cond, CAST(n, 'Dynamic'), cond > 1, CAST(m, 'Dynamic'), CAST(k, 'Dynamic'))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_multi_if_same_domain_accept; + +DROP TABLE IF EXISTS test_ttl_selector_different_domains_reject; + +-- The result of a selector carries the *union* of its branches' payload domains. A string-to-carrier cast +-- stays on the fail-closed static enumeration (it infers the stored payload from the row contents), so its +-- synthetic `AggregateFunction` candidate is in the union and keeps the whole selector rejected. +CREATE TABLE test_ttl_selector_different_domains_reject +( + cond UInt8, + n UInt32, + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), CAST(s, 'Dynamic'))) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_selector_different_domains_reject +( + cond UInt8, + n UInt32, + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), CAST(s, 'Dynamic'))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_different_domains_reject; + +SET allow_suspicious_ttl_expressions = 0; + +DROP TABLE IF EXISTS test_ttl_selector_literal_branches_accept; + +-- Branches whose candidate materializations differ only by *value* still describe the same payload domain: +-- both literals below can only ever produce the `UInt8` payload, so the union of the branch domains is +-- propagated and the expression is accepted. +CREATE TABLE test_ttl_selector_literal_branches_accept +( + cond UInt8, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(1, 'Dynamic'), CAST(2, 'Dynamic'))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_literal_branches_accept; + +DROP TABLE IF EXISTS test_ttl_selector_multi_if_literal_branches_accept; + +CREATE TABLE test_ttl_selector_multi_if_literal_branches_accept +( + cond UInt8, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(multiIf(cond, CAST(1, 'Dynamic'), cond > 1, CAST(2, 'Dynamic'), CAST(3, 'Dynamic'))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_multi_if_literal_branches_accept; + +DROP TABLE IF EXISTS test_ttl_selector_union_of_domains_accept; + +-- Branches with genuinely different payload domains are accepted when every payload in the *union* is +-- consumable: `toDateTime` handles both the `UInt32` payload of the first branch and the `UInt8` payload +-- of the second one. +CREATE TABLE test_ttl_selector_union_of_domains_accept +( + cond UInt8, + n UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), CAST(1, 'Dynamic'))) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_union_of_domains_accept; + +DROP TABLE IF EXISTS test_ttl_array_map_accept; + +-- `arrayMap` returns an array of the values its lambda body produces, so the body's narrowed domain +-- describes the elements of the result: the elements of `arrayMap(x -> CAST(x, 'Dynamic'), arr)` over +-- `arr Array(UInt32)` can only ever hold the `UInt32` payload, and a consumer of an element is accepted. +CREATE TABLE test_ttl_array_map_accept +( + arr Array(UInt32), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(arrayElement(arrayMap(x -> CAST(x, 'Dynamic'), arr), 1)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_array_map_accept; + +DROP TABLE IF EXISTS test_ttl_array_map_string_source_reject; + +-- The rules narrowing the lambda body's domain are the same as everywhere else, so a cast of a *string* +-- inside the lambda stays fail-closed (`cast_string_to_dynamic_use_inference` would parse the stored +-- alternative out of the row contents) and the consumer of an element is rejected. +CREATE TABLE test_ttl_array_map_string_source_reject +( + arr Array(String), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(arrayElement(arrayMap(x -> CAST(x, 'Dynamic'), arr), 1)) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +DROP TABLE IF EXISTS test_ttl_array_map_dynamic_input_reject; + +-- A lambda body that just passes a stored `Dynamic` column through keeps the static enumeration of the +-- payloads that column can hold, so a consumer that cannot handle all of them is rejected. +CREATE TABLE test_ttl_array_map_dynamic_input_reject +( + arr Array(Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(arrayElement(arrayMap(x -> x, arr), 1)) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_array_map_dynamic_input_reject +( + arr Array(Dynamic), + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(arrayElement(arrayMap(x -> x, arr), 1)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_array_map_dynamic_input_reject; + +SET allow_suspicious_ttl_expressions = 0; + +DROP TABLE IF EXISTS test_ttl_selector_lifted_branch_accept; + +-- A selector converts every value branch to its result type, so a branch that is no carrier at all still +-- only contributes the payloads that conversion produces from its values: `if(cond, CAST(n, 'Dynamic'), m)` +-- over `n`, `m UInt32` holds a numeric payload whichever branch is taken, and `toDateTime` consumes it. +CREATE TABLE test_ttl_selector_lifted_branch_accept +( + cond UInt8, + n UInt32, + m UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), m)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_lifted_branch_accept; + +DROP TABLE IF EXISTS test_ttl_selector_lifted_literal_accept; + +-- The same for a literal branch lifted to the carrier result type. +CREATE TABLE test_ttl_selector_lifted_literal_accept +( + cond UInt8, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(1, 'Dynamic'), 2)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_lifted_literal_accept; + +DROP TABLE IF EXISTS test_ttl_selector_lifted_state_branch_reject; + +-- Lifting a branch does not weaken the check: the aggregate state of the other branch stays in the union +-- of the domains, so a consumer that cannot handle it is still rejected. +CREATE TABLE test_ttl_selector_lifted_state_branch_reject +( + cond UInt8, + state AggregateFunction(max, UInt32), + m UInt32, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(state, 'Dynamic'), m)) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +DROP TABLE IF EXISTS test_ttl_selector_lifted_string_branch_reject; + +-- A *string* branch is lifted by a conversion that infers the payload out of the row contents, so its +-- domain is unknown and the selector falls back to the static enumeration of the result type. +CREATE TABLE test_ttl_selector_lifted_string_branch_reject +( + cond UInt8, + n UInt32, + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), s)) + INTERVAL 1 DAY; -- { serverError BAD_TTL_EXPRESSION } + +SET allow_suspicious_ttl_expressions = 1; + +CREATE TABLE test_ttl_selector_lifted_string_branch_reject +( + cond UInt8, + n UInt32, + s String, + d DateTime +) +ENGINE = MergeTree() +ORDER BY tuple() +TTL toDateTime(if(cond, CAST(n, 'Dynamic'), s)) + INTERVAL 1 DAY; + +DROP TABLE test_ttl_selector_lifted_string_branch_reject; + +SET allow_suspicious_ttl_expressions = 0; diff --git a/tests/queries/0_stateless/04339_ast_fuzzer_replicated_ddl_metadata_transaction.reference b/tests/queries/0_stateless/04339_ast_fuzzer_replicated_ddl_metadata_transaction.reference new file mode 100644 index 000000000000..d6a50236c639 --- /dev/null +++ b/tests/queries/0_stateless/04339_ast_fuzzer_replicated_ddl_metadata_transaction.reference @@ -0,0 +1,2 @@ +alive +fuzzer_skipped_in_replicated_ddl 1 diff --git a/tests/queries/0_stateless/04339_ast_fuzzer_replicated_ddl_metadata_transaction.sh b/tests/queries/0_stateless/04339_ast_fuzzer_replicated_ddl_metadata_transaction.sh new file mode 100755 index 000000000000..fba92f5a1ac7 --- /dev/null +++ b/tests/queries/0_stateless/04339_ast_fuzzer_replicated_ddl_metadata_transaction.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest +# no-fasttest: needs a Replicated database (ZooKeeper), not available in fast test. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# distributed_ddl_output_mode=none on every DDL: with the default 'throw' mode a replicated DDL +# prints a per-replica status row ("s1 r1 OK 0 0") to stdout, which is not part of this test's +# checked output. Pin it so the DDL below never pollutes stdout regardless of the CI profile. +ddl="${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none" + +# Reuse the implicit test database when it is already Replicated (the replicated-database CI +# variant); otherwise make a dedicated Replicated database. The dedicated one uses the unique +# per-test ZooKeeper prefix so the test is safe to run in parallel with itself. +db="${CLICKHOUSE_DATABASE}" +if [[ $(${CLICKHOUSE_CLIENT} -q "SELECT engine = 'Replicated' FROM system.databases WHERE name = '${CLICKHOUSE_DATABASE}'") != 1 ]]; then + db="rdb_${CLICKHOUSE_DATABASE}" + ${ddl} -q "DROP DATABASE IF EXISTS ${db} SYNC" + ${ddl} -q "CREATE DATABASE ${db} ENGINE = Replicated('/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/rdb', 's1', 'r1')" +fi + +${ddl} -q "CREATE TABLE ${db}.t (a UInt64) ENGINE = ReplicatedMergeTree ORDER BY a" + +# ast_fuzzer_any_query = 1 fuzzes this non-read-only ALTER, and the setting is serialized into the ZK +# DDL entry, so DatabaseReplicatedDDLWorker re-fuzzes it while re-executing the entry on the entry's +# live ZooKeeperMetadataTransaction. The unique query id attributes the ProfileEvent below to this +# statement (the client query and the DDLWorker re-execution share initial_query_id; fuzzed sub-queries +# get fresh ids). Fuzzed follow-ups log their own internal errors; the outer ALTER succeeds, so stdout +# stays empty here. +qid="04339_${CLICKHOUSE_DATABASE}_$$" +${ddl} --send_logs_level=fatal --query_id="${qid}" -q "ALTER TABLE ${db}.t ADD COLUMN IF NOT EXISTS b UInt64 SETTINGS ast_fuzzer_runs = 5, ast_fuzzer_any_query = 1" + +# The server must still be alive after the fuzzed replicated DDL. +${CLICKHOUSE_CLIENT} -q "SELECT 'alive'" + +# Build-mode-independent proof of the fix. 'alive' above only proves the server survived, which a +# release build does even without the fix (the LOGICAL_ERROR is caught inside the fuzzer). Instead +# assert the fuzzer declined to fuzz during the internal DDL re-execution: +# ASTFuzzerSkippedReplicatedDDLInternal is bumped only when executeASTFuzzerQueries returns early on a +# context that holds a live metadata transaction (the DDLWorker re-execution row). It is attributed to +# this ALTER by initial_query_id (client statement + DDLWorker re-execution; fuzzed sub-queries get +# fresh ids and are excluded). Without the fix the counter stays 0 and this assertion flips to 0. +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log" +# enable_parallel_replicas = 0: single-node introspection of system.query_log; keep the CI randomizer +# from turning it into a distributed read over the parallel_replicas cluster. +# current_database = currentDatabase(): required by the query_log style rule; the DDLWorker re-execution +# row (which carries the counter) runs in the current database, and initial_query_id already pins the +# result to this exact statement. +${CLICKHOUSE_CLIENT} -q " + SELECT 'fuzzer_skipped_in_replicated_ddl', + sum(ProfileEvents['ASTFuzzerSkippedReplicatedDDLInternal']) > 0 + FROM system.query_log + WHERE event_date >= today() - 1 + AND initial_query_id = '${qid}' + AND current_database = currentDatabase() + AND type = 'QueryFinish' + SETTINGS enable_parallel_replicas = 0" + +${ddl} -q "DROP TABLE IF EXISTS ${db}.t SYNC" +if [[ "${db}" != "${CLICKHOUSE_DATABASE}" ]]; then + ${ddl} -q "DROP DATABASE IF EXISTS ${db} SYNC" +fi diff --git a/tests/queries/0_stateless/04339_union_subquery_intersect_except_child_column_order.reference b/tests/queries/0_stateless/04339_union_subquery_intersect_except_child_column_order.reference new file mode 100644 index 000000000000..36637a4248d1 --- /dev/null +++ b/tests/queries/0_stateless/04339_union_subquery_intersect_except_child_column_order.reference @@ -0,0 +1,13 @@ +0 +0 +0 +0 +0 +0 +1 +1 +2 +2 +3 +4 +100 diff --git a/tests/queries/0_stateless/04339_union_subquery_intersect_except_child_column_order.sql b/tests/queries/0_stateless/04339_union_subquery_intersect_except_child_column_order.sql new file mode 100644 index 000000000000..01a37d725565 --- /dev/null +++ b/tests/queries/0_stateless/04339_union_subquery_intersect_except_child_column_order.sql @@ -0,0 +1,37 @@ +-- Outer query needs a subset of columns of a UNION whose children are INTERSECT/EXCEPT. +-- The old analyzer used to abort with LOGICAL_ERROR "Different order of columns in UNION subquery" +-- because INTERSECT/EXCEPT children ignore required_result_column_names and return the full header. + +SET enable_analyzer = 0; + +SELECT x FROM +( + SELECT 0 AS x, 1 AS y INTERSECT DISTINCT SELECT 0 AS x, 1 AS y + UNION ALL + SELECT 0 AS x, 1 AS y INTERSECT DISTINCT SELECT 0 AS x, 1 AS y +) +ORDER BY x; + +SELECT x FROM +( + SELECT 0 AS x, 1 AS y INTERSECT SELECT 0 AS x, 1 AS y + UNION ALL + SELECT 0 AS x, 1 AS y INTERSECT SELECT 0 AS x, 1 AS y +) +ORDER BY x; + +SELECT a FROM +( + (SELECT number AS a, number * 10 AS b FROM numbers(5) INTERSECT SELECT number AS a, number * 10 AS b FROM numbers(3)) + UNION ALL + (SELECT number AS a, number * 10 AS b FROM numbers(2)) +) +ORDER BY a; + +SELECT a FROM +( + (SELECT number AS a, number AS b FROM numbers(5) EXCEPT SELECT number AS a, number AS b FROM numbers(2)) + UNION ALL + (SELECT 100 AS a, 100 AS b) +) +ORDER BY a; diff --git a/tests/queries/0_stateless/04342_distributed_plan_read_with_projection.reference b/tests/queries/0_stateless/04342_distributed_plan_read_with_projection.reference new file mode 100644 index 000000000000..47b88792d110 --- /dev/null +++ b/tests/queries/0_stateless/04342_distributed_plan_read_with_projection.reference @@ -0,0 +1,4 @@ +-- distributed read over a projected table does not abort +499 +-- matches the single-node result +499 diff --git a/tests/queries/0_stateless/04342_distributed_plan_read_with_projection.sql b/tests/queries/0_stateless/04342_distributed_plan_read_with_projection.sql new file mode 100644 index 000000000000..a7c19602f8c0 --- /dev/null +++ b/tests/queries/0_stateless/04342_distributed_plan_read_with_projection.sql @@ -0,0 +1,57 @@ +-- Tags: no-old-analyzer +-- no-old-analyzer: make_distributed_plan requires the analyzer. + +-- Regression test: a distributed read (make_distributed_plan) over a table with a normal projection +-- used to abort with LOGICAL_ERROR 'Different list of shards in child plans'. The projection +-- optimization replaced the single read with a Union of (surviving-parts read, projection read), but +-- only the surviving-parts branch carried the distributed (sharded) flag, so the two Union branches +-- exposed different shard lists and makeDistributedPlan asserted on the mismatch. The projection +-- optimization now declines for distributed reads, keeping the read whole. + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; + +CREATE TABLE t1 (id UInt32, s String) ENGINE = MergeTree ORDER BY id; +-- Small granularity so the read spans several granules; the projection split must produce a +-- non-trivial read on both Union branches to reach the sharded-read decision (a single-granule read +-- does not reproduce). A few hundred granules is plenty and keeps the fixture cheap. +CREATE TABLE t2 (id1 UInt32, id2 UInt32) ENGINE = MergeTree ORDER BY id1 SETTINGS index_granularity = 16; + +-- Two inserts so some parts are served by the projection and some by the surviving parts; the +-- projection ADD between them leaves the first batch's parts without the projection. This mix of +-- projection / non-projection parts is what reproduces the shard-list mismatch, so keep it. +INSERT INTO t2 SELECT number, number % 10 FROM numbers(2000); +ALTER TABLE t2 ADD PROJECTION proj (SELECT id2 ORDER BY id2); +INSERT INTO t2 SELECT number, number % 10 FROM numbers(2000); + +INSERT INTO t1 SELECT number, toString(number) FROM numbers(100); + +-- Pin max_rows_to_group_by = 0: the outer count() is an AggregatingStep and a nonzero limit (which +-- randomized settings can set) makes make_distributed_plan reject the query before it reaches the +-- projection regression this test targets. +SET max_rows_to_group_by = 0; +-- Pin optimize_use_projections = 1. Without it, randomized settings can disable projection +-- optimization, so optimizeUseNormalProjections never runs, no Union split happens, and neither the +-- fixed nor the unfixed binary aborts, so the test would pass trivially and prove nothing. +SET optimize_use_projections = 1; +-- Pin distributed_plan_max_rows_to_broadcast low so t2's read is sharded (the bug path) without a +-- huge fixture; t1 stays under the threshold and is broadcast, as in the original bug. Otherwise +-- randomized settings could raise it above the selected row count and skip the sharded read. +SET make_distributed_plan = 1, enable_parallel_replicas = 0, distributed_plan_execute_locally = 1, + distributed_plan_default_shuffle_join_bucket_count = 3, distributed_plan_default_reader_bucket_count = 3, + distributed_plan_max_rows_to_broadcast = 10; + +-- t2's read is sharded; the projection match would split it into a Union. t1 is broadcast. +SELECT '-- distributed read over a projected table does not abort'; +SELECT count() FROM ( + SELECT s FROM t1 AS lhs LEFT JOIN (SELECT * FROM t2 PREWHERE id2 = 2 WHERE id2 = 2) AS rhs ON lhs.id = rhs.id2 +); + +-- Same query single-node, for an explicit value to compare against. +SELECT '-- matches the single-node result'; +SELECT count() FROM ( + SELECT s FROM t1 AS lhs LEFT JOIN (SELECT * FROM t2 PREWHERE id2 = 2 WHERE id2 = 2) AS rhs ON lhs.id = rhs.id2 +) SETTINGS make_distributed_plan = 0; + +DROP TABLE t1; +DROP TABLE t2; diff --git a/tests/queries/0_stateless/04365_bloom_filter_arrayJoin_in.reference b/tests/queries/0_stateless/04365_bloom_filter_arrayJoin_in.reference new file mode 100644 index 000000000000..74770921ebee --- /dev/null +++ b/tests/queries/0_stateless/04365_bloom_filter_arrayJoin_in.reference @@ -0,0 +1,57 @@ +1 +1 +1 +1 +1 +0 +1 +0 +2 +99999 +1 +99999 +1 +5000 +5000 +1 +5000 +5000 +1 +1 +1 +20000 +20000 +0 +[1,2] +[1,2] +[1,2] +[1,2] +[1] +[1] +1 +1 +0 +0 +[1] +[1] +1 +1 +1 +1 +[1] +[1] +[1] +[1] +0 +0 +1 +1 +0 +0 +1 +[1] +[1] +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/04365_bloom_filter_arrayJoin_in.sql b/tests/queries/0_stateless/04365_bloom_filter_arrayJoin_in.sql new file mode 100644 index 000000000000..d688bd79058a --- /dev/null +++ b/tests/queries/0_stateless/04365_bloom_filter_arrayJoin_in.sql @@ -0,0 +1,251 @@ +-- Tags: no-parallel-replicas +-- `arrayJoin(col) IN (set)` and `arrayJoin(col) = const` must use the Array bloom filter index, +-- like `hasAny(col, set)` and `has(col, const)` already do. +-- Issues: https://github.com/ClickHouse/ClickHouse/issues/109516 +-- https://github.com/ClickHouse/ClickHouse/issues/109844 + +DROP TABLE IF EXISTS t_arrayjoin_bf; + +CREATE TABLE t_arrayjoin_bf +( + id UInt64, + tags Array(String), + INDEX idx_tags tags TYPE bloom_filter GRANULARITY 1 +) +-- Pin granule layout (query-level SETTINGS override CI-randomized merge tree settings) so the +-- Granules: X/Y counts below are deterministic: 100000 rows / 8192 = 13 granules. +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +-- Each tag is unique, so a given tag lives in exactly one granule: pruning is observable. +INSERT INTO t_arrayjoin_bf SELECT number, [concat('tag_', toString(number))] FROM numbers(100000); + +-- Baseline: hasAny prunes to the single matching granule (a bloom filter false positive keeps one extra -> 2/13). +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE hasAny(tags, ['tag_42'])) WHERE explain ILIKE '%Granules: 2/13%'; + +-- arrayJoin(tags) IN (const set) now uses the index and prunes identically to hasAny. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) IN ('tag_42')) WHERE explain ILIKE '%Granules: 2/13%'; + +-- arrayJoin(tags) IN (subquery set). +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) IN (SELECT 'tag_42')) WHERE explain ILIKE '%Granules: 2/13%'; + +-- arrayJoin(tags) GLOBAL IN (subquery set). +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) GLOBAL IN (SELECT 'tag_42')) WHERE explain ILIKE '%Granules: 2/13%'; + +-- Multi-element set: two tags in two distinct granules -> more granules read than single-element. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) IN ('tag_42', 'tag_99999')) WHERE explain ILIKE '%Granules: 3/13%'; + +-- Safety: NOT IN must NOT prune (a granule with the set element can still yield rows outside the set). +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) NOT IN ('tag_42')) WHERE explain ILIKE '%Name: idx_tags%'; + +-- arrayJoin(tags) = const now uses the index and prunes identically to has(tags, const). +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) = 'tag_42') WHERE explain ILIKE '%Granules: 2/13%'; + +-- Safety: != must NOT prune (a granule with the value can still yield rows whose arrayJoined value differs). +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) != 'tag_42') WHERE explain ILIKE '%Name: idx_tags%'; + +-- Correctness: results are unaffected by index usage. +SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) IN ('tag_42', 'tag_99999'); +SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) NOT IN ('tag_42'); +SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) = 'tag_42'; +SELECT count() FROM t_arrayjoin_bf WHERE arrayJoin(tags) != 'tag_42'; + +DROP TABLE t_arrayjoin_bf; + +-- An empty array produces no row for the inner `arrayJoin(col)`, so only a granule holding the +-- default as a real element is kept. +DROP TABLE IF EXISTS t_arrayjoin_bf_default; + +CREATE TABLE t_arrayjoin_bf_default +( + id UInt64, + tags Array(String), + INDEX idx_tags tags TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +-- First 5000 rows: arrays that really contain the default value '' as an element. +INSERT INTO t_arrayjoin_bf_default SELECT number, ['', concat('x_', toString(number))] FROM numbers(5000); +-- Remaining rows: unique non-default tags, no empty string. +INSERT INTO t_arrayjoin_bf_default SELECT number + 5000, [concat('tag_', toString(number))] FROM numbers(95000); + +-- Default value is a real element in one granule -> pruning still fires (2/13) and the result is correct. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) IN ('')) WHERE explain ILIKE '%Granules: 2/13%'; +SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) IN ('') SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) IN ('') SETTINGS use_skip_indexes = 0; +-- Pinned: with `optimize_empty_string_comparisons = 1` the predicate becomes `empty(s)`, which no +-- longer reaches the derivation. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) = '' SETTINGS optimize_empty_string_comparisons = 0) WHERE explain ILIKE '%Granules: 2/13%'; +-- Results are identical with the skip index on and off. +SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) = '' SETTINGS use_skip_indexes = 1, optimize_empty_string_comparisons = 0; +SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) = '' SETTINGS use_skip_indexes = 0, optimize_empty_string_comparisons = 0; +-- A non-default value that is a real element in one granule -> pruning fires (1/13), result correct. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) = 'x_1') WHERE explain ILIKE '%Granules: 1/13%'; +SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) = 'x_1' SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_default WHERE arrayJoin(tags) = 'x_1' SETTINGS use_skip_indexes = 0; + +DROP TABLE t_arrayjoin_bf_default; + +-- LEFT ARRAY JOIN expands an empty array into a default-valued row, and that predicate sits above +-- the ARRAY JOIN step, so it never reaches the skip index. +DROP TABLE IF EXISTS t_arrayjoin_bf_left; + +CREATE TABLE t_arrayjoin_bf_left +( + id UInt64, + tags Array(String), + INDEX idx_tags tags TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +INSERT INTO t_arrayjoin_bf_left SELECT number, [] FROM numbers(20000); +INSERT INTO t_arrayjoin_bf_left SELECT number + 20000, [concat('tag_', toString(number))] FROM numbers(80000); + +-- 20000 empty-array rows are each expanded to one default-value row -> 20000 matches, index or not. +SELECT count() FROM t_arrayjoin_bf_left LEFT ARRAY JOIN tags WHERE tags IN ('') SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_left LEFT ARRAY JOIN tags WHERE tags IN ('') SETTINGS use_skip_indexes = 0; +-- The skip index must be absent from the plan, not merely agree on the count. +SELECT count() FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_left LEFT ARRAY JOIN tags WHERE tags IN ('')) WHERE explain ILIKE '%Name: idx_tags%'; + +DROP TABLE t_arrayjoin_bf_left; + +-- Hash-domain gate: comparison coerces more widely than the conversion the derivations hash +-- through, so they only fire where the two agree. Every case must match index on/off, and not raise. +DROP TABLE IF EXISTS t_arrayjoin_bf_domain_str; + +CREATE TABLE t_arrayjoin_bf_domain_str +( + id UInt64, + s Array(String), + INDEX idx_s s TYPE bloom_filter(0.0001) GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +-- Row 2 stores a trailing NUL, which compares equal to the unpadded FixedString constant. +INSERT INTO t_arrayjoin_bf_domain_str VALUES (1, ['abc']), (2, ['abc\0']); + +-- String element vs FixedString constant: comparison strips the padding the conversion keeps. +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_str WHERE arrayJoin(s) = toFixedString('abc', 5) SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_str WHERE arrayJoin(s) = toFixedString('abc', 5) SETTINGS use_skip_indexes = 0); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_str WHERE arrayJoin(s) IN (SELECT toFixedString('abc', 5)) SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_str WHERE arrayJoin(s) IN (SELECT toFixedString('abc', 5)) SETTINGS use_skip_indexes = 0); +-- String element vs Enum constant: the field carries the number, the element stores the label. +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_str WHERE arrayJoin(s) = CAST('abc', 'Enum8(\'abc\' = 1)') SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_str WHERE arrayJoin(s) = CAST('abc', 'Enum8(\'abc\' = 1)') SETTINGS use_skip_indexes = 0); + +DROP TABLE t_arrayjoin_bf_domain_str; + +DROP TABLE IF EXISTS t_arrayjoin_bf_domain_fixed; + +CREATE TABLE t_arrayjoin_bf_domain_fixed +( + id UInt64, + f Array(FixedString(3)), + INDEX idx_f f TYPE bloom_filter(0.0001) GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +INSERT INTO t_arrayjoin_bf_domain_fixed VALUES (1, ['V0']); + +-- A wider FixedString set element: the narrowing conversion would throw TOO_LARGE_STRING_SIZE. +SELECT count() FROM t_arrayjoin_bf_domain_fixed WHERE arrayJoin(f) IN (SELECT toFixedString('V0', 5)) SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_domain_fixed WHERE arrayJoin(f) IN (SELECT toFixedString('V0', 5)) SETTINGS use_skip_indexes = 0; + +DROP TABLE t_arrayjoin_bf_domain_fixed; + +DROP TABLE IF EXISTS t_arrayjoin_bf_domain_num; + +CREATE TABLE t_arrayjoin_bf_domain_num +( + id UInt64, + u Array(UInt8), + f Array(Float64), + INDEX idx_u u TYPE bloom_filter(0.0001) GRANULARITY 1, + INDEX idx_f f TYPE bloom_filter(0.0001) GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +-- Negative zero compares equal to positive zero but hashes differently. +INSERT INTO t_arrayjoin_bf_domain_num VALUES (1, [5], [-0.0]), (2, [7], [1.25]); + +-- Numeric element vs unparsable String set: the conversion would throw CANNOT_PARSE_TEXT. +SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) IN (SELECT 'not-a-number') SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) IN (SELECT 'not-a-number') SETTINGS use_skip_indexes = 0; +-- Float element: -0.0 must still be found by an equality against +0.0. +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(f) = 0.0 SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(f) = 0.0 SETTINGS use_skip_indexes = 0); +-- Float element vs Decimal constant: the conversion would throw TYPE_MISMATCH. +SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(f) = toDecimal64(1.25, 2) SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(f) = toDecimal64(1.25, 2) SETTINGS use_skip_indexes = 0; +-- Cross-integer stays admitted: a UInt8 element against a UInt64 constant still prunes, both forms. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) = toUInt64(5)) WHERE explain ILIKE '%Granules: 1/2%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) IN (SELECT toUInt64(5))) WHERE explain ILIKE '%Granules: 1/2%'; +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) = toUInt64(5) SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) = toUInt64(5) SETTINGS use_skip_indexes = 0); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) IN (SELECT toUInt64(5)) SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) IN (SELECT toUInt64(5)) SETTINGS use_skip_indexes = 0); +-- A value outside the element's range matches nothing, with or without the index. +SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) = 100000 SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_domain_num WHERE arrayJoin(u) = 100000 SETTINGS use_skip_indexes = 0; + +DROP TABLE t_arrayjoin_bf_domain_num; + +DROP TABLE IF EXISTS t_arrayjoin_bf_domain_ip; + +CREATE TABLE t_arrayjoin_bf_domain_ip +( + id UInt64, + v Array(IPv4), + INDEX idx_v v TYPE bloom_filter(0.0001) GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +INSERT INTO t_arrayjoin_bf_domain_ip VALUES (1, ['1.2.3.4']); + +-- IPv4 element vs the equivalent mapped IPv6 constant: the conversion would throw TYPE_MISMATCH. +SELECT count() FROM t_arrayjoin_bf_domain_ip WHERE arrayJoin(v) = toIPv6('::ffff:1.2.3.4') SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_domain_ip WHERE arrayJoin(v) = toIPv6('::ffff:1.2.3.4') SETTINGS use_skip_indexes = 0; + +DROP TABLE t_arrayjoin_bf_domain_ip; + +DROP TABLE IF EXISTS t_arrayjoin_bf_domain_enum; + +CREATE TABLE t_arrayjoin_bf_domain_enum +( + id UInt64, + e Array(Enum8('a' = 1, 'b' = 2)), + INDEX idx_e e TYPE bloom_filter(0.0001) GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +-- Row 2 holds a different label, so a granule can be pruned. +INSERT INTO t_arrayjoin_bf_domain_enum VALUES (1, ['a']), (2, ['b']); + +-- An unknown label with validation disabled: the conversion would throw UNKNOWN_ELEMENT_OF_ENUM. +SELECT count() FROM t_arrayjoin_bf_domain_enum WHERE arrayJoin(e) = 'missing' SETTINGS use_skip_indexes = 1, validate_enum_literals_in_operators = 0; +SELECT count() FROM t_arrayjoin_bf_domain_enum WHERE arrayJoin(e) = 'missing' SETTINGS use_skip_indexes = 0, validate_enum_literals_in_operators = 0; +-- The matching label still prunes: an identical element and constant type is admitted. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_domain_enum WHERE arrayJoin(e) = CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')) WHERE explain ILIKE '%Granules: 1/2%'; +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_enum WHERE arrayJoin(e) = CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)') SETTINGS use_skip_indexes = 1); +SELECT groupArray(id) FROM (SELECT id FROM t_arrayjoin_bf_domain_enum WHERE arrayJoin(e) = CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)') SETTINGS use_skip_indexes = 0); + +DROP TABLE t_arrayjoin_bf_domain_enum; + +DROP TABLE IF EXISTS t_arrayjoin_bf_domain_lc; + +CREATE TABLE t_arrayjoin_bf_domain_lc +( + id UInt64, + tags Array(LowCardinality(String)), + INDEX idx_tags tags TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 8192, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +INSERT INTO t_arrayjoin_bf_domain_lc SELECT number, [concat('tag_', toString(number))] FROM numbers(100000); + +-- A LowCardinality wrapper is stripped before the comparison, so both derivations still prune. +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_domain_lc WHERE arrayJoin(tags) = 'tag_42') WHERE explain ILIKE '%Granules: 2/13%'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_arrayjoin_bf_domain_lc WHERE arrayJoin(tags) IN ('tag_42')) WHERE explain ILIKE '%Granules: 2/13%'; +SELECT count() FROM t_arrayjoin_bf_domain_lc WHERE arrayJoin(tags) = 'tag_42' SETTINGS use_skip_indexes = 1; +SELECT count() FROM t_arrayjoin_bf_domain_lc WHERE arrayJoin(tags) = 'tag_42' SETTINGS use_skip_indexes = 0; + +DROP TABLE t_arrayjoin_bf_domain_lc; diff --git a/tests/queries/0_stateless/04368_mergetree_projection_table_function_row_policy.reference b/tests/queries/0_stateless/04368_mergetree_projection_table_function_row_policy.reference new file mode 100644 index 000000000000..c370fcd93350 --- /dev/null +++ b/tests/queries/0_stateless/04368_mergetree_projection_table_function_row_policy.reference @@ -0,0 +1,45 @@ +-- no policy: table function returns all rows +Alice +Bob +Carol +Dave +-- base table honours the policy +Alice +Carol +-- projection stores the policy column: filter is applied +Alice +Carol +-- filter applied even when the policy column is selected explicitly +Alice engineering +Carol engineering +-- projection lacks the policy column: read is refused +-- bare-column policy is enforced (expect 1, 3) +1 +3 +-- policy on the _partition_id virtual is enforced (expect 1, 3) +1 +3 +-- DEFAULT column stored: filtered on the stored value (expect 1, 2) +1 +2 +-- user PREWHERE does not observe policy-hidden rows (expect 1, 3) +1 +3 +-- LIMIT returns visible rows, not rows hidden by the policy (expect 1) +1 +-- read-in-order + LIMIT skips the policy-hidden leading rows (expect 51) +51 +-- row policy and additional_table_filters both apply (expect 1, 4) +1 +4 +-- policy on an ALIAS column: read is refused +-- policy on a DEFAULT column with only its dependency stored: read is refused +-- policy on a MATERIALIZED column with only its dependency stored: read is refused +-- policy on a position-relative virtual (_part_offset): read is refused +-- same position-relative virtual wrapped in a SQL UDF: read is refused +-- policy on a column shadowed by a projection expression: read is refused +-- aggregate projection under a row policy: read is refused +-- policy on the _block_number virtual: read is refused +-- policy on the projection-only _parent_part_offset name: read is refused +-- table-function row policy combined with a parent policy: read is refused +-- pending on-the-fly mutation under a row policy: read is refused diff --git a/tests/queries/0_stateless/04368_mergetree_projection_table_function_row_policy.sql b/tests/queries/0_stateless/04368_mergetree_projection_table_function_row_policy.sql new file mode 100644 index 000000000000..e1e4e5c452b5 --- /dev/null +++ b/tests/queries/0_stateless/04368_mergetree_projection_table_function_row_policy.sql @@ -0,0 +1,368 @@ +-- Tags: no-parallel +-- ^ the UDF case creates a global SQL UDF (CREATE FUNCTION), which cannot run concurrently. +-- mergeTreeProjection used to ignore the parent table's row policy (clickhouse-private#53773). It now +-- resolves the policy against the projection with the analyzer and refuses when it can't be enforced. + +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS users_rls_proj; +DROP ROW POLICY IF EXISTS rp_users_rls_proj ON users_rls_proj; + +CREATE TABLE users_rls_proj (id UInt64, name String, department String, salary UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO users_rls_proj VALUES (1, 'Alice', 'engineering', 100000), (2, 'Bob', 'finance', 120000), (3, 'Carol', 'engineering', 110000), (4, 'Dave', 'hr', 90000); + +-- proj_with_dept stores the policy column `department`; proj_no_dept does not. +ALTER TABLE users_rls_proj ADD PROJECTION proj_with_dept (SELECT id, name, salary ORDER BY department); +ALTER TABLE users_rls_proj ADD PROJECTION proj_no_dept (SELECT id, name ORDER BY id); +ALTER TABLE users_rls_proj MATERIALIZE PROJECTION proj_with_dept SETTINGS mutations_sync = 2; +ALTER TABLE users_rls_proj MATERIALIZE PROJECTION proj_no_dept SETTINGS mutations_sync = 2; + +SELECT '-- no policy: table function returns all rows'; +SELECT name FROM mergeTreeProjection(currentDatabase(), 'users_rls_proj', 'proj_with_dept') ORDER BY name; + +CREATE ROW POLICY rp_users_rls_proj ON users_rls_proj FOR SELECT USING department = 'engineering' TO ALL; + +SELECT '-- base table honours the policy'; +SELECT name FROM users_rls_proj ORDER BY name; + +SELECT '-- projection stores the policy column: filter is applied'; +SELECT name FROM mergeTreeProjection(currentDatabase(), 'users_rls_proj', 'proj_with_dept') ORDER BY name; + +SELECT '-- filter applied even when the policy column is selected explicitly'; +SELECT name, department FROM mergeTreeProjection(currentDatabase(), 'users_rls_proj', 'proj_with_dept') ORDER BY name; + +SELECT '-- projection lacks the policy column: read is refused'; +SELECT name FROM mergeTreeProjection(currentDatabase(), 'users_rls_proj', 'proj_no_dept') ORDER BY name; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_users_rls_proj ON users_rls_proj; +DROP TABLE users_rls_proj; + +-- A bare-column policy (`USING flag`). +DROP TABLE IF EXISTS bare_rls_proj; +DROP ROW POLICY IF EXISTS rp_bare_rls_proj ON bare_rls_proj; + +CREATE TABLE bare_rls_proj (id UInt64, visible UInt8) ENGINE = MergeTree ORDER BY id; +INSERT INTO bare_rls_proj VALUES (1, 1), (2, 0), (3, 1); + +ALTER TABLE bare_rls_proj ADD PROJECTION p (SELECT id, visible ORDER BY id); +ALTER TABLE bare_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_bare_rls_proj ON bare_rls_proj FOR SELECT USING visible TO ALL; + +SELECT '-- bare-column policy is enforced (expect 1, 3)'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'bare_rls_proj', 'p') ORDER BY id; + +DROP ROW POLICY rp_bare_rls_proj ON bare_rls_proj; +DROP TABLE bare_rls_proj; + +-- A part-identity virtual (`_partition_id`) has the same value in the projection as in the parent. +DROP TABLE IF EXISTS pid_rls_proj; +DROP ROW POLICY IF EXISTS rp_pid_rls_proj ON pid_rls_proj; + +CREATE TABLE pid_rls_proj (id UInt64, val UInt64) ENGINE = MergeTree PARTITION BY (id % 2) ORDER BY id; +INSERT INTO pid_rls_proj VALUES (1, 10), (2, 20), (3, 30), (4, 40); + +ALTER TABLE pid_rls_proj ADD PROJECTION p (SELECT id, val ORDER BY val); +ALTER TABLE pid_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_pid_rls_proj ON pid_rls_proj FOR SELECT USING _partition_id = '1' TO ALL; + +SELECT '-- policy on the _partition_id virtual is enforced (expect 1, 3)'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'pid_rls_proj', 'p') ORDER BY id; + +DROP ROW POLICY rp_pid_rls_proj ON pid_rls_proj; +DROP TABLE pid_rls_proj; + +-- DEFAULT column stored in the projection: filtered on the stored value, not `b + 1` (c = 999 stays hidden). +DROP TABLE IF EXISTS default_rls_proj; +DROP ROW POLICY IF EXISTS rp_default_rls_proj ON default_rls_proj; + +CREATE TABLE default_rls_proj (a UInt64, b UInt64, c UInt64 DEFAULT b + 1) ENGINE = MergeTree ORDER BY a; +INSERT INTO default_rls_proj (a, b) VALUES (1, 10), (2, 20); +INSERT INTO default_rls_proj (a, b, c) VALUES (3, 30, 999); + +ALTER TABLE default_rls_proj ADD PROJECTION p (SELECT a, c ORDER BY a); +ALTER TABLE default_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_default_rls_proj ON default_rls_proj FOR SELECT USING c < 100 TO ALL; + +SELECT '-- DEFAULT column stored: filtered on the stored value (expect 1, 2)'; +SELECT a FROM mergeTreeProjection(currentDatabase(), 'default_rls_proj', 'p') ORDER BY a; + +DROP ROW POLICY rp_default_rls_proj ON default_rls_proj; +DROP TABLE default_rls_proj; + +-- A user PREWHERE must not observe rows the policy hides: the policy filter runs first. +DROP TABLE IF EXISTS prewhere_rls_proj; +DROP ROW POLICY IF EXISTS rp_prewhere_rls_proj ON prewhere_rls_proj; + +CREATE TABLE prewhere_rls_proj (id UInt64, secret String, val UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO prewhere_rls_proj VALUES (1, 'public', 10), (2, 'private', 20), (3, 'public', 30); + +ALTER TABLE prewhere_rls_proj ADD PROJECTION p (SELECT id, secret, val ORDER BY secret); +ALTER TABLE prewhere_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_prewhere_rls_proj ON prewhere_rls_proj FOR SELECT USING secret = 'public' TO ALL; + +SELECT '-- user PREWHERE does not observe policy-hidden rows (expect 1, 3)'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'prewhere_rls_proj', 'p') +PREWHERE throwIf(secret = 'private', 'row policy leak') = 0 ORDER BY id; + +DROP ROW POLICY rp_prewhere_rls_proj ON prewhere_rls_proj; +DROP TABLE prewhere_rls_proj; + +-- A LIMIT must not stop the read before the policy filters (hiding the first 50 rows, asking for 1). +DROP TABLE IF EXISTS limit_rls_proj; +DROP ROW POLICY IF EXISTS rp_limit_rls_proj ON limit_rls_proj; + +CREATE TABLE limit_rls_proj (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO limit_rls_proj SELECT number FROM numbers(1, 100); + +ALTER TABLE limit_rls_proj ADD PROJECTION p (SELECT id ORDER BY id); +ALTER TABLE limit_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_limit_rls_proj ON limit_rls_proj FOR SELECT USING id > 50 TO ALL; + +SELECT '-- LIMIT returns visible rows, not rows hidden by the policy (expect 1)'; +SELECT count() FROM (SELECT id FROM mergeTreeProjection(currentDatabase(), 'limit_rls_proj', 'p') LIMIT 1); + +DROP ROW POLICY rp_limit_rls_proj ON limit_rls_proj; +DROP TABLE limit_rls_proj; + +-- read-in-order + LIMIT: the in-order limit is soft, so hidden leading rows are skipped (expect 51, not empty). +DROP TABLE IF EXISTS order_limit_rls_proj; +DROP ROW POLICY IF EXISTS rp_order_limit_rls_proj ON order_limit_rls_proj; + +CREATE TABLE order_limit_rls_proj (id UInt64, visible UInt8) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 2; +INSERT INTO order_limit_rls_proj SELECT number + 1, number >= 50 FROM numbers(200); + +ALTER TABLE order_limit_rls_proj ADD PROJECTION p (SELECT id, visible ORDER BY id); +ALTER TABLE order_limit_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_order_limit_rls_proj ON order_limit_rls_proj FOR SELECT USING visible TO ALL; + +SELECT '-- read-in-order + LIMIT skips the policy-hidden leading rows (expect 51)'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'order_limit_rls_proj', 'p') ORDER BY id LIMIT 1 +SETTINGS optimize_read_in_order = 1; + +DROP ROW POLICY rp_order_limit_rls_proj ON order_limit_rls_proj; +DROP TABLE order_limit_rls_proj; + +-- row policy and additional_table_filters must both apply, even when the filtered column is not selected. +DROP TABLE IF EXISTS addfilter_rls_proj; +DROP ROW POLICY IF EXISTS rp_addfilter_rls_proj ON addfilter_rls_proj; + +CREATE TABLE addfilter_rls_proj (id UInt64, tenant String) ENGINE = MergeTree ORDER BY id; +INSERT INTO addfilter_rls_proj VALUES (1, 'a'), (2, 'x'), (3, 'y'), (4, 'a'); + +ALTER TABLE addfilter_rls_proj ADD PROJECTION p (SELECT id, tenant ORDER BY tenant); +ALTER TABLE addfilter_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_addfilter_rls_proj ON addfilter_rls_proj FOR SELECT USING tenant != 'x' TO ALL; + +SELECT '-- row policy and additional_table_filters both apply (expect 1, 4)'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'addfilter_rls_proj', 'p') AS p ORDER BY id +SETTINGS additional_table_filters = {'p' : 'tenant != ''y'''}; + +DROP ROW POLICY rp_addfilter_rls_proj ON addfilter_rls_proj; +DROP TABLE addfilter_rls_proj; + +-- The remaining cases cannot be enforced against the projection, so the read is refused. + +-- ALIAS column: the projection stores `b`, not the alias `c`, so `c` is unknown when resolved there. +DROP TABLE IF EXISTS alias_rls_proj; +DROP ROW POLICY IF EXISTS rp_alias_rls_proj ON alias_rls_proj; + +CREATE TABLE alias_rls_proj (a UInt64, b UInt64, c UInt64 ALIAS b + 1) ENGINE = MergeTree ORDER BY a; +INSERT INTO alias_rls_proj (a, b) VALUES (1, 10), (2, 20), (3, 30); + +ALTER TABLE alias_rls_proj ADD PROJECTION p (SELECT a, b ORDER BY a); +ALTER TABLE alias_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_alias_rls_proj ON alias_rls_proj FOR SELECT USING c > 21 TO ALL; + +SELECT '-- policy on an ALIAS column: read is refused'; +SELECT a FROM mergeTreeProjection(currentDatabase(), 'alias_rls_proj', 'p') ORDER BY a; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_alias_rls_proj ON alias_rls_proj; +DROP TABLE alias_rls_proj; + +-- DEFAULT column, only its dependency stored: the stored value can differ from `b + 1`, so can't rebuild it. +DROP TABLE IF EXISTS default_dep_rls_proj; +DROP ROW POLICY IF EXISTS rp_default_dep_rls_proj ON default_dep_rls_proj; + +CREATE TABLE default_dep_rls_proj (a UInt64, b UInt64, c UInt64 DEFAULT b + 1) ENGINE = MergeTree ORDER BY a; +INSERT INTO default_dep_rls_proj (a, b) VALUES (1, 10), (2, 20); +INSERT INTO default_dep_rls_proj (a, b, c) VALUES (3, 30, 5); + +ALTER TABLE default_dep_rls_proj ADD PROJECTION p (SELECT a, b ORDER BY a); +ALTER TABLE default_dep_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_default_dep_rls_proj ON default_dep_rls_proj FOR SELECT USING c > 20 TO ALL; + +SELECT '-- policy on a DEFAULT column with only its dependency stored: read is refused'; +SELECT a FROM mergeTreeProjection(currentDatabase(), 'default_dep_rls_proj', 'p') ORDER BY a; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_default_dep_rls_proj ON default_dep_rls_proj; +DROP TABLE default_dep_rls_proj; + +-- MATERIALIZED column, only its dependency stored: stored value can diverge (e.g. after ALTER MODIFY). +DROP TABLE IF EXISTS materialized_dep_rls_proj; +DROP ROW POLICY IF EXISTS rp_materialized_dep_rls_proj ON materialized_dep_rls_proj; + +CREATE TABLE materialized_dep_rls_proj (x UInt64, m UInt64 MATERIALIZED x + 1) ENGINE = MergeTree ORDER BY x; +INSERT INTO materialized_dep_rls_proj (x) VALUES (1), (2), (3); + +ALTER TABLE materialized_dep_rls_proj ADD PROJECTION p (SELECT x ORDER BY x); +ALTER TABLE materialized_dep_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_materialized_dep_rls_proj ON materialized_dep_rls_proj FOR SELECT USING m > 2 TO ALL; + +SELECT '-- policy on a MATERIALIZED column with only its dependency stored: read is refused'; +SELECT x FROM mergeTreeProjection(currentDatabase(), 'materialized_dep_rls_proj', 'p') ORDER BY x; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_materialized_dep_rls_proj ON materialized_dep_rls_proj; +DROP TABLE materialized_dep_rls_proj; + +-- `_part_offset`: the projection reorders rows, so its value isn't the parent's - refused, direct and via UDF. +DROP TABLE IF EXISTS virt_rls_proj; +DROP ROW POLICY IF EXISTS rp_virt_rls_proj ON virt_rls_proj; +DROP FUNCTION IF EXISTS rp_visible_04368; + +CREATE TABLE virt_rls_proj (id UInt64, val UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO virt_rls_proj VALUES (1, 30), (2, 20), (3, 10); + +ALTER TABLE virt_rls_proj ADD PROJECTION p (SELECT _part_offset, id, val ORDER BY val); +ALTER TABLE virt_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_virt_rls_proj ON virt_rls_proj FOR SELECT USING _part_offset < 1 TO ALL; + +SELECT '-- policy on a position-relative virtual (_part_offset): read is refused'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'virt_rls_proj', 'p') ORDER BY id; -- { serverError ACCESS_DENIED } + +CREATE FUNCTION rp_visible_04368 AS (x) -> x < 1; +DROP ROW POLICY rp_virt_rls_proj ON virt_rls_proj; +CREATE ROW POLICY rp_virt_rls_proj ON virt_rls_proj FOR SELECT USING rp_visible_04368(_part_offset) TO ALL; + +SELECT '-- same position-relative virtual wrapped in a SQL UDF: read is refused'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'virt_rls_proj', 'p') ORDER BY id; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_virt_rls_proj ON virt_rls_proj; +DROP FUNCTION rp_visible_04368; +DROP TABLE virt_rls_proj; + +-- a projection expression bound to a parent column name isn't exposed under that name, so it can't resolve. +DROP TABLE IF EXISTS shadow_rls_proj; +DROP ROW POLICY IF EXISTS rp_shadow_rls_proj ON shadow_rls_proj; + +CREATE TABLE shadow_rls_proj (a UInt8, c UInt8) ENGINE = MergeTree ORDER BY a; +INSERT INTO shadow_rls_proj VALUES (1, 0), (2, 1), (3, 0); + +ALTER TABLE shadow_rls_proj ADD PROJECTION p (SELECT a, (a = 1) AS c ORDER BY a); +ALTER TABLE shadow_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_shadow_rls_proj ON shadow_rls_proj FOR SELECT USING c TO ALL; + +SELECT '-- policy on a column shadowed by a projection expression: read is refused'; +SELECT a FROM mergeTreeProjection(currentDatabase(), 'shadow_rls_proj', 'p') ORDER BY a; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_shadow_rls_proj ON shadow_rls_proj; +DROP TABLE shadow_rls_proj; + +-- aggregate projection: a per-row policy can't be enforced after aggregation, even on the GROUP BY key. +DROP TABLE IF EXISTS agg_rls_proj; +DROP ROW POLICY IF EXISTS rp_agg_rls_proj ON agg_rls_proj; + +CREATE TABLE agg_rls_proj (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; +INSERT INTO agg_rls_proj VALUES (1, 10), (1, 20), (2, 30); + +ALTER TABLE agg_rls_proj ADD PROJECTION p (SELECT key, sum(value) GROUP BY key); +ALTER TABLE agg_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_agg_rls_proj ON agg_rls_proj FOR SELECT USING key = 1 TO ALL; + +SELECT '-- aggregate projection under a row policy: read is refused'; +SELECT key FROM mergeTreeProjection(currentDatabase(), 'agg_rls_proj', 'p'); -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_agg_rls_proj ON agg_rls_proj; +DROP TABLE agg_rls_proj; + +-- `_block_number` isn't preserved by the projection (synthesized from the parent part), so it can't be enforced. +DROP TABLE IF EXISTS blocknum_rls_proj; +DROP ROW POLICY IF EXISTS rp_blocknum_rls_proj ON blocknum_rls_proj; + +CREATE TABLE blocknum_rls_proj (id UInt64, val UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO blocknum_rls_proj VALUES (1, 10), (2, 20), (3, 30); + +ALTER TABLE blocknum_rls_proj ADD PROJECTION p (SELECT id, val ORDER BY val); +ALTER TABLE blocknum_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_blocknum_rls_proj ON blocknum_rls_proj FOR SELECT USING _block_number = 1 TO ALL; + +SELECT '-- policy on the _block_number virtual: read is refused'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'blocknum_rls_proj', 'p') ORDER BY id; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_blocknum_rls_proj ON blocknum_rls_proj; +DROP TABLE blocknum_rls_proj; + +-- `_parent_part_offset` is a projection-only name absent on the parent, so a policy on it can't be enforced. +DROP TABLE IF EXISTS pparent_rls_proj; +DROP ROW POLICY IF EXISTS rp_pparent_rls_proj ON pparent_rls_proj; + +CREATE TABLE pparent_rls_proj (id UInt64, val UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO pparent_rls_proj VALUES (1, 10), (2, 20), (3, 30); + +ALTER TABLE pparent_rls_proj ADD PROJECTION p (SELECT _part_offset, id, val ORDER BY val); +ALTER TABLE pparent_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_pparent_rls_proj ON pparent_rls_proj FOR SELECT USING _parent_part_offset = 0 TO ALL; + +SELECT '-- policy on the projection-only _parent_part_offset name: read is refused'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'pparent_rls_proj', 'p') ORDER BY id; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_pparent_rls_proj ON pparent_rls_proj; +DROP TABLE pparent_rls_proj; + +-- a policy on the table function itself (_table_function.*) must not be dropped; can't combine, so refuse. +DROP TABLE IF EXISTS tf_rls_proj; +DROP ROW POLICY IF EXISTS rp_tf_rls_proj ON _table_function.*; +DROP ROW POLICY IF EXISTS rp_tf_parent_rls_proj ON tf_rls_proj; + +CREATE TABLE tf_rls_proj (id UInt64, dept String) ENGINE = MergeTree ORDER BY id; +INSERT INTO tf_rls_proj VALUES (1, 'eng'), (2, 'fin'), (3, 'eng'); + +ALTER TABLE tf_rls_proj ADD PROJECTION p (SELECT id, dept ORDER BY dept); +ALTER TABLE tf_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +CREATE ROW POLICY rp_tf_rls_proj ON _table_function.* FOR SELECT USING 0 TO ALL; +CREATE ROW POLICY rp_tf_parent_rls_proj ON tf_rls_proj FOR SELECT USING dept = 'eng' TO ALL; + +SELECT '-- table-function row policy combined with a parent policy: read is refused'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'tf_rls_proj', 'p') ORDER BY id; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_tf_rls_proj ON _table_function.*; +DROP ROW POLICY rp_tf_parent_rls_proj ON tf_rls_proj; +DROP TABLE tf_rls_proj; + +-- an on-the-fly mutation applies to the parent read but not the projection, so the projection is stale - refused. +DROP TABLE IF EXISTS onfly_rls_proj; +DROP ROW POLICY IF EXISTS rp_onfly_rls_proj ON onfly_rls_proj; + +CREATE TABLE onfly_rls_proj (id UInt64, visible UInt8) ENGINE = MergeTree ORDER BY id; +INSERT INTO onfly_rls_proj VALUES (1, 1), (2, 1), (3, 1); + +ALTER TABLE onfly_rls_proj ADD PROJECTION p (SELECT id, visible ORDER BY id); +ALTER TABLE onfly_rls_proj MATERIALIZE PROJECTION p SETTINGS mutations_sync = 2; + +SYSTEM STOP MERGES onfly_rls_proj; +ALTER TABLE onfly_rls_proj UPDATE visible = 0 WHERE id = 1 SETTINGS mutations_sync = 0; + +CREATE ROW POLICY rp_onfly_rls_proj ON onfly_rls_proj FOR SELECT USING visible TO ALL; + +SELECT '-- pending on-the-fly mutation under a row policy: read is refused'; +SELECT id FROM mergeTreeProjection(currentDatabase(), 'onfly_rls_proj', 'p') ORDER BY id +SETTINGS apply_mutations_on_fly = 1; -- { serverError ACCESS_DENIED } + +DROP ROW POLICY rp_onfly_rls_proj ON onfly_rls_proj; +SYSTEM START MERGES onfly_rls_proj; +DROP TABLE onfly_rls_proj; diff --git a/tests/queries/0_stateless/04408_text_index_serialization_version_setting.reference b/tests/queries/0_stateless/04408_text_index_serialization_version_setting.reference new file mode 100644 index 000000000000..961882d95f0d --- /dev/null +++ b/tests/queries/0_stateless/04408_text_index_serialization_version_setting.reference @@ -0,0 +1,33 @@ +-- default value +v1_with_codec +-- invalid value is rejected +-- v0_initial version: round-trip read +1024 +512 +0 +-- v0_initial version: merge keeps the format readable +1024 +512 +-- v1_with_codec version: round-trip read +512 +512 +-- v2_with_positions version: phrase search round-trip +512 +512 +0 +-- v2_with_positions version: merge keeps the format readable +512 +512 +-- a posting list codec setting overrides the v0_initial preference +512 +-- altering into the same combination also keeps the index writable +512 +-- a posting list codec index argument also overrides the v0_initial preference +512 +-- a phrase search index overrides an older version preference on CREATE +512 +0 +-- adding a phrase search index on a table pinned to an older version +512 +-- pinning the version on an existing phrase search table keeps the index writable +512 diff --git a/tests/queries/0_stateless/04408_text_index_serialization_version_setting.sql b/tests/queries/0_stateless/04408_text_index_serialization_version_setting.sql new file mode 100644 index 000000000000..59a50ebd9d8c --- /dev/null +++ b/tests/queries/0_stateless/04408_text_index_serialization_version_setting.sql @@ -0,0 +1,174 @@ +-- Tags: no-parallel-replicas + +-- Tests the `text_index_serialization_version` MergeTree setting that selects the on-disk text index format +-- version ('v0_initial', 'v1_with_codec' or 'v2_with_positions'), used to preserve forward compatibility +-- during upgrades. + +SET enable_analyzer = 1; + +SELECT '-- default value'; +SELECT value FROM system.merge_tree_settings WHERE name = 'text_index_serialization_version'; + +SELECT '-- invalid value is rejected'; +DROP TABLE IF EXISTS tab_bad; +CREATE TABLE tab_bad +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha') +) +ENGINE = MergeTree() ORDER BY id +SETTINGS text_index_serialization_version = 'nonsense'; -- { serverError BAD_ARGUMENTS } + +SELECT '-- v0_initial version: round-trip read'; +DROP TABLE IF EXISTS tab_v0_initial; +CREATE TABLE tab_v0_initial +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha') +) +ENGINE = MergeTree() ORDER BY id +-- Pin the codec to 'none': a randomized non-'none' codec would silently bump the format to 'v1_with_codec'. +SETTINGS index_granularity = 64, text_index_serialization_version = 'v0_initial', text_index_posting_list_codec = 'none'; + +INSERT INTO tab_v0_initial SELECT number, 'foo bar' FROM numbers(512); +INSERT INTO tab_v0_initial SELECT number, 'foo baz' FROM numbers(512); + +SELECT count() FROM tab_v0_initial WHERE hasToken(str, 'foo'); +SELECT count() FROM tab_v0_initial WHERE hasToken(str, 'bar'); +SELECT count() FROM tab_v0_initial WHERE hasToken(str, 'qux'); + +SELECT '-- v0_initial version: merge keeps the format readable'; +OPTIMIZE TABLE tab_v0_initial FINAL; +SELECT count() FROM tab_v0_initial WHERE hasToken(str, 'foo'); +SELECT count() FROM tab_v0_initial WHERE hasToken(str, 'baz'); + +SELECT '-- v1_with_codec version: round-trip read'; +DROP TABLE IF EXISTS tab_v1_with_codec; +CREATE TABLE tab_v1_with_codec +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha') +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_serialization_version = 'v1_with_codec', text_index_posting_list_codec = 'bitpacking'; + +INSERT INTO tab_v1_with_codec SELECT number, 'foo bar' FROM numbers(512); +SELECT count() FROM tab_v1_with_codec WHERE hasToken(str, 'foo'); +SELECT count() FROM tab_v1_with_codec WHERE hasToken(str, 'bar'); + +SELECT '-- v2_with_positions version: phrase search round-trip'; +DROP TABLE IF EXISTS tab_v2_with_positions; +CREATE TABLE tab_v2_with_positions +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha', positions = 1) +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_serialization_version = 'v2_with_positions', allow_experimental_text_index_positions = 1; + +INSERT INTO tab_v2_with_positions SELECT number, 'foo bar baz' FROM numbers(512); +SELECT count() FROM tab_v2_with_positions WHERE hasPhrase(str, 'foo bar'); +SELECT count() FROM tab_v2_with_positions WHERE hasPhrase(str, 'bar baz'); +SELECT count() FROM tab_v2_with_positions WHERE hasPhrase(str, 'baz bar'); + +SELECT '-- v2_with_positions version: merge keeps the format readable'; +INSERT INTO tab_v2_with_positions SELECT number, 'foo baz bar' FROM numbers(512); +OPTIMIZE TABLE tab_v2_with_positions FINAL; +SELECT count() FROM tab_v2_with_positions WHERE hasPhrase(str, 'foo bar'); +SELECT count() FROM tab_v2_with_positions WHERE hasPhrase(str, 'baz bar'); + +SELECT '-- a posting list codec setting overrides the v0_initial preference'; +-- The version setting is only a preference: the codec cannot be represented in 'v0_initial', +-- so the index is silently written in 'v1_with_codec' and stays readable. +DROP TABLE IF EXISTS tab_codec_override; +CREATE TABLE tab_codec_override +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha') +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_serialization_version = 'v0_initial', text_index_posting_list_codec = 'bitpacking'; + +INSERT INTO tab_codec_override SELECT number, 'foo bar' FROM numbers(512); +SELECT count() FROM tab_codec_override WHERE hasToken(str, 'foo'); + +SELECT '-- altering into the same combination also keeps the index writable'; +DROP TABLE IF EXISTS tab_alter; +CREATE TABLE tab_alter +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha') +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_posting_list_codec = 'bitpacking'; +ALTER TABLE tab_alter MODIFY SETTING text_index_serialization_version = 'v0_initial'; +INSERT INTO tab_alter SELECT number, 'foo bar' FROM numbers(512); +SELECT count() FROM tab_alter WHERE hasToken(str, 'bar'); + +SELECT '-- a posting list codec index argument also overrides the v0_initial preference'; +-- The table-level codec setting is pinned to 'none', so the bump to 'v1_with_codec' comes from the index argument alone. +DROP TABLE IF EXISTS tab_codec_arg; +CREATE TABLE tab_codec_arg +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha', posting_list_codec = 'bitpacking') +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_serialization_version = 'v0_initial', text_index_posting_list_codec = 'none'; + +INSERT INTO tab_codec_arg SELECT number, 'foo bar' FROM numbers(512); +SELECT count() FROM tab_codec_arg WHERE hasToken(str, 'foo'); + +SELECT '-- a phrase search index overrides an older version preference on CREATE'; +-- Positions cannot be represented in 'v1_with_codec', so the index +-- is silently written in 'v2_with_positions' and phrase search works. +DROP TABLE IF EXISTS tab_positions_pinned; +CREATE TABLE tab_positions_pinned +( + id UInt32, + str String, + INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha', positions = 1) +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_serialization_version = 'v1_with_codec', allow_experimental_text_index_positions = 1; + +INSERT INTO tab_positions_pinned SELECT number, 'foo bar baz' FROM numbers(512); +SELECT count() FROM tab_positions_pinned WHERE hasPhrase(str, 'foo bar'); +SELECT count() FROM tab_positions_pinned WHERE hasPhrase(str, 'baz bar'); + +SELECT '-- adding a phrase search index on a table pinned to an older version'; +DROP TABLE IF EXISTS tab_add_index; +CREATE TABLE tab_add_index +( + id UInt32, + str String +) +ENGINE = MergeTree() ORDER BY id +SETTINGS index_granularity = 64, text_index_serialization_version = 'v1_with_codec', allow_experimental_text_index_positions = 1; +ALTER TABLE tab_add_index ADD INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha', positions = 1); +INSERT INTO tab_add_index SELECT number, 'foo bar' FROM numbers(512); +SELECT count() FROM tab_add_index WHERE hasPhrase(str, 'foo bar'); + +SELECT '-- pinning the version on an existing phrase search table keeps the index writable'; +-- On an existing table the setting is only a preference: the index keeps being written +-- in the 'v2_with_positions' format it requires, so inserts and merges never fail. +ALTER TABLE tab_v2_with_positions MODIFY SETTING text_index_serialization_version = 'v1_with_codec'; +INSERT INTO tab_v2_with_positions SELECT number, 'foo bar qux' FROM numbers(512); +OPTIMIZE TABLE tab_v2_with_positions FINAL; +SELECT count() FROM tab_v2_with_positions WHERE hasPhrase(str, 'bar qux'); + +DROP TABLE tab_v0_initial; +DROP TABLE tab_v1_with_codec; +DROP TABLE tab_v2_with_positions; +DROP TABLE tab_codec_override; +DROP TABLE tab_alter; +DROP TABLE tab_codec_arg; +DROP TABLE tab_positions_pinned; +DROP TABLE tab_add_index; diff --git a/tests/queries/0_stateless/04409_variant_escape_filename_fallback.reference b/tests/queries/0_stateless/04409_variant_escape_filename_fallback.reference new file mode 100644 index 000000000000..21f029baedb3 --- /dev/null +++ b/tests/queries/0_stateless/04409_variant_escape_filename_fallback.reference @@ -0,0 +1,10 @@ +(1,2) 1 2 +(1,2) 1 2 +(1,2) 1 2 +(3,4) 3 4 +(5,6) 5 6 +(5,6) 5 6 +(5,6) 5 6 +(7,8) 7 8 +(1,2) 1 2 +(3,4) 3 4 diff --git a/tests/queries/0_stateless/04409_variant_escape_filename_fallback.sql b/tests/queries/0_stateless/04409_variant_escape_filename_fallback.sql new file mode 100644 index 000000000000..484613855230 --- /dev/null +++ b/tests/queries/0_stateless/04409_variant_escape_filename_fallback.sql @@ -0,0 +1,55 @@ +-- Test bidirectional fallback for escape_variant_subcolumn_filenames setting. +-- When replicas in a hydra group have different values of this setting, +-- parts written with escaping enabled must be readable by replicas with escaping disabled and vice versa. + +set enable_variant_type=1; + +-- Case 1: Start with escaping disabled, then switch to enabled. +-- Parts written without escaping should still be readable after enabling escaping. +drop table if exists test_fallback; +create table test_fallback (v Variant(Tuple(a UInt32, b UInt32))) engine=MergeTree order by tuple() settings min_rows_for_wide_part=0, min_bytes_for_wide_part=0, escape_variant_subcolumn_filenames=0, replace_long_file_name_to_hash=0; +insert into test_fallback select tuple(1, 2)::Tuple(a UInt32, b UInt32); +select v, v.`Tuple(a UInt32, b UInt32)`.a, v.`Tuple(a UInt32, b UInt32)`.b from test_fallback; + +alter table test_fallback modify setting escape_variant_subcolumn_filenames=1; +-- Old part (unescaped filenames) must still be readable with the new setting. +select v, v.`Tuple(a UInt32, b UInt32)`.a, v.`Tuple(a UInt32, b UInt32)`.b from test_fallback; +-- Insert new data with escaping enabled. +insert into test_fallback select tuple(3, 4)::Tuple(a UInt32, b UInt32); +select v, v.`Tuple(a UInt32, b UInt32)`.a, v.`Tuple(a UInt32, b UInt32)`.b from test_fallback order by v.`Tuple(a UInt32, b UInt32)`.a; +drop table test_fallback; + +-- Case 2: Start with escaping enabled, then switch to disabled. +-- Parts written with escaping should still be readable after disabling escaping. +drop table if exists test_fallback; +create table test_fallback (v Variant(Tuple(a UInt32, b UInt32))) engine=MergeTree order by tuple() settings min_rows_for_wide_part=0, min_bytes_for_wide_part=0, escape_variant_subcolumn_filenames=1, replace_long_file_name_to_hash=0; +insert into test_fallback select tuple(5, 6)::Tuple(a UInt32, b UInt32); +select v, v.`Tuple(a UInt32, b UInt32)`.a, v.`Tuple(a UInt32, b UInt32)`.b from test_fallback; + +alter table test_fallback modify setting escape_variant_subcolumn_filenames=0; +-- Old part (escaped filenames) must still be readable with the new setting. +select v, v.`Tuple(a UInt32, b UInt32)`.a, v.`Tuple(a UInt32, b UInt32)`.b from test_fallback; +-- Insert new data with escaping disabled. +insert into test_fallback select tuple(7, 8)::Tuple(a UInt32, b UInt32); +select v, v.`Tuple(a UInt32, b UInt32)`.a, v.`Tuple(a UInt32, b UInt32)`.b from test_fallback order by v.`Tuple(a UInt32, b UInt32)`.a; +drop table test_fallback; + +-- Case 3: RENAME COLUMN after switching escaping from disabled to enabled. +-- Parts written without escaping must survive a column rename after enabling escaping. +drop table if exists test_fallback; +create table test_fallback (v Variant(Tuple(a UInt32, b UInt32))) engine=MergeTree order by tuple() settings min_rows_for_wide_part=0, min_bytes_for_wide_part=0, escape_variant_subcolumn_filenames=0, replace_long_file_name_to_hash=0; +insert into test_fallback select tuple(1, 2)::Tuple(a UInt32, b UInt32); +alter table test_fallback modify setting escape_variant_subcolumn_filenames=1; +alter table test_fallback rename column v to w; +select w, w.`Tuple(a UInt32, b UInt32)`.a, w.`Tuple(a UInt32, b UInt32)`.b from test_fallback; +drop table test_fallback; + +-- Case 4: RENAME COLUMN after switching escaping from enabled to disabled. +-- Parts written with escaping must survive a column rename after disabling escaping. +drop table if exists test_fallback; +create table test_fallback (v Variant(Tuple(a UInt32, b UInt32))) engine=MergeTree order by tuple() settings min_rows_for_wide_part=0, min_bytes_for_wide_part=0, escape_variant_subcolumn_filenames=1, replace_long_file_name_to_hash=0; +insert into test_fallback select tuple(3, 4)::Tuple(a UInt32, b UInt32); +alter table test_fallback modify setting escape_variant_subcolumn_filenames=0; +alter table test_fallback rename column v to w; +select w, w.`Tuple(a UInt32, b UInt32)`.a, w.`Tuple(a UInt32, b UInt32)`.b from test_fallback; +drop table test_fallback; diff --git a/tests/queries/0_stateless/04490_parallel_view_processing_values.reference b/tests/queries/0_stateless/04490_parallel_view_processing_values.reference new file mode 100644 index 000000000000..dcc128d5fa59 --- /dev/null +++ b/tests/queries/0_stateless/04490_parallel_view_processing_values.reference @@ -0,0 +1,4 @@ +pvp_test_parallel_select parallel +pvp_test_parallel_values parallel +pvp_test_serial_select serial +pvp_test_serial_values serial diff --git a/tests/queries/0_stateless/04490_parallel_view_processing_values.sql b/tests/queries/0_stateless/04490_parallel_view_processing_values.sql new file mode 100644 index 000000000000..b2318860641a --- /dev/null +++ b/tests/queries/0_stateless/04490_parallel_view_processing_values.sql @@ -0,0 +1,78 @@ +-- Tags: no-object-storage, no-parallel, no-fasttest +-- no-object-storage: extra S3 threads affect peak_threads_usage +-- no-parallel: peak_threads_usage is sensitive to concurrent queries +-- Regression test for https://github.com/ClickHouse/ClickHouse/issues/106845 + +SET max_threads = 10; +SET max_block_size = 10; + +DROP TABLE IF EXISTS pvp_source; +DROP TABLE IF EXISTS pvp_target; + +CREATE TABLE pvp_source (n UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE pvp_target (n UInt64, mv UInt8, s UInt8) ENGINE = MergeTree ORDER BY tuple(); + +CREATE MATERIALIZED VIEW pvp_mv1 TO pvp_target AS SELECT n, 1 AS mv, sleep(0.1) AS s FROM pvp_source; +CREATE MATERIALIZED VIEW pvp_mv2 TO pvp_target AS SELECT n, 2 AS mv, sleep(0.1) AS s FROM pvp_source; +CREATE MATERIALIZED VIEW pvp_mv3 TO pvp_target AS SELECT n, 3 AS mv, sleep(0.1) AS s FROM pvp_source; + +-- serial: parallel_view_processing=0 +INSERT INTO pvp_source SETTINGS + log_queries = 1, + async_insert = 0, + insert_deduplication_token = 'pvp_test_serial_values', + parallel_view_processing = 0 +VALUES (1); + +-- parallel: parallel_view_processing=1 +INSERT INTO pvp_source SETTINGS + log_queries = 1, + async_insert = 0, + insert_deduplication_token = 'pvp_test_parallel_values', + parallel_view_processing = 1 +VALUES (2); + +-- INSERT ... SELECT with parallel_view_processing=0 → serial +INSERT INTO pvp_source SETTINGS + log_queries = 1, + async_insert = 0, + insert_deduplication_token = 'pvp_test_serial_select', + parallel_view_processing = 0 +SELECT number FROM numbers(1); + +-- INSERT ... SELECT with parallel_view_processing=1 → parallel +INSERT INTO pvp_source SETTINGS + log_queries = 1, + async_insert = 0, + insert_deduplication_token = 'pvp_test_parallel_select', + parallel_view_processing = 1 +SELECT number FROM numbers(1); + +SYSTEM FLUSH LOGS system.query_log; + +-- `peak_threads_usage` is an exact count of threads that concurrently attached to the query's +-- thread group (see `ThreadGroup::linkThread`), not a sampled or timed value, so unlike +-- wall-clock duration it is not affected by interpreter/sanitizer overhead on debug/ASan/MSan/TSan +-- builds: serial (views run one after another) attaches 1-2 threads, parallel (3 views fanned +-- out concurrently) attaches at least 3. +SELECT + Settings['insert_deduplication_token'] AS token, + if(max(peak_threads_usage) >= 3, 'parallel', 'serial') AS mode +FROM system.query_log +WHERE + event_date >= yesterday() + AND event_time >= now() - 600 + AND current_database = currentDatabase() + AND type != 'QueryStart' + AND query_kind = 'Insert' + AND Settings['insert_deduplication_token'] IN ( + 'pvp_test_serial_values', 'pvp_test_parallel_values', + 'pvp_test_serial_select', 'pvp_test_parallel_select') +GROUP BY token +ORDER BY token; + +DROP VIEW pvp_mv1; +DROP VIEW pvp_mv2; +DROP VIEW pvp_mv3; +DROP TABLE pvp_target; +DROP TABLE pvp_source; diff --git a/tests/queries/0_stateless/04492_create_or_replace_materialized_view_populate_keeps_subscription.reference b/tests/queries/0_stateless/04492_create_or_replace_materialized_view_populate_keeps_subscription.reference new file mode 100644 index 000000000000..2a429f36a078 --- /dev/null +++ b/tests/queries/0_stateless/04492_create_or_replace_materialized_view_populate_keeps_subscription.reference @@ -0,0 +1,9 @@ +after first create 1 +after replace, populated 1 +after inserts following replace 1 +after inserts following replace 2 +after inserts following replace 3 +after second replace and insert 1 +after second replace and insert 2 +after second replace and insert 3 +after second replace and insert 4 diff --git a/tests/queries/0_stateless/04492_create_or_replace_materialized_view_populate_keeps_subscription.sql b/tests/queries/0_stateless/04492_create_or_replace_materialized_view_populate_keeps_subscription.sql new file mode 100644 index 000000000000..68560e0253dc --- /dev/null +++ b/tests/queries/0_stateless/04492_create_or_replace_materialized_view_populate_keeps_subscription.sql @@ -0,0 +1,37 @@ +-- Tags: no-ordinary-database, no-replicated-database +-- no-ordinary-database: CREATE OR REPLACE MATERIALIZED VIEW requires an Atomic database. +-- no-replicated-database: POPULATE is not supported in a Replicated database. + +-- `CREATE OR REPLACE MATERIALIZED VIEW ... POPULATE` used to leave the new view unsubscribed from +-- its source table. The replace creates a temporary view, populates it (which caches the temporary +-- name -> temporary storage in the query context), then atomically swaps it with the target via +-- EXCHANGE. The internal DROP of the old table then resolved the temporary name through the stale +-- cache and shut down the live (new) view instead, detaching it from its source. As a result every +-- insert after the replace was silently dropped. See https://github.com/ClickHouse/ClickHouse/issues/108726 + +DROP TABLE IF EXISTS src SYNC; +DROP TABLE IF EXISTS mv SYNC; + +CREATE TABLE src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO src VALUES (1); + +-- The view must already exist so that CREATE OR REPLACE actually replaces it (EXCHANGE), not just creates it. +CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM src; +SELECT 'after first create', count() FROM mv; + +CREATE OR REPLACE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM src; +-- POPULATE re-captured the existing source data. +SELECT 'after replace, populated', count() FROM mv; + +-- The crucial part: the view must still be subscribed to its source after the replace. +INSERT INTO src VALUES (2); +INSERT INTO src VALUES (3); +SELECT 'after inserts following replace', id FROM mv ORDER BY id; + +-- A second replace must keep the subscription working as well. +CREATE OR REPLACE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM src; +INSERT INTO src VALUES (4); +SELECT 'after second replace and insert', id FROM mv ORDER BY id; + +DROP TABLE mv SYNC; +DROP TABLE src SYNC; diff --git a/tests/queries/0_stateless/04493_create_or_replace_materialized_view_populate_keeps_subscription_concurrent.reference b/tests/queries/0_stateless/04493_create_or_replace_materialized_view_populate_keeps_subscription_concurrent.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/04493_create_or_replace_materialized_view_populate_keeps_subscription_concurrent.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/04493_create_or_replace_materialized_view_populate_keeps_subscription_concurrent.sh b/tests/queries/0_stateless/04493_create_or_replace_materialized_view_populate_keeps_subscription_concurrent.sh new file mode 100755 index 000000000000..d1c2047bd2cc --- /dev/null +++ b/tests/queries/0_stateless/04493_create_or_replace_materialized_view_populate_keeps_subscription_concurrent.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Tags: no-ordinary-database, no-replicated-database +# no-ordinary-database: CREATE OR REPLACE MATERIALIZED VIEW requires an Atomic database. +# no-replicated-database: POPULATE is not supported in a Replicated database. + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/108726 +# `CREATE OR REPLACE MATERIALIZED VIEW ... POPULATE` used to leave the new view unsubscribed from +# its source table, so every row inserted after the replace was silently dropped. This is the +# concurrent variant of 04492: the source is hammered with inserts while the replace is in progress +# (`merge_tree_storage_snapshot_sleep_ms` widens the replace's snapshot window so the inserts +# reliably overlap it), and once the replace has completed the new view must still be subscribed. +# +# We prove the subscription is live with a sentinel row inserted *after* everything settles: it must +# reach the view. We deliberately do not assert anything about the rows inserted *during* the +# replace: the source-to-view dependency transfer inside the internal `EXCHANGE` is not yet atomic, +# so a row inserted in that narrow window can be missed, and an insert that lands exactly mid-swap +# can even fail outright with `UNKNOWN_TABLE` (the new view's target table is momentarily +# unreachable). Closing that window is a separate change. This test covers only the deterministic +# fix: the new view stays subscribed once the replace completes. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS src SYNC" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS mv SYNC" + +${CLICKHOUSE_CLIENT} --query "CREATE TABLE src (id UInt64) ENGINE = MergeTree ORDER BY id" +${CLICKHOUSE_CLIENT} --query "INSERT INTO src SELECT number FROM numbers(5000)" + +# The view must already exist so that CREATE OR REPLACE actually replaces it (EXCHANGE), not just creates it. +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM src" + +# Hammer the source with inserts while the replace is in progress. An insert that lands in the brief +# instant the replace swaps the tables can transiently fail with `UNKNOWN_TABLE` (the new view's +# target table is momentarily unreachable, because the source-to-view dependency transfer inside the +# internal `EXCHANGE` is not yet atomic). That residual window is out of scope for this test (see the +# note above), so we tolerate that specific error on the concurrent inserts; any other insert failure +# still fails the test. The contract we assert is the sentinel below, after everything has settled. +( + for j in $(seq 0 9); do + if ! err=$(${CLICKHOUSE_CLIENT} --query "INSERT INTO src SELECT number FROM numbers(100000 + ${j} * 1000, 1000)" 2>&1); then + echo "$err" | grep -qF "UNKNOWN_TABLE" || { echo "$err" >&2; exit 1; } + fi + done +) & +inserts_pid=$! + +${CLICKHOUSE_CLIENT} --merge_tree_storage_snapshot_sleep_ms=150 --query "CREATE OR REPLACE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM src" + +# A concurrent insert may have hit the tolerated residual window above; any other failure fails here. +wait "$inserts_pid" + +# The crucial part: once the (concurrent) replace has completed, the new view must still be +# subscribed to its source - a sentinel row inserted now must reach the view. On the buggy version +# the view was detached by the replace, so this row would never arrive and the count would be 0. +${CLICKHOUSE_CLIENT} --query "INSERT INTO src VALUES (999999999)" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM mv WHERE id = 999999999" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE mv SYNC" +${CLICKHOUSE_CLIENT} --query "DROP TABLE src SYNC" diff --git a/tests/queries/0_stateless/04501_iceberg_datetime64_partition_pruning.reference b/tests/queries/0_stateless/04501_iceberg_datetime64_partition_pruning.reference new file mode 100644 index 000000000000..3d6a5ff111dc --- /dev/null +++ b/tests/queries/0_stateless/04501_iceberg_datetime64_partition_pruning.reference @@ -0,0 +1,6 @@ +--- full read --- +2024-06-15 12:30:00.654321 a +--- with prunning --- +a +--- without prunning --- +a diff --git a/tests/queries/0_stateless/04501_iceberg_datetime64_partition_pruning.sh b/tests/queries/0_stateless/04501_iceberg_datetime64_partition_pruning.sh new file mode 100755 index 000000000000..4efc849bca12 --- /dev/null +++ b/tests/queries/0_stateless/04501_iceberg_datetime64_partition_pruning.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) +# +# Regression test: manifests written by ClickHouse declared timestamp partition fields as a +# bare Avro `long`, without the `timestamp-micros` logical type that Iceberg requires. On +# read such a value parses into an `Int64` field instead of a `DateTime64` one, so partition +# pruning compared it against a `Decimal64` predicate constant that never matches. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}_dt64_pruning" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (t DateTime64, v String) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') + PARTITION BY (t) +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query " + INSERT INTO ${TABLE} VALUES ('2024-06-15 12:30:00.654321', 'a') +" + +echo "--- full read ---" +${CLICKHOUSE_CLIENT} --query "SELECT t, v FROM ${TABLE} ORDER BY t FORMAT TSV" + +echo "--- with prunning ---" +${CLICKHOUSE_CLIENT} --query " + SELECT v FROM ${TABLE} WHERE t = '2024-06-15 12:30:00.654321' SETTINGS use_iceberg_partition_pruning = 1 FORMAT TSV" + +echo "--- without prunning ---" +${CLICKHOUSE_CLIENT} --query " + SELECT v FROM ${TABLE} WHERE t = '2024-06-15 12:30:00.654321' SETTINGS use_iceberg_partition_pruning = 0 FORMAT TSV" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" diff --git a/tests/queries/0_stateless/04507_variant_escape_filename_rename_consistency.reference b/tests/queries/0_stateless/04507_variant_escape_filename_rename_consistency.reference new file mode 100644 index 000000000000..792c35f18836 --- /dev/null +++ b/tests/queries/0_stateless/04507_variant_escape_filename_rename_consistency.reference @@ -0,0 +1,76 @@ +Case 1: escaping 0 -> 1 with RENAME COLUMN +Before rename .bin files: +v.Tuple(a UInt32, b UInt32)%2Ea.bin +v.Tuple(a UInt32, b UInt32)%2Eb.bin +v.variant_discr.bin +Before rename columns_substreams.txt: +columns substreams version: 1 +1 columns: +3 substreams for column `v`: + v.variant_discr + v.Tuple(a UInt32, b UInt32)%2Ea + v.Tuple(a UInt32, b UInt32)%2Eb +After rename .bin files: +w.Tuple(a UInt32, b UInt32)%2Ea.bin +w.Tuple(a UInt32, b UInt32)%2Eb.bin +w.variant_discr.bin +After rename columns_substreams.txt: +columns substreams version: 1 +1 columns: +3 substreams for column `w`: + w.variant_discr + w.Tuple(a UInt32, b UInt32)%2Ea + w.Tuple(a UInt32, b UInt32)%2Eb +CHECK TABLE: +1 +Data: +(1,2) 1 2 +Case 2: escaping 1 -> 0 with RENAME COLUMN +Before rename .bin files: +v.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Ea.bin +v.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Eb.bin +v.variant_discr.bin +Before rename columns_substreams.txt: +columns substreams version: 1 +1 columns: +3 substreams for column `v`: + v.variant_discr + v.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Ea + v.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Eb +After rename .bin files: +w.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Ea.bin +w.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Eb.bin +w.variant_discr.bin +After rename columns_substreams.txt: +columns substreams version: 1 +1 columns: +3 substreams for column `w`: + w.variant_discr + w.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Ea + w.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Eb +CHECK TABLE: +1 +Data: +(3,4) 3 4 +Case 3: escaping 0 -> 1 with RENAME COLUMN, no columns_substreams.txt +Before rename .bin files: +v.Tuple(a UInt32, b UInt32)%2Ea.bin +v.Tuple(a UInt32, b UInt32)%2Eb.bin +v.variant_discr.bin +After rename .bin files: +w.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Ea.bin +w.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Eb.bin +w.variant_discr.bin +Data: +(5,6) 5 6 +Case 4: escaping 1 -> 0 with RENAME COLUMN, no columns_substreams.txt +Before rename .bin files: +v.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Ea.bin +v.Tuple%28a%20UInt32%2C%20b%20UInt32%29%2Eb.bin +v.variant_discr.bin +After rename .bin files: +w.Tuple(a UInt32, b UInt32)%2Ea.bin +w.Tuple(a UInt32, b UInt32)%2Eb.bin +w.variant_discr.bin +Data: +(7,8) 7 8 diff --git a/tests/queries/0_stateless/04507_variant_escape_filename_rename_consistency.sh b/tests/queries/0_stateless/04507_variant_escape_filename_rename_consistency.sh new file mode 100755 index 000000000000..f8894ae912a5 --- /dev/null +++ b/tests/queries/0_stateless/04507_variant_escape_filename_rename_consistency.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Tags: no-darwin, no-object-storage, no-shared-merge-tree +# +# no-darwin: the macOS filesystem (APFS) is case-insensitive, so `MergeTree` hashes stream +# filenames unconditionally and the expected `.bin` names never appear on disk. +# no-object-storage, no-shared-merge-tree: the test inspects the part directory directly, but for +# object storage the local part directory contains metadata files, not the data itself, so +# reading `columns_substreams.txt` from it returns blob metadata instead of the substream list. +# +# The tables pin `min_bytes_for_full_part_storage=0` in addition to the wide-part settings: with +# packed part storage every file of the part lives inside a single `data.cmrk3`-style blob, so the +# individual `.bin` files and `columns_substreams.txt` do not exist as separate files on disk. CI +# randomizes `min_bytes_for_full_part_storage` to a large value, which would otherwise make the +# test fail. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CH="$CLICKHOUSE_CLIENT --enable_variant_type=1" + +# Verify that after RENAME COLUMN with a flipped escape_variant_subcolumn_filenames setting, +# the .bin filenames on disk stay consistent with columns_substreams.txt and CHECK TABLE passes. + +echo "Case 1: escaping 0 -> 1 with RENAME COLUMN" +$CH -q "DROP TABLE IF EXISTS test_rename_escape" +$CH -q "CREATE TABLE test_rename_escape (v Variant(Tuple(a UInt32, b UInt32))) ENGINE=MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, escape_variant_subcolumn_filenames=0, replace_long_file_name_to_hash=0" +$CH -q "INSERT INTO test_rename_escape SELECT tuple(1, 2)::Tuple(a UInt32, b UInt32)" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "Before rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort +echo "Before rename columns_substreams.txt:" +cat "${part_path}columns_substreams.txt" + +$CH -q "ALTER TABLE test_rename_escape MODIFY SETTING escape_variant_subcolumn_filenames=1" +$CH -q "ALTER TABLE test_rename_escape RENAME COLUMN v TO w SETTINGS mutations_sync=2" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "After rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort +echo "After rename columns_substreams.txt:" +cat "${part_path}columns_substreams.txt" + +echo "CHECK TABLE:" +$CH -q "CHECK TABLE test_rename_escape" | cut -f2 + +echo "Data:" +$CH -q "SELECT w, w.\`Tuple(a UInt32, b UInt32)\`.a, w.\`Tuple(a UInt32, b UInt32)\`.b FROM test_rename_escape" + +$CH -q "DROP TABLE test_rename_escape" + +echo "Case 2: escaping 1 -> 0 with RENAME COLUMN" +$CH -q "DROP TABLE IF EXISTS test_rename_escape" +$CH -q "CREATE TABLE test_rename_escape (v Variant(Tuple(a UInt32, b UInt32))) ENGINE=MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, escape_variant_subcolumn_filenames=1, replace_long_file_name_to_hash=0" +$CH -q "INSERT INTO test_rename_escape SELECT tuple(3, 4)::Tuple(a UInt32, b UInt32)" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "Before rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort +echo "Before rename columns_substreams.txt:" +cat "${part_path}columns_substreams.txt" + +$CH -q "ALTER TABLE test_rename_escape MODIFY SETTING escape_variant_subcolumn_filenames=0" +$CH -q "ALTER TABLE test_rename_escape RENAME COLUMN v TO w SETTINGS mutations_sync=2" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "After rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort +echo "After rename columns_substreams.txt:" +cat "${part_path}columns_substreams.txt" + +echo "CHECK TABLE:" +$CH -q "CHECK TABLE test_rename_escape" | cut -f2 + +echo "Data:" +$CH -q "SELECT w, w.\`Tuple(a UInt32, b UInt32)\`.a, w.\`Tuple(a UInt32, b UInt32)\`.b FROM test_rename_escape" + +$CH -q "DROP TABLE test_rename_escape" + +# Cases 3 and 4: same as above but with columns_substreams.txt manually removed +# to exercise the enumerateStreams fallback path in collectFilesForRenames. + +echo "Case 3: escaping 0 -> 1 with RENAME COLUMN, no columns_substreams.txt" +$CH -q "DROP TABLE IF EXISTS test_rename_escape" +$CH -q "CREATE TABLE test_rename_escape (v Variant(Tuple(a UInt32, b UInt32))) ENGINE=MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, escape_variant_subcolumn_filenames=0, replace_long_file_name_to_hash=0" +$CH -q "INSERT INTO test_rename_escape SELECT tuple(5, 6)::Tuple(a UInt32, b UInt32)" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "Before rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort + +$CH -q "DETACH TABLE test_rename_escape" +rm "${part_path}columns_substreams.txt" +$CH -q "ATTACH TABLE test_rename_escape" + +$CH -q "ALTER TABLE test_rename_escape MODIFY SETTING escape_variant_subcolumn_filenames=1" +$CH -q "ALTER TABLE test_rename_escape RENAME COLUMN v TO w SETTINGS mutations_sync=2" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "After rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort + +echo "Data:" +$CH -q "SELECT w, w.\`Tuple(a UInt32, b UInt32)\`.a, w.\`Tuple(a UInt32, b UInt32)\`.b FROM test_rename_escape" + +$CH -q "DROP TABLE test_rename_escape" + +echo "Case 4: escaping 1 -> 0 with RENAME COLUMN, no columns_substreams.txt" +$CH -q "DROP TABLE IF EXISTS test_rename_escape" +$CH -q "CREATE TABLE test_rename_escape (v Variant(Tuple(a UInt32, b UInt32))) ENGINE=MergeTree ORDER BY tuple() SETTINGS min_rows_for_wide_part=0, min_bytes_for_wide_part=0, min_bytes_for_full_part_storage=0, escape_variant_subcolumn_filenames=1, replace_long_file_name_to_hash=0" +$CH -q "INSERT INTO test_rename_escape SELECT tuple(7, 8)::Tuple(a UInt32, b UInt32)" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "Before rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort + +$CH -q "DETACH TABLE test_rename_escape" +rm "${part_path}columns_substreams.txt" +$CH -q "ATTACH TABLE test_rename_escape" + +$CH -q "ALTER TABLE test_rename_escape MODIFY SETTING escape_variant_subcolumn_filenames=0" +$CH -q "ALTER TABLE test_rename_escape RENAME COLUMN v TO w SETTINGS mutations_sync=2" + +part_path=$($CH -q "SELECT path FROM system.parts WHERE table = 'test_rename_escape' AND database = currentDatabase() AND active") + +echo "After rename .bin files:" +ls "$part_path" | grep '\.bin$' | sort + +echo "Data:" +$CH -q "SELECT w, w.\`Tuple(a UInt32, b UInt32)\`.a, w.\`Tuple(a UInt32, b UInt32)\`.b FROM test_rename_escape" + +$CH -q "DROP TABLE test_rename_escape" diff --git a/tests/queries/0_stateless/04510_accurateCastOrDefault_settings.reference b/tests/queries/0_stateless/04510_accurateCastOrDefault_settings.reference new file mode 100644 index 000000000000..91cc3243c903 --- /dev/null +++ b/tests/queries/0_stateless/04510_accurateCastOrDefault_settings.reference @@ -0,0 +1,12 @@ +accurateCastOrDefault with input_format_try_infer_dates=0 +{"d":"2020-01-01"} String +accurateCastOrDefault with input_format_try_infer_dates=1 +{"d":"2020-01-01"} Date +accurateCastOrNull with input_format_try_infer_dates=0 +{"d":"2020-01-01"} String +CAST with input_format_try_infer_dates=0 +{"d":"2020-01-01"} String +accurateCastOrDefault timezone substitution +DateTime(\'Europe/Moscow\') +DateTime64(3, \'Europe/Moscow\') +1000 diff --git a/tests/queries/0_stateless/04510_accurateCastOrDefault_settings.sql b/tests/queries/0_stateless/04510_accurateCastOrDefault_settings.sql new file mode 100644 index 000000000000..7a3de1c5dfe3 --- /dev/null +++ b/tests/queries/0_stateless/04510_accurateCastOrDefault_settings.sql @@ -0,0 +1,24 @@ +-- Accessing the json.d subcolumn requires the analyzer. +SET enable_analyzer = 1; + +-- Test that accurateCastOrDefault respects format settings like input_format_try_infer_dates +SELECT 'accurateCastOrDefault with input_format_try_infer_dates=0'; +SELECT accurateCastOrDefault('{"d" : "2020-01-01"}', 'JSON') AS json, dynamicType(json.d) SETTINGS input_format_try_infer_dates=0, input_format_try_infer_datetimes=0; + +SELECT 'accurateCastOrDefault with input_format_try_infer_dates=1'; +SELECT accurateCastOrDefault('{"d" : "2020-01-01"}', 'JSON') AS json, dynamicType(json.d) SETTINGS input_format_try_infer_dates=1; + +SELECT 'accurateCastOrNull with input_format_try_infer_dates=0'; +SELECT accurateCastOrNull('{"d" : "2020-01-01"}', 'JSON') AS json, dynamicType(json.d) SETTINGS input_format_try_infer_dates=0, input_format_try_infer_datetimes=0; + +SELECT 'CAST with input_format_try_infer_dates=0'; +SELECT CAST('{"d" : "2020-01-01"}', 'JSON') AS json, dynamicType(json.d) SETTINGS input_format_try_infer_dates=0, input_format_try_infer_datetimes=0; + +-- Test that accurateCastOrDefault applies timezone substitution from DateTime source +SELECT 'accurateCastOrDefault timezone substitution'; +SELECT toTypeName(accurateCastOrDefault(toDateTime('2020-01-01', 'Europe/Moscow'), 'DateTime')); +SELECT toTypeName(accurateCastOrDefault(toDateTime('2020-01-01', 'Europe/Moscow'), 'DateTime64')); + +-- Test that accurateCastOrDefault respects DataTypeValidationSettings (forbidden types) +SELECT accurateCastOrDefault('hello', 'FixedString(1000)') SETTINGS allow_suspicious_fixed_string_types=0; -- { serverError ILLEGAL_COLUMN } +SELECT length(accurateCastOrDefault('hello', 'FixedString(1000)')) SETTINGS allow_suspicious_fixed_string_types=1; diff --git a/tests/queries/0_stateless/04512_accurateCastOrDefault_nullable_target.reference b/tests/queries/0_stateless/04512_accurateCastOrDefault_nullable_target.reference new file mode 100644 index 000000000000..3a83183c65db --- /dev/null +++ b/tests/queries/0_stateless/04512_accurateCastOrDefault_nullable_target.reference @@ -0,0 +1,25 @@ +\N +\N +\N +\N +\N +true +123 +\N +\N +42 +42 +\N +\N +\N +42 +42 +42 +\N +\N +\N +\N +\N +\N +\N +\N diff --git a/tests/queries/0_stateless/04512_accurateCastOrDefault_nullable_target.sql b/tests/queries/0_stateless/04512_accurateCastOrDefault_nullable_target.sql new file mode 100644 index 000000000000..87a2ea4ced89 --- /dev/null +++ b/tests/queries/0_stateless/04512_accurateCastOrDefault_nullable_target.sql @@ -0,0 +1,55 @@ +-- When the target type is Nullable, accurateCastOrDefault should return NULL on +-- cast failure, not the inner type's default value. +SELECT accurateCastOrDefault('test', 'Nullable(Bool)'); +SELECT accurateCastOrDefault('not_a_number', 'Nullable(UInt32)'); +SELECT accurateCastOrDefault('bad', 'Nullable(Int64)'); +SELECT accurateCastOrDefault('bad', 'Nullable(Float64)'); +SELECT accurateCastOrDefault('bad', 'Nullable(Date)'); + +-- Successful casts should return the actual value. +SELECT accurateCastOrDefault('1', 'Nullable(Bool)'); +SELECT accurateCastOrDefault('123', 'Nullable(UInt32)'); + +-- NULL input should produce NULL output for Nullable targets. +SELECT accurateCastOrDefault(NULL, 'Nullable(UInt32)'); + +-- A NULL input is a successful cast to a Nullable target, not a failure that +-- should be replaced with an explicit default. +SELECT accurateCastOrDefault(NULL, 'Nullable(UInt32)', CAST(42, 'Nullable(UInt32)')); + +-- A NULL input for a non-nullable target is a failed conversion and must use +-- the caller-supplied default. +SELECT accurateCastOrDefault(NULL, 'UInt32', 42::UInt32); +SELECT toUInt32OrDefault(NULL, 42::UInt32); + +-- The source NULL must be preserved when it is encoded in a low-cardinality +-- nullable column, rather than replaced with the explicit default. +SELECT accurateCastOrDefault(CAST(NULL, 'LowCardinality(Nullable(String))'), 'Nullable(UInt32)', CAST(42, 'Nullable(UInt32)')); + +-- Dynamic and Variant encode NULL with a discriminator rather than a physical +-- null map, but it is still a successful cast to a Nullable target. +SELECT accurateCastOrDefault(CAST(NULL, 'Dynamic'), 'Nullable(UInt32)', CAST(42, 'Nullable(UInt32)')); +SELECT accurateCastOrDefault(CAST(NULL, 'Variant(UInt8, String, Nothing)'), 'Nullable(UInt32)', CAST(42, 'Nullable(UInt32)')); + +-- Targets with native NULL representations must use their own null carrier +-- rather than being forced into an outer Nullable column. +SELECT accurateCastOrDefault(42, 'Dynamic'); +SELECT accurateCastOrDefault(42, 'Variant(UInt8, String)'); +SELECT accurateCastOrDefault(42, 'LowCardinality(Nullable(UInt32))') SETTINGS allow_suspicious_low_cardinality_types = 1; + +-- `Dynamic` and `Variant` source NULLs are preserved for non-Nullable targets +-- when `cast_keep_nullable` is enabled, just like physically Nullable sources. +SET cast_keep_nullable = 1; +SELECT accurateCastOrDefault(CAST(NULL, 'Dynamic'), 'UInt32'); +SELECT accurateCastOrDefault(CAST(NULL, 'Variant(UInt8, String, Nothing)'), 'UInt32'); +SELECT toUInt32OrDefault(CAST(NULL, 'Dynamic')); +SELECT toUInt32OrDefault(CAST(NULL, 'Variant(UInt8, String, Nothing)')); +SELECT toUInt32OrDefault(CAST(NULL, 'LowCardinality(Nullable(String))')); + +-- Native NULL carriers cannot be wrapped in an outer Nullable column. +SELECT accurateCastOrDefault(NULL, 'Dynamic'); +SELECT accurateCastOrDefault(NULL, 'Variant(UInt8, String)'); +SELECT accurateCastOrDefault(CAST(NULL, 'LowCardinality(Nullable(String))'), 'LowCardinality(Nullable(UInt32))') + SETTINGS allow_suspicious_low_cardinality_types = 1; + +SET cast_keep_nullable = 0; diff --git a/tests/queries/0_stateless/04512_format_row_no_newline_empty_row_underflow.reference b/tests/queries/0_stateless/04512_format_row_no_newline_empty_row_underflow.reference new file mode 100644 index 000000000000..1b34d3678143 --- /dev/null +++ b/tests/queries/0_stateless/04512_format_row_no_newline_empty_row_underflow.reference @@ -0,0 +1,16 @@ +0 +2 + +610A +62 +0 +0 +0 +1 +1 +50 +44 +1\t2\tgood +0,"good" +1,"good" +2,"good" diff --git a/tests/queries/0_stateless/04512_format_row_no_newline_empty_row_underflow.sql b/tests/queries/0_stateless/04512_format_row_no_newline_empty_row_underflow.sql new file mode 100644 index 000000000000..af4df55206ce --- /dev/null +++ b/tests/queries/0_stateless/04512_format_row_no_newline_empty_row_underflow.sql @@ -0,0 +1,31 @@ +-- Tags: no-fasttest +-- no-fasttest: the RawBLOB format is not available in fast test builds. + +-- `formatRowNoNewline` strips a trailing newline from each row. It must never rewind past the start of the +-- current row into the previous row's bytes. Otherwise a row that emits no bytes (e.g. an empty string with the +-- `RawBLOB` format) produces non-monotonic `ColumnString` offsets and a `size_t` underflow in the string size. + +-- An empty row right after a non-empty one that ended with a newline must stay empty (length 0), not underflow. +SELECT length(formatRowNoNewline('RawBLOB', s)) AS len +FROM (SELECT arrayJoin(['a\n\n', '']) AS s) +ORDER BY ALL; + +-- The bytes of a following row must be exactly those the row emitted (no cross-row bleed). +SELECT hex(formatRowNoNewline('RawBLOB', s)) AS bytes +FROM (SELECT arrayJoin(['a\n\n', '', 'b']) AS s) +ORDER BY ALL; + +-- Several consecutive empty rows around non-empty ones stay empty and keep offsets monotonic. +SELECT length(formatRowNoNewline('RawBLOB', s)) AS len +FROM (SELECT arrayJoin(['x\n', '', '', 'y\n', '']) AS s) +ORDER BY ALL; + +-- Regression guard for the newline-stripping itself: it must keep working for rows after the internal write +-- buffer has grown past its initial size and been flushed at least once (which happens from the second row on +-- for these sizes). Every row must have its trailing newline stripped, so all lengths are equal. +SELECT DISTINCT length(formatRowNoNewline('TSV', repeat('z', 50))) AS len FROM numbers(5); +SELECT DISTINCT length(formatRowNoNewline('CSV', number, repeat('w', 40))) AS len FROM numbers(5); + +-- Normal row formats keep their usual behavior. +SELECT formatRowNoNewline('TSV', 1, 2, 'good') AS f; +SELECT formatRowNoNewline('CSV', number, 'good') FROM numbers(3); diff --git a/tests/queries/0_stateless/04545_bloom_filter_index_transform_null_in.reference b/tests/queries/0_stateless/04545_bloom_filter_index_transform_null_in.reference new file mode 100644 index 000000000000..d72422fb9ba6 --- /dev/null +++ b/tests/queries/0_stateless/04545_bloom_filter_index_transform_null_in.reference @@ -0,0 +1,53 @@ +String: IN null-free set prunes with transform_null_in=1 +1 +2 +String: GLOBAL IN null-free set prunes with transform_null_in=1 +1 +2 +String: IN subquery of the same type prunes with transform_null_in=1 +1 +2 +String: GLOBAL IN subquery of the same type prunes with transform_null_in=1 +1 +2 +String: `=` prunes with transform_null_in=1 (was already working) +1 +1 +Nullable: IN null-free set prunes with transform_null_in=1 +1 +1 +LowCardinality: IN null-free set prunes with transform_null_in=1 +1 +2 +LowCardinality(Nullable): IN null-free set prunes with transform_null_in=1 +1 +1 +Nullable: IN set with NULL does not prune, result includes NULL rows +0 +11 +Correctness: null-free set, results equal for transform_null_in 0 vs 1 +1 +Nullable: hand-written nullIn null-free set prunes with transform_null_in=0 +1 +1 +Nullable: hand-written globalNullIn null-free set prunes with transform_null_in=0 +1 +1 +Nullable: hand-written nullIn with a NULL literal keeps skip-index and full-scan results equal at transform_null_in=0 +1 +1 +Array: IN does not prune with transform_null_in=1 (unsound array hashing) +0 +Array: IN empty array result is correct with transform_null_in=1 +1 +Type mismatch: String index vs integer set does not prune with transform_null_in=1 +0 +Type mismatch: query still raises the conversion error with transform_null_in=1 +Lossy cast: results are identical with and without the skip index +1 +Lossy cast: both matching rows are returned +2 +Tuple: IN does not prune with transform_null_in=1 +0 +Tuple: NULL-carrying set returns the same rows with and without the skip index +1 diff --git a/tests/queries/0_stateless/04545_bloom_filter_index_transform_null_in.sql b/tests/queries/0_stateless/04545_bloom_filter_index_transform_null_in.sql new file mode 100644 index 000000000000..a7fd80cc92ca --- /dev/null +++ b/tests/queries/0_stateless/04545_bloom_filter_index_transform_null_in.sql @@ -0,0 +1,170 @@ +-- Tags: no-parallel-replicas +-- https://github.com/ClickHouse/ClickHouse/issues/111311 +-- With transform_null_in=1 the analyzer rewrites `x IN (...)` to `nullIn(x, ...)`. When the +-- IN-set has no NULL element, the bloom_filter skip index must still be used (nullIn selects +-- the same rows as in). When the set contains a NULL, the index is not used (no pruning). +-- The contract is full-scan avoidance, so every "prunes" assertion checks that the skip index +-- actually reduced the read granule count (read < total), not merely that it was analyzed. + +DROP TABLE IF EXISTS t_bf_null_in; +CREATE TABLE t_bf_null_in (x String, INDEX idx_x x TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO t_bf_null_in SELECT toString(number) FROM numbers(1000); + +SELECT 'String: IN null-free set prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in WHERE x IN ('5', '500') SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in WHERE x IN ('5', '500') SETTINGS transform_null_in = 1; + +-- globalNullIn is classified separately from nullIn, so it needs its own pruning assertion. +SELECT 'String: GLOBAL IN null-free set prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in WHERE x GLOBAL IN ('5', '500') SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in WHERE x GLOBAL IN ('5', '500') SETTINGS transform_null_in = 1; + +-- A subquery set takes its element types from the subquery header, a literal set from the tuple, +-- so the type check sees a differently-built set on this path. +SELECT 'String: IN subquery of the same type prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in WHERE x IN (SELECT toString(arrayJoin(['5', '500']))) SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in WHERE x IN (SELECT toString(arrayJoin(['5', '500']))) SETTINGS transform_null_in = 1; + +SELECT 'String: GLOBAL IN subquery of the same type prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in WHERE x GLOBAL IN (SELECT toString(arrayJoin(['5', '500']))) SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in WHERE x GLOBAL IN (SELECT toString(arrayJoin(['5', '500']))) SETTINGS transform_null_in = 1; + +SELECT 'String: `=` prunes with transform_null_in=1 (was already working)'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in WHERE x = '5' SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in WHERE x = '5' SETTINGS transform_null_in = 1; + +DROP TABLE t_bf_null_in; + +-- Nullable / LowCardinality / LowCardinality(Nullable) type-wrapper matrix. +DROP TABLE IF EXISTS t_bf_null_in_n; +CREATE TABLE t_bf_null_in_n +( + a Nullable(String), + b LowCardinality(String), + c LowCardinality(Nullable(String)), + INDEX idx_a a TYPE bloom_filter GRANULARITY 1, + INDEX idx_b b TYPE bloom_filter GRANULARITY 1, + INDEX idx_c c TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO t_bf_null_in_n +SELECT if(number % 100 = 0, NULL, toString(number)), toString(number), if(number % 100 = 0, NULL, toString(number)) +FROM numbers(1000); + +SELECT 'Nullable: IN null-free set prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_n WHERE a IN ('5', '500') SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in_n WHERE a IN ('5', '500') SETTINGS transform_null_in = 1; + +SELECT 'LowCardinality: IN null-free set prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_n WHERE b IN ('5', '500') SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in_n WHERE b IN ('5', '500') SETTINGS transform_null_in = 1; + +SELECT 'LowCardinality(Nullable): IN null-free set prunes with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_n WHERE c IN ('5', '500') SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in_n WHERE c IN ('5', '500') SETTINGS transform_null_in = 1; + +-- When the set contains a NULL, nullIn also matches NULL rows: the index must NOT prune, +-- and the result must include the NULL rows (10 rows: number % 100 = 0 -> {0,100,...,900}). +SELECT 'Nullable: IN set with NULL does not prune, result includes NULL rows'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_n WHERE a IN ('5', NULL) SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in_n WHERE a IN ('5', NULL) SETTINGS transform_null_in = 1; + +-- Correctness cross-check: results are identical with transform_null_in=0 and =1 for a +-- null-free set over a non-null column value. +SELECT 'Correctness: null-free set, results equal for transform_null_in 0 vs 1'; +SELECT + (SELECT count() FROM t_bf_null_in_n WHERE b IN ('5', '500') SETTINGS transform_null_in = 0) = + (SELECT count() FROM t_bf_null_in_n WHERE b IN ('5', '500') SETTINGS transform_null_in = 1); + +-- A hand-written `nullIn` reaches this branch regardless of transform_null_in, so the set it +-- carries is NULL-free whenever the setting is off and pruning is then sound. +SELECT 'Nullable: hand-written nullIn null-free set prunes with transform_null_in=0'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_n WHERE nullIn(a, ('5', '500')) SETTINGS transform_null_in = 0) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in_n WHERE nullIn(a, ('5', '500')) SETTINGS transform_null_in = 0; + +SELECT 'Nullable: hand-written globalNullIn null-free set prunes with transform_null_in=0'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_n WHERE globalNullIn(a, ('5', '500')) SETTINGS transform_null_in = 0) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT count() FROM t_bf_null_in_n WHERE globalNullIn(a, ('5', '500')) SETTINGS transform_null_in = 0; + +-- A NULL literal is not a set element when transform_null_in = 0, so no NULL row can be pruned away. +SELECT 'Nullable: hand-written nullIn with a NULL literal keeps skip-index and full-scan results equal at transform_null_in=0'; +SELECT + (SELECT count() FROM t_bf_null_in_n WHERE nullIn(a, ('5', NULL)) SETTINGS transform_null_in = 0, use_skip_indexes = 1) = + (SELECT count() FROM t_bf_null_in_n WHERE nullIn(a, ('5', NULL)) SETTINGS transform_null_in = 0, use_skip_indexes = 0); +SELECT count() FROM t_bf_null_in_n WHERE nullIn(a, ('5', NULL)) SETTINGS transform_null_in = 0; + +DROP TABLE t_bf_null_in_n; + +-- Array column: whole-array equality bloom filter hashing is not sound for granules that mix +-- empty and non-empty arrays, so the index must NOT be used for `nullIn` on Array columns. +-- Regression: without the Array guard, `x IN ([])` would wrongly prune the granule holding []. +DROP TABLE IF EXISTS t_bf_null_in_arr; +CREATE TABLE t_bf_null_in_arr (x Array(UInt32), INDEX idx_x x TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 2; +INSERT INTO t_bf_null_in_arr VALUES ([]), ([1]), ([2]), ([3]); + +SELECT 'Array: IN does not prune with transform_null_in=1 (unsound array hashing)'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_arr WHERE x IN ([]) SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT 'Array: IN empty array result is correct with transform_null_in=1'; +SELECT count() FROM t_bf_null_in_arr WHERE x IN ([]) SETTINGS transform_null_in = 1; + +DROP TABLE t_bf_null_in_arr; + +-- Type-incompatible set: the index hashes the set value cast to the index type, while execution +-- casts each column value to the set type. Those two casts are not inverse, so a matching row can +-- hash differently from what the index searches for ('01' -> UInt8 1 -> '1'). Pruning such a +-- granule loses that row, so the index must NOT be used. Types are compared modulo Nullable / +-- LowCardinality, so the wrapper cases above are unaffected. +DROP TABLE IF EXISTS t_bf_null_in_ty; +CREATE TABLE t_bf_null_in_ty (x String, INDEX idx_x x TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO t_bf_null_in_ty SELECT toString(number) FROM numbers(1000); + +SELECT 'Type mismatch: String index vs integer set does not prune with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_ty WHERE x IN (SELECT toUInt8(1)) SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT 'Type mismatch: query still raises the conversion error with transform_null_in=1'; +SELECT count() FROM t_bf_null_in_ty WHERE x IN (SELECT toUInt8(1)) SETTINGS transform_null_in = 1; -- { serverError CANNOT_PARSE_TEXT } + +DROP TABLE t_bf_null_in_ty; + +-- Lossy round trip: every value below parses as UInt8, so nothing throws, and '01' / '+1' both +-- equal 1 at execution while hashing differently from the index's '1'. Pruning would silently +-- drop them, so this arm fails if the type check above is ever removed. +DROP TABLE IF EXISTS t_bf_null_in_lossy; +CREATE TABLE t_bf_null_in_lossy (x String, INDEX idx_x x TYPE bloom_filter GRANULARITY 1) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO t_bf_null_in_lossy VALUES ('01'), ('+1'), ('2'), ('3'); + +SELECT 'Lossy cast: results are identical with and without the skip index'; +SELECT + (SELECT count() FROM t_bf_null_in_lossy WHERE x IN (SELECT toUInt8(1)) SETTINGS transform_null_in = 1, use_skip_indexes = 0) = + (SELECT count() FROM t_bf_null_in_lossy WHERE x IN (SELECT toUInt8(1)) SETTINGS transform_null_in = 1); +SELECT 'Lossy cast: both matching rows are returned'; +SELECT count() FROM t_bf_null_in_lossy WHERE x IN (SELECT toUInt8(1)) SETTINGS transform_null_in = 1; + +DROP TABLE t_bf_null_in_lossy; + +-- Tuple lhs: the NULL-free and type checks above are per-column, and the recursive `tuple(...)` +-- branch matches each element against its own index without the set, so those checks cannot be +-- applied there. Reusing the single-column check would prune granules holding a row that a +-- NULL-carrying tuple set matches. `(a, b)` holds 2 in both granules, so only `b` discriminates. +DROP TABLE IF EXISTS t_bf_null_in_tup; +CREATE TABLE t_bf_null_in_tup +( + a Nullable(Int32), + b Nullable(Int32), + INDEX idx_a a TYPE bloom_filter GRANULARITY 1, + INDEX idx_b b TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 2; +INSERT INTO t_bf_null_in_tup VALUES (2, 50), (9, 51), (2, NULL), (9, 52); + +SELECT 'Tuple: IN does not prune with transform_null_in=1'; +SELECT count() > 0 FROM (EXPLAIN indexes = 1 SELECT count() FROM t_bf_null_in_tup WHERE (a, b) IN ((2, NULL)) SETTINGS transform_null_in = 1) WHERE explain LIKE '%Granules: %/%' AND toUInt64OrZero(extract(explain, 'Granules: (\d+)/')) < toUInt64OrZero(extract(explain, 'Granules: \d+/(\d+)')); +SELECT 'Tuple: NULL-carrying set returns the same rows with and without the skip index'; +SELECT + (SELECT count() FROM t_bf_null_in_tup WHERE (a, b) IN ((2, NULL)) SETTINGS transform_null_in = 1, use_skip_indexes = 0) = + (SELECT count() FROM t_bf_null_in_tup WHERE (a, b) IN ((2, NULL)) SETTINGS transform_null_in = 1); + +DROP TABLE t_bf_null_in_tup; diff --git a/tests/queries/0_stateless/04550_restore_fsync_after_insert.reference b/tests/queries/0_stateless/04550_restore_fsync_after_insert.reference new file mode 100644 index 000000000000..a1f56bd020f6 --- /dev/null +++ b/tests/queries/0_stateless/04550_restore_fsync_after_insert.reference @@ -0,0 +1,6 @@ +has zero-byte part file: 1 +count on: 1000 +count off: 1000 +restore fsync delta covers all part files: 1 +encrypted incremental count: 1000 +encrypted incremental restore with fsync_after_insert=1, all part files fsynced: 1 diff --git a/tests/queries/0_stateless/04550_restore_fsync_after_insert.sh b/tests/queries/0_stateless/04550_restore_fsync_after_insert.sh new file mode 100755 index 000000000000..12fba381da15 --- /dev/null +++ b/tests/queries/0_stateless/04550_restore_fsync_after_insert.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-object-storage, no-random-merge-tree-settings, no-replicated-database, no-shared-merge-tree +# no-fasttest: the encrypted case below needs the encrypted disk type, which is built only with SSL. +# no-object-storage: object storage does not fsync file contents (the fix is gated on !isRemote()). +# no-random-merge-tree-settings: the test asserts on FileSync counts, which depend on the part layout. +# no-replicated-database, no-shared-merge-tree: the encrypted case below pins a custom local disk. + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/111321 +# RESTORE must fsync the restored part file contents when the table enables fsync_after_insert, +# otherwise a power loss right after RESTORE returns leaves the parts torn and the table empty. +# We assert on the RESTORE query's FileSync ProfileEvent (parallel-safe: filtered by query_id + +# current_database). RESTORE also performs a few backup-side FileSync events unrelated to the part +# files, so the discriminating signal is the on-vs-off FileSync DELTA: that constant backup-side +# noise cancels out, and the remaining delta must cover every physical file of the restored part +# (an empty Array column contributes a zero-byte .bin, which INSERT fsyncs too). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `arr` is Array(UInt32) left empty for every row, so its `.bin` is a required zero-byte part file - +# INSERT fsyncs it, so RESTORE must too. Two tables with identical data, differing only in fsync_after_insert. +$CLICKHOUSE_CLIENT -m -q " + DROP TABLE IF EXISTS t_restore_fsync_on; + DROP TABLE IF EXISTS t_restore_fsync_off; + + CREATE TABLE t_restore_fsync_on (id UInt64, s String, arr Array(UInt32)) ENGINE = MergeTree ORDER BY id + SETTINGS fsync_after_insert = 1, fsync_part_directory = 1, min_bytes_for_wide_part = 0; + INSERT INTO t_restore_fsync_on SELECT number, toString(number), [] FROM numbers(1000); + + CREATE TABLE t_restore_fsync_off (id UInt64, s String, arr Array(UInt32)) ENGINE = MergeTree ORDER BY id + SETTINGS fsync_after_insert = 0, fsync_part_directory = 0, min_bytes_for_wide_part = 0; + INSERT INTO t_restore_fsync_off SELECT number, toString(number), [] FROM numbers(1000); +" + +# Count the physical files RESTORE actually copies (and therefore must fsync). This is the real target, +# larger than system.parts.files (= checksums entries) - it includes checksums.txt, columns.txt and the +# zero-byte arr.bin - but excludes the version-metadata files RESTORE deliberately skips (see +# restorePartFromBackup: txn_version.txt[.tmp] and metadata_version.txt are not copied). The restored +# part has the same on-disk file set. +part_path=$($CLICKHOUSE_CLIENT -q "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 't_restore_fsync_on' AND active") +copied_files=$(find "$part_path" -type f \ + ! -name 'txn_version.txt' ! -name 'txn_version.txt.tmp' ! -name 'metadata_version.txt' | wc -l) +# Sanity: there is a required zero-byte file in the part (the empty Array's .bin), which INSERT fsyncs too. +zero_byte_files=$(find "$part_path" -type f -size 0 | wc -l) +echo "has zero-byte part file: $([ "$zero_byte_files" -ge 1 ] && echo 1 || echo 0)" + +$CLICKHOUSE_CLIENT -q "BACKUP TABLE t_restore_fsync_on TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_on')" > /dev/null +$CLICKHOUSE_CLIENT -q "BACKUP TABLE t_restore_fsync_off TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_off')" > /dev/null +$CLICKHOUSE_CLIENT -m -q "DROP TABLE t_restore_fsync_on SYNC; DROP TABLE t_restore_fsync_off SYNC;" + +qid_on="restore-on-$CLICKHOUSE_DATABASE" +qid_off="restore-off-$CLICKHOUSE_DATABASE" +$CLICKHOUSE_CLIENT --query_id "$qid_on" -q "RESTORE TABLE t_restore_fsync_on FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_on')" > /dev/null +$CLICKHOUSE_CLIENT --query_id "$qid_off" -q "RESTORE TABLE t_restore_fsync_off FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_off')" > /dev/null + +# Data must be intact after restore. +echo "count on: $($CLICKHOUSE_CLIENT -q "SELECT count() FROM t_restore_fsync_on")" +echo "count off: $($CLICKHOUSE_CLIENT -q "SELECT count() FROM t_restore_fsync_off")" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" + +# The on/off FileSync delta cancels the constant backup-side syncs that both restores perform and +# isolates the restored part-file syncs. With fsync_after_insert=1 every restore-copied file (incl. the +# zero-byte arr.bin) is fsynced, so the delta must be >= the number of files restore copied. Before the +# fix no part file was synced and the delta was ~0. +$CLICKHOUSE_CLIENT --param_qid_on "$qid_on" --param_qid_off "$qid_off" --param_copied "$copied_files" -q " + WITH + (SELECT ProfileEvents['FileSync'] FROM system.query_log + WHERE query_id = {qid_on:String} AND type = 'QueryFinish' AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC LIMIT 1) AS sync_on, + (SELECT ProfileEvents['FileSync'] FROM system.query_log + WHERE query_id = {qid_off:String} AND type = 'QueryFinish' AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC LIMIT 1) AS sync_off + SELECT 'restore fsync delta covers all part files: ', (toInt64(sync_on) - toInt64(sync_off)) >= {copied:UInt64}" + +$CLICKHOUSE_CLIENT -m -q "DROP TABLE t_restore_fsync_on SYNC; DROP TABLE t_restore_fsync_off SYNC;" + +# Encrypted incremental restore: files that come entirely from the base backup are copied via +# getBaseBackup()->copyFileToDisk(..., sync). That branch must forward the encrypted read (else the +# restore fails with CANNOT_RESTORE_TO_NONENCRYPTED_DISK) and still fsync the files when requested. +enc_disk="disk(type = encrypted, disk = disk(type = local, path = '${CLICKHOUSE_DISKS_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}_enc/'), key = '1234567812345678')" +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS t_restore_fsync_enc; + CREATE TABLE t_restore_fsync_enc (id UInt64, s String, arr Array(UInt32)) ENGINE = MergeTree ORDER BY id + SETTINGS fsync_after_insert = 1, fsync_part_directory = 1, min_bytes_for_wide_part = 0, disk = $enc_disk; + INSERT INTO t_restore_fsync_enc SELECT number, toString(number), [] FROM numbers(1000); +" +enc_files=$($CLICKHOUSE_CLIENT -q "SELECT files FROM system.parts WHERE database = currentDatabase() AND table = 't_restore_fsync_enc' AND active") + +# Full backup, then an unchanged incremental backup so every file is served by the base backup. +$CLICKHOUSE_CLIENT -q "BACKUP TABLE t_restore_fsync_enc TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_enc_base')" > /dev/null +$CLICKHOUSE_CLIENT -q "BACKUP TABLE t_restore_fsync_enc TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_enc_incr') SETTINGS base_backup = Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_enc_base')" > /dev/null +$CLICKHOUSE_CLIENT -q "DROP TABLE t_restore_fsync_enc SYNC" + +qid_enc="restore-enc-$CLICKHOUSE_DATABASE" +$CLICKHOUSE_CLIENT --query_id "$qid_enc" -q "RESTORE TABLE t_restore_fsync_enc FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_enc_incr') SETTINGS base_backup = Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_enc_base')" > /dev/null + +# Before the fix the restore threw and the table stayed empty; now it restores every row. +echo "encrypted incremental count: $($CLICKHOUSE_CLIENT -q "SELECT count() FROM t_restore_fsync_enc")" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT --param_query_id "$qid_enc" --param_files "$enc_files" -q " + SELECT 'encrypted incremental restore with fsync_after_insert=1, all part files fsynced: ', + ProfileEvents['FileSync'] >= {files:UInt64} + FROM system.query_log + WHERE query_id = {query_id:String} AND type = 'QueryFinish' AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC LIMIT 1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_restore_fsync_enc SYNC" diff --git a/tests/queries/0_stateless/04603_analyzer_compatibility_multiple_joins_qualify_column_names.reference b/tests/queries/0_stateless/04603_analyzer_compatibility_multiple_joins_qualify_column_names.reference new file mode 100644 index 000000000000..953ec5a4c3bc --- /dev/null +++ b/tests/queries/0_stateless/04603_analyzer_compatibility_multiple_joins_qualify_column_names.reference @@ -0,0 +1,142 @@ +=== family2: repro1 ll.Date, setting OFF (throws) === +=== family2: repro2 a.x, setting OFF (throws) === +=== family2: repro1 ll.Date, setting ON === +D +=== family2: repro2 a.x, setting ON === +X +=== family2: repro1 GROUP BY ALL, setting ON === +D 1 +=== describe: 2-join star, setting ON === +ll.k UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +=== describe: qualified matcher ll.*, setting ON === +ll.k UInt8 +ll.Date String +=== describe: comma-join trigger, setting ON === +ll.k UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +=== describe: explicit unique qualified identifier a.x, setting ON === +a.x String +=== describe: explicit bare identifier x stays bare, setting ON === +x String +=== describe: alias always wins a.x AS y, setting ON === +y String +=== describe: real tables without aliases, setting ON === +ta.x UInt8 +ta.ka UInt8 +tb.kb UInt8 +tc.kc UInt8 +=== describe: real tables, explicit qualified column ta.x, setting ON === +ta.x UInt8 +=== describe: 1 join only -- unchanged even with setting ON === +k UInt8 +Date String +t1.k UInt8 +=== describe: function projection name unchanged (ll.k + 1), setting ON === +plus(ll.k, 1) UInt16 +=== describe: SELECT * over JOIN ... USING keeps merge semantics, setting ON === +k UInt8 +ll.Date String +t2.k UInt8 +=== describe: unaliased joined subquery gets no prefix (old parity), setting ON === +k UInt8 +Date String +t1.k2 UInt8 +t2.k3 UInt8 +=== describe: 2-join star, setting OFF (today unchanged) === +ll.k UInt8 +Date String +t1.k UInt8 +t2.k UInt8 +=== describe: comma-join trigger, setting OFF (today unchanged) === +ll.k UInt8 +Date String +t1.k UInt8 +t2.k UInt8 +=== describe: real tables without aliases, setting OFF (today unchanged) === +x UInt8 +ka UInt8 +kb UInt8 +kc UInt8 +=== nested outer scope, setting ON === +D +=== describe: ARRAY JOIN mixed with 2 joins, setting ON === +ll.k UInt8 +arr UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +=== distributed: remote() two-shard, family2 shape, setting ON === +D +D +=== describe: COLUMNS(col) identifier-list form stays bare, setting ON === +Date String +=== describe: COLUMNS(regexp) form is qualified like * (old parity), setting ON === +ll.Date String +=== describe: *, COLUMNS(col) -- order-independent, setting ON === +ll.k UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +Date String +=== describe: COLUMNS(col), * -- order-independent, setting ON === +Date String +ll.k UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +=== outer scope cannot see the bare COLUMNS(col) name, setting ON === +=== describe: CTE joined in multiple joins gets CTE-name qualifier, setting ON === +ll.k UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +=== family2: outer ref into CTE-based derived table, setting ON === +D +=== describe: UNION CTE joined in multiple joins gets CTE-name qualifier, setting ON === +ll.k UInt8 +ll.Date String +t1.k UInt8 +t2.k UInt8 +=== family2: outer ref into UNION-CTE derived table, setting ON === +D +E +=== describe: two USING chains, setting ON === +k UInt8 +ll.Date String +=== two USING chains: outer ref to a qualified column, setting ON === +D +=== two USING chains: merged key is referenced bare, setting ON === +1 +=== two USING chains: merged key is not qualified, setting ON === +=== describe: COLUMNS(qualified) keeps the written name, setting ON === +ll.Date String +=== family2: outer ref into a COLUMNS(qualified) derived table, setting ON === +D +=== describe: COLUMNS(qualified, qualified) keeps both written names, setting ON === +ll.Date String +t1.k UInt8 +=== describe: EXCEPT still matches the bare column name, setting ON === +ll.Date String +=== describe: COLUMNS(qualified) is untouched with the setting OFF === +Date String +=== describe: COLUMNS(qualified) is untouched at a single JOIN, setting ON === +Date String +=== describe: COLUMNS(qualified, qualified) APPLY(toString), setting ON === +toString(a.x) String +toString(a.y) String +=== describe: COLUMNS(qualified), unrelated toString(qualified) does not leak the qualifier, setting ON === +ll.Date String +toString(Date) String +=== describe: COLUMNS(qualified) with group_by_use_nulls + ROLLUP, setting ON === +ll.Date Nullable(String) +=== family2: outer ref, COLUMNS(qualified) with group_by_use_nulls + ROLLUP, setting ON === +D +\N +=== describe: COLUMNS(alias) keeps the written alias, setting ON === +z String +z String diff --git a/tests/queries/0_stateless/04603_analyzer_compatibility_multiple_joins_qualify_column_names.sql b/tests/queries/0_stateless/04603_analyzer_compatibility_multiple_joins_qualify_column_names.sql new file mode 100644 index 000000000000..81df204581c6 --- /dev/null +++ b/tests/queries/0_stateless/04603_analyzer_compatibility_multiple_joins_qualify_column_names.sql @@ -0,0 +1,259 @@ +-- Compatibility setting `analyzer_compatibility_multiple_joins_qualify_column_names` +-- makes the analyzer mimic the old analyzer's multiple-joins column-naming +-- rewrite (when the `FROM` clause has two or more `JOIN`s), so that outer queries +-- referencing hidden inner aliases (e.g. `SELECT ll.Date FROM (SELECT * FROM t AS ll +-- LEFT JOIN x ... LEFT JOIN y ...)`) resolve again. + +SET enable_analyzer = 1; + +-- ============================================================ +-- Family 2 reproducers (the point of the feature) +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 0; + +SELECT '=== family2: repro1 ll.Date, setting OFF (throws) ==='; +SELECT ll.Date FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); -- { serverError UNKNOWN_IDENTIFIER } + +SELECT '=== family2: repro2 a.x, setting OFF (throws) ==='; +SELECT a.x FROM (SELECT a.x FROM (SELECT 1 AS k, 'X' AS x) AS a LEFT JOIN (SELECT 1 AS k) AS b ON a.k = b.k LEFT JOIN (SELECT 1 AS k) AS c ON a.k = c.k); -- { serverError UNKNOWN_IDENTIFIER } + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +SELECT '=== family2: repro1 ll.Date, setting ON ==='; +SELECT ll.Date FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== family2: repro2 a.x, setting ON ==='; +SELECT a.x FROM (SELECT a.x FROM (SELECT 1 AS k, 'X' AS x) AS a LEFT JOIN (SELECT 1 AS k) AS b ON a.k = b.k LEFT JOIN (SELECT 1 AS k) AS c ON a.k = c.k); + +SELECT '=== family2: repro1 GROUP BY ALL, setting ON ==='; +SELECT ll.Date AS Date, count() FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k) GROUP BY ALL; + +-- ============================================================ +-- DESCRIBE parity matrix (setting ON) -- compare against old analyzer's names +-- ============================================================ + +SELECT '=== describe: 2-join star, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: qualified matcher ll.*, setting ON ==='; +DESCRIBE (SELECT ll.* FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: comma-join trigger, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll, (SELECT 2 AS k) AS t1, (SELECT 3 AS k) AS t2); + +SELECT '=== describe: explicit unique qualified identifier a.x, setting ON ==='; +DESCRIBE (SELECT a.x FROM (SELECT 1 AS k, 'X' AS x) AS a LEFT JOIN (SELECT 1 AS k) AS b ON a.k = b.k LEFT JOIN (SELECT 1 AS k) AS c ON a.k = c.k); + +SELECT '=== describe: explicit bare identifier x stays bare, setting ON ==='; +DESCRIBE (SELECT x FROM (SELECT 1 AS k, 'X' AS x) AS a LEFT JOIN (SELECT 1 AS k) AS b ON a.k = b.k LEFT JOIN (SELECT 1 AS k) AS c ON a.k = c.k); + +SELECT '=== describe: alias always wins a.x AS y, setting ON ==='; +DESCRIBE (SELECT a.x AS y FROM (SELECT 1 AS k, 'X' AS x) AS a LEFT JOIN (SELECT 1 AS k) AS b ON a.k = b.k LEFT JOIN (SELECT 1 AS k) AS c ON a.k = c.k); + +DROP TABLE IF EXISTS ta; +DROP TABLE IF EXISTS tb; +DROP TABLE IF EXISTS tc; +CREATE TABLE ta (x UInt8, ka UInt8) ENGINE = Memory; +CREATE TABLE tb (kb UInt8) ENGINE = Memory; +CREATE TABLE tc (kc UInt8) ENGINE = Memory; + +SELECT '=== describe: real tables without aliases, setting ON ==='; +DESCRIBE (SELECT * FROM ta JOIN tb ON ta.ka = tb.kb JOIN tc ON ta.ka = tc.kc); + +SELECT '=== describe: real tables, explicit qualified column ta.x, setting ON ==='; +DESCRIBE (SELECT ta.x FROM ta JOIN tb ON ta.ka = tb.kb JOIN tc ON ta.ka = tc.kc); + +SELECT '=== describe: 1 join only -- unchanged even with setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k); + +-- ============================================================ +-- Documented deviations from the old analyzer (setting ON -- pin the NEW behavior) +-- ============================================================ + +SELECT '=== describe: function projection name unchanged (ll.k + 1), setting ON ==='; +DESCRIBE (SELECT ll.k + 1 FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: SELECT * over JOIN ... USING keeps merge semantics, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 USING (k) LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SET joined_subquery_requires_alias = 0; + +SELECT '=== describe: unaliased joined subquery gets no prefix (old parity), setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) LEFT JOIN (SELECT 1 AS k2) AS t1 ON k = t1.k2 LEFT JOIN (SELECT 1 AS k3) AS t2 ON k = t2.k3); + +SET joined_subquery_requires_alias = 1; + +-- ============================================================ +-- Setting OFF (default) -- byte-identical to today's default analyzer behavior +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 0; + +SELECT '=== describe: 2-join star, setting OFF (today unchanged) ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: comma-join trigger, setting OFF (today unchanged) ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll, (SELECT 2 AS k) AS t1, (SELECT 3 AS k) AS t2); + +SELECT '=== describe: real tables without aliases, setting OFF (today unchanged) ==='; +DESCRIBE (SELECT * FROM ta JOIN tb ON ta.ka = tb.kb JOIN tc ON ta.ka = tc.kc); + +DROP TABLE ta; +DROP TABLE tb; +DROP TABLE tc; + +-- ============================================================ +-- Nested / outer-scope usage +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +SELECT '=== nested outer scope, setting ON ==='; +SELECT ll.Date FROM (SELECT * FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k)); + +-- ============================================================ +-- ARRAY JOIN mixed with 2 joins (setting ON). The old analyzer throws +-- (`Multiple JOIN does not support mix with ARRAY JOINs`); the analyzer already +-- supports this combination today. The exact qualified name for the ARRAY JOIN result +-- column `arr` is uncertain until the analyzer hooks land -- see report if it differs. +-- ============================================================ + +SELECT '=== describe: ARRAY JOIN mixed with 2 joins, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, [1, 2] AS arr, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k ARRAY JOIN arr); + +-- ============================================================ +-- Distributed: the hidden-alias outer reference must survive going through `remote` +-- (aliases are attached in `toAST`, so the qualified projection name is preserved +-- across the shard boundary). +-- ============================================================ + +SELECT '=== distributed: remote() two-shard, family2 shape, setting ON ==='; +SELECT ll.Date FROM remote('127.0.0.{1,2}', view(SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k)) ORDER BY ll.Date SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +-- ============================================================ +-- `COLUMNS` matcher: the identifier-list form `COLUMNS(col1, col2)` is not a +-- matcher expansion (it is resolved as a plain list of column references), so +-- with the setting on each column keeps the name exactly as its identifier was +-- written. The setting never adds a qualifier here, unlike the regexp form +-- `COLUMNS('')`, which goes through the same expansion as `*`. An item +-- written without a qualifier therefore stays bare, on the old analyzer as well. +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +SELECT '=== describe: COLUMNS(col) identifier-list form stays bare, setting ON ==='; +DESCRIBE (SELECT COLUMNS(Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: COLUMNS(regexp) form is qualified like * (old parity), setting ON ==='; +DESCRIBE (SELECT COLUMNS('^Date$') FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: *, COLUMNS(col) -- order-independent, setting ON ==='; +DESCRIBE (SELECT *, COLUMNS(Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: COLUMNS(col), * -- order-independent, setting ON ==='; +DESCRIBE (SELECT COLUMNS(Date), * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== outer scope cannot see the bare COLUMNS(col) name, setting ON ==='; +-- fails on the old analyzer as well, since the identifier-list form never qualifies its columns +SELECT ll.Date FROM (SELECT COLUMNS(Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); -- { serverError UNKNOWN_IDENTIFIER } + +-- ============================================================ +-- CTE used as a joined table expression: the qualifier is the CTE name +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +SELECT '=== describe: CTE joined in multiple joins gets CTE-name qualifier, setting ON ==='; +DESCRIBE (WITH ll AS (SELECT 1 AS k, 'D' AS Date) + SELECT * FROM ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k + LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== family2: outer ref into CTE-based derived table, setting ON ==='; +SELECT ll.Date FROM (WITH ll AS (SELECT 1 AS k, 'D' AS Date) + SELECT * FROM ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k + LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: UNION CTE joined in multiple joins gets CTE-name qualifier, setting ON ==='; +DESCRIBE (WITH ll AS (SELECT 1 AS k, 'D' AS Date UNION ALL SELECT 2 AS k, 'E' AS Date) + SELECT * FROM ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k + LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== family2: outer ref into UNION-CTE derived table, setting ON ==='; +SELECT ll.Date FROM (WITH ll AS (SELECT 1 AS k, 'D' AS Date UNION ALL SELECT 2 AS k, 'E' AS Date) + SELECT * FROM ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k + LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k) +ORDER BY ll.Date; + +-- ============================================================ +-- Two `USING` chains: the old analyzer rejected this shape +-- (`NOT_IMPLEMENTED: Multiple USING statements are not supported`); the analyzer +-- supports it and applies the same naming as elsewhere, except that the merged +-- `USING` key belongs to the join rather than to a single table and therefore +-- keeps its bare name. +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +SELECT '=== describe: two USING chains, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 USING (k) LEFT JOIN (SELECT 1 AS k) AS t2 USING (k)); + +SELECT '=== two USING chains: outer ref to a qualified column, setting ON ==='; +SELECT ll.Date FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 USING (k) LEFT JOIN (SELECT 1 AS k) AS t2 USING (k)); + +SELECT '=== two USING chains: merged key is referenced bare, setting ON ==='; +SELECT k FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 USING (k) LEFT JOIN (SELECT 1 AS k) AS t2 USING (k)); + +SELECT '=== two USING chains: merged key is not qualified, setting ON ==='; +SELECT ll.k FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 USING (k) LEFT JOIN (SELECT 1 AS k) AS t2 USING (k)); -- { serverError UNKNOWN_IDENTIFIER } + +-- ============================================================ +-- Identifier-list `COLUMNS`: a qualified item keeps the written qualifier. +-- The old analyzer's rewrite left such items spelled as written, so +-- `COLUMNS(a.x)` produced a column named `a.x` once there were two or more +-- `JOIN`s. Unqualified items are unaffected. +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; + +SELECT '=== describe: COLUMNS(qualified) keeps the written name, setting ON ==='; +DESCRIBE (SELECT COLUMNS(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== family2: outer ref into a COLUMNS(qualified) derived table, setting ON ==='; +SELECT ll.Date FROM (SELECT COLUMNS(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: COLUMNS(qualified, qualified) keeps both written names, setting ON ==='; +DESCRIBE (SELECT COLUMNS(ll.Date, t1.k) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: EXCEPT still matches the bare column name, setting ON ==='; +DESCRIBE (SELECT COLUMNS(ll.k, ll.Date) EXCEPT (k) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +SELECT '=== describe: COLUMNS(qualified) is untouched with the setting OFF ==='; +DESCRIBE (SELECT COLUMNS(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k) SETTINGS analyzer_compatibility_multiple_joins_qualify_column_names = 0; + +SELECT '=== describe: COLUMNS(qualified) is untouched at a single JOIN, setting ON ==='; +DESCRIBE (SELECT COLUMNS(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k); + +-- The old analyzer gives `toString(x)`/`toString(y)` here; the star and regexp +-- matcher forms already produce `toString(a.x)` under the setting, so this keeps +-- the list form consistent with them rather than adding a new deviation. +SELECT '=== describe: COLUMNS(qualified, qualified) APPLY(toString), setting ON ==='; +DESCRIBE (SELECT COLUMNS(a.x, a.y) APPLY(toString) FROM (SELECT 1 AS k, 'X' AS x, 'Y' AS y) AS a LEFT JOIN (SELECT 1 AS k) AS b ON a.k = b.k LEFT JOIN (SELECT 1 AS k) AS c ON a.k = c.k); + +-- The matcher must not leak its written qualifier into an unrelated sibling +-- expression referencing the same column. +SELECT '=== describe: COLUMNS(qualified), unrelated toString(qualified) does not leak the qualifier, setting ON ==='; +DESCRIBE (SELECT COLUMNS(ll.Date), toString(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); + +-- The previous commit on this branch fixed matcher projection names being dropped by the +-- `group_by_use_nulls` rewrite; the clone this branch introduces must survive it too. +SELECT '=== describe: COLUMNS(qualified) with group_by_use_nulls + ROLLUP, setting ON ==='; +DESCRIBE (SELECT COLUMNS(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k GROUP BY ll.Date WITH ROLLUP) SETTINGS group_by_use_nulls = 1; + +SELECT '=== family2: outer ref, COLUMNS(qualified) with group_by_use_nulls + ROLLUP, setting ON ==='; +SELECT ll.Date FROM (SELECT COLUMNS(ll.Date) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k GROUP BY ll.Date WITH ROLLUP) ORDER BY 1 NULLS LAST SETTINGS group_by_use_nulls = 1; + +-- A list item that resolves through a `SELECT`-list alias also keeps the written name, which is +-- what the old analyzer did; without the setting it would be named after the underlying column. +SELECT '=== describe: COLUMNS(alias) keeps the written alias, setting ON ==='; +DESCRIBE (SELECT ll.Date AS z, COLUMNS(z) FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k); diff --git a/tests/queries/0_stateless/04603_join_using_nested_projection_alias.reference b/tests/queries/0_stateless/04603_join_using_nested_projection_alias.reference new file mode 100644 index 000000000000..335942c848fa --- /dev/null +++ b/tests/queries/0_stateless/04603_join_using_nested_projection_alias.reference @@ -0,0 +1,6 @@ +2 +2 +v2 +0 +11 +0 diff --git a/tests/queries/0_stateless/04603_join_using_nested_projection_alias.sql b/tests/queries/0_stateless/04603_join_using_nested_projection_alias.sql new file mode 100644 index 000000000000..c0f72caf30e8 --- /dev/null +++ b/tests/queries/0_stateless/04603_join_using_nested_projection_alias.sql @@ -0,0 +1,73 @@ +-- Related: https://github.com/ClickHouse/clickhouse-private/issues/55715 (Cluster C) +-- Resolving JOIN USING identifiers from aliases nested inside SELECT-list +-- expressions under analyzer_compatibility_join_using_top_level_identifier = 1. + +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS events; + +CREATE TABLE events +( + event_date Date, platform String, advertising_id String, idfv String, + event_name String, event_revenue_usd String +) ENGINE = MergeTree ORDER BY event_date; +INSERT INTO events VALUES ('2024-01-01', 'android', 'aid1', '', 'install', '0'), ('2024-01-01', 'android', 'aid1', '', 'af_purchase', '5.0'), ('2024-01-02', 'ios', '', 'idfv1', 'install', '0'), ('2024-01-02', 'ios', '', 'idfv1', 'af_purchase', '3.0'); + +-- case 1: Cluster C production repro. Default (setting = 0) cannot resolve the alias. +SET analyzer_compatibility_join_using_top_level_identifier = 0; +WITH dateDiff('day', InstallDate, event_date) AS lifetime +SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay +FROM events AS iap +INNER JOIN ( + SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id, min(event_date) AS InstallDate + FROM events WHERE event_name = 'install' GROUP BY id +) AS sub USING (id) +WHERE event_name LIKE 'af_%' AND toFloat64OrZero(event_revenue_usd) > 0; -- { serverError UNKNOWN_IDENTIFIER } + +SET analyzer_compatibility_join_using_top_level_identifier = 1; +WITH dateDiff('day', InstallDate, event_date) AS lifetime +SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay +FROM events AS iap +INNER JOIN ( + SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id, min(event_date) AS InstallDate + FROM events WHERE event_name = 'install' GROUP BY id +) AS sub USING (id) +WHERE event_name LIKE 'af_%' AND toFloat64OrZero(event_revenue_usd) > 0; + +DROP TABLE events; + +-- case 2: minimal nested-in-aggregate alias. +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT sum(x + 1 AS id) FROM (SELECT 1 AS x) t1 JOIN (SELECT 2 AS id) t2 USING (id); +SET analyzer_compatibility_join_using_top_level_identifier = 0; +SELECT sum(x + 1 AS id) FROM (SELECT 1 AS x) t1 JOIN (SELECT 2 AS id) t2 USING (id); -- { serverError UNKNOWN_IDENTIFIER } + +-- case 3: alias nested in a plain function. +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT concat('v', toString(x + 1 AS id)) FROM (SELECT 1 AS x) t1 JOIN (SELECT 2 AS id) t2 USING (id); + +-- case 4: nested alias takes priority over a real left column (old-analyzer-compatible). +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT sum(x + 10 AS id) FROM (SELECT 1 AS x, 2 AS id) t1 JOIN (SELECT 2 AS id) t2 USING (id); +SET analyzer_compatibility_join_using_top_level_identifier = 0; +SELECT sum(x + 10 AS id) FROM (SELECT 1 AS x, 2 AS id) t1 JOIN (SELECT 2 AS id) t2 USING (id); + +-- case 5: alias defined only in WHERE is out of scope. +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT sum(x) FROM (SELECT 1 AS x) t1 JOIN (SELECT 2 AS id) t2 USING (id) WHERE (x + 1 AS id) > 0; -- { serverError UNKNOWN_IDENTIFIER } + +-- case 6: nested alias expression references a column absent from the left table. +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT sum(y + 1 AS id) FROM (SELECT 1 AS x) t1 JOIN (SELECT 2 AS id, 3 AS y) t2 USING (id); -- { serverError UNKNOWN_IDENTIFIER } + +-- case 7: duplicated nested alias with different expressions must not be picked arbitrarily. +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT sum(x + 1 AS id) + sum(x + 2 AS id) FROM (SELECT 1 AS x) t1 JOIN (SELECT 2 AS id) t2 USING (id); -- { serverError UNKNOWN_IDENTIFIER } + +-- case 6b: with the setting on, the nested alias takes priority even when a real left +-- column exists; its expression must resolve from the left table, so this throws +-- (old-analyzer-compatible), while without the setting the column resolves. +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SELECT sum(y + 1 AS id) FROM (SELECT 1 AS x, 5 AS id) t1 JOIN (SELECT 2 AS id, 3 AS y) t2 USING (id); -- { serverError UNKNOWN_IDENTIFIER } +SET analyzer_compatibility_join_using_top_level_identifier = 0; +SELECT sum(y + 1 AS id) FROM (SELECT 1 AS x, 5 AS id) t1 JOIN (SELECT 2 AS id, 3 AS y) t2 USING (id); diff --git a/tests/queries/0_stateless/04604_join_using_nested_projection_alias_remote.reference b/tests/queries/0_stateless/04604_join_using_nested_projection_alias_remote.reference new file mode 100644 index 000000000000..165abcfd9d12 --- /dev/null +++ b/tests/queries/0_stateless/04604_join_using_nested_projection_alias_remote.reference @@ -0,0 +1,8 @@ +2 +2 +0 +2 +\N v +\N v +b_1 w +b_1 w diff --git a/tests/queries/0_stateless/04604_join_using_nested_projection_alias_remote.sql b/tests/queries/0_stateless/04604_join_using_nested_projection_alias_remote.sql new file mode 100644 index 000000000000..31dc2499aeb0 --- /dev/null +++ b/tests/queries/0_stateless/04604_join_using_nested_projection_alias_remote.sql @@ -0,0 +1,108 @@ +-- Related: https://github.com/ClickHouse/clickhouse-private/issues/55715 (Cluster C, distributed/parallel replicas) +-- A nested-alias `USING` key cannot be re-resolved by a remote server (the rendered query loses the nested +-- alias), so parallel replicas are downgraded at analysis time and `Distributed`/`remote` shipping fails loudly. + +SET enable_analyzer = 1; +SET analyzer_compatibility_join_using_top_level_identifier = 1; + +CREATE TABLE events +( + event_date Date, platform String, advertising_id String, idfv String, + event_name String, event_revenue_usd String +) ENGINE = MergeTree ORDER BY event_date; +INSERT INTO events VALUES ('2024-01-01', 'android', 'aid1', '', 'install', '0'), ('2024-01-01', 'android', 'aid1', '', 'af_purchase', '5.0'), ('2024-01-02', 'ios', '', 'idfv1', 'install', '0'), ('2024-01-02', 'ios', '', 'idfv1', 'af_purchase', '3.0'); + +-- P1: parallel replicas silently downgraded (hook fires before any cluster lookup), so the query still returns. +SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay +FROM events AS iap +INNER JOIN ( + SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id, min(event_date) AS InstallDate + FROM events WHERE event_name = 'install' GROUP BY id +) AS sub USING (id) +WHERE event_name LIKE 'af_%' AND toFloat64OrZero(event_revenue_usd) > 0 +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree = 1, automatic_parallel_replicas_mode = 0; + +-- P1b: the same unshippable JOIN nested in a `FROM` subquery is downgraded too. The subquery is planned +-- from its own context copy, so disabling parallel replicas only on the root would not cover it. +SELECT max(users_pay) FROM +( + SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay + FROM events AS iap + INNER JOIN ( + SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id, min(event_date) AS InstallDate + FROM events WHERE event_name = 'install' GROUP BY id + ) AS sub USING (id) + WHERE event_name LIKE 'af_%' AND toFloat64OrZero(event_revenue_usd) > 0 +) +SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree = 1, automatic_parallel_replicas_mode = 0; + +-- P2: force mode throws instead of downgrading. +SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay +FROM events AS iap +INNER JOIN ( + SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id, min(event_date) AS InstallDate + FROM events WHERE event_name = 'install' GROUP BY id +) AS sub USING (id) +WHERE event_name LIKE 'af_%' AND toFloat64OrZero(event_revenue_usd) > 0 +SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', automatic_parallel_replicas_mode = 0; -- { serverError SUPPORT_IS_DISABLED } + +-- R1: the nested-alias `USING` key is rejected on the initiator before shipping (guard fires first), not a shard-side re-analysis failure. +SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay +FROM remote('127.0.0.{1,2}', currentDatabase(), events) AS iap +INNER JOIN ( + SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id, min(event_date) AS InstallDate + FROM events WHERE event_name = 'install' GROUP BY id +) AS sub USING (id) +WHERE event_name LIKE 'af_%' AND toFloat64OrZero(event_revenue_usd) > 0; -- { serverError UNSUPPORTED_METHOD } + +-- R2: the `join_use_nulls` LEFT variant is rejected the same way, on the initiator. +SELECT uniqExact(lower(if(platform = 'android', advertising_id, idfv)) AS id) AS users_pay +FROM remote('127.0.0.{1,2}', currentDatabase(), events) AS iap +LEFT JOIN (SELECT lower(if(platform = 'ios', idfv, advertising_id)) AS id FROM events WHERE event_name = 'install' GROUP BY id) AS sub USING (id) +WHERE event_name LIKE 'af_%' +SETTINGS join_use_nulls = 1; -- { serverError UNSUPPORTED_METHOD } + +-- R3: minimal repro from the PR review; the nested-alias `id` in `USING` over `remote` is rejected on the initiator. +CREATE TABLE t_shadow (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_shadow VALUES (1); + +SELECT sum(x + 10 AS id) +FROM remote('127.0.0.{1,2}', currentDatabase(), t_shadow) AS tsh +JOIN (SELECT 11 AS id) t2 USING (id); -- { serverError UNSUPPORTED_METHOD } + +-- R4: shadowing variant; the alias `id` shadows the real column `id`, so the shard joins by the real column and returns `0`, deliberately differing from the local result of `11`. +CREATE TABLE t_shadow2 (x UInt64, id UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_shadow2 VALUES (1, 5); + +SELECT sum(x + 10 AS id) +FROM remote('127.0.0.{1,2}', currentDatabase(), t_shadow2) AS tsh +JOIN (SELECT 11 AS id) t2 USING (id); + +-- R5: `ALIAS`-column positive control; a table `ALIAS` column is excluded from the guard, so this ships and returns. +CREATE TABLE t_aliascol (x UInt64, id UInt64 ALIAS x + 100) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_aliascol VALUES (1); +SELECT count() FROM remote('127.0.0.{1,2}', currentDatabase(), t_aliascol) AS ta JOIN (SELECT 101 AS id) t2 USING (id); + +-- T1: top-level aliases keep working over `remote` (control; no downgrade, no error). +-- Each left row is read once per shard, so the single-shard result is duplicated; ORDER BY makes it deterministic. +CREATE TABLE t1 (id String, val String) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE t2 (id String, code String) ENGINE = MergeTree() ORDER BY id; +CREATE TABLE t3 (id String, code String) ENGINE = MergeTree() ORDER BY id; +INSERT INTO t1 VALUES ('a', 'v'), ('b', 'w'); +INSERT INTO t2 VALUES ('b', 'c'); +INSERT INTO t3 VALUES ('a_1', 'c'), ('b_1', 'd'); + +SELECT t2.id || '_1' AS id, t1.val +FROM remote('127.0.0.{1,2}', currentDatabase(), t1) AS t1 +LEFT JOIN t2 ON t1.id = t2.id +LEFT JOIN t3 USING (id) +ORDER BY t1.val, id +SETTINGS join_use_nulls = 1; + +DROP TABLE events; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t_shadow; +DROP TABLE t_shadow2; +DROP TABLE t_aliascol; diff --git a/tests/queries/0_stateless/04612_key_condition_nested_cast_lowcardinality.reference b/tests/queries/0_stateless/04612_key_condition_nested_cast_lowcardinality.reference new file mode 100644 index 000000000000..f6f9fc763320 --- /dev/null +++ b/tests/queries/0_stateless/04612_key_condition_nested_cast_lowcardinality.reference @@ -0,0 +1,20 @@ +1 +1 +1 +1 +35 +1 +1 +1 +1 +1 +1 +1 +1 +1 +3 +1 +1 +1 +2 +1 diff --git a/tests/queries/0_stateless/04612_key_condition_nested_cast_lowcardinality.sql b/tests/queries/0_stateless/04612_key_condition_nested_cast_lowcardinality.sql new file mode 100644 index 000000000000..07c81a660e93 --- /dev/null +++ b/tests/queries/0_stateless/04612_key_condition_nested_cast_lowcardinality.sql @@ -0,0 +1,175 @@ +-- Regression test for a LOGICAL_ERROR ("Bad cast from ColumnString to ColumnLowCardinality") during +-- primary-key index analysis over a LowCardinality key wrapped in a nested CAST chain that +-- re-introduces LowCardinality mid-chain, e.g. CAST(CAST(s, 'LowCardinality(String)'), 'String'). +-- Each chain function is built against the previous one's result type, so an inner +-- CAST(..., 'LowCardinality(String)') legitimately makes the next wrapper declare a LowCardinality +-- argument. applyFunction used to cache every intermediate result with LowCardinality stripped from +-- both the type and the column, so that next wrapper (which has +-- useDefaultImplementationForLowCardinalityColumns = false) received a full ColumnString and its +-- checkAndGetColumn aborted (in debug/sanitizer) or failed the query (in +-- release). The cache now keeps each function's own result type and representation, so every function +-- receives exactly the argument type it was built for. + +SET allow_suspicious_low_cardinality_types = 1; + +DROP TABLE IF EXISTS t_04612; + +CREATE TABLE t_04612 (s LowCardinality(Nullable(Int32))) + ENGINE = MergeTree ORDER BY s + SETTINGS index_granularity = 8, allow_nullable_key = 1; +INSERT INTO t_04612 SELECT number FROM numbers(20); +INSERT INTO t_04612 SELECT number + 1000 FROM numbers(20); + +-- Each of these previously aborted in KeyCondition. The PK-pruned count (WHERE) is checked against a +-- brute-force scan (countIf over the same predicate) so pruning stays correct, not just non-crashing. +SELECT count() = (SELECT countIf(CAST(CAST(s, 'LowCardinality(String)'), 'String') < '5') FROM t_04612) + FROM t_04612 WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') < '5'; +SELECT count() = (SELECT countIf(CAST(CAST(s, 'LowCardinality(String)'), 'Nullable(String)') < '5') FROM t_04612) + FROM t_04612 WHERE CAST(CAST(s, 'LowCardinality(String)'), 'Nullable(String)') < '5'; +SELECT count() = (SELECT countIf(CAST(CAST(CAST(s, 'LowCardinality(String)'), 'String'), 'String') < '5') FROM t_04612) + FROM t_04612 WHERE CAST(CAST(CAST(s, 'LowCardinality(String)'), 'String'), 'String') < '5'; +SELECT count() = (SELECT countIf(CAST(CAST(s, 'LowCardinality(String)'), 'Nullable(FixedString(8))') < '5') FROM t_04612) + FROM t_04612 WHERE CAST(CAST(s, 'LowCardinality(String)'), 'Nullable(FixedString(8))') < '5'; + +-- The concrete pruned count, for a stable reference. +SELECT count() FROM t_04612 WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') < '5'; + +-- Pruning must still fire (a "safe fallback" that silently disables the index would be a regression +-- that a correct-results check cannot catch): a selective nested-cast predicate over a numeric +-- LowCardinality key must read only a small fraction of granules. Assert read_granules < total_granules. +DROP TABLE IF EXISTS t_04612_prune; +CREATE TABLE t_04612_prune (s LowCardinality(Int64)) ENGINE = MergeTree ORDER BY s SETTINGS index_granularity = 8; +INSERT INTO t_04612_prune SELECT number FROM numbers(1000); +SELECT + toUInt32(extract(g, '^(\d+)')) < toUInt32(extract(g, '/(\d+)$')) AS pruning_fired +FROM ( + SELECT extract(trimLeft(explain), 'Granules: (\d+/\d+)') AS g + FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_04612_prune WHERE toInt64(CAST(s, 'LowCardinality(Int64)')) BETWEEN 40 AND 45 + ) + WHERE explain ILIKE '%Granules: %/%' +); + +DROP TABLE t_04612; +DROP TABLE t_04612_prune; + +-- Same regression through a typed LowCardinality ALIAS over a numeric key (plus a skip index), +-- distinct from the String CAST variants above. +DROP TABLE IF EXISTS t_04612_alias; +CREATE TABLE t_04612_alias + (a UInt64, x LowCardinality(UInt64) ALIAS a + 1, y UInt64 ALIAS x * 2, + INDEX idx y TYPE bloom_filter GRANULARITY 1) + ENGINE = MergeTree ORDER BY a + SETTINGS index_granularity = 8, allow_suspicious_low_cardinality_types = 1; +INSERT INTO t_04612_alias SELECT number FROM numbers(1000); + +-- Previously aborted in KeyCondition; PK-pruned count checked against a brute-force scan. +SELECT count() = (SELECT countIf(y = 1048576) FROM t_04612_alias) FROM t_04612_alias WHERE y = 1048576; +SELECT count() = (SELECT countIf(y = 1000) FROM t_04612_alias) FROM t_04612_alias WHERE y = 1000; +SELECT count() FROM t_04612_alias WHERE y = 1000; + +-- Pruning must still fire over the typed-ALIAS chain (not a silent index-disabling fallback). +SELECT + toUInt32(extract(g, '^(\d+)')) < toUInt32(extract(g, '/(\d+)$')) AS pruning_fired +FROM ( + SELECT extract(trimLeft(explain), 'Granules: (\d+/\d+)') AS g + FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_04612_alias WHERE y = 1000 + ) + WHERE explain ILIKE '%Granules: %/%' +); + +DROP TABLE t_04612_alias; + +-- Same nested-cast chain over a LowCardinality PARTITION key. Unlike the WHERE cases above (which +-- run through applyFunction's cached-column branch), partition/minmax pruning feeds explicit Field +-- bounds through applyFunctionForField, so this covers the second half of the fix. +DROP TABLE IF EXISTS t_04612_part; +CREATE TABLE t_04612_part (s LowCardinality(Nullable(Int32)), v UInt32) + ENGINE = MergeTree PARTITION BY s ORDER BY v + SETTINGS allow_nullable_key = 1; +INSERT INTO t_04612_part SELECT number, number FROM numbers(20); +INSERT INTO t_04612_part SELECT number + 1000, number FROM numbers(20); + +SELECT count() = (SELECT countIf(CAST(CAST(s, 'LowCardinality(String)'), 'String') < '5') FROM t_04612_part) + FROM t_04612_part WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') < '5'; + +-- At least one index (partition/minmax) must prune parts (a "Parts: X/Y" line with X < Y). +SELECT max(toUInt32(extract(g, '^(\d+)')) < toUInt32(extract(g, '/(\d+)$'))) AS pruning_fired +FROM ( + SELECT extract(trimLeft(explain), 'Parts: (\d+/\d+)') AS g + FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_04612_part WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') < '5' + SETTINGS optimize_use_implicit_projections = 0, optimize_trivial_count_query = 0 + ) + WHERE explain ILIKE '%Parts: %/%' +); + +-- The set-index path (IN, and `has` over a constant array) applies the same chain through +-- MergeTreeSetIndex::checkInRange, which passes the key column's raw (still LowCardinality) type. Both +-- of these aborted with "Bad cast from ColumnLowCardinality to ColumnNullable" until +-- applyMonotonicFunctionsChainToRange normalized the incoming type itself. +SELECT count() = (SELECT countIf(CAST(CAST(s, 'LowCardinality(String)'), 'String') IN ('1', '2', '3')) FROM t_04612_part) + FROM t_04612_part WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') IN ('1', '2', '3'); +SELECT count() = (SELECT countIf(has(['1', '2', '3'], CAST(CAST(s, 'LowCardinality(String)'), 'String'))) FROM t_04612_part) + FROM t_04612_part WHERE has(['1', '2', '3'], CAST(CAST(s, 'LowCardinality(String)'), 'String')); +SELECT count() FROM t_04612_part WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') IN ('1', '2', '3'); + +-- The set-index path must still prune parts, not merely return the right count: a chain whose types +-- disagree makes MergeTreeSetIndex decline silently and scan everything, which count equality alone +-- cannot detect. +SELECT max(toUInt32(extract(g, '^(\d+)')) < toUInt32(extract(g, '/(\d+)$'))) AS pruning_fired +FROM ( + SELECT extract(trimLeft(explain), 'Parts: (\d+/\d+)') AS g + FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_04612_part WHERE CAST(CAST(s, 'LowCardinality(String)'), 'String') IN ('1', '2', '3') + SETTINGS optimize_use_implicit_projections = 0, optimize_trivial_count_query = 0 + ) + WHERE explain ILIKE '%Parts: %/%' +); +SELECT max(toUInt32(extract(g, '^(\d+)')) < toUInt32(extract(g, '/(\d+)$'))) AS pruning_fired +FROM ( + SELECT extract(trimLeft(explain), 'Parts: (\d+/\d+)') AS g + FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_04612_part WHERE has(['1', '2', '3'], CAST(CAST(s, 'LowCardinality(String)'), 'String')) + SETTINGS optimize_use_implicit_projections = 0, optimize_trivial_count_query = 0 + ) + WHERE explain ILIKE '%Parts: %/%' +); + +DROP TABLE t_04612_part; + +-- A comparison whose constant needs a supertype cast appended after the chain. `extractAtomFromTree` +-- strips LowCardinality from the key type to pick that supertype, but the chain's last function still +-- returns LowCardinality, so the appended cast must declare the type it is actually given. Declaring +-- the stripped type instead made this fail with "Illegal column LowCardinality(Int32) of first +-- argument of function toDateTime64" (ILLEGAL_COLUMN). A LowCardinality PARTITION key routes this +-- through applyFunctionForField, the explicit-Field path. +DROP TABLE IF EXISTS t_04612_super; +CREATE TABLE t_04612_super (d LowCardinality(Date), v UInt32) + ENGINE = MergeTree ORDER BY tuple() PARTITION BY d + SETTINGS index_granularity = 8; +INSERT INTO t_04612_super SELECT toDate('2020-01-01') + number, number FROM numbers(4); + +SELECT count() = (SELECT countIf(CAST(d, 'LowCardinality(Date32)') < toDateTime('2020-01-03 00:00:00')) FROM t_04612_super) + FROM t_04612_super WHERE CAST(d, 'LowCardinality(Date32)') < toDateTime('2020-01-03 00:00:00'); +SELECT count() FROM t_04612_super WHERE CAST(d, 'LowCardinality(Date32)') < toDateTime('2020-01-03 00:00:00'); + +-- Partition pruning must still fire for that predicate. +SELECT max(toUInt32(extract(g, '^(\d+)')) < toUInt32(extract(g, '/(\d+)$'))) AS pruning_fired +FROM ( + SELECT extract(trimLeft(explain), 'Parts: (\d+/\d+)') AS g + FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM t_04612_super WHERE CAST(d, 'LowCardinality(Date32)') < toDateTime('2020-01-03 00:00:00') + SETTINGS optimize_use_implicit_projections = 0, optimize_trivial_count_query = 0 + ) + WHERE explain ILIKE '%Parts: %/%' +); + +DROP TABLE t_04612_super; diff --git a/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python b/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python new file mode 100644 index 000000000000..b9aac4e38b04 --- /dev/null +++ b/tests/queries/0_stateless/04616_replicated_serialization_oob_index_native_protocol.python @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Regression test for a post-auth out-of-bounds read over the native TCP protocol. +# +# A crafted `REPLICATED`-serialized column can trigger the same out-of-bounds read +# (nested_column[index]) via two independent paths that SerializationReplicated must +# reject with INCORRECT_DATA (code 117), while the server stays alive: +# +# 1. out-of-bounds index: per-row indexes point past the nested ("elements") column. +# Before the fix, only the index row count was validated, not the index values. +# 2. truncated elements: indexes are all in range for the advertised num_elements, but +# the elements payload is cut short. Bulk readers of primitive types (e.g. +# SerializationNumber::deserializeBinaryBulk) short-read on EOF instead of throwing, +# and NativeReader only checks the top-level column size (the index count for +# ColumnReplicated, not the nested column size), so without an explicit post-read +# size check the nested column would silently end up smaller than num_elements. +# +# The client speaks the raw native protocol pinned to revision 54482 +# (DBMS_MIN_REVISION_WITH_REPLICATED_SERIALIZATION, the minimum that enables REPLICATED), +# so the set of exchanged, revision-gated fields is fixed and does not drift. + +import os +import socket +import struct +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "replicated oob poc" + +# DBMS_MIN_REVISION_WITH_REPLICATED_SERIALIZATION +REVISION = 54482 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt8(s): + return readUInt(s) + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) # database + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version (>= 54471) + readStringBinary(s) # timezone (>= 54058) + readStringBinary(s) # display name (>= 54372) + readVarUInt(s) # version patch (>= 54401) + readStringBinary(s) # proto_send chunked (>= 54470) + readStringBinary(s) # proto_recv chunked (>= 54470) + for _ in range(readVarUInt(s)): # password complexity rules (>= 54461) + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce, UInt64 (>= 54462) + # Server settings in STRINGS_WITH_FLAGS format: (name, flags, value)* terminated + # by an empty name (>= 54474). The server sends its changed settings here. + while True: + if readStringBinary(s) == "": # setting name + break + readVarUInt(s) # flags + readStringBinary(s) # value + readVarUInt(s) # query plan serialization version (>= 54477) + readVarUInt(s) # cluster function protocol version (>= 54479) + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key (>= 54458) + writeStringBinary("notchunked", ba) # proto_send chunked (>= 54470) + writeStringBinary("notchunked", ba) # proto_recv chunked (>= 54470) + writeVarUInt(0, ba) # parallel replicas protocol version (>= 54471) + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds (>= 54449) + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key (>= 54060) + writeVarUInt(0, ba) # distributed_depth (>= 54448) + writeVarUInt(1, ba) # client version patch (>= 54401) + ba.append(0) # opentelemetry: no trace id (>= 54442) + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator (>= 54453) + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number (>= 54475) + writeVarUInt(0, ba) # script line number (>= 54475) + ba.append(0) # jwt: none (>= 54476) + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings (they are set in the query text) + writeStringBinary("", ba) # interserver externally granted roles (>= 54472) + writeStringBinary("", ba) # interserver secret (>= 54441) + writeVarUInt(2, ba) # stage = Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator (>= 54459) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) # field 1 + ba.append(0) # is_overflows = false + writeVarUInt(2, ba) # field 2 + ba.extend(struct.pack("= 54480) + for _ in range(readVarUInt(s)): + readUInt32(s) + else: + raise RuntimeError("Unknown BlockInfo field {}".format(field_num)) + + +def sendEmptyBlock(s): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) + serializeBlockInfo(ba) + writeVarUInt(0, ba) # columns + writeVarUInt(0, ba) # rows + s.sendall(ba) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readUInt8(s) # has_nested + return code, text + + +def readKindStack(s): + kind = readUInt8(s) + if kind == 5: # COMBINATION: number of kinds and the kinds themselves + readStrict(s, readVarUInt(s)) + + +def readHeader(s): + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + code, text = readException(s) + raise RuntimeError("Unexpected exception {}: {}".format(code, text)) + assert packet_type == 1, "Expected Data (header) packet, got {}".format(packet_type) + readStringBinary(s) # external table name + readBlockInfo(s) + columns = readVarUInt(s) + rows = readVarUInt(s) + assert rows == 0, "Expected an empty header block, got {} rows".format(rows) + # Consume the per-column descriptor so the stream stays aligned. + for _ in range(columns): + readStringBinary(s) # column name + readStringBinary(s) # column type + if readUInt8(s): # has custom serialization (>= 54454) + readKindStack(s) + + +def sendCraftedReplicatedBlock(s): + limit = 4 + num_elements = 1 + oob_index = 0xFFFFFFFF + + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) # table name + serializeBlockInfo(ba) + writeVarUInt(1, ba) # columns + writeVarUInt(limit, ba) # rows + writeStringBinary("x", ba) + writeStringBinary("UInt64", ba) + ba.append(1) # has_custom serialization + ba.append(4) # KindStackBinarySerializationType::REPLICATED + + # ReplicatedIndexes substream: row count, size of index type, index values. + writeVarUInt(limit, ba) + ba.append(8) # size_of_index_type = sizeof(UInt64) + for _ in range(limit): + ba.extend(struct.pack("&1 | grep -m1 -c "NOT_IMPLEMENTED" -$CLICKHOUSE_CLIENT -q "SELECT count() FROM part_dst" +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO part_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM part_dst), (SELECT count(DISTINCT _partition_id) FROM part_dst)" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO part_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM part_dst), (SELECT count(DISTINCT _partition_id) FROM part_dst)" $CLICKHOUSE_CLIENT -q "DROP TABLE part_mv2" $CLICKHOUSE_CLIENT -q "DROP TABLE part_mv1" diff --git a/tests/queries/0_stateless/04623_union_aggregate_state_in_tuple_header.reference b/tests/queries/0_stateless/04623_union_aggregate_state_in_tuple_header.reference new file mode 100644 index 000000000000..8ad3478a0ca0 --- /dev/null +++ b/tests/queries/0_stateless/04623_union_aggregate_state_in_tuple_header.reference @@ -0,0 +1,11 @@ +2 +2 +2 +1 +4 +1 +2 +1 +2 +[1] +[2] diff --git a/tests/queries/0_stateless/04623_union_aggregate_state_in_tuple_header.sql b/tests/queries/0_stateless/04623_union_aggregate_state_in_tuple_header.sql new file mode 100644 index 000000000000..eb291425d7e1 --- /dev/null +++ b/tests/queries/0_stateless/04623_union_aggregate_state_in_tuple_header.sql @@ -0,0 +1,120 @@ +-- Found by AST fuzzer. An expression over a `UNION ALL` of aggregate-state columns with the same +-- state representation but different functions (`quantileState` vs `quantilesState(0.9)`) wraps +-- the states into a `Tuple`. The per-branch headers then differ only by the aggregate function +-- nested inside the tuple, and the block structure checks (e.g. after the `liftUpUnion` +-- optimization and in `QueryPipeline`) compared such nested columns strictly by name, failing +-- debug and sanitizer builds with a logical error "Block structure mismatch". +SELECT tuple(s) FROM +( + SELECT quantileState(number) AS s FROM numbers(7) + UNION ALL + SELECT quantilesState(0.9)(number) FROM numbers(5) +) FORMAT Null; + +SELECT count() FROM +( + SELECT tuple(s) FROM + ( + SELECT quantileState(number) AS s FROM numbers(7) + UNION ALL + SELECT quantilesState(0.9)(number) FROM numbers(5) + ) +); + +-- Constant aggregate-state columns. The constant-value comparison in the block structure check +-- must relax the aggregate-state leaves: comparing aggregate states as `Field` throws when the +-- aggregate function type names differ, even though the states are compatible by state +-- representation. +SELECT count() FROM +( + SELECT arrayReduce('quantileState(0.5)', [1]) AS s + UNION ALL + SELECT arrayReduce('quantilesState(0.9)', [1]) AS s +); + +-- The same, but the constant aggregate state is nested inside a `Tuple`. +SELECT count() FROM +( + SELECT tuple(arrayReduce('quantileState(0.5)', [1])) AS s + UNION ALL + SELECT tuple(arrayReduce('quantilesState(0.9)', [1])) AS s +); + +-- The queries below build a `Variant` from a set operation over unrelated types, which only the +-- analyzer does (the old one fails to find a common type for the branches). +SET enable_analyzer = 1; + +-- The same, but the aggregate state is nested inside a `Variant`: a set operation over two +-- unrelated `Tuple` types builds a `Variant` of both, and the aggregate state lives inside one of +-- the alternatives. The `Variant` columns then differ only by the nested aggregate function, so the +-- block structure check must descend into `Variant` alternatives too. +SELECT count() IGNORE NULLS FROM +( + (SELECT tuple(3, quantileState(number)) FROM numbers(7)) + EXCEPT ALL + (SELECT tuple(quantilesState(0.9)(number), toInt128(2)) FROM numbers(5)) +); + +-- The alternatives inside a `Variant` column are stored in a local order that may differ from the +-- global (type) order and between the two sides of a `UNION`. The check must compare the +-- alternatives by global discriminator (the order the column name lists them in), not in the +-- storage order, otherwise same-typed `Variant` columns are reported as a structure mismatch. +SELECT count() FROM +( + SELECT n, m FROM + ( + (SELECT 2 AS n, map('z', 'a') AS m FROM numbers(2)) + EXCEPT ALL + (SELECT map(toFixedString('z', 1), 'a') AS m, 2 AS n FROM numbers(2)) + ) + UNION ALL + SELECT toLowCardinality(1) AS n, map('-1', 'b') AS m FROM numbers(2) +) +SETTINGS allow_suspicious_types_in_order_by = 1; + +-- The relaxation of the constant-value comparison must apply only to the aggregate-state leaves. +-- A constant `Tuple` whose aggregate-state element is compatible between the branches but whose +-- scalar element differs holds genuinely different constants: they must not be collapsed into a +-- single header constant, and the scalar element must keep the per-branch values. +SELECT t.2 AS scalar FROM +( + SELECT tuple(arrayReduce('quantileState(0.5)', [1]), 1) AS t + UNION ALL + SELECT tuple(arrayReduce('quantilesState(0.9)', [1]), 2) AS t +) +ORDER BY scalar; + +-- The same for top-level constant aggregate states with different serialized state bytes: the +-- states are compatible by state representation, but they are different constants and must keep +-- the per-branch values. +SELECT finalizeAggregation(s) AS v FROM +( + SELECT arrayReduce('quantileState(0.5)', [1]) AS s + UNION ALL + SELECT arrayReduce('quantileState(0.7)', [2]) AS s +) +ORDER BY v; + +-- Shadowing a constant aggregate-state column with a compatible constant of a different aggregate +-- function under the same alias. The planner compares the constant values of the same-name INPUT +-- and COLUMN nodes when finalizing an actions chain step, and the plain `Field` comparison of the +-- aggregate states throws for different function names. Such constants must compare as different +-- without throwing, so that the redefinition is preserved. +SELECT finalizeAggregation(s) FROM +( + SELECT arrayReduce('quantilesState(0.9)', [1]) AS s + FROM + ( + SELECT arrayReduce('quantileState(0.5)', [1]) AS s + ) +); + +-- The same with different serialized state bytes: the outer redefinition must win. +SELECT finalizeAggregation(s) FROM +( + SELECT arrayReduce('quantilesState(0.9)', [2]) AS s + FROM + ( + SELECT arrayReduce('quantileState(0.5)', [1]) AS s + ) +); diff --git a/tests/queries/0_stateless/04627_object_storage_lazy_hive_partitioning.reference b/tests/queries/0_stateless/04627_object_storage_lazy_hive_partitioning.reference new file mode 100644 index 000000000000..5078afda1eed --- /dev/null +++ b/tests/queries/0_stateless/04627_object_storage_lazy_hive_partitioning.reference @@ -0,0 +1,4 @@ +1 +2 +1 A +2 B diff --git a/tests/queries/0_stateless/04627_object_storage_lazy_hive_partitioning.sql b/tests/queries/0_stateless/04627_object_storage_lazy_hive_partitioning.sql new file mode 100644 index 000000000000..94d11ce42550 --- /dev/null +++ b/tests/queries/0_stateless/04627_object_storage_lazy_hive_partitioning.sql @@ -0,0 +1,41 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: Depends on S3 + +-- CREATE and ATTACH of an object storage table with an explicit schema and format must not access +-- the endpoint. The hive partitioning sample path is resolved lazily on the first use of the table. + +DROP TABLE IF EXISTS 04627_unreachable, 04627_hive, 04627_wrong_creds; + +CREATE TABLE 04627_unreachable (id UInt64, val String) +ENGINE = S3('http://localhost:1/no-such-bucket/*.parquet', 'test', 'testtest', 'Parquet'); + +DETACH TABLE 04627_unreachable; +ATTACH TABLE 04627_unreachable; + +DROP TABLE 04627_unreachable; + +-- Hive partition columns are still detected, on the first query over the table. +SET s3_truncate_on_insert = 1; + +INSERT INTO FUNCTION s3(s3_conn, url = 'http://localhost:11111/test/04627_hive/key=A/data.parquet', format = Parquet) SELECT 1 AS id; +INSERT INTO FUNCTION s3(s3_conn, url = 'http://localhost:11111/test/04627_hive/key=B/data.parquet', format = Parquet) SELECT 2 AS id; + +CREATE TABLE 04627_hive (id UInt64) +ENGINE = S3(s3_conn, url = 'http://localhost:11111/test/04627_hive/**.parquet', format = Parquet); + +-- The resolution follows the construction context, the triggering query settings do not override it. +SELECT id FROM 04627_hive ORDER BY id SETTINGS use_hive_partitioning = 0; + +SELECT id, key FROM 04627_hive ORDER BY id; + +DROP TABLE 04627_hive; + +-- Wrong credentials make the resolution fail fast. By default the triggering query runs +-- with only a warning, with `throw_on_hive_partitioning_resolution_failure` it fails. +CREATE TABLE 04627_wrong_creds (id UInt64) +ENGINE = S3('http://localhost:11111/test/04627_hive/**.parquet', 'invalid', 'invalid', 'Parquet'); + +DESCRIBE TABLE 04627_wrong_creds FORMAT Null; +DESCRIBE TABLE 04627_wrong_creds SETTINGS throw_on_hive_partitioning_resolution_failure = 1; -- {serverError S3_ERROR} + +DROP TABLE 04627_wrong_creds; diff --git a/tests/queries/0_stateless/04628_read_nothing_step_serializable.reference b/tests/queries/0_stateless/04628_read_nothing_step_serializable.reference new file mode 100644 index 000000000000..ae03cf29d129 --- /dev/null +++ b/tests/queries/0_stateless/04628_read_nothing_step_serializable.reference @@ -0,0 +1,16 @@ +aggregation over an empty table +0 +0 +group by over an empty table +aggregation over an empty table distributes +group by over an empty table distributes +empty side unioned with a populated table +4999900000 +4999950000 +5000000000 +5000050000 +join with an empty side +0 +wrapped and compound header types +0 \N [] {} (0,'') \N +projection above an empty source diff --git a/tests/queries/0_stateless/04628_read_nothing_step_serializable.sql b/tests/queries/0_stateless/04628_read_nothing_step_serializable.sql new file mode 100644 index 000000000000..17643611c6f4 --- /dev/null +++ b/tests/queries/0_stateless/04628_read_nothing_step_serializable.sql @@ -0,0 +1,67 @@ +-- Tags: no-old-analyzer +-- no-old-analyzer: make_distributed_plan requires the analyzer. + +DROP TABLE IF EXISTS t_read_nothing; +DROP TABLE IF EXISTS t_read_nothing_full; +DROP TABLE IF EXISTS t_read_nothing_types; + +-- Left empty on purpose: an empty MergeTree table is planned as a `ReadNothing` source. +CREATE TABLE t_read_nothing (x UInt64) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t_read_nothing_full (x UInt64) ENGINE = MergeTree ORDER BY tuple() AS SELECT number FROM numbers(200000); +CREATE TABLE t_read_nothing_types +( + lc LowCardinality(Nullable(String)), + arr Array(UInt64), + m Map(String, Nullable(Int32)), + t Tuple(a UInt8, b String), + n Nullable(Float64) +) ENGINE = MergeTree ORDER BY tuple(); + +SET distributed_plan_default_shuffle_join_bucket_count = 3, distributed_plan_default_reader_bucket_count = 3; +-- Distributed aggregation cannot enforce a global max_rows_to_group_by, and the functional-test +-- profile sets it nonzero, so pin it off. Trivial-count would fold the aggregation away. +SET make_distributed_plan = 1, enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, + distributed_plan_execute_locally = 1, distributed_plan_max_rows_to_broadcast = 0, + max_rows_to_group_by = 0, optimize_trivial_count_query = 0; + +-- All of the following previously failed with +-- SUPPORT_IS_DISABLED: step 'ReadNothing' is not serializable for remote execution. +SELECT 'aggregation over an empty table'; +SELECT sum(x) FROM t_read_nothing; +SELECT count() FROM t_read_nothing; +SELECT 'group by over an empty table'; +SELECT x % 8, sum(x) FROM t_read_nothing GROUP BY 1 ORDER BY 1; + +-- The serializability check only runs once a plan splits into more than one stage, so a query that +-- stays single-stage would pass without ever reaching `ReadNothingStep::deserialize`. Assert that +-- these plans really do distribute. The row is absent unless an exchange was planted, so removing +-- `make_distributed_plan` above makes the test fail instead of silently covering nothing. The +-- oracle must not aggregate: an aggregating query over `EXPLAIN` is itself distributed. +SELECT 'aggregation over an empty table distributes' +FROM (EXPLAIN PIPELINE SELECT sum(x) FROM t_read_nothing) +WHERE explain LIKE '%ReadFromDistributedPlanSource%' LIMIT 1; +SELECT 'group by over an empty table distributes' +FROM (EXPLAIN PIPELINE SELECT x % 8, sum(x) FROM t_read_nothing GROUP BY 1 ORDER BY 1) +WHERE explain LIKE '%ReadFromDistributedPlanSource%' LIMIT 1; + +SELECT 'empty side unioned with a populated table'; +SELECT sum(x) FROM (SELECT x FROM t_read_nothing UNION ALL SELECT x FROM t_read_nothing_full) +GROUP BY x % 4 ORDER BY 1; + +SELECT 'join with an empty side'; +SELECT count() FROM t_read_nothing_full a INNER JOIN t_read_nothing b ON a.x = b.x; + +-- The output header is the step's whole state and travels through the generic per-node preamble, +-- so exercise wrapped and compound types explicitly. +SELECT 'wrapped and compound header types'; +SELECT count(), min(lc), max(arr), min(m), max(t), sum(n) FROM t_read_nothing_types; +SELECT lc, groupArray(arr) FROM t_read_nothing_types GROUP BY lc ORDER BY lc; + +-- Only column names and types travel in the serialized header, so a projection above the source +-- does not change what `ReadNothing` carries. Kept as a plain shape check. +SELECT 'projection above an empty source'; +SELECT c, sum(x) FROM (SELECT 42 AS c, x FROM t_read_nothing) GROUP BY c ORDER BY c; + +DROP TABLE t_read_nothing; +DROP TABLE t_read_nothing_full; +DROP TABLE t_read_nothing_types; diff --git a/tests/queries/0_stateless/04628_split_filter_name_clash_header_leak.reference b/tests/queries/0_stateless/04628_split_filter_name_clash_header_leak.reference new file mode 100644 index 000000000000..f312e4a4bae6 --- /dev/null +++ b/tests/queries/0_stateless/04628_split_filter_name_clash_header_leak.reference @@ -0,0 +1,9 @@ +1 1 +intersect +except +union +1 +values +1 +1 7 107 +\N 7 107 diff --git a/tests/queries/0_stateless/04628_split_filter_name_clash_header_leak.sql b/tests/queries/0_stateless/04628_split_filter_name_clash_header_leak.sql new file mode 100644 index 000000000000..1850bb909879 --- /dev/null +++ b/tests/queries/0_stateless/04628_split_filter_name_clash_header_leak.sql @@ -0,0 +1,34 @@ +SET enable_analyzer = 1; +SET query_plan_enable_optimizations = 1; +SET query_plan_split_filter = 1; +-- The '[split]' step marker is only emitted when this is non-zero. +SET query_plan_max_step_description_length = 500; + +-- The split must fire, and the split filter column must not survive in the branch output header. +SELECT countSubstrings(explain, '[split]') > 0 AS split_fired, + length(JSONExtractArrayRaw(explain, 1, 'Plan', 'Header')) AS branch_header_columns +FROM (EXPLAIN json = 1, header = 1 + SELECT x FROM (SELECT arrayJoin([materialize(1), NULL]) AS x GROUP BY NULL) WHERE NULL); + +SELECT 'intersect'; +SELECT x FROM (SELECT arrayJoin([materialize(1), NULL]) AS x GROUP BY NULL) WHERE NULL +INTERSECT DISTINCT +SELECT 1; + +SELECT 'except'; +SELECT x FROM (SELECT arrayJoin([materialize(1), NULL]) AS x GROUP BY NULL) WHERE NULL +EXCEPT DISTINCT +SELECT 1; + +SELECT 'union'; +SELECT x FROM (SELECT arrayJoin([materialize(1), NULL]) AS x GROUP BY NULL) WHERE NULL +UNION ALL +SELECT 1; + +-- The filter column is also an input name here and is consumed downstream, so a mis-resolved +-- name would surface as wrong values or NOT_FOUND_COLUMN_IN_BLOCK, not as a header leak. +SELECT 'values'; +SELECT countSubstrings(explain, '[split]') > 0 AS split_fired +FROM (EXPLAIN json = 1, header = 1 + SELECT x, materialize(7) AS k, k + 100 FROM (SELECT arrayJoin([materialize(1), NULL]) AS x GROUP BY materialize(7)) WHERE materialize(7)); +SELECT x, materialize(7) AS k, k + 100 FROM (SELECT arrayJoin([materialize(1), NULL]) AS x GROUP BY materialize(7)) WHERE materialize(7) ORDER BY x NULLS LAST; diff --git a/tests/queries/0_stateless/04630_text_index_serialization_version_compatibility.reference b/tests/queries/0_stateless/04630_text_index_serialization_version_compatibility.reference new file mode 100644 index 000000000000..fad6d3770b60 --- /dev/null +++ b/tests/queries/0_stateless/04630_text_index_serialization_version_compatibility.reference @@ -0,0 +1,12 @@ +512 +-- reads work under the pin +512 +-- insert works under the pin +512 +-- merge works under the pin +512 +512 +-- the part merged under the pin contains the positions substream +positions file exists +-- without the pin writes and merges keep working +512 diff --git a/tests/queries/0_stateless/04630_text_index_serialization_version_compatibility.sh b/tests/queries/0_stateless/04630_text_index_serialization_version_compatibility.sh new file mode 100755 index 000000000000..e7318721b928 --- /dev/null +++ b/tests/queries/0_stateless/04630_text_index_serialization_version_compatibility.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `text_index_serialization_version` setting is a preference, not a hard constraint: an index +# with `positions` is always written in the 'v2_with_positions' format because older +# formats cannot represent positions. The `compatibility` setting may pin an older default on an +# existing table, but that must not make the index unwritable: inserts and merges keep working +# and keep writing the format the index requires, and readers take the format version from the +# on-disk header of each part. + +data_path="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +$CLICKHOUSE_LOCAL --path "$data_path" -m -q " +CREATE TABLE tab (id UInt32, str String, INDEX text_idx str TYPE text(tokenizer = 'splitByNonAlpha', positions = 1)) +ENGINE = MergeTree ORDER BY id +SETTINGS allow_experimental_text_index_positions = 1; +INSERT INTO tab SELECT number, 'foo bar' FROM numbers(512); +INSERT INTO tab SELECT number + 512, 'foo baz' FROM numbers(512); +SELECT count() FROM tab WHERE hasPhrase(str, 'foo bar'); +" + +# `compatibility` below the version that introduced positions resolves the table's effective +# `text_index_serialization_version` to 'v0_initial' without any change to the table metadata. + +echo '-- reads work under the pin' +$CLICKHOUSE_LOCAL --path "$data_path" --compatibility '26.5' -q "SELECT count() FROM tab WHERE hasPhrase(str, 'foo bar')" + +echo '-- insert works under the pin' +$CLICKHOUSE_LOCAL --path "$data_path" --compatibility '26.5' -m -q " +INSERT INTO tab SELECT number + 1024, 'qux quux' FROM numbers(512); +SELECT count() FROM tab WHERE hasPhrase(str, 'qux quux'); +" + +echo '-- merge works under the pin' +$CLICKHOUSE_LOCAL --path "$data_path" --compatibility '26.5' -m -q " +OPTIMIZE TABLE tab FINAL; +SELECT count() FROM tab WHERE hasPhrase(str, 'foo bar'); +SELECT count() FROM tab WHERE hasPhrase(str, 'qux quux'); +" + +echo '-- the part merged under the pin contains the positions substream' +part_path=$($CLICKHOUSE_LOCAL --path "$data_path" -q "SELECT path FROM system.parts WHERE database = currentDatabase() AND table = 'tab' AND active") +if [ -f "${part_path}skp_idx_text_idx.pos.idx" ]; then + echo 'positions file exists' +else + echo "no positions file in $part_path:" + ls "$part_path" +fi + +echo '-- without the pin writes and merges keep working' +$CLICKHOUSE_LOCAL --path "$data_path" -m -q " +INSERT INTO tab SELECT number + 1536, 'corge grault' FROM numbers(512); +OPTIMIZE TABLE tab FINAL; +SELECT count() FROM tab WHERE hasPhrase(str, 'corge grault'); +" + +rm -rf "${data_path:?}" diff --git a/tests/queries/0_stateless/04652_direct_join_dictionary_sparse_key.reference b/tests/queries/0_stateless/04652_direct_join_dictionary_sparse_key.reference new file mode 100644 index 000000000000..88fe6660ee1f --- /dev/null +++ b/tests/queries/0_stateless/04652_direct_join_dictionary_sparse_key.reference @@ -0,0 +1,19 @@ +The join keys are really serialized Sparse +probe_sparse 1 +probe_sparse_arr 1 +Sparse key, key presence only +4000 3200 800 160 3200 800 +Sparse key, dictionary attribute read +336000 4000 3200 800 +Sparse default key 0 is really found +800 3200 160 +Sparse key, aggregation in order +1 1 50 +Sparse key replicated by ARRAY JOIN, attribute read +378000 4500 900 3600 +Sparse mapping equals dense mapping +1 +Sparse mapping equals hash join mapping +1 +Direct join is still chosen for the sparse key +1 diff --git a/tests/queries/0_stateless/04652_direct_join_dictionary_sparse_key.sql b/tests/queries/0_stateless/04652_direct_join_dictionary_sparse_key.sql new file mode 100644 index 000000000000..ab8e3bb12fa6 --- /dev/null +++ b/tests/queries/0_stateless/04652_direct_join_dictionary_sparse_key.sql @@ -0,0 +1,128 @@ +-- Tags: no-parallel-replicas +-- DirectKeyValueJoin (the algorithm under test) cannot be chosen with parallel replicas, so the +-- ParallelReplicas runner variant would throw NOT_IMPLEMENTED instead of exercising the fix. +-- getColumnVectorData returned a reference into a column owned only by a function-local ColumnPtr +-- whenever the key column had to be materialized, so a Sparse-serialized left join key read freed +-- memory in FlatDictionary::hasKeys / ::getColumn. + +DROP DICTIONARY IF EXISTS dict_sparse_key; +DROP TABLE IF EXISTS probe_sparse; +DROP TABLE IF EXISTS probe_dense; +DROP TABLE IF EXISTS probe_sparse_arr; +DROP TABLE IF EXISTS dict_source; + +-- The attribute is never 0 and join_use_nulls is pinned to 0 below, so `r.v = 0` in the results +-- below unambiguously means "key not found": that is what a lookup blind to the sparse default +-- key 0 would produce. +CREATE TABLE dict_source (j UInt64, v UInt64) ENGINE = MergeTree ORDER BY j; +INSERT INTO dict_source SELECT number, (number + 1) * 10 FROM numbers(20); + +CREATE DICTIONARY dict_sparse_key (j UInt64, v UInt64) PRIMARY KEY j +SOURCE(CLICKHOUSE(TABLE 'dict_source' DB currentDatabase())) LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 0); + +-- `j` is default-heavy, so it is serialized Sparse and reaches the dictionary lookup unmaterialized. +-- 25 keys x 160 rows: keys 0..19 are present in the dictionary, keys 20..24 are absent. +CREATE TABLE probe_sparse (k UInt32, j UInt64) ENGINE = MergeTree ORDER BY k +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.0; +INSERT INTO probe_sparse SELECT number % 50, number % 25 FROM numbers(4000); + +-- Same data written densely: the reference results must agree with the sparse table. +CREATE TABLE probe_dense (k UInt32, j UInt64) ENGINE = MergeTree ORDER BY k +SETTINGS ratio_of_defaults_for_sparse_serialization = 1.0; +INSERT INTO probe_dense SELECT number % 50, number % 25 FROM numbers(4000); + +-- `j` is carried (not array-joined) through ARRAY JOIN over a Sparse base column, so it reaches the +-- lookup as a column that has to be materialized. The serialization guard below is what pins the +-- sparse base; the query does not distinguish lazy from eager replication. +CREATE TABLE probe_sparse_arr (j UInt64, arr Array(UInt8)) ENGINE = MergeTree ORDER BY tuple() +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.0; +INSERT INTO probe_sparse_arr SELECT number % 25, [1, 2, 3] FROM numbers(1500); + +SELECT 'The join keys are really serialized Sparse'; +SELECT table, countIf(serialization_kind = 'Sparse') > 0 FROM system.parts_columns +WHERE database = currentDatabase() AND table IN ('probe_sparse', 'probe_sparse_arr') + AND column = 'j' AND active +GROUP BY table ORDER BY table; + +SET join_algorithm = 'direct'; +-- The assertions below distinguish a dictionary hit from a miss by the attribute value, so the +-- representation of a miss must be fixed: with join_use_nulls = 1 a miss would be NULL instead of +-- the default 0. The Stress check injects join_use_nulls = 1 on some threads. +SET join_use_nulls = 0; + +-- FlatDictionary::hasKeys is the site that read the freed buffer. Its result is the presence mask, +-- which getByKeys applies to the returned right KEY column only, so r.j is what observes the mask +-- directly: the attribute counts would not, because attributes are fetched independently of it. The +-- two counts below catch a misclassified key whose value is not 0; key 0 needs the nullable-key +-- query further down, because a blanked right key is 0 as well. +SELECT 'Sparse key, key presence only'; +SELECT count(), countIf(r.v != 0), countIf(r.v = 0), countIf(l.j = 0 AND r.v = 10), + countIf(l.j < 20 AND r.j = l.j), countIf(l.j >= 20 AND r.j = 0) +FROM probe_sparse AS l LEFT JOIN dict_sparse_key AS r ON l.j = r.j; + +-- FlatDictionary::getColumn is a second site reached through the same getByKeys call. +-- countIf(r.v = (l.j + 1) * 10) is the dictionary contents restated inline: it equals the number of +-- present-key rows only if every one of them carries the value belonging to its own key, so a +-- lookup that permutes values between keys is caught even though it preserves sum(r.v). +SELECT 'Sparse key, dictionary attribute read'; +SELECT sum(r.v), count(), countIf(r.v = (l.j + 1) * 10), countIf(r.v = 0 AND l.j >= 20) +FROM probe_sparse AS l LEFT JOIN dict_sparse_key AS r ON l.j = r.j; + +-- Key 0 is the sparse default, and a blanked right key is also 0, so the assertion above cannot +-- tell a found key 0 from a lost one. Under join_use_nulls = 1 an unmatched row yields NULL for the +-- right key, which separates the two. This statement asks for that value explicitly, so the setting +-- the Stress check injects cannot change what it measures. +SELECT 'Sparse default key 0 is really found'; +SELECT countIf(r.j IS NULL), countIf(l.j < 20 AND r.j IS NOT NULL), countIf(l.j = 0 AND r.j = 0) +FROM probe_sparse AS l LEFT JOIN dict_sparse_key AS r ON l.j = r.j +SETTINGS join_use_nulls = 1; + +-- Aggregating in order over the sparse key: the shape observed failing in CI. +SELECT 'Sparse key, aggregation in order'; +SELECT max(u), min(u), count() FROM +( + SELECT l.k, uniqExact(l.k) AS u FROM probe_sparse AS l LEFT JOIN dict_sparse_key AS r ON l.j = r.j GROUP BY l.k +) +SETTINGS optimize_aggregation_in_order = 1, max_threads = 1; + +SELECT 'Sparse key replicated by ARRAY JOIN, attribute read'; +SELECT sum(r.v), count(), countIf(r.v = 0), countIf(r.v = (l.j + 1) * 10) +FROM (SELECT j FROM probe_sparse_arr ARRAY JOIN arr) AS l +LEFT JOIN dict_sparse_key AS r ON l.j = r.j +SETTINGS enable_lazy_columns_replication = 1; + +-- The whole per-key mapping, not just its total, must equal what a dense key and a non-direct join +-- produce on the same data: a total is invariant under any permutation of values between keys. +-- Grouping by (l.j, r.v) also exposes a key whose rows disagree with each other as extra tuples. +SELECT 'Sparse mapping equals dense mapping'; +SELECT + (SELECT arraySort(groupArray((j, v, c))) FROM + (SELECT l.j AS j, r.v AS v, count() AS c FROM probe_sparse AS l + LEFT JOIN dict_sparse_key AS r ON l.j = r.j GROUP BY l.j, r.v)) + = (SELECT arraySort(groupArray((j, v, c))) FROM + (SELECT l.j AS j, r.v AS v, count() AS c FROM probe_dense AS l + LEFT JOIN dict_sparse_key AS r ON l.j = r.j GROUP BY l.j, r.v)); + +SELECT 'Sparse mapping equals hash join mapping'; +SELECT + (SELECT arraySort(groupArray((j, v, c))) FROM + (SELECT l.j AS j, r.v AS v, count() AS c FROM probe_sparse AS l + LEFT JOIN dict_sparse_key AS r ON l.j = r.j GROUP BY l.j, r.v)) + = (SELECT arraySort(groupArray((j, v, c))) FROM + (SELECT l.j AS j, r.v AS v, count() AS c FROM probe_sparse AS l + LEFT JOIN dict_sparse_key AS r ON l.j = r.j GROUP BY l.j, r.v + SETTINGS join_algorithm = 'hash')); + +SELECT 'Direct join is still chosen for the sparse key'; +SELECT count() > 0 FROM +( + EXPLAIN actions = 1 + SELECT count() FROM probe_sparse AS l LEFT JOIN dict_sparse_key AS r ON l.j = r.j +) +WHERE explain ILIKE '%Algorithm: DirectKeyValueJoin%'; + +DROP DICTIONARY dict_sparse_key; +DROP TABLE probe_sparse; +DROP TABLE probe_dense; +DROP TABLE probe_sparse_arr; +DROP TABLE dict_source; diff --git a/tests/queries/0_stateless/04654_parquet_bloom_filter_bitset_out_of_bounds.reference b/tests/queries/0_stateless/04654_parquet_bloom_filter_bitset_out_of_bounds.reference new file mode 100644 index 000000000000..06eb5c37dfd1 --- /dev/null +++ b/tests/queries/0_stateless/04654_parquet_bloom_filter_bitset_out_of_bounds.reference @@ -0,0 +1,3 @@ +1 +1 +4950 diff --git a/tests/queries/0_stateless/04654_parquet_bloom_filter_bitset_out_of_bounds.sh b/tests/queries/0_stateless/04654_parquet_bloom_filter_bitset_out_of_bounds.sh new file mode 100755 index 000000000000..c09132ddbc2a --- /dev/null +++ b/tests/queries/0_stateless/04654_parquet_bloom_filter_bitset_out_of_bounds.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DATA_FILE=$CUR_DIR/data_parquet/04654_bloom_filter_bitset_out_of_bounds.parquet + +# The `s` column's BloomFilterHeader in this file claims a 1 GiB bitset, while its column metadata +# declares only 272 bytes of bloom filter data. Deriving bloom filter block byte ranges from the +# claimed size used to read far outside the buffer holding the bloom filter. +${CLICKHOUSE_LOCAL} --query="SELECT count() FROM file('$DATA_FILE') WHERE s = '42' + SETTINGS input_format_parquet_bloom_filter_push_down = 1" 2>&1 | grep -c "INCORRECT_DATA" + +# The bloom filter is only read when it can prune, so the same file reads fine otherwise. +${CLICKHOUSE_LOCAL} --query="SELECT count() FROM file('$DATA_FILE') WHERE s = '42' + SETTINGS input_format_parquet_bloom_filter_push_down = 0" +${CLICKHOUSE_LOCAL} --query="SELECT sum(n) FROM file('$DATA_FILE')" diff --git a/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.reference b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.reference new file mode 100644 index 000000000000..ced195aff7fd --- /dev/null +++ b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.reference @@ -0,0 +1,4 @@ +400 100 2 +800 100 2 +200 2 +200 2 diff --git a/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.sh b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.sh new file mode 100755 index 000000000000..d755328e22c1 --- /dev/null +++ b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# The scenarios pin the deduplication path exactly (the squash thresholds, the insert/thread +# counts), so settings randomization is disabled, as in 04621_deduplication_alias_hop_partitioned_target. +# +# The production crash from a plain materialized view - NO `Alias` engine anywhere. A row-count +# changing inner query (`GROUP BY`) feeds a PARTITIONED deduplicating target. The target sink splits +# the view-output block by partition and DeduplicationInfo::filterToPartition attributes each token's +# source-row range to the partitions via the scatter selector. But the view changed the row count, so +# the tokens describe the source rows while the selector describes the smaller view-output block: +# there is no source-row -> partition mapping. filterToPartition must keep every token in every +# partition (the target may still deduplicate a repeated token) instead of reading out of the +# selector's bounds. +# See https://github.com/ClickHouse/clickhouse-core-incidents/issues/2006 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv_drift_mv" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv_drift_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv_drift_src" + +# Only dst deduplicates, and it is partitioned, so the sink splits every insert by partition before +# deduplicating. The view's GROUP BY reduces the row count between src and dst. +$CLICKHOUSE_CLIENT -q "CREATE TABLE mv_drift_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE mv_drift_dst (x UInt64) ENGINE = MergeTree PARTITION BY x % 2 ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW mv_drift_mv TO mv_drift_dst AS SELECT x FROM mv_drift_src GROUP BY x" + +# A single data-fed insert carries one deduplication token, so the partition split keeps the whole +# info for every partition (single-token fast path) and only the cached data hash is used: the +# repeated insert must deduplicate in both partitions of dst. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_src), (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_src), (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE mv_drift_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE mv_drift_dst" + +# Two deduplication tokens in one sync insert: the source-side squashing (min_insert_block_size_rows, +# with max_insert_block_size as the parser cap) re-blocks the 800 input rows into two 400-row source +# blocks, each carrying its own token, and the view-input squashing concatenates them into one block +# before the row-count-changing GROUP BY. The partitioned sink then sees two tokens over a block that +# no longer matches the tokens' source rows. filterToPartition keeps both tokens in both partitions +# (pre-fix it walked the source-row ranges over the smaller partition selector: an out-of-bounds +# read), so the insert fills dst and the repeated insert is deduplicated. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +$CLICKHOUSE_CLIENT -q "DROP TABLE mv_drift_mv" +$CLICKHOUSE_CLIENT -q "DROP TABLE mv_drift_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE mv_drift_src" diff --git a/tests/queries/0_stateless/04661_refreshable_mv_cancel_during_planning.reference b/tests/queries/0_stateless/04661_refreshable_mv_cancel_during_planning.reference new file mode 100644 index 000000000000..c3a7783786f6 --- /dev/null +++ b/tests/queries/0_stateless/04661_refreshable_mv_cancel_during_planning.reference @@ -0,0 +1 @@ +dropped diff --git a/tests/queries/0_stateless/04661_refreshable_mv_cancel_during_planning.sh b/tests/queries/0_stateless/04661_refreshable_mv_cancel_during_planning.sh new file mode 100755 index 000000000000..d3c11dfbebe0 --- /dev/null +++ b/tests/queries/0_stateless/04661_refreshable_mv_cancel_during_planning.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + create table src (k UInt64) engine MergeTree order by k as select number from numbers(10);" + +# `k IN (subquery)` over the primary key materializes the set during query planning +# (ReadFromMergeTree::buildIndexes -> KeyCondition -> FutureSetFromSubquery::buildOrderedSetInplace), +# so the refresh blocks in a nested pipeline that it does not own an executor for yet. +$CLICKHOUSE_CLIENT -q " + create materialized view rmv refresh every 1 second (k UInt64) engine MergeTree order by k as + select k from src where k in (select number from numbers(30) where sleepEachRow(1) = 0) + settings max_block_size = 1;" + +# Wait until the refresh is inside that nested pipeline. Fail hard on timeout: a drop that never +# meets a blocked refresh returns quickly for the wrong reason, and the test would then match the +# reference without exercising the cancellation path it covers. +i=0 +while [ "$($CLICKHOUSE_CLIENT -q " + select count() from system.processes + where current_database = currentDatabase() and query like 'INSERT INTO%sleepEachRow%' and elapsed > 2")" -ne 1 ]; do + sleep 0.3 + i=$((i + 1)) + if [ "$i" -gt 200 ]; then + echo "Refresh did not reach the planning stage in time" >&2 + exit 1 + fi +done + +# The drop must cancel the refresh, not wait for the blocked planning to finish. +if timeout 10 $CLICKHOUSE_CLIENT -q "drop table rmv"; then + echo "dropped" +else + echo "drop did not finish" +fi + +$CLICKHOUSE_CLIENT -q "drop table src" diff --git a/tests/queries/0_stateless/04662_kafka_num_consumers_attach.reference b/tests/queries/0_stateless/04662_kafka_num_consumers_attach.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/04662_kafka_num_consumers_attach.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/04662_kafka_num_consumers_attach.sql b/tests/queries/0_stateless/04662_kafka_num_consumers_attach.sql new file mode 100644 index 000000000000..d0c7ed371eaa --- /dev/null +++ b/tests/queries/0_stateless/04662_kafka_num_consumers_attach.sql @@ -0,0 +1,38 @@ +-- Tags: no-fasttest +-- The `kafka_num_consumers` limit is derived from the number of CPU cores available to the server, +-- so a stored table definition must not be re-validated against it when the table is loaded back. + +-- Suppress expected Kafka consumer connection errors from reaching client stderr. +SET send_logs_level = 'fatal'; + +DROP TABLE IF EXISTS kafka_many_consumers; + +SET kafka_disable_num_consumers_limit = 1; + +CREATE TABLE kafka_many_consumers (a UInt64) + ENGINE = Kafka + SETTINGS + kafka_broker_list = 'localhost:10000', + kafka_topic_list = 'foo', + kafka_group_name = 'foo', + kafka_format = 'JSONEachRow', + kafka_num_consumers = 1000; + +SET kafka_disable_num_consumers_limit = 0; + +DETACH TABLE kafka_many_consumers; +ATTACH TABLE kafka_many_consumers; + +SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 'kafka_many_consumers'; + +-- A freshly introduced definition is still validated. +CREATE TABLE kafka_many_consumers_2 (a UInt64) + ENGINE = Kafka + SETTINGS + kafka_broker_list = 'localhost:10000', + kafka_topic_list = 'foo', + kafka_group_name = 'foo', + kafka_format = 'JSONEachRow', + kafka_num_consumers = 1000; -- { serverError BAD_ARGUMENTS } + +DROP TABLE kafka_many_consumers; diff --git a/tests/queries/0_stateless/04669_query_condition_cache_lightweight_delete.reference b/tests/queries/0_stateless/04669_query_condition_cache_lightweight_delete.reference new file mode 100644 index 000000000000..0dd62f98559a --- /dev/null +++ b/tests/queries/0_stateless/04669_query_condition_cache_lightweight_delete.reference @@ -0,0 +1,27 @@ +--- the delete is materialized, nothing pending +0 +0 +0 +--- prime reads everything, reuse prunes +04669_lwd_prime 0 0 +04669_lwd_reuse 1 1 +--- apply_deleted_mask = 0 must not consume entries written by a normal read +0 +1 +--- and the reverse direction is also unaffected +1 +0 +--- apply_deleted_mask = 0 neither writes nor consumes the cache +0 +0 +04669_lwd_mask0_prime 0 0 +04669_lwd_mask0_reuse 0 0 +--- results stay correct with the cache warm +999999 +9 +--- a pending mutation on an unread column must not disable the cache +0 +0 +--- pending-mutation prime reads everything, reuse prunes +04669_lwd_pending_prime 0 0 +04669_lwd_pending_reuse 1 1 diff --git a/tests/queries/0_stateless/04669_query_condition_cache_lightweight_delete.sql b/tests/queries/0_stateless/04669_query_condition_cache_lightweight_delete.sql new file mode 100644 index 000000000000..84b9f89589c1 --- /dev/null +++ b/tests/queries/0_stateless/04669_query_condition_cache_lightweight_delete.sql @@ -0,0 +1,142 @@ +-- Tags: no-parallel, no-parallel-replicas +-- no-parallel: drops the (instance-wide) query condition cache +-- no-parallel-replicas: the query condition cache is populated per replica, so the granule +-- accounting below is deterministic only on a single replica + +-- A materialized lightweight delete must not disable the query condition cache. The cache write +-- path used to skip any part with a non-empty `mutation_steps`, which also holds the step applying +-- the committed `_row_exists` mask - so one deleted row disabled the cache for the whole table. + +SET use_query_condition_cache = 1; +-- The cache needs the analyzer on both the write and the read side. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_qcc_lwd; + +-- auto_statistics_types = '': randomized auto statistics would prune the whole part for the +-- never-matching predicates below, leaving nothing to read and the granule counts below vacuous. +CREATE TABLE t_qcc_lwd (id UInt64, v UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0, auto_statistics_types = ''; + +INSERT INTO t_qcc_lwd SELECT number, number FROM numbers(1000000); + +-- Materialize the delete so the part carries `_row_exists` and no mutation is left pending. +DELETE FROM t_qcc_lwd WHERE id = 0 SETTINGS mutations_sync = 2; + +SELECT '--- the delete is materialized, nothing pending'; +SELECT count() FROM system.mutations +WHERE database = currentDatabase() AND table = 't_qcc_lwd' AND NOT is_done; + +SYSTEM DROP QUERY CONDITION CACHE; + +-- `v = 123456789` matches no row, so after the first (priming) run every granule of every part is +-- known not to match and the second run must read no marks at all. Asserting zero rather than +-- "fewer" matters: only the part holding `id = 0` carries the mask, so if the insert ever lands in +-- more than one part, a weaker assertion would be satisfied by the other parts pruning. +SELECT count() FROM t_qcc_lwd WHERE v = 123456789 SETTINGS log_comment = '04669_lwd_prime'; +SELECT count() FROM t_qcc_lwd WHERE v = 123456789 SETTINGS log_comment = '04669_lwd_reuse'; + +SYSTEM FLUSH LOGS query_log; + +SELECT '--- prime reads everything, reuse prunes'; +-- Columns: (any QCC hit), (read no marks at all). Expected: prime = 0 0, reuse = 1 1. +SELECT + log_comment, + ProfileEvents['QueryConditionCacheHits'] > 0, + ProfileEvents['SelectedMarks'] = 0 +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND current_database = currentDatabase() + AND log_comment IN ('04669_lwd_prime', '04669_lwd_reuse') +ORDER BY event_time_microseconds; + +-- The unmaterialized (on-fly) direction, which must keep the cache write disabled, is covered by +-- 03229_query_condition_cache_on_fly_mutations. + +SELECT '--- apply_deleted_mask = 0 must not consume entries written by a normal read'; +-- `id = 0` matches only the deleted row, so a normal read may record the granule as non-matching. +-- An `apply_deleted_mask = 0` read must still return that row instead of reusing the verdict. +SYSTEM DROP QUERY CONDITION CACHE; +SELECT count() FROM t_qcc_lwd WHERE id = 0; +SELECT count() FROM t_qcc_lwd WHERE id = 0 SETTINGS apply_deleted_mask = 0; + +SELECT '--- and the reverse direction is also unaffected'; +SYSTEM DROP QUERY CONDITION CACHE; +SELECT count() FROM t_qcc_lwd WHERE id = 0 SETTINGS apply_deleted_mask = 0; +SELECT count() FROM t_qcc_lwd WHERE id = 0; + +SELECT '--- apply_deleted_mask = 0 neither writes nor consumes the cache'; +-- Such queries are kept out of the cache entirely instead of getting their own key space, so a +-- repeated `apply_deleted_mask = 0` query does not prune. Both runs must miss and read every mark. +-- Pinned here because it is the one behaviour this change gives up; a follow-up that keys entries by +-- `apply_deleted_mask` instead of disabling them has to update this block deliberately. +SYSTEM DROP QUERY CONDITION CACHE; +SELECT count() FROM t_qcc_lwd WHERE v = 123456789 +SETTINGS apply_deleted_mask = 0, log_comment = '04669_lwd_mask0_prime'; +SELECT count() FROM t_qcc_lwd WHERE v = 123456789 +SETTINGS apply_deleted_mask = 0, log_comment = '04669_lwd_mask0_reuse'; + +SYSTEM FLUSH LOGS query_log; + +SELECT + log_comment, + ProfileEvents['QueryConditionCacheHits'] > 0, + ProfileEvents['SelectedMarks'] = 0 +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND current_database = currentDatabase() + AND log_comment IN ('04669_lwd_mask0_prime', '04669_lwd_mask0_reuse') +ORDER BY event_time_microseconds; + +SELECT '--- results stay correct with the cache warm'; +SELECT count() FROM t_qcc_lwd; +SELECT count() FROM t_qcc_lwd WHERE v < 10; + +DROP TABLE t_qcc_lwd; + +-- A pending on-fly mutation of a column the query does not read produces no read step, so it must +-- not disable the cache write either. Only an `apply_mutations_on_fly = 0` query can consume the +-- entry: the read path skips the cache while a data mutation is pending. + +SELECT '--- a pending mutation on an unread column must not disable the cache'; + +DROP TABLE IF EXISTS t_qcc_lwd_pending; + +CREATE TABLE t_qcc_lwd_pending (id UInt64, v UInt64, w UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0, auto_statistics_types = ''; + +INSERT INTO t_qcc_lwd_pending SELECT number, number, number FROM numbers(1000000); + +DELETE FROM t_qcc_lwd_pending WHERE id = 0 SETTINGS mutations_sync = 2; + +SYSTEM STOP MERGES t_qcc_lwd_pending; +ALTER TABLE t_qcc_lwd_pending UPDATE w = 0 WHERE id = 1 SETTINGS mutations_sync = 0; + +SYSTEM DROP QUERY CONDITION CACHE; + +-- The prime reads only `v`, so the pending `UPDATE` of `w` is irrelevant to it and the write must +-- still happen; the reuse with `apply_mutations_on_fly = 0` must consume it and prune. +SELECT count() FROM t_qcc_lwd_pending WHERE v = 123456789 +SETTINGS apply_mutations_on_fly = 1, log_comment = '04669_lwd_pending_prime'; +SELECT count() FROM t_qcc_lwd_pending WHERE v = 123456789 +SETTINGS apply_mutations_on_fly = 0, log_comment = '04669_lwd_pending_reuse'; + +SYSTEM FLUSH LOGS query_log; + +SELECT '--- pending-mutation prime reads everything, reuse prunes'; +SELECT + log_comment, + ProfileEvents['QueryConditionCacheHits'] > 0, + ProfileEvents['SelectedMarks'] = 0 +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND current_database = currentDatabase() + AND log_comment IN ('04669_lwd_pending_prime', '04669_lwd_pending_reuse') +ORDER BY event_time_microseconds; + +DROP TABLE t_qcc_lwd_pending; diff --git a/tests/queries/0_stateless/04670_query_condition_cache_unique_key.reference b/tests/queries/0_stateless/04670_query_condition_cache_unique_key.reference new file mode 100644 index 000000000000..3302a8badc3a --- /dev/null +++ b/tests/queries/0_stateless/04670_query_condition_cache_unique_key.reference @@ -0,0 +1,10 @@ +--- a UNIQUE KEY part cannot carry a materialized _row_exists mask +0 +--- and a UNIQUE KEY read does not use the cache at all +0 +0 +04670_uk_prime 0 0 +04670_uk_reuse 0 0 +--- results are correct +100000 +10 diff --git a/tests/queries/0_stateless/04670_query_condition_cache_unique_key.sql b/tests/queries/0_stateless/04670_query_condition_cache_unique_key.sql new file mode 100644 index 000000000000..c13f37e8981b --- /dev/null +++ b/tests/queries/0_stateless/04670_query_condition_cache_unique_key.sql @@ -0,0 +1,71 @@ +-- Tags: no-parallel, no-parallel-replicas, no-fasttest, no-ordinary-database, no-replicated-database, no-shared-merge-tree, no-object-storage, no-s3-storage, no-async-insert +-- no-parallel: drops the (instance-wide) query condition cache +-- no-parallel-replicas: the cache is populated per replica, so the mark counts below are +-- deterministic only on a single replica +-- no-fasttest: a UNIQUE KEY insert writes the dense-index SST, which needs RocksDB +-- no-object-storage, no-s3-storage: UNIQUE KEY requires a storage policy of local disks +-- no-ordinary-database, no-replicated-database, no-shared-merge-tree: UNIQUE KEY is only supported +-- on plain MergeTree in an Atomic database + +-- Why the query condition cache and UNIQUE KEY tables do not interact, pinned so that changing +-- either half is deliberate. +-- +-- 1. A UNIQUE KEY read never uses the cache. `ReadFromMergeTree` turns it off for both the write and +-- the consult side, because the cache is CSN-oblivious while the delete bitmap is not: a mark +-- recorded as non-matching after a bitmap drop could be skipped by a reader pinned at an older +-- snapshot whose rows are still live. +-- 2. No UNIQUE KEY part can carry a materialized `_row_exists` mask, because mutation-class commands +-- are rejected on such tables - `DELETE FROM` included, as it is executed as an +-- `UPDATE _row_exists`. +-- +-- Either one on its own is enough to keep such tables away from the materialized-mask handling in +-- `appliesMutationsBeforePrewhere`. Re-enabling the cache for UNIQUE KEY reads (there is a TODO for +-- a snapshot-aware cache) makes this test fail, which is the point: that work has to look at the +-- mask and bitmap interaction rather than just flipping the flag. + +SET allow_experimental_unique_key = 1; +SET async_insert = 0; +SET use_query_condition_cache = 1; +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_qcc_uk; + +CREATE TABLE t_qcc_uk (id UInt64, v UInt64) +ENGINE = MergeTree ORDER BY id UNIQUE KEY (id) +SETTINGS index_granularity = 8192, min_bytes_for_wide_part = 0, auto_statistics_types = ''; + +INSERT INTO t_qcc_uk SELECT number, number FROM numbers(100000); + +SELECT '--- a UNIQUE KEY part cannot carry a materialized _row_exists mask'; +DELETE FROM t_qcc_uk WHERE id = 0; -- { serverError SUPPORT_IS_DISABLED } + +SELECT sum(has_lightweight_delete) FROM system.parts +WHERE database = currentDatabase() AND table = 't_qcc_uk' AND active; + +SELECT '--- and a UNIQUE KEY read does not use the cache at all'; +SYSTEM DROP QUERY CONDITION CACHE; + +-- `v = 123456789` matches no row. On a plain MergeTree table the second run would hit the cache and +-- read no marks (see 04669_query_condition_cache_lightweight_delete); here neither run may. +SELECT count() FROM t_qcc_uk WHERE v = 123456789 SETTINGS log_comment = '04670_uk_prime'; +SELECT count() FROM t_qcc_uk WHERE v = 123456789 SETTINGS log_comment = '04670_uk_reuse'; + +SYSTEM FLUSH LOGS query_log; + +-- Columns: (any QCC hit), (read no marks at all). Expected: both runs = 0 0. +SELECT + log_comment, + ProfileEvents['QueryConditionCacheHits'] > 0, + ProfileEvents['SelectedMarks'] = 0 +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 + AND type = 'QueryFinish' + AND current_database = currentDatabase() + AND log_comment IN ('04670_uk_prime', '04670_uk_reuse') +ORDER BY event_time_microseconds; + +SELECT '--- results are correct'; +SELECT count() FROM t_qcc_uk; +SELECT count() FROM t_qcc_uk WHERE v < 10; + +DROP TABLE t_qcc_uk; diff --git a/tests/queries/0_stateless/04671_join_const_propagation_partition_pruning.reference b/tests/queries/0_stateless/04671_join_const_propagation_partition_pruning.reference new file mode 100644 index 000000000000..2eea2b7c3a55 --- /dev/null +++ b/tests/queries/0_stateless/04671_join_const_propagation_partition_pruning.reference @@ -0,0 +1,16 @@ +bounds in JOIN ON +10 1555 +bounds as plain literals +10 1555 +pruned parts +Parts: 1/14 +Parts: 1/1 +Parts: 1/1 +equi key plus bounds in JOIN ON +1 395 +left join keeps all left rows +400 399 +left join with bounds in WHERE +10 1555 +equality against a constant column +1 155 diff --git a/tests/queries/0_stateless/04671_join_const_propagation_partition_pruning.sql b/tests/queries/0_stateless/04671_join_const_propagation_partition_pruning.sql new file mode 100644 index 000000000000..7db5fea05e12 --- /dev/null +++ b/tests/queries/0_stateless/04671_join_const_propagation_partition_pruning.sql @@ -0,0 +1,78 @@ +-- Tags: no-parallel-replicas, no-old-analyzer + +-- A column that is constant in a JOIN input is invisible inside the join condition and in a filter +-- above the join, because both see it as an ordinary column. Constants are substituted so that +-- predicates like `t.d >= bounds.lo` become single-sided and reach index analysis. + +DROP TABLE IF EXISTS t_join_const_prune; + +CREATE TABLE t_join_const_prune (d Date, v UInt32) +ENGINE = MergeTree +PARTITION BY toYYYYMM(d) +ORDER BY d; + +-- 400 days starting at 2025-01-01, i.e. 14 monthly partitions; June 2025 holds 30 rows. +INSERT INTO t_join_const_prune SELECT toDate('2025-01-01') + number, number FROM numbers(400); + +-- The `max_rows_to_read` limits below only tell a pruned read apart from a full scan of all 400 rows. +-- They are not tight: the same row can be accounted for more than once, for example when mark ranges +-- are split into intersecting and non-intersecting ones. + +SELECT 'bounds in JOIN ON'; +WITH bounds AS (SELECT toDate('2025-06-01') AS lo, toDate('2025-06-10') AS hi) +SELECT count(), sum(v) +FROM t_join_const_prune AS t +JOIN bounds ON t.d >= bounds.lo AND t.d <= bounds.hi +SETTINGS max_rows_to_read = 150; + +SELECT 'bounds as plain literals'; +SELECT count(), sum(v) +FROM t_join_const_prune AS t +WHERE t.d >= toDate('2025-06-01') AND t.d <= toDate('2025-06-10') +SETTINGS max_rows_to_read = 150; + +SELECT 'pruned parts'; +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 + WITH bounds AS (SELECT toDate('2025-06-01') AS lo, toDate('2025-06-10') AS hi) + SELECT count() + FROM t_join_const_prune AS t + JOIN bounds ON t.d >= bounds.lo AND t.d <= bounds.hi +) WHERE explain LIKE '%Parts: %'; + +-- The bound reaches index analysis next to the equi key, so only the two 2026 partitions are read. +SELECT 'equi key plus bounds in JOIN ON'; +WITH bounds AS (SELECT 395 AS k, toDate('2026-01-01') AS lo) +SELECT count(), sum(v) +FROM t_join_const_prune AS t +JOIN bounds ON t.v = bounds.k AND t.d >= bounds.lo +SETTINGS max_rows_to_read = 150; + +-- A LEFT JOIN keeps every left row, so a condition on the left side cannot be applied to the left +-- input: it decides matching only. Row 395 is 2026-01-31, the single row that matches. +SELECT 'left join keeps all left rows'; +WITH bounds AS (SELECT 395 AS k, toDate('2026-01-01') AS lo) +SELECT count(), countIf(lo IS NULL) +FROM t_join_const_prune AS t +LEFT JOIN bounds ON t.v = bounds.k AND t.d >= bounds.lo +SETTINGS join_use_nulls = 1; + +-- A WHERE that references the right side rejects the NULL-extended rows, so the join is an INNER +-- join by the time constants are substituted. Without that rewrite the join has no equi key at all +-- and is not supported, hence `query_plan_convert_outer_join_to_inner_join` is pinned here. +SELECT 'left join with bounds in WHERE'; +WITH bounds AS (SELECT toDate('2025-06-01') AS lo, toDate('2025-06-10') AS hi) +SELECT count(), sum(v) +FROM t_join_const_prune AS t +LEFT JOIN bounds ON t.d >= bounds.lo AND t.d <= bounds.hi +WHERE bounds.hi >= t.d +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1; + +-- An equality between the two sides stays a join key, it is not turned into a constant filter. +SELECT 'equality against a constant column'; +WITH bounds AS (SELECT toDate('2025-06-05') AS dd) +SELECT count(), sum(v) +FROM t_join_const_prune AS t +JOIN bounds ON t.d = bounds.dd; + +DROP TABLE t_join_const_prune; diff --git a/tests/queries/0_stateless/04672_read_in_order_virtual_row_sort_prefix.reference b/tests/queries/0_stateless/04672_read_in_order_virtual_row_sort_prefix.reference new file mode 100644 index 000000000000..70f3e1602bee --- /dev/null +++ b/tests/queries/0_stateless/04672_read_in_order_virtual_row_sort_prefix.reference @@ -0,0 +1,43 @@ +--- single table: sorting key (name, code) +-- plain key +Prefix sort description: name ASC, code ASC +Read type: InOrder +Virtual row conversions +-- cast on the only column +Prefix sort description: CAST(name AS Nullable(String)) ASC +Read type: InOrder +Virtual row conversions +-- cast on the first of two columns +Prefix sort description: CAST(name AS Nullable(String)) ASC, code ASC +Read type: InOrder +Virtual row conversions +-- cast on the last of two columns +Prefix sort description: name ASC, CAST(code AS Nullable(String)) ASC +Read type: InOrder +Virtual row conversions +-- toString on the first of two columns +Prefix sort description: toString(name) ASC, code ASC +Read type: InOrder +Virtual row conversions +-- non-key column last +Prefix sort description: name ASC, code ASC +Read type: InOrder +Virtual row conversions +--- non-strictly monotonic function: the prefix is cut after it +Prefix sort description: toStartOfMonth(d) ASC +Read type: InOrder +Virtual row conversions +--- INNER JOIN: read-in-order needs the virtual row +-- cast on the first of two key columns +Prefix sort description: CAST(name AS Nullable(String)) ASC, code ASC +Read type: InOrder +Virtual row conversions +Read type: Default +-- right table column last +Prefix sort description: name ASC, code ASC +Read type: InOrder +Virtual row conversions +Read type: Default +--- results stay correctly ordered +1 +1 diff --git a/tests/queries/0_stateless/04672_read_in_order_virtual_row_sort_prefix.sql b/tests/queries/0_stateless/04672_read_in_order_virtual_row_sort_prefix.sql new file mode 100644 index 000000000000..da290bc2e149 --- /dev/null +++ b/tests/queries/0_stateless/04672_read_in_order_virtual_row_sort_prefix.sql @@ -0,0 +1,79 @@ +-- Tags: no-parallel-replicas + +-- The `ORDER BY` prefix that can be served from the sorting key is cut short at the first +-- non-strictly-monotonic function, and the virtual row optimization is only built for a prefix. +-- When the virtual row is not built, read-in-order is refused for `INNER JOIN` altogether. + +DROP TABLE IF EXISTS ev; +DROP TABLE IF EXISTS dict; +DROP TABLE IF EXISTS by_day; + +CREATE TABLE ev (name String, code String, ref String) ENGINE = MergeTree ORDER BY (name, code); +CREATE TABLE dict (ref String, label String) ENGINE = MergeTree ORDER BY ref; +CREATE TABLE by_day (d Date, num UInt32) ENGINE = MergeTree ORDER BY (d, num); + +INSERT INTO ev VALUES ('n1', 'c1', 'r1'), ('n1', 'c2', 'r2'), ('n2', 'c1', 'r1'); +INSERT INTO ev VALUES ('n2', 'c2', 'r3'), ('n3', 'c1', 'r2'), ('n3', 'c2', 'r3'); +INSERT INTO dict VALUES ('r1', 'l1'), ('r2', 'l2'), ('r3', 'l3'); +INSERT INTO by_day VALUES ('2020-01-01', 1), ('2020-01-01', 2), ('2020-01-02', 1); + +SET optimize_read_in_order = 1, read_in_order_use_virtual_row = 1; +SET max_bytes_ratio_before_external_join = 0, max_bytes_before_external_join = 0, query_plan_read_in_order_through_join = 1; +SET query_plan_optimize_join_order_limit = 1, query_plan_optimize_join_order_randomize = 0, query_plan_join_swap_table = 0; + +SELECT '--- single table: sorting key (name, code)'; + +SELECT '-- plain key'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM ev ORDER BY name, code LIMIT 5) WHERE e != ''; + +SELECT '-- cast on the only column'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM ev ORDER BY name::Nullable(String) LIMIT 5) WHERE e != ''; + +SELECT '-- cast on the first of two columns'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM ev ORDER BY name::Nullable(String), code LIMIT 5) WHERE e != ''; + +SELECT '-- cast on the last of two columns'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM ev ORDER BY name, code::Nullable(String) LIMIT 5) WHERE e != ''; + +SELECT '-- toString on the first of two columns'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM ev ORDER BY toString(name), code LIMIT 5) WHERE e != ''; + +SELECT '-- non-key column last'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM ev ORDER BY name, code, ref LIMIT 5) WHERE e != ''; + +SELECT '--- non-strictly monotonic function: the prefix is cut after it'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 SELECT * FROM by_day ORDER BY toStartOfMonth(d), num LIMIT 5) WHERE e != ''; + +SELECT '--- INNER JOIN: read-in-order needs the virtual row'; + +SELECT '-- cast on the first of two key columns'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 + SELECT ev.name, ev.code, dict.label FROM ev INNER JOIN dict ON ev.ref = dict.ref + ORDER BY ev.name::Nullable(String), ev.code LIMIT 5) WHERE e != ''; + +SELECT '-- right table column last'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0, compact = 1, pretty = 1 + SELECT ev.name, ev.code, dict.label FROM ev INNER JOIN dict ON ev.ref = dict.ref + ORDER BY ev.name, ev.code, dict.label LIMIT 5) WHERE e != ''; + +SELECT '--- results stay correctly ordered'; + +SELECT groupArray(t) = arraySort(x -> x, groupArray(t)) FROM ( + SELECT (name::Nullable(String), code) AS t FROM ev ORDER BY name::Nullable(String), code LIMIT 10); + +SELECT groupArray(t) = arraySort(x -> x, groupArray(t)) FROM ( + SELECT (ev.name, ev.code, dict.label) AS t FROM ev INNER JOIN dict ON ev.ref = dict.ref + ORDER BY ev.name, ev.code, dict.label LIMIT 10); + +DROP TABLE ev; +DROP TABLE dict; +DROP TABLE by_day; diff --git a/tests/queries/0_stateless/04692_insert_dedup_json_path_names.reference b/tests/queries/0_stateless/04692_insert_dedup_json_path_names.reference new file mode 100644 index 000000000000..7598b2224ca7 --- /dev/null +++ b/tests/queries/0_stateless/04692_insert_dedup_json_path_names.reference @@ -0,0 +1,12 @@ +identical object deduplicated 1 +renamed path kept 2 +renamed path contents {"a":1} +renamed path contents {"b":1} +both paths renamed kept 2 +renamed String path kept 2 +renamed nested path kept 2 +renamed path with other value kept 2 +reordered keys deduplicated 1 +renamed path kept without async_insert 2 +renamed path kept for INSERT SELECT 2 +insert without deduplication appends 2 diff --git a/tests/queries/0_stateless/04692_insert_dedup_json_path_names.sql b/tests/queries/0_stateless/04692_insert_dedup_json_path_names.sql new file mode 100644 index 000000000000..df6a4b808d75 --- /dev/null +++ b/tests/queries/0_stateless/04692_insert_dedup_json_path_names.sql @@ -0,0 +1,110 @@ +-- Insert deduplication hashes a JSON object by walking its paths in sorted order and hashing each +-- path name next to its values. Hashing the values alone left the hash blind to which path they +-- belong to, so `{"a":1}` and `{"b":1}` collided and the second insert was silently dropped. + +SET enable_json_type = 1; +SET max_insert_threads = 1; + +DROP TABLE IF EXISTS t_dedup_json_same; +DROP TABLE IF EXISTS t_dedup_json_renamed; +DROP TABLE IF EXISTS t_dedup_json_renamed_pair; +DROP TABLE IF EXISTS t_dedup_json_renamed_string; +DROP TABLE IF EXISTS t_dedup_json_renamed_nested; +DROP TABLE IF EXISTS t_dedup_json_other_value; +DROP TABLE IF EXISTS t_dedup_json_key_order; +DROP TABLE IF EXISTS t_dedup_json_sync; +DROP TABLE IF EXISTS t_dedup_json_select; +DROP TABLE IF EXISTS t_dedup_json_no_dedup; + +-- The same object twice must still deduplicate. +CREATE TABLE t_dedup_json_same (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_same VALUES (1, '{"a":1}'); +INSERT INTO t_dedup_json_same VALUES (1, '{"a":1}'); +SELECT 'identical object deduplicated', count() FROM t_dedup_json_same; + +-- Same value under a different path name: two different objects, both must be kept. +CREATE TABLE t_dedup_json_renamed (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_renamed VALUES (1, '{"a":1}'); +INSERT INTO t_dedup_json_renamed VALUES (1, '{"b":1}'); +SELECT 'renamed path kept', count() FROM t_dedup_json_renamed; +SELECT 'renamed path contents', data FROM t_dedup_json_renamed ORDER BY toString(data); + +-- Every path renamed, values unchanged. +CREATE TABLE t_dedup_json_renamed_pair (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_renamed_pair VALUES (1, '{"a":1,"b":2}'); +INSERT INTO t_dedup_json_renamed_pair VALUES (1, '{"c":1,"d":2}'); +SELECT 'both paths renamed kept', count() FROM t_dedup_json_renamed_pair; + +-- The same with a String value, which hashes through a different nested column. +CREATE TABLE t_dedup_json_renamed_string (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_renamed_string VALUES (1, '{"x":"s"}'); +INSERT INTO t_dedup_json_renamed_string VALUES (1, '{"y":"s"}'); +SELECT 'renamed String path kept', count() FROM t_dedup_json_renamed_string; + +-- A renamed path one level down, so the differing path names are nested ones. +CREATE TABLE t_dedup_json_renamed_nested (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_renamed_nested VALUES (1, '{"a":{"b":1}}'); +INSERT INTO t_dedup_json_renamed_nested VALUES (1, '{"c":{"d":1}}'); +SELECT 'renamed nested path kept', count() FROM t_dedup_json_renamed_nested; + +-- Control: a differing value was always caught, because the value is hashed. +CREATE TABLE t_dedup_json_other_value (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_other_value VALUES (1, '{"a":1}'); +INSERT INTO t_dedup_json_other_value VALUES (1, '{"b":2}'); +SELECT 'renamed path with other value kept', count() FROM t_dedup_json_other_value; + +-- Paths are hashed in sorted order, so the key order of the input text must not matter. +CREATE TABLE t_dedup_json_key_order (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_key_order VALUES (1, '{"a":1,"b":2}'); +INSERT INTO t_dedup_json_key_order VALUES (1, '{"b":2,"a":1}'); +SELECT 'reordered keys deduplicated', count() FROM t_dedup_json_key_order; + +-- The hash is shared by the async and the synchronous insert path, so it must hold with +-- async_insert disabled too. +CREATE TABLE t_dedup_json_sync (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_sync SETTINGS async_insert = 0 VALUES (1, '{"a":1}'); +INSERT INTO t_dedup_json_sync SETTINGS async_insert = 0 VALUES (1, '{"b":1}'); +SELECT 'renamed path kept without async_insert', count() FROM t_dedup_json_sync; + +-- INSERT SELECT reaches the same hash once deduplication is not declined for an unordered query. +CREATE TABLE t_dedup_json_select (id UInt64, data JSON) +ENGINE = MergeTree ORDER BY id SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO t_dedup_json_select SETTINGS deduplicate_insert_select = 'enable_even_for_bad_queries' +SELECT 1, materialize('{"a":1}')::JSON; +INSERT INTO t_dedup_json_select SETTINGS deduplicate_insert_select = 'enable_even_for_bad_queries' +SELECT 1, materialize('{"b":1}')::JSON; +SELECT 'renamed path kept for INSERT SELECT', count() FROM t_dedup_json_select; + +-- Without a deduplication window the same object must append. +CREATE TABLE t_dedup_json_no_dedup (id UInt64, data JSON) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_dedup_json_no_dedup VALUES (1, '{"a":1}'); +INSERT INTO t_dedup_json_no_dedup VALUES (1, '{"a":1}'); +SELECT 'insert without deduplication appends', count() FROM t_dedup_json_no_dedup; + +DROP TABLE t_dedup_json_no_dedup; +DROP TABLE t_dedup_json_select; +DROP TABLE t_dedup_json_sync; +DROP TABLE t_dedup_json_key_order; +DROP TABLE t_dedup_json_other_value; +DROP TABLE t_dedup_json_renamed_nested; +DROP TABLE t_dedup_json_renamed_string; +DROP TABLE t_dedup_json_renamed_pair; +DROP TABLE t_dedup_json_renamed; +DROP TABLE t_dedup_json_same; diff --git a/tests/queries/0_stateless/04695_analyzer_group_by_use_nulls_matcher_projection_names.reference b/tests/queries/0_stateless/04695_analyzer_group_by_use_nulls_matcher_projection_names.reference new file mode 100644 index 000000000000..b62ceca601e1 --- /dev/null +++ b/tests/queries/0_stateless/04695_analyzer_group_by_use_nulls_matcher_projection_names.reference @@ -0,0 +1,58 @@ +=== describe: single join, ROLLUP === +k Nullable(UInt8) +Date Nullable(String) +t1.k Nullable(UInt8) +=== outer ref t1.k, single join, ROLLUP === +1 +\N +\N +\N +=== describe: single join, CUBE === +k Nullable(UInt8) +Date Nullable(String) +t1.k Nullable(UInt8) +=== outer ref t1.k, single join, CUBE === +1 +1 +1 +1 +\N +\N +\N +\N +=== describe: single join, GROUPING SETS === +k Nullable(UInt8) +Date Nullable(String) +t1.k Nullable(UInt8) +=== outer ref t1.k, single join, GROUPING SETS === +1 +\N +=== describe: single join, plain GROUP BY === +k UInt8 +Date String +t1.k UInt8 +=== describe: single join, ROLLUP, group_by_use_nulls = 0 === +k UInt8 +Date String +t1.k UInt8 +=== describe: two joins, ROLLUP, setting ON === +ll.k Nullable(UInt8) +ll.Date Nullable(String) +t1.k Nullable(UInt8) +t2.k Nullable(UInt8) +=== outer ref ll.Date, two joins, ROLLUP, setting ON === +D +D +D +\N +\N +=== describe: two joins, CUBE, setting ON === +ll.k Nullable(UInt8) +ll.Date Nullable(String) +t1.k Nullable(UInt8) +t2.k Nullable(UInt8) +=== describe: two joins, GROUPING SETS, setting ON === +ll.k Nullable(UInt8) +ll.Date Nullable(String) +t1.k Nullable(UInt8) +t2.k Nullable(UInt8) diff --git a/tests/queries/0_stateless/04695_analyzer_group_by_use_nulls_matcher_projection_names.sql b/tests/queries/0_stateless/04695_analyzer_group_by_use_nulls_matcher_projection_names.sql new file mode 100644 index 000000000000..4619af200506 --- /dev/null +++ b/tests/queries/0_stateless/04695_analyzer_group_by_use_nulls_matcher_projection_names.sql @@ -0,0 +1,63 @@ +-- Columns expanded from a matcher (`*`) keep their projection names when the +-- `group_by_use_nulls` rewrite turns them into nullable copies for `ROLLUP`, +-- `CUBE` and `GROUPING SETS`. Without that, the qualification a matcher assigns +-- to columns of joined table expressions is lost: the result columns collide +-- (two columns named `k`) and an outer query can no longer reference them, +-- while the old analyzer resolves such references fine. + +SET enable_analyzer = 1; + +-- ============================================================ +-- Default behavior: qualification added to disambiguate joined columns +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 0; +SET group_by_use_nulls = 1; + +SELECT '=== describe: single join, ROLLUP ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY ROLLUP(ll.k, ll.Date, t1.k)); + +SELECT '=== outer ref t1.k, single join, ROLLUP ==='; +SELECT t1.k FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY ROLLUP(ll.k, ll.Date, t1.k)) ORDER BY t1.k NULLS LAST; + +SELECT '=== describe: single join, CUBE ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY CUBE(ll.k, ll.Date, t1.k)); + +SELECT '=== outer ref t1.k, single join, CUBE ==='; +SELECT t1.k FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY CUBE(ll.k, ll.Date, t1.k)) ORDER BY t1.k NULLS LAST; + +SELECT '=== describe: single join, GROUPING SETS ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY GROUPING SETS ((ll.k, ll.Date), (t1.k))); + +SELECT '=== outer ref t1.k, single join, GROUPING SETS ==='; +SELECT t1.k FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY GROUPING SETS ((ll.k, ll.Date), (t1.k))) ORDER BY t1.k NULLS LAST; + +-- Control: a plain `GROUP BY` never enters the nullable rewrite. +SELECT '=== describe: single join, plain GROUP BY ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY ll.k, ll.Date, t1.k); + +-- Control: the same `ROLLUP` query without `group_by_use_nulls`. +SET group_by_use_nulls = 0; + +SELECT '=== describe: single join, ROLLUP, group_by_use_nulls = 0 ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k GROUP BY ROLLUP(ll.k, ll.Date, t1.k)); + +-- ============================================================ +-- The same, with the qualification forced by +-- `analyzer_compatibility_multiple_joins_qualify_column_names` +-- ============================================================ + +SET analyzer_compatibility_multiple_joins_qualify_column_names = 1; +SET group_by_use_nulls = 1; + +SELECT '=== describe: two joins, ROLLUP, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k GROUP BY ROLLUP(ll.k, ll.Date, t1.k, t2.k)); + +SELECT '=== outer ref ll.Date, two joins, ROLLUP, setting ON ==='; +SELECT ll.Date FROM (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k GROUP BY ROLLUP(ll.k, ll.Date, t1.k, t2.k)) ORDER BY ll.Date NULLS LAST; + +SELECT '=== describe: two joins, CUBE, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k GROUP BY CUBE(ll.k, ll.Date, t1.k, t2.k)); + +SELECT '=== describe: two joins, GROUPING SETS, setting ON ==='; +DESCRIBE (SELECT * FROM (SELECT 1 AS k, 'D' AS Date) AS ll LEFT JOIN (SELECT 1 AS k) AS t1 ON ll.k = t1.k LEFT JOIN (SELECT 1 AS k) AS t2 ON ll.k = t2.k GROUP BY GROUPING SETS ((ll.k, ll.Date), (t1.k, t2.k))); diff --git a/tests/queries/0_stateless/04695_filesystem_cache_wait_for_concurrent_download_timeout.reference b/tests/queries/0_stateless/04695_filesystem_cache_wait_for_concurrent_download_timeout.reference new file mode 100644 index 000000000000..84125c47a13f --- /dev/null +++ b/tests/queries/0_stateless/04695_filesystem_cache_wait_for_concurrent_download_timeout.reference @@ -0,0 +1,4 @@ +1 +499999500000 +499999500000 +1 diff --git a/tests/queries/0_stateless/04695_filesystem_cache_wait_for_concurrent_download_timeout.sh b/tests/queries/0_stateless/04695_filesystem_cache_wait_for_concurrent_download_timeout.sh new file mode 100755 index 000000000000..e853ee18d164 --- /dev/null +++ b/tests/queries/0_stateless/04695_filesystem_cache_wait_for_concurrent_download_timeout.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-random-settings, no-replicated-database, no-object-storage, no-parallel-replicas +# no-parallel: enables a global pauseable failpoint which pauses every filesystem cache write on the server. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A private cache keeps the test independent from other cache users on the server (e.g. background +# merges of `test.hits_s3` write through the shared `s3_cache` and can fill it with non-releasable +# segments, failing this table's space reservations). The whole data file must fit in one file +# segment, so that the waiter below never becomes a downloader itself (it would pause on its own +# cache write and deadlock, because the failpoint is disabled only after it returns). +cache_name="cache_04695_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS t_download_wait_timeout; + CREATE TABLE t_download_wait_timeout (a UInt64) ENGINE = MergeTree ORDER BY a + SETTINGS disk = disk( + type = cache, + name = '$cache_name', + path = '$cache_name/', + max_size = '1Gi', + max_file_segment_size = '256Mi', + cache_on_write_operations = 0, + load_metadata_asynchronously = 0, + disk = 'local_disk'); + INSERT INTO t_download_wait_timeout SELECT number FROM numbers(1000000); +" + +# Warm up the in-memory mark cache, so that both queries below take marks from it and their only +# filesystem cache accesses are for the data file. Otherwise the paused downloader would hold the +# mark cache load token and the waiter would block on it instead of `FileSegment::wait`. +# Strictly synchronous, no prefetches and no background download: a cache write still in flight +# after this query would pause on the failpoint while holding a segment the queries below need. +$CLICKHOUSE_CLIENT --max_threads 1 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --query "SELECT a FROM t_download_wait_timeout LIMIT 1 FORMAT Null" + +$CLICKHOUSE_CLIENT --query " + SYSTEM DROP FILESYSTEM CACHE '$cache_name'; + SYSTEM ENABLE FAILPOINT file_segment_pause_before_write; +" + +# The failpoint is server-global, so it must be disabled even if the script aborts before reaching +# the explicit disable below. Otherwise every filesystem cache write on the server stays paused and +# the following tests hang behind it, masking the failure that happened here. +trap '$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT file_segment_pause_before_write" > /dev/null 2>&1 || true' EXIT + +# The downloader pauses inside its first cache write, holding the file segment in DOWNLOADING state. +$CLICKHOUSE_CLIENT --max_threads 1 \ + --enable_filesystem_cache 1 \ + --read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --query "SELECT sum(a) FROM t_download_wait_timeout" & + +# The failpoint pauses every cache write on the server, so the first pause is not necessarily the +# downloader's: additionally wait until the downloader holds the segment of this test's cache. +$CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT file_segment_pause_before_write PAUSE" +downloading=0 +for _ in {1..600}; do + downloading=$($CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.filesystem_cache WHERE cache_name = '$cache_name' AND state = 'DOWNLOADING'") + [[ "$downloading" == "1" ]] && break + sleep 0.1 +done +echo "$downloading" + +# The waiter needs the paused segment, gives up waiting after 100 ms and bypasses the cache. +waiter_query_id="04695_waiter_${CLICKHOUSE_DATABASE}_${RANDOM}" +$CLICKHOUSE_CLIENT --query_id "$waiter_query_id" --max_threads 1 \ + --enable_filesystem_cache 1 \ + --read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --use_uncompressed_cache 0 \ + --filesystem_cache_wait_for_concurrent_download_timeout_milliseconds 100 \ + --query "SELECT sum(a) FROM t_download_wait_timeout" + +$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT file_segment_pause_before_write" +wait + +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT --query " + SELECT ProfileEvents['FileSegmentWaitTimeouts'] >= 1 + FROM system.query_log + WHERE current_database = currentDatabase() AND query_id = '$waiter_query_id' AND type = 'QueryFinish' +" + +$CLICKHOUSE_CLIENT --query "DROP TABLE t_download_wait_timeout" diff --git a/tests/queries/0_stateless/04695_to_string_monotonicity.reference b/tests/queries/0_stateless/04695_to_string_monotonicity.reference new file mode 100644 index 000000000000..4b818857bcd8 --- /dev/null +++ b/tests/queries/0_stateless/04695_to_string_monotonicity.reference @@ -0,0 +1,19 @@ +1 +-01:00:00 +-27:46:40 +-83:20:00 +-99:00:00 +2021-11-07 01:59:59 2021-11-07 01:00:00 +5 +2021-11-07 01:00:00 +2021-11-07 01:00:01 +2021-11-07 01:00:02 +2021-11-07 01:00:03 +2021-11-07 01:00:04 +2021-11-07 01:59:55 +2021-11-07 01:59:56 +2021-11-07 01:59:57 +2021-11-07 01:59:58 +2021-11-07 01:59:59 +│ Prefix sort description: toString(d) ASC, y ASC +│ Result sort description: toString(d) ASC, y ASC diff --git a/tests/queries/0_stateless/04695_to_string_monotonicity.sql b/tests/queries/0_stateless/04695_to_string_monotonicity.sql new file mode 100644 index 000000000000..1bf4d6c0f27f --- /dev/null +++ b/tests/queries/0_stateless/04695_to_string_monotonicity.sql @@ -0,0 +1,33 @@ +-- Tags: no-parallel-replicas +-- ^ parallel replicas change the plan asserted in this test + +-- Monotonicity of `toString` for date and time types, see `ToStringMonotonicity`. + +DROP TABLE IF EXISTS t_time; +CREATE TABLE t_time (x Time) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_time VALUES ('-99:00:00'), ('-83:20:00'), ('-27:46:40'), ('-01:00:00'); + +-- `'-01:00:00'` is lexicographically less than `'-99:00:00'`, so the primary key must not exclude the part. +SELECT count() FROM t_time WHERE toString(x) >= '-99:00:00'; +SELECT toString(x) FROM t_time ORDER BY toString(x) SETTINGS optimize_read_in_order = 1; + +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (x DateTime('America/New_York')) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_dst SELECT toDateTime(1636264795 + number, 'America/New_York') FROM numbers(10); + +-- Local time moves back from `01:59:59` to `01:00:00` in the middle of the part. +SELECT toString(toDateTime(1636264799, 'America/New_York')), toString(toDateTime(1636264800, 'America/New_York')); +SELECT count() FROM t_dst WHERE toString(x) >= '2021-11-07 01:59:00'; +SELECT toString(x) FROM t_dst ORDER BY toString(x) SETTINGS optimize_read_in_order = 1; + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date, y UInt32) ENGINE = MergeTree ORDER BY (d, y); +INSERT INTO t_date SELECT toDate('2021-01-01') + intDiv(number, 4), number % 4 FROM numbers(20); + +SELECT trimLeft(explain) FROM ( + EXPLAIN PLAN actions = 1, compact = 1, pretty = 1 SELECT * FROM t_date ORDER BY toString(d), y SETTINGS optimize_read_in_order = 1 +) WHERE explain LIKE '%sort description%'; + +DROP TABLE t_time; +DROP TABLE t_dst; +DROP TABLE t_date; diff --git a/tests/queries/0_stateless/04700_merge_temporary_database.reference b/tests/queries/0_stateless/04700_merge_temporary_database.reference new file mode 100644 index 000000000000..44e0be8e3569 --- /dev/null +++ b/tests/queries/0_stateless/04700_merge_temporary_database.reference @@ -0,0 +1,4 @@ +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/04700_merge_temporary_database.sql b/tests/queries/0_stateless/04700_merge_temporary_database.sql new file mode 100644 index 000000000000..5ef3443fcb05 --- /dev/null +++ b/tests/queries/0_stateless/04700_merge_temporary_database.sql @@ -0,0 +1,31 @@ +-- The `_temporary_and_external_tables` database holds the temporary tables of all sessions and all users, +-- and it is not covered by access control, so it must not be reachable through `Merge`. + +CREATE TEMPORARY TABLE t_merge_temporary (dummy UInt8) ENGINE = Memory; +INSERT INTO t_merge_temporary VALUES (42); + +-- A database regexp skips it, but still reads the other databases it matches. +-- Temporary tables are stored under generated names starting with `_tmp_`. +SELECT * FROM merge(REGEXP('^(_temporary_and_external_tables|system)$'), '^(one|_tmp_)') ORDER BY dummy; +SELECT * FROM merge(REGEXP('^(_temporary_and_external_tables|system)$'), '^(one|_tmp_)') ORDER BY dummy SETTINGS enable_analyzer = 0; + +-- The same for the `Merge` table engine. +CREATE TABLE t_merge_temporary_engine (dummy UInt8) + ENGINE = Merge(REGEXP('^(_temporary_and_external_tables|system)$'), '^(one|_tmp_)'); + +SELECT * FROM t_merge_temporary_engine ORDER BY dummy; +SELECT * FROM t_merge_temporary_engine ORDER BY dummy SETTINGS enable_analyzer = 0; + +DROP TABLE t_merge_temporary_engine; + +-- When it is the only database that matches, there is nothing to read. +SELECT * FROM merge(REGEXP('^_temporary_and_external_tables$'), '^_tmp_'); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE } + +-- Naming it explicitly is denied, the same way as direct access to it is. +SELECT * FROM merge('_temporary_and_external_tables', '^_tmp_'); -- { serverError DATABASE_ACCESS_DENIED } +SELECT * FROM _temporary_and_external_tables.t_merge_temporary; -- { serverError DATABASE_ACCESS_DENIED } + +-- The same at `CREATE` time for the `Merge` engine: with an explicit column list, no read happens during `CREATE`, +-- so the unusable table definition would be stored otherwise. +CREATE TABLE t_merge_temporary_explicit (dummy UInt8) + ENGINE = Merge('_temporary_and_external_tables', '^_tmp_'); -- { serverError DATABASE_ACCESS_DENIED } diff --git a/tests/queries/0_stateless/04715_distributed_infer_columns_access.reference b/tests/queries/0_stateless/04715_distributed_infer_columns_access.reference new file mode 100644 index 000000000000..02b56be1f5d8 --- /dev/null +++ b/tests/queries/0_stateless/04715_distributed_infer_columns_access.reference @@ -0,0 +1,16 @@ +-- the user cannot describe the target directly +1 +-- 1. omitted columns, no rights on the target: rejected +1 +-- 1. nothing was created, so nothing was disclosed +0 +-- 2. omitted columns, with SHOW COLUMNS on the target: allowed and inferred +x UInt64 +secret_column String +-- 3. explicit columns, no rights on the target: still allowed +1 +-- 4. temporary table, omitted columns, no rights on the target: rejected +1 +-- 5. a detached table with an inferred structure re-attaches without the grant +x UInt64 +secret_column String diff --git a/tests/queries/0_stateless/04715_distributed_infer_columns_access.sh b/tests/queries/0_stateless/04715_distributed_infer_columns_access.sh new file mode 100755 index 000000000000..7d52b00387b1 --- /dev/null +++ b/tests/queries/0_stateless/04715_distributed_infer_columns_access.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Tags: shard, no-replicated-database +# no-replicated-database: on a replicated / shared-catalog database the DDL runs with no user, so the +# in-storage access check asserted here is a no-op and the deny path silently allows. +# Blocked on https://github.com/ClickHouse/ClickHouse/issues/111561 - re-enable when fixed. + +# Regression coverage for the access check applied when a `Distributed` table omits its structure: +# 1. Inferring the structure from a local-shard target requires `SHOW_COLUMNS` on that target, so a +# user who cannot describe the target cannot learn its columns by creating a `Distributed` over it. +# 2. With `SHOW_COLUMNS` granted, the structure is still inferred (the check does not break inference). +# 3. An explicit column list infers nothing and stays allowed without any rights on the target. +# 4. The same check applies to a temporary table, which reaches the engine through its own path. +# 5. A table whose structure was inferred still detaches and re-attaches after the grant is +# revoked: the inferred columns are persisted into its metadata, so the short `ATTACH` carries +# them and never re-infers. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +db=${CLICKHOUSE_DATABASE} +user="user_04715_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} <&1 \ + | grep -c -m1 "ACCESS_DENIED\|Not enough privileges" + +echo "-- 1. omitted columns, no rights on the target: rejected" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.d ENGINE = Distributed('test_shard_localhost', '$db', 'protected_target')" 2>&1 \ + | grep -c -m1 "ACCESS_DENIED\|Not enough privileges" + +echo "-- 1. nothing was created, so nothing was disclosed" +${CLICKHOUSE_CLIENT} --query \ + "SELECT count() FROM system.tables WHERE database = '$db' AND name = 'd'" + +echo "-- 2. omitted columns, with SHOW COLUMNS on the target: allowed and inferred" +${CLICKHOUSE_CLIENT} --query "GRANT SHOW COLUMNS ON $db.protected_target TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.d ENGINE = Distributed('test_shard_localhost', '$db', 'protected_target')" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "SELECT name, type FROM system.columns WHERE database = '$db' AND table = 'd' ORDER BY position" + +echo "-- 3. explicit columns, no rights on the target: still allowed" +${CLICKHOUSE_CLIENT} --query "REVOKE SHOW COLUMNS ON $db.protected_target FROM $user" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.d_explicit (x UInt64) ENGINE = Distributed('test_shard_localhost', '$db', 'protected_target')" +${CLICKHOUSE_CLIENT} --query \ + "SELECT count() FROM system.tables WHERE database = '$db' AND name = 'd_explicit'" + +echo "-- 4. temporary table, omitted columns, no rights on the target: rejected" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TEMPORARY TABLE tmp_d ENGINE = Distributed('test_shard_localhost', '$db', 'protected_target')" 2>&1 \ + | grep -c -m1 "ACCESS_DENIED\|Not enough privileges" + +echo "-- 5. a detached table with an inferred structure re-attaches without the grant" +${CLICKHOUSE_CLIENT} --query "GRANT DROP TABLE, UNDROP TABLE ON $db.* TO $user" +${CLICKHOUSE_CLIENT} --user "$user" <&1 \ + | grep -c -m1 "ACCESS_DENIED\|Not enough privileges" + +echo "-- 1. omitted columns, no rights on the destination: rejected" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.b ENGINE = Buffer('$db', 'protected_target', 1, 10, 100, 10000, 1000000, 10000000, 100000000)" 2>&1 \ + | grep -c -m1 "ACCESS_DENIED\|Not enough privileges" + +echo "-- 2. omitted columns, with SHOW COLUMNS: allowed and inferred from the destination" +${CLICKHOUSE_CLIENT} --query "GRANT SHOW COLUMNS ON $db.protected_target TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.b ENGINE = Buffer('$db', 'protected_target', 1, 10, 100, 10000, 1000000, 10000000, 100000000)" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "SELECT name, type FROM system.columns WHERE database = '$db' AND table = 'b' ORDER BY position" + +echo "-- 3. a short ATTACH still works after the grant is revoked" +${CLICKHOUSE_CLIENT} --query "REVOKE SHOW COLUMNS ON $db.protected_target FROM $user" +${CLICKHOUSE_CLIENT} --user "$user" --query "DETACH TABLE $db.b" +${CLICKHOUSE_CLIENT} --user "$user" --query "ATTACH TABLE $db.b" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "SELECT name, type FROM system.columns WHERE database = '$db' AND table = 'b' ORDER BY position" +${CLICKHOUSE_CLIENT} --query "DROP TABLE $db.b" + +echo "-- 4. explicit columns, no rights on the destination: still allowed" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.b_explicit (x UInt64) ENGINE = Buffer('$db', 'protected_target', 1, 10, 100, 10000, 1000000, 10000000, 100000000)" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "SELECT name, type FROM system.columns WHERE database = '$db' AND table = 'b_explicit' ORDER BY position" +${CLICKHOUSE_CLIENT} --query "DROP TABLE $db.b_explicit" + +echo "-- 5. a temporary table with omitted columns is checked too" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TEMPORARY TABLE tmp_b ENGINE = Buffer('$db', 'protected_target', 1, 10, 100, 10000, 1000000, 10000000, 100000000)" 2>&1 \ + | grep -c -m1 "ACCESS_DENIED\|Not enough privileges" + +echo "-- 6. SELECT on the destination implies SHOW COLUMNS, so reading users are unaffected" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON $db.protected_target TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --query \ + "CREATE TABLE $db.b_select ENGINE = Buffer('$db', 'protected_target', 1, 10, 100, 10000, 1000000, 10000000, 100000000)" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT x, secret_column FROM $db.b_select ORDER BY x" +${CLICKHOUSE_CLIENT} --query "DROP TABLE $db.b_select" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS $user" +${CLICKHOUSE_CLIENT} --query "DROP TABLE $db.protected_target" diff --git a/tests/queries/0_stateless/04743_trivial_count_array_join_argument.reference b/tests/queries/0_stateless/04743_trivial_count_array_join_argument.reference new file mode 100644 index 000000000000..002f592ac5ce --- /dev/null +++ b/tests/queries/0_stateless/04743_trivial_count_array_join_argument.reference @@ -0,0 +1,22 @@ +6 +6 +6 +5 +11 +6 +6 +6 +5 +11 +3 +3 +3 +3 +6 +0 +1 +1 +1 +6 +6 +3 diff --git a/tests/queries/0_stateless/04743_trivial_count_array_join_argument.sh b/tests/queries/0_stateless/04743_trivial_count_array_join_argument.sh new file mode 100755 index 000000000000..bc57a6627328 --- /dev/null +++ b/tests/queries/0_stateless/04743_trivial_count_array_join_argument.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# no-old-analyzer: The plan assertions describe applyTrivialCountIfPossible; the old analyzer decides trivial count in TreeRewriter + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q " +DROP TABLE IF EXISTS t_04743; +CREATE TABLE t_04743 (A Array(UInt32), B Array(UInt32), n UInt32) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_04743 VALUES ([1,2,3],[1,2],1), ([4,5],[],2), ([6],[7,8,9],3); + +-- arrayJoin in the aggregate argument multiplies rows, so the stored row count (3) is not the answer +SELECT count(arrayJoin(A)) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(unnest(A)) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(arrayJoin(A) + 1) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(arrayJoin(B)) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(arrayJoin(arrayJoin([A, B]))) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; + +-- the same values with the optimization off +SELECT count(arrayJoin(A)) FROM t_04743 SETTINGS optimize_trivial_count_query = 0; +SELECT count(unnest(A)) FROM t_04743 SETTINGS optimize_trivial_count_query = 0; +SELECT count(arrayJoin(A) + 1) FROM t_04743 SETTINGS optimize_trivial_count_query = 0; +SELECT count(arrayJoin(B)) FROM t_04743 SETTINGS optimize_trivial_count_query = 0; +SELECT count(arrayJoin(arrayJoin([A, B]))) FROM t_04743 SETTINGS optimize_trivial_count_query = 0; + +-- aggregates without arrayJoin keep the optimization +SELECT count() FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(*) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(1) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count(n) FROM t_04743 SETTINGS optimize_trivial_count_query = 1; +SELECT count() FROM t_04743 ARRAY JOIN A SETTINGS optimize_trivial_count_query = 1; + +-- plans: the optimization is refused for the arrayJoin argument and kept otherwise +SELECT count() > 0 FROM (EXPLAIN SELECT count(arrayJoin(A)) FROM t_04743 SETTINGS optimize_trivial_count_query = 1) +WHERE explain ILIKE '%Optimized trivial count%'; +SELECT count() > 0 FROM (EXPLAIN SELECT count(arrayJoin(A)) FROM t_04743 SETTINGS optimize_trivial_count_query = 1) +WHERE explain ILIKE '%ReadFromMergeTree%'; +SELECT count() > 0 FROM (EXPLAIN SELECT count() FROM t_04743 SETTINGS optimize_trivial_count_query = 1) +WHERE explain ILIKE '%Optimized trivial count%'; +SELECT count() > 0 FROM (EXPLAIN SELECT count(n) FROM t_04743 SETTINGS optimize_trivial_count_query = 1) +WHERE explain ILIKE '%Optimized trivial count%'; + +DROP TABLE t_04743; +" + +# file() counts inside read() when the flag is set, and it reaches that path even though +# totalRows() is unknown, so it distinguishes the guard's position from a later one. +unique_name=${CLICKHOUSE_TEST_UNIQUE_NAME} +tmp_dir=${USER_FILES_PATH}/${unique_name} +mkdir -p "${tmp_dir}" +rm -rf "${tmp_dir:?}"/* + +cat > "${tmp_dir}/arr.csv" <<'EOF' +"[1,2,3]" +"[4,5]" +"[6]" +EOF + +chmod 777 "${tmp_dir}" +chmod 777 "${tmp_dir}/arr.csv" + +${CLICKHOUSE_CLIENT} -q " +SELECT count(arrayJoin(A)) FROM file('${unique_name}/arr.csv', 'CSV', 'A Array(UInt32)') +SETTINGS optimize_trivial_count_query = 1, optimize_count_from_files = 1; +SELECT count(arrayJoin(A)) FROM file('${unique_name}/arr.csv', 'CSV', 'A Array(UInt32)') +SETTINGS optimize_trivial_count_query = 0, optimize_count_from_files = 1; +SELECT count() FROM file('${unique_name}/arr.csv', 'CSV', 'A Array(UInt32)') +SETTINGS optimize_trivial_count_query = 1, optimize_count_from_files = 1; +" + +rm -rf "${tmp_dir:?}" diff --git a/tests/queries/0_stateless/04745_join_low_cardinality_wide_int_key.reference b/tests/queries/0_stateless/04745_join_low_cardinality_wide_int_key.reference new file mode 100644 index 000000000000..07f11a62f2db --- /dev/null +++ b/tests/queries/0_stateless/04745_join_low_cardinality_wide_int_key.reference @@ -0,0 +1,22 @@ +lc_uint128 using 1 +lc_uint128 on 1 1 +lc_uint128 hash 1 +lc_uint128 parallel_hash 1 +lc_uint128 grace_hash 1 +lc_uint128 full_sorting_merge 1 +lc_int128 1 +lc_uint256 1 +lc_int256 1 +lc_uint8 1 +lc_uint64 1 +plain_uint128 1 +lc_string 1 +lc_uuid 1 +lc_nullable_uint128 1 +lc_nullable_uint128 rows 1 1 +lc_uint128_multikey 1 +lc_uint128_asof 1 1 +lc_uint128 highbits 18446744073709551617 +lc_uint256 highbits 340282366920938463463374607431768211457 +lc_int128 highbits -1 +lc_int256 highbits -1 diff --git a/tests/queries/0_stateless/04745_join_low_cardinality_wide_int_key.sql b/tests/queries/0_stateless/04745_join_low_cardinality_wide_int_key.sql new file mode 100644 index 000000000000..d3def1ac6069 --- /dev/null +++ b/tests/queries/0_stateless/04745_join_low_cardinality_wide_int_key.sql @@ -0,0 +1,150 @@ +SET allow_suspicious_low_cardinality_types = 1; + +-- A single `LowCardinality` key wider than 8 bytes must match by value, not collapse to one bucket. + +DROP TABLE IF EXISTS t_l; +DROP TABLE IF EXISTS t_r; + +-- `LowCardinality(UInt128)`: 1 must match only 1, not 5 or 7. +CREATE TABLE t_l (id LowCardinality(UInt128)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UInt128)) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'lc_uint128 using', count() FROM t_l JOIN t_r USING (id); +SELECT 'lc_uint128 on', a.id, b.id FROM t_l a JOIN t_r b ON a.id = b.id ORDER BY a.id, b.id; +SELECT 'lc_uint128 hash', count() FROM t_l JOIN t_r USING (id) SETTINGS join_algorithm = 'hash'; +SELECT 'lc_uint128 parallel_hash', count() FROM t_l JOIN t_r USING (id) SETTINGS join_algorithm = 'parallel_hash', max_threads = 8; +SELECT 'lc_uint128 grace_hash', count() FROM t_l JOIN t_r USING (id) SETTINGS join_algorithm = 'grace_hash'; +SELECT 'lc_uint128 full_sorting_merge', count() FROM t_l JOIN t_r USING (id) SETTINGS join_algorithm = 'full_sorting_merge'; +DROP TABLE t_l; +DROP TABLE t_r; + +-- Sibling affected widths. +CREATE TABLE t_l (id LowCardinality(Int128)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(Int128)) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'lc_int128', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(UInt256)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UInt256)) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'lc_uint256', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(Int256)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(Int256)) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'lc_int256', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +-- Unaffected shapes: regression guards, all correct before the fix. +CREATE TABLE t_l (id LowCardinality(UInt8)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UInt8)) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'lc_uint8', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(UInt64)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UInt64)) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'lc_uint64', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id UInt128) ENGINE = Memory; +CREATE TABLE t_r (id UInt128) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1); +INSERT INTO t_r VALUES (7), (0), (1); +SELECT 'plain_uint128', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(String)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(String)) ENGINE = Memory; +INSERT INTO t_l VALUES ('5'), ('1'); +INSERT INTO t_r VALUES ('7'), ('0'), ('1'); +SELECT 'lc_string', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(UUID)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UUID)) ENGINE = Memory; +INSERT INTO t_l VALUES ('00000000-0000-0000-0000-000000000005'), ('00000000-0000-0000-0000-000000000001'); +INSERT INTO t_r VALUES ('00000000-0000-0000-0000-000000000007'), ('00000000-0000-0000-0000-000000000000'), ('00000000-0000-0000-0000-000000000001'); +SELECT 'lc_uuid', count() FROM t_l JOIN t_r USING (id); +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(Nullable(UInt128))) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(Nullable(UInt128))) ENGINE = Memory; +INSERT INTO t_l VALUES (5), (1), (NULL); +INSERT INTO t_r VALUES (7), (0), (1), (NULL); +-- A `NULL` key matches nothing, so the count stays 1. +SELECT 'lc_nullable_uint128', count() FROM t_l JOIN t_r USING (id); +SELECT 'lc_nullable_uint128 rows', a.id, b.id FROM t_l a JOIN t_r b ON a.id = b.id ORDER BY a.id; +DROP TABLE t_l; +DROP TABLE t_r; + +-- Multi-column key containing a wide `LowCardinality` column. +CREATE TABLE t_l (a LowCardinality(UInt128), b UInt8) ENGINE = Memory; +CREATE TABLE t_r (a LowCardinality(UInt128), b UInt8) ENGINE = Memory; +INSERT INTO t_l VALUES (5, 1), (1, 2); +INSERT INTO t_r VALUES (7, 1), (0, 2), (1, 2); +SELECT 'lc_uint128_multikey', count() FROM t_l JOIN t_r USING (a, b); +DROP TABLE t_l; +DROP TABLE t_r; + +-- `ASOF` join over a wide `LowCardinality` equality key. +CREATE TABLE t_l (k LowCardinality(UInt128), t UInt64) ENGINE = Memory; +CREATE TABLE t_r (k LowCardinality(UInt128), t UInt64) ENGINE = Memory; +INSERT INTO t_l VALUES (1, 10), (5, 10); +INSERT INTO t_r VALUES (1, 5), (7, 5); +SELECT 'lc_uint128_asof', a.k, b.k FROM t_l a ASOF JOIN t_r b ON a.k = b.k AND a.t >= b.t ORDER BY a.k; +DROP TABLE t_l; +DROP TABLE t_r; + +-- Below, the two left values share their low half and differ only above, so a comparison narrower +-- than the key type makes them collide and the single right value matches both. + +CREATE TABLE t_l (id LowCardinality(UInt128)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UInt128)) ENGINE = Memory; +INSERT INTO t_l VALUES (toUInt128(1) + bitShiftLeft(toUInt128(1), 64)), (toUInt128(1) + bitShiftLeft(toUInt128(2), 64)); +INSERT INTO t_r VALUES (toUInt128(1) + bitShiftLeft(toUInt128(1), 64)); +SELECT 'lc_uint128 highbits', a.id FROM t_l a JOIN t_r b ON a.id = b.id ORDER BY a.id; +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(UInt256)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(UInt256)) ENGINE = Memory; +INSERT INTO t_l VALUES (toUInt256(1) + bitShiftLeft(toUInt256(1), 128)), (toUInt256(1) + bitShiftLeft(toUInt256(2), 128)); +INSERT INTO t_r VALUES (toUInt256(1) + bitShiftLeft(toUInt256(1), 128)); +SELECT 'lc_uint256 highbits', a.id FROM t_l a JOIN t_r b ON a.id = b.id ORDER BY a.id; +DROP TABLE t_l; +DROP TABLE t_r; + +-- Signed high half: -1 is all-ones, so it shares its low bytes with the positive 2^64-1 / 2^128-1. +CREATE TABLE t_l (id LowCardinality(Int128)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(Int128)) ENGINE = Memory; +INSERT INTO t_l VALUES (toInt128(-1)), (toInt128(bitShiftLeft(toUInt128(1), 64) - 1)); +INSERT INTO t_r VALUES (toInt128(-1)); +SELECT 'lc_int128 highbits', a.id FROM t_l a JOIN t_r b ON a.id = b.id ORDER BY a.id; +DROP TABLE t_l; +DROP TABLE t_r; + +CREATE TABLE t_l (id LowCardinality(Int256)) ENGINE = Memory; +CREATE TABLE t_r (id LowCardinality(Int256)) ENGINE = Memory; +INSERT INTO t_l VALUES (toInt256(-1)), (toInt256(bitShiftLeft(toUInt256(1), 128) - 1)); +INSERT INTO t_r VALUES (toInt256(-1)); +SELECT 'lc_int256 highbits', a.id FROM t_l a JOIN t_r b ON a.id = b.id ORDER BY a.id; +DROP TABLE t_l; +DROP TABLE t_r; diff --git a/tests/queries/0_stateless/04757_paimon_nullable_composite_types.reference b/tests/queries/0_stateless/04757_paimon_nullable_composite_types.reference new file mode 100644 index 000000000000..7e3af3ce0c5b --- /dev/null +++ b/tests/queries/0_stateless/04757_paimon_nullable_composite_types.reference @@ -0,0 +1,6 @@ +id Int32 +arr Array(Nullable(Int32)) +m Map(String, Nullable(Int32)) +=== +1 [1,2] {'k':1} +2 [] {} diff --git a/tests/queries/0_stateless/04757_paimon_nullable_composite_types.sql b/tests/queries/0_stateless/04757_paimon_nullable_composite_types.sql new file mode 100644 index 000000000000..ea0055acb283 --- /dev/null +++ b/tests/queries/0_stateless/04757_paimon_nullable_composite_types.sql @@ -0,0 +1,9 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: Depends on AWS/MinIO paimon_nullable_composites dataset + +-- A nullable Paimon ARRAY/MAP column must not be wrapped in Nullable, which made the +-- whole table unreadable. https://github.com/ClickHouse/ClickHouse/issues/113337 + +desc paimonS3(s3_conn, filename='paimon_nullable_composites'); +select '==='; +select id, arr, m from paimonS3(s3_conn, filename='paimon_nullable_composites') order by id; diff --git a/tests/queries/0_stateless/04780_json_subcolumn_index_match_not_quadratic.reference b/tests/queries/0_stateless/04780_json_subcolumn_index_match_not_quadratic.reference new file mode 100644 index 000000000000..d971dab4560d --- /dev/null +++ b/tests/queries/0_stateless/04780_json_subcolumn_index_match_not_quadratic.reference @@ -0,0 +1,6 @@ +plain OK +longidx OK +withjson OK +tokens OK +0 +Granules: 0/1000 diff --git a/tests/queries/0_stateless/04780_json_subcolumn_index_match_not_quadratic.sh b/tests/queries/0_stateless/04780_json_subcolumn_index_match_not_quadratic.sh new file mode 100755 index 000000000000..bdfdfbfec127 --- /dev/null +++ b/tests/queries/0_stateless/04780_json_subcolumn_index_match_not_quadratic.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Tags: long, no-fasttest, no-parallel-replicas, no-flaky-check +# Test for https://github.com/ClickHouse/ClickHouse/issues/113003 +# Skip-index condition building matched a filter column name against JSONAllPaths(...) index +# columns by enumerating every dot split of the name and formatting a lookup key per split. The +# name embeds the text of a folded constant, so a large dotted constant made index analysis +# quadratic in the constant's length. +# Each arm compares a dotted constant against a no-dots constant of the same length. The oracle is +# the allocated-bytes counter rather than wall clock: it is exactly reproducible, whereas the +# ~2.6s planning delta is smaller than debug-build client startup jitter. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +REPEATS=100000 +# Measured on a debug build: pre-fix the dotted arm allocates 2.8x-3.0x the control in every arm, +# post-fix 1.0003x. 1.5 sits an order of magnitude away from the fixed behavior and ~1.9x below +# the regression, so it discriminates with margin on both sides. +MAX_RATIO_PERCENT=150 + +LONG_SUFFIX=$(printf 'a%.0s' $(seq 1 170)) + +$CLICKHOUSE_CLIENT -nm -q " + SET enable_json_type = 1, allow_suspicious_indices = 1; + + -- index_granularity is pinned on every table: the granule counts asserted below are + -- ceil(rows / index_granularity), which the test runner otherwise randomizes. + + -- No JSON column at all, so the matcher can never succeed: the reported shape. + CREATE TABLE plain (s String, INDEX ix s TYPE bloom_filter GRANULARITY 1) + ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1; + + -- index_columns carries a long NON-JSON entry. Bounding the split enumeration by the longest + -- index column would leave this arm quadratic. + CREATE TABLE longidx (s String, INDEX ilong concat(s, '${LONG_SUFFIX}') TYPE bloom_filter GRANULARITY 1) + ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1; + + -- A JSONAllPaths index IS present, so an early-out keyed on its absence cannot fire. + CREATE TABLE withjson (s String, j JSON, + INDEX ix s TYPE bloom_filter GRANULARITY 1, + INDEX jx JSONAllPaths(j) TYPE bloom_filter GRANULARITY 1) + ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1; + + -- Token indexes reach the same matcher through a different condition class. + CREATE TABLE tokens (s String, INDEX ix s TYPE tokenbf_v1(256, 2, 0) GRANULARITY 1) + ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1; + + INSERT INTO plain SELECT 'v' || toString(number % 10) FROM numbers(1000); + INSERT INTO longidx SELECT 'v' || toString(number % 10) FROM numbers(1000); + INSERT INTO withjson SELECT 'v' || toString(number % 10), '{\"a\":1}' FROM numbers(1000); + INSERT INTO tokens SELECT 'v' || toString(number % 10) FROM numbers(1000); +" + +# Sets ALLOC_BYTES to the bytes the server allocated while planning one query. +# EXPLAIN runs index analysis without reading data, so the measurement is the matcher's cost. +alloc_bytes_for() { + local table="$1" unit="$2" + local query_id="04780-${CLICKHOUSE_DATABASE}-${table}-${unit}-${RANDOM}" + $CLICKHOUSE_CLIENT --query_id "$query_id" --max_query_size 1048576 --max_execution_time 300 -q " + SELECT count() FROM ( + EXPLAIN indexes = 1 + SELECT count() FROM ${table} WHERE position(repeat('${unit}', ${REPEATS}), s) = 1 + )" >/dev/null + $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" >/dev/null + ALLOC_BYTES=$($CLICKHOUSE_CLIENT -q " + SELECT ProfileEvents['MemoryAllocatedWithoutCheckBytes'] + FROM system.query_log + WHERE current_database = currentDatabase() AND query_id = '${query_id}' AND type = 'QueryFinish'") + [ -n "$ALLOC_BYTES" ] && [ "$ALLOC_BYTES" -gt 0 ] +} + +for table in plain longidx withjson tokens; do + alloc_bytes_for "$table" 'a.' || { echo "FAIL: no query_log row for $table dotted arm" >&2; exit 1; } + dotted=$ALLOC_BYTES + + alloc_bytes_for "$table" 'ab' || { echo "FAIL: no query_log row for $table control arm" >&2; exit 1; } + control=$ALLOC_BYTES + + if [ $((dotted * 100)) -gt $((control * MAX_RATIO_PERCENT)) ]; then + echo "FAIL: $table index analysis over a constant with ${REPEATS} dots allocated ${dotted} bytes," \ + "more than ${MAX_RATIO_PERCENT}% of the no-dots control (${control} bytes)" >&2 + exit 1 + fi + echo "$table OK" +done + +# A dotted constant must not change which granules are read, and a real JSON subcolumn filter on +# the same table must still prune. +$CLICKHOUSE_CLIENT -nm -q " + SET enable_json_type = 1; + SELECT count() FROM withjson WHERE position(repeat('a.', 100), s) = 1; + SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT count() FROM withjson WHERE j.absent_path = 'zzz' + ) WHERE explain LIKE '%Granules:%'; +" + +$CLICKHOUSE_CLIENT -nm -q " + DROP TABLE plain; DROP TABLE longidx; DROP TABLE withjson; DROP TABLE tokens; +" diff --git a/tests/queries/0_stateless/04811_merge_temporary_database_attach_full_definition.reference b/tests/queries/0_stateless/04811_merge_temporary_database_attach_full_definition.reference new file mode 100644 index 000000000000..3c96e57467f6 --- /dev/null +++ b/tests/queries/0_stateless/04811_merge_temporary_database_attach_full_definition.reference @@ -0,0 +1,3 @@ +DATABASE_ACCESS_DENIED +0 +0 diff --git a/tests/queries/0_stateless/04811_merge_temporary_database_attach_full_definition.sh b/tests/queries/0_stateless/04811_merge_temporary_database_attach_full_definition.sh new file mode 100755 index 000000000000..12fb07680e23 --- /dev/null +++ b/tests/queries/0_stateless/04811_merge_temporary_database_attach_full_definition.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# A full-definition `ATTACH TABLE` is CREATE-like user input, so naming the internal database +# of temporary tables in the `Merge` engine must be denied there as well, the same way as in +# `CREATE TABLE`; otherwise the forbidden and unusable definition would be persisted. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Generate a random UUID to avoid collisions in Atomic databases. +UUID=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") + +# -m1 because the error message may contain the error code name multiple times. +$CLICKHOUSE_CLIENT -q "ATTACH TABLE t_merge_tmp_attach_full UUID '${UUID}' (dummy UInt8) ENGINE = Merge('_temporary_and_external_tables', '^_tmp_');" 2>&1 | grep -m 1 -o -F 'DATABASE_ACCESS_DENIED' + +# A short ATTACH (stored metadata) of a legitimate definition still works. +# `send_logs_level=fatal` suppresses the "full table definition is not recommended" warning. +$CLICKHOUSE_CLIENT --send_logs_level fatal -q "ATTACH TABLE t_merge_tmp_attach_full UUID '${UUID}' (dummy UInt8) ENGINE = Merge('system', '^one$');" +$CLICKHOUSE_CLIENT -q "SELECT * FROM t_merge_tmp_attach_full;" +$CLICKHOUSE_CLIENT -q "DETACH TABLE t_merge_tmp_attach_full;" +$CLICKHOUSE_CLIENT -q "ATTACH TABLE t_merge_tmp_attach_full;" +$CLICKHOUSE_CLIENT -q "SELECT * FROM t_merge_tmp_attach_full;" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_merge_tmp_attach_full;" diff --git a/tests/queries/0_stateless/04812_merge_alias_in_prepared_set.reference b/tests/queries/0_stateless/04812_merge_alias_in_prepared_set.reference new file mode 100644 index 000000000000..22bdad5a20dd --- /dev/null +++ b/tests/queries/0_stateless/04812_merge_alias_in_prepared_set.reference @@ -0,0 +1,19 @@ +constant tuple +[0,1,7] +explicit column +[0,1,7] +with filter +[1] +set table +[0,1,7] +not in +[0,1,7] +tuple key +[0,1,7] +nullable +[1,7,255] 1 +query info cache +[0,1,7] +identical children +[0,1] +ALIAS y IN (1, 2, 3) diff --git a/tests/queries/0_stateless/04812_merge_alias_in_prepared_set.sql b/tests/queries/0_stateless/04812_merge_alias_in_prepared_set.sql new file mode 100644 index 000000000000..04408c54d44b --- /dev/null +++ b/tests/queries/0_stateless/04812_merge_alias_in_prepared_set.sql @@ -0,0 +1,100 @@ +-- Reading through `merge()` over a child table whose ALIAS column contains `IN` used to abort with +-- `Logical error: No set is registered for key ...`. It triggers only when the children disagree +-- about that column's default, which makes the Merge-level column look physical. + +-- The old analyzer reads an ALIAS column through `TreeRewriter`, which never consults the set +-- registry, so every case below produces its expected value on an unfixed server unless this is set. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t04812_phys; +DROP TABLE IF EXISTS t04812_alias; +DROP TABLE IF EXISTS t04812_set; + +-- Constant tuple: the `findTuple` branch. +CREATE TABLE t04812_phys (x UInt8) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t04812_alias (y UInt8, x UInt8 ALIAS y IN (1, 2, 3)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t04812_phys VALUES (7); +INSERT INTO t04812_alias VALUES (2), (9); +SELECT 'constant tuple'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^t04812_(phys|alias)$'); + +-- The same shape with an explicit column list, not `SELECT *`. +SELECT 'explicit column'; +SELECT arraySort(groupArray(x)) FROM (SELECT x FROM merge(currentDatabase(), '^t04812_(phys|alias)$')); + +-- A filter reaching the child read, which is how the failure was first seen in CI. +SELECT 'with filter'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^t04812_(phys|alias)$') WHERE x = 1; + +-- `Set` table: the `findStorage` branch, whose registry key carries no element types. +DROP TABLE t04812_alias; +CREATE TABLE t04812_set (k UInt8) ENGINE = Set; +INSERT INTO t04812_set VALUES (1), (2); +CREATE TABLE t04812_alias (y UInt8, x UInt8 ALIAS y IN t04812_set) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t04812_alias VALUES (2), (9); +SELECT 'set table'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^t04812_(phys|alias)$'); + +-- `NOT IN` reaches the same lookup. +DROP TABLE t04812_alias; +CREATE TABLE t04812_alias (y UInt8, x UInt8 ALIAS y NOT IN (1, 2, 3)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t04812_alias VALUES (2), (9); +SELECT 'not in'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^t04812_(phys|alias)$'); + +-- A tuple key, so the registry key carries two element types. +DROP TABLE t04812_alias; +CREATE TABLE t04812_alias (a UInt8, b UInt8, x UInt8 ALIAS (a, b) IN ((1, 2), (3, 4))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t04812_alias VALUES (1, 2), (9, 9); +SELECT 'tuple key'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^t04812_(phys|alias)$'); + +-- A Nullable operand, where the alias itself is Nullable. +DROP TABLE t04812_phys; +DROP TABLE t04812_alias; +CREATE TABLE t04812_phys (x Nullable(UInt8)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t04812_alias (y Nullable(UInt8), x Nullable(UInt8) ALIAS y IN (1, 2, NULL)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t04812_phys VALUES (7); +INSERT INTO t04812_alias VALUES (2), (NULL); +SELECT 'nullable'; +SELECT arraySort(groupArray(ifNull(x, 255))), countIf(x IS NULL) FROM merge(currentDatabase(), '^t04812_(phys|alias)$'); + +DROP TABLE t04812_phys; +DROP TABLE t04812_alias; +DROP TABLE t04812_set; + +-- A third child identical to the second takes the query info cache path, which skips +-- `getModifiedQueryInfo` and evaluates its alias in `convertAndFilterSourceStream` instead. +-- The values are asserted so that a silently unevaluated alias fails this case. +DROP TABLE IF EXISTS u04812_phys; +DROP TABLE IF EXISTS u04812_a; +DROP TABLE IF EXISTS u04812_b; +CREATE TABLE u04812_phys (x UInt8) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE u04812_a (y UInt8, x UInt8 ALIAS y IN (1, 2, 3)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE u04812_b (y UInt8, x UInt8 ALIAS y IN (1, 2, 3)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO u04812_phys VALUES (7); +INSERT INTO u04812_a VALUES (2); +INSERT INTO u04812_b VALUES (9); +SELECT 'query info cache'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^u04812_'); + +DROP TABLE u04812_phys; +DROP TABLE u04812_a; +DROP TABLE u04812_b; + +-- Control: with both children declaring the same ALIAS there is no disagreement, the Merge column +-- keeps its ALIAS, and this case already passed before the fix. It is what shows that the +-- discriminating condition of the cases above is the disagreement between children. +DROP TABLE IF EXISTS v04812_a; +DROP TABLE IF EXISTS v04812_b; +CREATE TABLE v04812_a (y UInt8, x UInt8 ALIAS y IN (1, 2, 3)) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE v04812_b (y UInt8, x UInt8 ALIAS y IN (1, 2, 3)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO v04812_a VALUES (2); +INSERT INTO v04812_b VALUES (9); +SELECT 'identical children'; +SELECT arraySort(groupArray(x)) FROM merge(currentDatabase(), '^v04812_'); +SELECT default_kind, default_expression FROM system.columns +WHERE database = currentDatabase() AND table = 'v04812_a' AND name = 'x'; + +DROP TABLE v04812_a; +DROP TABLE v04812_b; diff --git a/tests/queries/0_stateless/04812_merge_row_policy_shared_ast.reference b/tests/queries/0_stateless/04812_merge_row_policy_shared_ast.reference new file mode 100644 index 000000000000..2ec92d18c89b --- /dev/null +++ b/tests/queries/0_stateless/04812_merge_row_policy_shared_ast.reference @@ -0,0 +1,4 @@ +4 +4 +8 +8 diff --git a/tests/queries/0_stateless/04812_merge_row_policy_shared_ast.sh b/tests/queries/0_stateless/04812_merge_row_policy_shared_ast.sh new file mode 100755 index 000000000000..5303466321fb --- /dev/null +++ b/tests/queries/0_stateless/04812_merge_row_policy_shared_ast.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `RowPolicyFilter::expression` is the parsed policy condition owned by `RowPolicyCache` and is shared by +# every query that reads the table. `ReadFromMerge::RowPolicyData` used to hand it straight to +# `TreeRewriter`, which rewrites the AST it is given in place and substitutes the results of scalar +# subqueries for the subqueries themselves. Reading a `Merge` table therefore froze the value of a scalar +# subquery inside the policy in the cache, for the rest of the server's lifetime and for every user. +# The same in-place rewrite was reported by ThreadSanitizer as a data race on the shared AST. +# +# This is a shell test because the table inside the policy's subquery has to be qualified with the +# database name: the policy condition is analyzed anew for every read, including reads on the remote +# side of a parallel-replicas query, where the session default database is not the test database. + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS t_04812_src; + DROP TABLE IF EXISTS t_04812_limit; + DROP TABLE IF EXISTS t_04812_merge; + DROP ROW POLICY IF EXISTS p_04812 ON t_04812_src; + + CREATE TABLE t_04812_src (x UInt64) ENGINE = MergeTree ORDER BY x; + INSERT INTO t_04812_src SELECT number FROM numbers(10); + + CREATE TABLE t_04812_limit (v UInt64) ENGINE = MergeTree ORDER BY v; + INSERT INTO t_04812_limit VALUES (3); + + CREATE ROW POLICY p_04812 ON t_04812_src USING x <= (SELECT max(v) FROM ${CLICKHOUSE_DATABASE}.t_04812_limit) TO ALL; + CREATE TABLE t_04812_merge (x UInt64) ENGINE = Merge(currentDatabase(), '^t_04812_src\$'); + + -- The policy admits 0, 1, 2, 3. + SELECT count() FROM t_04812_merge; + SELECT count() FROM t_04812_src; + + INSERT INTO t_04812_limit VALUES (7); + + -- The policy now admits 0 .. 7. Reading through Merge used to keep answering 4, and so did the direct + -- read, because the Merge read above had replaced the subquery with its value in the cached policy AST. + SELECT count() FROM t_04812_merge; + SELECT count() FROM t_04812_src; + + DROP ROW POLICY p_04812 ON t_04812_src; + DROP TABLE t_04812_merge; + DROP TABLE t_04812_limit; + DROP TABLE t_04812_src; +" diff --git a/tests/queries/0_stateless/04812_row_policy_top_k_optimization.reference b/tests/queries/0_stateless/04812_row_policy_top_k_optimization.reference new file mode 100644 index 000000000000..97ac9d1fb76c --- /dev/null +++ b/tests/queries/0_stateless/04812_row_policy_top_k_optimization.reference @@ -0,0 +1,7 @@ +100 +101 +102 +-- +100 +101 +102 diff --git a/tests/queries/0_stateless/04812_row_policy_top_k_optimization.sql b/tests/queries/0_stateless/04812_row_policy_top_k_optimization.sql new file mode 100644 index 000000000000..853287ccc6ea --- /dev/null +++ b/tests/queries/0_stateless/04812_row_policy_top_k_optimization.sql @@ -0,0 +1,46 @@ +-- Regression test for the top-K `ORDER BY ... LIMIT` optimization with a row policy. +-- +-- A row policy restricts rows inside the reader, just like a `WHERE` / `PREWHERE`, but `tryOptimizeTopK` +-- decided `where_clause` from the plan-visible filters only, so a query filtered by a policy alone took +-- the unfiltered fast path: `perform_top_k_optimization` narrowed the read to the marks holding the +-- smallest sort key values, the policy discarded every row in them, and the query returned fewer rows +-- than the `LIMIT` - here nothing at all instead of 100, 101, 102. + +DROP ROW POLICY IF EXISTS rp_04812 ON t_04812; +DROP TABLE IF EXISTS t_04812; + +CREATE TABLE t_04812 (key UInt64, INDEX mm_key key TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8; + +INSERT INTO t_04812 SELECT number FROM numbers(300); + +-- Drops the first 100 rows by the sort key `key`, so the first surviving row is `key` = 100. +CREATE ROW POLICY rp_04812 ON t_04812 FOR SELECT USING key >= 100 TO ALL; + +-- Must return the first three surviving rows in `key` order, never fewer, on both analyzers. +-- +-- Every setting the narrowing depends on is pinned, because the test runner randomizes them and the +-- bug only shows with the values below: `use_skip_indexes_on_data_read = 0` skips the narrowing +-- altogether, and `query_plan_max_limit_for_top_k_optimization` below the `LIMIT` disables the +-- optimization. All of them are pinned to their default values, so the query is the one a user runs. +-- +-- `max_rows_to_read = 0` is pinned because the stateless-test user profile sets it to 20000000, and +-- `ReadFromMergeTree::supportsSkipIndexesOnDataRead` disables skip indexes on data read - including +-- the top-K narrowing - whenever a throwing `max_rows_to_read` / `max_rows_to_read_leaf` limit is set +-- (row estimation does not work when granules are skipped during the scan). Without the pin the bug +-- cannot manifest in the CI environment at all, and bugfix validation reports "bug does not reproduce +-- on master". `0` is the default, so the query still is the one a user runs. +SELECT key FROM t_04812 ORDER BY key LIMIT 3 + SETTINGS enable_analyzer = 0, use_skip_indexes = 1, use_skip_indexes_for_top_k = 1, use_skip_indexes_on_data_read = 1, + query_plan_max_limit_for_top_k_optimization = 1000, max_threads = 1, enable_parallel_replicas = 0, + max_rows_to_read = 0, max_rows_to_read_leaf = 0; + +SELECT '--'; + +SELECT key FROM t_04812 ORDER BY key LIMIT 3 + SETTINGS enable_analyzer = 1, use_skip_indexes = 1, use_skip_indexes_for_top_k = 1, use_skip_indexes_on_data_read = 1, + query_plan_max_limit_for_top_k_optimization = 1000, max_threads = 1, enable_parallel_replicas = 0, + max_rows_to_read = 0, max_rows_to_read_leaf = 0; + +DROP ROW POLICY rp_04812 ON t_04812; +DROP TABLE t_04812; diff --git a/tests/queries/0_stateless/04813_merge_temporary_database_restore.reference b/tests/queries/0_stateless/04813_merge_temporary_database_restore.reference new file mode 100644 index 000000000000..481524060819 --- /dev/null +++ b/tests/queries/0_stateless/04813_merge_temporary_database_restore.reference @@ -0,0 +1,2 @@ +CREATE TABLE default.m\n(\n `x` UInt8\n)\nENGINE = Merge(\'_temporary_and_external_tables\', \'^src$\') +x UInt8 0 diff --git a/tests/queries/0_stateless/04813_merge_temporary_database_restore.sh b/tests/queries/0_stateless/04813_merge_temporary_database_restore.sh new file mode 100755 index 000000000000..46cf2ad17f6b --- /dev/null +++ b/tests/queries/0_stateless/04813_merge_temporary_database_restore.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# A `Merge` table over `_temporary_and_external_tables` created before the database became +# forbidden must remain restorable from a backup: `RESTORE` (like replicated-database DDL +# replay) brings back previously stored metadata, not fresh user input, so only reading +# from the table is denied. The backup is prepared from a legitimate definition and then +# edited on disk, because such a table can no longer be created directly. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORK_DIR=${CLICKHOUSE_TMP}/04813_merge_temporary_database_restore_${CLICKHOUSE_DATABASE} +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}/backups" + +CONFIG="${WORK_DIR}/config.xml" +cat > "${CONFIG}" < + + ${WORK_DIR}/backups + + +EOF + +# The source database name has the same length as `_temporary_and_external_tables`, +# so the substitution below preserves the file size recorded in the backup metadata. +SRC_DB=db_0123456789012345678901234567 + +${CLICKHOUSE_LOCAL} --config-file "${CONFIG}" --path "${WORK_DIR}/data" -q " +CREATE DATABASE ${SRC_DB}; +CREATE TABLE ${SRC_DB}.src (x UInt8) ENGINE = MergeTree ORDER BY x; +CREATE TABLE m (x UInt8) ENGINE = Merge('${SRC_DB}', '^src\$'); +BACKUP TABLE m TO File('${WORK_DIR}/backups/b1') FORMAT Null; +" + +sed -i "s/${SRC_DB}/_temporary_and_external_tables/" "${WORK_DIR}/backups/b1/metadata/default/m.sql" + +${CLICKHOUSE_LOCAL} --config-file "${CONFIG}" --path "${WORK_DIR}/data_restored" -q " +RESTORE TABLE default.m FROM File('${WORK_DIR}/backups/b1') FORMAT Null; +SHOW CREATE TABLE m; +-- Introspection stays best-effort for the restored table: the size columns of \`system.columns\` +-- go through \`StorageMerge::tryGetColumnSizes\`, which must not throw for the forbidden database. +SELECT name, type, data_compressed_bytes FROM system.columns WHERE database = currentDatabase() AND table = 'm'; +SELECT * FROM m; -- { serverError DATABASE_ACCESS_DENIED } +" + +rm -rf "${WORK_DIR}" diff --git a/tests/queries/0_stateless/04815_parallel_replicas_custom_key_merge_distributed_child.reference b/tests/queries/0_stateless/04815_parallel_replicas_custom_key_merge_distributed_child.reference new file mode 100644 index 000000000000..ba18b0fc3c08 --- /dev/null +++ b/tests/queries/0_stateless/04815_parallel_replicas_custom_key_merge_distributed_child.reference @@ -0,0 +1,4 @@ +300000 +14999850000 +47 9 +300000 diff --git a/tests/queries/0_stateless/04815_parallel_replicas_custom_key_merge_distributed_child.sql b/tests/queries/0_stateless/04815_parallel_replicas_custom_key_merge_distributed_child.sql new file mode 100644 index 000000000000..57f1540043c9 --- /dev/null +++ b/tests/queries/0_stateless/04815_parallel_replicas_custom_key_merge_distributed_child.sql @@ -0,0 +1,33 @@ +-- A `Merge` table over a `Distributed` child plans all of its children up to `WithMergeableState` +-- through an interpreter. The custom-key parallel replicas read replaces a child plan with a remote +-- read at the fixed stage `WithMergeableStateAfterAggregationAndLimit`, so the parent received +-- finalized (post-aggregation, post-LIMIT) data where it expected partial aggregation states: +-- `CANNOT_CONVERT_TYPE` for `count`, and an exception about a missing `AggregatedChunkInfo` in +-- `GroupingAggregatedTransform` when the finalized type coincides with the state type structurally. +-- https://github.com/ClickHouse/ClickHouse/issues/113741 + +DROP TABLE IF EXISTS t_mrg_ck_1; +DROP TABLE IF EXISTS t_mrg_ck_2; +DROP TABLE IF EXISTS t_mrg_ck_3; + +CREATE TABLE t_mrg_ck_1 (k UInt64) ENGINE = MergeTree ORDER BY k AS SELECT number FROM numbers(100000); +CREATE TABLE t_mrg_ck_2 (k UInt64) ENGINE = MergeTree ORDER BY k AS SELECT number FROM numbers(100000); +CREATE TABLE t_mrg_ck_3 (k UInt64) ENGINE = Distributed('test_shard_localhost', currentDatabase(), 't_mrg_ck_1'); + +SET enable_analyzer = 1; +SET enable_parallel_replicas = 1, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1, + parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_custom_key = 'k'; + +SELECT count() FROM merge(currentDatabase(), '^t_mrg_ck_'); +SELECT sum(k) FROM merge(currentDatabase(), '^t_mrg_ck_') GROUP BY ALL; +SELECT 47, quantileExactInclusive(visibleWidth(['1', '2'])) IGNORE NULLS FROM merge(currentDatabase(), '^t_mrg_ck_') GROUP BY ALL LIMIT 973; + +SET parallel_replicas_mode = 'custom_key_range', parallel_replicas_custom_key_range_upper = 100000; + +SELECT count() FROM merge(currentDatabase(), '^t_mrg_ck_'); + +DROP TABLE t_mrg_ck_1; +DROP TABLE t_mrg_ck_2; +DROP TABLE t_mrg_ck_3; diff --git a/tests/queries/0_stateless/04816_direct_join_virtual_column.reference b/tests/queries/0_stateless/04816_direct_join_virtual_column.reference new file mode 100644 index 000000000000..7eb7a0a11229 --- /dev/null +++ b/tests/queries/0_stateless/04816_direct_join_virtual_column.reference @@ -0,0 +1,11 @@ +VALA t_04816_rocks +VALA t_04816_rocks +one d_04816 +one d_04816 +VALA 1 +VALA 1 +VALA t_04816_rocks +VALA t_04816_rocks +VALA +1 +0 diff --git a/tests/queries/0_stateless/04816_direct_join_virtual_column.sql b/tests/queries/0_stateless/04816_direct_join_virtual_column.sql new file mode 100644 index 000000000000..7ded34036e13 --- /dev/null +++ b/tests/queries/0_stateless/04816_direct_join_virtual_column.sql @@ -0,0 +1,49 @@ +-- Tags: use-rocksdb +-- Direct join published a storage data column under a right-side virtual column's name. + +DROP TABLE IF EXISTS t_04816_rocks; +DROP TABLE IF EXISTS t_04816_src; +DROP DICTIONARY IF EXISTS d_04816; + +CREATE TABLE t_04816_rocks (k LowCardinality(String), v String) ENGINE = EmbeddedRocksDB PRIMARY KEY k; +INSERT INTO t_04816_rocks VALUES ('KEYA', 'VALA'); + +CREATE TABLE t_04816_src (k Nullable(UInt64), v String) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_04816_src VALUES (1, 'one'); +CREATE DICTIONARY d_04816 (k Nullable(UInt64), v String) PRIMARY KEY k +SOURCE(CLICKHOUSE(TABLE 't_04816_src' DB currentDatabase())) LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(0); + +SELECT dd.v, dd._table FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k; +SELECT dd.v, dd._table FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k SETTINGS join_algorithm = 'hash'; + +SELECT dd.v, dd._table FROM (SELECT CAST(1, 'Nullable(UInt64)') AS pref) AS t +LEFT JOIN d_04816 AS dd ON t.pref = dd.k; +SELECT dd.v, dd._table FROM (SELECT CAST(1, 'Nullable(UInt64)') AS pref) AS t +LEFT JOIN d_04816 AS dd ON t.pref = dd.k SETTINGS join_algorithm = 'hash'; + +SELECT dd.v, dd._database = currentDatabase() FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k; +SELECT dd.v, dd._database = currentDatabase() FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k SETTINGS join_algorithm = 'hash'; + +SELECT dd.v, dd._table FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +INNER JOIN t_04816_rocks AS dd ON t.pref = dd.k; +SELECT dd.v, dd._table FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +INNER JOIN t_04816_rocks AS dd ON t.pref = dd.k SETTINGS join_algorithm = 'hash'; + +SELECT dd.v FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k; + +SELECT dd.v, dd._table FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k SETTINGS join_algorithm = 'direct'; -- { serverError NOT_IMPLEMENTED } + +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT dd.v FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k) WHERE explain ILIKE '%Algorithm: DirectKeyValueJoin%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT dd.v, dd._table FROM (SELECT CAST('KEYA', 'LowCardinality(String)') AS pref) AS t +LEFT JOIN t_04816_rocks AS dd ON t.pref = dd.k) WHERE explain ILIKE '%Algorithm: DirectKeyValueJoin%'; + +DROP DICTIONARY d_04816; +DROP TABLE t_04816_src; +DROP TABLE t_04816_rocks; diff --git a/tests/queries/0_stateless/04836_parenthesized_definitions_attach_partition_from.reference b/tests/queries/0_stateless/04836_parenthesized_definitions_attach_partition_from.reference new file mode 100644 index 000000000000..689995b089d5 --- /dev/null +++ b/tests/queries/0_stateless/04836_parenthesized_definitions_attach_partition_from.reference @@ -0,0 +1,5 @@ +1 2 3 +1 2 +0 +CREATE TABLE default.t_nested_parens_src\n(\n `x` UInt64,\n `y` UInt64\n)\nENGINE = MergeTree\nPARTITION BY (x + (1))\nORDER BY y\nSETTINGS index_granularity = 8192 +CREATE TABLE default.t_parens_ttl\n(\n `x` UInt64,\n `d` Date\n)\nENGINE = MergeTree\nORDER BY x\nTTL (d + toIntervalYear(10))\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/04836_parenthesized_definitions_attach_partition_from.sql b/tests/queries/0_stateless/04836_parenthesized_definitions_attach_partition_from.sql new file mode 100644 index 000000000000..d3289ddfc4f9 --- /dev/null +++ b/tests/queries/0_stateless/04836_parenthesized_definitions_attach_partition_from.sql @@ -0,0 +1,57 @@ +-- Tags: no-random-merge-tree-settings +-- Tag no-random-merge-tree-settings: the test shows the definition of a table, and the randomized +-- settings would be printed with it. + +-- Whether the user wrote redundant parentheses around a definition expression is not a property of +-- the table: `PARTITION BY (a)` and `PARTITION BY a` are the same key. `ATTACH PARTITION FROM` +-- compares the definitions of the two tables as text, and that text must not see the parentheses. + +DROP TABLE IF EXISTS t_parens_src; +DROP TABLE IF EXISTS t_parens_dst; + +CREATE TABLE t_parens_src (x UInt64, y UInt64, z UInt64, + INDEX ix (y * z) TYPE minmax, + PROJECTION p (SELECT (y) ORDER BY z)) +ENGINE = MergeTree PARTITION BY (x) PRIMARY KEY (y) ORDER BY (y, z); + +CREATE TABLE t_parens_dst (x UInt64, y UInt64, z UInt64, + INDEX ix y * z TYPE minmax, + PROJECTION p (SELECT y ORDER BY z)) +ENGINE = MergeTree PARTITION BY x PRIMARY KEY y ORDER BY (y, z); + +INSERT INTO t_parens_src VALUES (1, 2, 3); +ALTER TABLE t_parens_dst ATTACH PARTITION 1 FROM t_parens_src; +SELECT * FROM t_parens_dst; + +-- The parentheses do not have to be at the top level of the expression. +DROP TABLE IF EXISTS t_nested_parens_src; +DROP TABLE IF EXISTS t_nested_parens_dst; + +CREATE TABLE t_nested_parens_src (x UInt64, y UInt64) ENGINE = MergeTree PARTITION BY (x + (1)) ORDER BY y; +CREATE TABLE t_nested_parens_dst (x UInt64, y UInt64) ENGINE = MergeTree PARTITION BY x + 1 ORDER BY y; + +INSERT INTO t_nested_parens_src VALUES (1, 2); +ALTER TABLE t_nested_parens_dst ATTACH PARTITION 2 FROM t_nested_parens_src; +SELECT * FROM t_nested_parens_dst; + +-- Definitions that differ in more than the parentheses are still rejected. +DROP TABLE IF EXISTS t_other_key; +CREATE TABLE t_other_key (x UInt64, y UInt64) ENGINE = MergeTree PARTITION BY x + 2 ORDER BY y; +ALTER TABLE t_other_key ATTACH PARTITION 2 FROM t_nested_parens_src; -- { serverError BAD_ARGUMENTS } + +DROP TABLE IF EXISTS t_other_index; +CREATE TABLE t_other_index (x UInt64, y UInt64, z UInt64, + INDEX ix y + z TYPE minmax, + PROJECTION p (SELECT y ORDER BY z)) +ENGINE = MergeTree PARTITION BY x PRIMARY KEY y ORDER BY (y, z); +ALTER TABLE t_other_index ATTACH PARTITION 1 FROM t_parens_src; -- { serverError BAD_ARGUMENTS } + +-- Restating a `TTL` with parentheses added is not a change, so it schedules no mutation. +DROP TABLE IF EXISTS t_parens_ttl; +CREATE TABLE t_parens_ttl (x UInt64, d Date) ENGINE = MergeTree ORDER BY x TTL d + INTERVAL 10 YEAR; +ALTER TABLE t_parens_ttl MODIFY TTL (d + INTERVAL 10 YEAR); +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_parens_ttl'; + +-- The stored definition still shows the parentheses exactly as they were written. +SHOW CREATE TABLE t_nested_parens_src; +SHOW CREATE TABLE t_parens_ttl; diff --git a/tests/queries/0_stateless/04836_statistics_in_set_selectivity_estimation.reference b/tests/queries/0_stateless/04836_statistics_in_set_selectivity_estimation.reference new file mode 100644 index 000000000000..d13ae7a7840c --- /dev/null +++ b/tests/queries/0_stateless/04836_statistics_in_set_selectivity_estimation.reference @@ -0,0 +1,13 @@ +--- statistics are materialized (merged part, level >= 1) --- +1 +--- size-based estimate used only above the limit --- +1 1 +--- the estimate never changes the result --- +19994 1 1 +--- same PREWHERE as the exact ranges --- +1 +1 +--- expression on the left: same PREWHERE above and below the limit --- +1 +--- an unbuilt set is skipped rather than filled --- +1 diff --git a/tests/queries/0_stateless/04836_statistics_in_set_selectivity_estimation.sh b/tests/queries/0_stateless/04836_statistics_in_set_selectivity_estimation.sh new file mode 100755 index 000000000000..efc9281ca37e --- /dev/null +++ b/tests/queries/0_stateless/04836_statistics_in_set_selectivity_estimation.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `ConditionSelectivityEstimator` must bound what it spends on `col IN (set)`: it must not run a +# subquery to fill a set, and above `statistics_max_set_size_for_exact_selectivity_estimation` it +# must estimate from the size and bounds of the set instead of deriving every range. + +RUN="${CLICKHOUSE_DATABASE}_$$_${RANDOM}" + +# Statistics are only materialized by a merge, so two parts and an OPTIMIZE are required - with a +# single level-0 part the estimator has nothing to load and none of this code runs. +$CLICKHOUSE_CLIENT -m --query " +DROP TABLE IF EXISTS probe_tbl; +DROP TABLE IF EXISTS set_tbl; +CREATE TABLE probe_tbl (k1 UInt64, k2 UInt64, payload String) ENGINE = MergeTree ORDER BY k1 +SETTINGS auto_statistics_types = 'basic, uniq'; +CREATE TABLE set_tbl (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO probe_tbl SELECT number, number, repeat('x', 20) FROM numbers(20000); +INSERT INTO probe_tbl SELECT number + 20000, number, repeat('x', 20) FROM numbers(20000); +INSERT INTO set_tbl SELECT number FROM numbers(20000); +OPTIMIZE TABLE probe_tbl FINAL; +" + +echo '--- statistics are materialized (merged part, level >= 1) ---' +$CLICKHOUSE_CLIENT --query " +SELECT max(level) >= 1 FROM system.parts WHERE database = currentDatabase() AND table = 'probe_tbl' AND active; +" + +QUERY="SELECT count() FROM probe_tbl WHERE k1 IN (SELECT id FROM set_tbl) AND k2 > 5" + +# The 20000-element set is above the limit in the first run and below it in the second, so the same +# query takes the size-based path and then the exact-range path. +$CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --statistics_max_set_size_for_exact_selectivity_estimation=10000 \ + --log_comment="capped_${RUN}" --query "$QUERY FORMAT Null" +$CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --statistics_max_set_size_for_exact_selectivity_estimation=0 \ + --log_comment="exact_${RUN}" --query "$QUERY FORMAT Null" + +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log" + +# Above the limit the size-based estimate is used; with the limit disabled it never is. +echo '--- size-based estimate used only above the limit ---' +$CLICKHOUSE_CLIENT -m --query " +SELECT + sum(ProfileEvents['SelectivityEstimatorInSetEstimatedFromSize']) FILTER (WHERE log_comment = 'capped_${RUN}') > 0 AS capped_uses_size, + sum(ProfileEvents['SelectivityEstimatorInSetEstimatedFromSize']) FILTER (WHERE log_comment = 'exact_${RUN}') = 0 AS exact_never_uses_size +FROM system.query_log +WHERE type = 'QueryFinish' AND query_kind = 'Select' AND current_database = currentDatabase() + AND log_comment IN ('capped_${RUN}', 'exact_${RUN}'); +" + +# Whichever path the estimator takes, it only ranks PREWHERE candidates - the answer cannot change. +echo '--- the estimate never changes the result ---' +$CLICKHOUSE_CLIENT -m --query " +SELECT + (SELECT count() FROM probe_tbl WHERE k1 IN (SELECT id FROM set_tbl) AND k2 > 5 + SETTINGS use_statistics = 1, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, allow_reorder_prewhere_conditions = 1, statistics_max_set_size_for_exact_selectivity_estimation = 10000) AS capped, + capped = (SELECT count() FROM probe_tbl WHERE k1 IN (SELECT id FROM set_tbl) AND k2 > 5 + SETTINGS use_statistics = 1, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, allow_reorder_prewhere_conditions = 1, statistics_max_set_size_for_exact_selectivity_estimation = 0) AS same_as_exact, + capped = (SELECT count() FROM probe_tbl WHERE k1 IN (SELECT id FROM set_tbl) AND k2 > 5 + SETTINGS use_statistics = 0, optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, allow_reorder_prewhere_conditions = 1) AS same_as_no_statistics; +" + +# The size-based estimate must agree with the exact ranges well enough to pick the same PREWHERE. +echo '--- same PREWHERE as the exact ranges ---' +capped_pw=$($CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --statistics_max_set_size_for_exact_selectivity_estimation=10000 \ + --query "EXPLAIN actions=1 $QUERY" | grep -F 'Prewhere filter column:') +exact_pw=$($CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --statistics_max_set_size_for_exact_selectivity_estimation=0 \ + --query "EXPLAIN actions=1 $QUERY" | grep -F 'Prewhere filter column:') +[ -n "$capped_pw" ] && echo 1 || echo "no prewhere chosen" +[ "$capped_pw" = "$exact_pw" ] && echo 1 || { echo "PREWHERE differs"; echo "$capped_pw"; echo "$exact_pw"; } + +# An expression on the left has no statistics of its own, and below the limit it is given a flat +# default. The size-based path must not turn that into "matches everything" just because the set is +# large, or crossing an internal cost limit would silently change the plan. A literal list is used +# because a subquery set is never built for a non-indexed expression, so it would not reach the path. +echo '--- expression on the left: same PREWHERE above and below the limit ---' +LITERALS=$($CLICKHOUSE_CLIENT --query "SELECT arrayStringConcat(groupArray(toString(number)), ',') FROM numbers(500)") +EXPR_QUERY="SELECT count() FROM probe_tbl WHERE bitXor(k1, 42) IN ($LITERALS) AND k2 > 5" +expr_capped=$($CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --statistics_max_set_size_for_exact_selectivity_estimation=100 \ + --query "EXPLAIN actions=1 $EXPR_QUERY" | grep -F 'Prewhere filter column:') +expr_exact=$($CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --statistics_max_set_size_for_exact_selectivity_estimation=0 \ + --query "EXPLAIN actions=1 $EXPR_QUERY" | grep -F 'Prewhere filter column:') +[ "$expr_capped" = "$expr_exact" ] && echo 1 || { echo "PREWHERE differs"; echo "$expr_capped"; echo "$expr_exact"; } + +# A set nobody has built cannot be analysed, because filling it would mean running the subquery +# during planning. `k1` is not the sort key here, so no index analysis builds the set first. +echo '--- an unbuilt set is skipped rather than filled ---' +$CLICKHOUSE_CLIENT -m --query " +DROP TABLE IF EXISTS probe_unindexed; +CREATE TABLE probe_unindexed (k1 UInt64, k2 UInt64, payload String) ENGINE = MergeTree ORDER BY tuple() +SETTINGS auto_statistics_types = 'basic, uniq'; +INSERT INTO probe_unindexed SELECT number, number, repeat('x', 20) FROM numbers(20000); +INSERT INTO probe_unindexed SELECT number + 20000, number, repeat('x', 20) FROM numbers(20000); +OPTIMIZE TABLE probe_unindexed FINAL; +" +$CLICKHOUSE_CLIENT --use_statistics=1 --optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1 --allow_reorder_prewhere_conditions=1 --log_comment="unbuilt_${RUN}" \ + --query "SELECT count() FROM probe_unindexed WHERE k1 IN (SELECT id FROM set_tbl) AND k2 > 5 FORMAT Null" +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT --query " +SELECT sum(ProfileEvents['SelectivityEstimatorInSetNotBuilt']) > 0 +FROM system.query_log +WHERE type = 'QueryFinish' AND query_kind = 'Select' AND current_database = currentDatabase() + AND log_comment = 'unbuilt_${RUN}'; +" + +$CLICKHOUSE_CLIENT -m --query " +DROP TABLE probe_tbl; +DROP TABLE probe_unindexed; +DROP TABLE set_tbl; +" diff --git a/tests/queries/0_stateless/04837_parenthesized_definitions_replicated_metadata.reference b/tests/queries/0_stateless/04837_parenthesized_definitions_replicated_metadata.reference new file mode 100644 index 000000000000..b1be77ba94a6 --- /dev/null +++ b/tests/queries/0_stateless/04837_parenthesized_definitions_replicated_metadata.reference @@ -0,0 +1,3 @@ +joined +joined +CREATE TABLE default.t_parens_zk2_r1\n(\n `x` UInt64,\n `d` Date,\n INDEX ix (x) TYPE minmax GRANULARITY 1\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/t_parens_zk2\', \'r1\')\nORDER BY (d)\nTTL (d + toIntervalYear(10))\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/04837_parenthesized_definitions_replicated_metadata.sql b/tests/queries/0_stateless/04837_parenthesized_definitions_replicated_metadata.sql new file mode 100644 index 000000000000..327cbd664455 --- /dev/null +++ b/tests/queries/0_stateless/04837_parenthesized_definitions_replicated_metadata.sql @@ -0,0 +1,51 @@ +-- Tags: zookeeper, no-shared-merge-tree, no-replicated-database, no-random-merge-tree-settings +-- Tag no-shared-merge-tree, no-replicated-database: the test joins two explicit replicas of one +-- ReplicatedMergeTree table to compare what each of them has written to ZooKeeper. +-- Tag no-random-merge-tree-settings: the test shows the definition of a table, and the randomized +-- settings would be printed with it. + +-- A replica compares its own definitions with the ones stored in ZooKeeper, which may have been +-- written by a server that did not remember the redundant parentheses the user wrote (they became +-- a part of the AST only in 26.5). Neither the comparison nor the stored form may depend on them. + +DROP TABLE IF EXISTS t_parens_zk_r1 SYNC; +DROP TABLE IF EXISTS t_parens_zk_r2 SYNC; + +CREATE TABLE t_parens_zk_r1 (x UInt64, d Date, y UInt64 DEFAULT x + 1, + INDEX ix x * y TYPE minmax, + PROJECTION p (SELECT x ORDER BY d), + CONSTRAINT c CHECK x > 0) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_parens_zk', 'r1') +PARTITION BY x + 1 ORDER BY (x, d) SAMPLE BY x TTL d + INTERVAL 10 YEAR; + +-- The same table, written with redundant parentheses in every definition. +CREATE TABLE t_parens_zk_r2 (x UInt64, d Date, y UInt64 DEFAULT (x + 1), + INDEX ix (x * y) TYPE minmax, + PROJECTION p (SELECT (x) ORDER BY (d)), + CONSTRAINT c CHECK (x > 0)) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_parens_zk', 'r2') +PARTITION BY (x + (1)) ORDER BY ((x), (d)) SAMPLE BY (x) TTL (d + INTERVAL 10 YEAR); + +SELECT 'joined'; + +-- The other direction: the parenthesized definitions are the ones already in ZooKeeper. +DROP TABLE IF EXISTS t_parens_zk2_r1 SYNC; +DROP TABLE IF EXISTS t_parens_zk2_r2 SYNC; + +CREATE TABLE t_parens_zk2_r1 (x UInt64, d Date, INDEX ix (x) TYPE minmax) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_parens_zk2', 'r1') +ORDER BY (d) TTL (d + INTERVAL 10 YEAR); + +CREATE TABLE t_parens_zk2_r2 (x UInt64, d Date, INDEX ix x TYPE minmax) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_parens_zk2', 'r2') +ORDER BY d TTL d + INTERVAL 10 YEAR; + +SELECT 'joined'; + +-- The stored definitions still show the parentheses exactly as they were written. +SHOW CREATE TABLE t_parens_zk2_r1; + +-- A replica whose definition differs in more than the parentheses is still rejected. +CREATE TABLE t_parens_zk2_r3 (x UInt64, d Date, INDEX ix (x + 1) TYPE minmax) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_parens_zk2', 'r3') +ORDER BY d TTL d + INTERVAL 10 YEAR; -- { serverError METADATA_MISMATCH } diff --git a/tests/queries/0_stateless/04838_top_k_dynamic_filter_empty_tuple.reference b/tests/queries/0_stateless/04838_top_k_dynamic_filter_empty_tuple.reference new file mode 100644 index 000000000000..2075358d1c1b --- /dev/null +++ b/tests/queries/0_stateless/04838_top_k_dynamic_filter_empty_tuple.reference @@ -0,0 +1,42 @@ +direct_has_filter 0 +normal_has_filter 1 +nested_has_filter 1 +nullable_nested_has_filter 1 +mixed_has_filter 1 +array_nested_has_filter 1 +direct_results +() +() +() +() +() +normal_results +(99999) +(99998) +(99997) +(99996) +(99995) +nested_results +(()) +(()) +(()) +(()) +(()) +nullable_nested_results +(NULL) +(NULL) +(NULL) +(NULL) +(()) +mixed_results +(99999,()) +(99998,()) +(99997,()) +(99996,()) +(99995,()) +array_nested_results +[(),()] +[(),()] +[(),()] +[(),()] +[()] diff --git a/tests/queries/0_stateless/04838_top_k_dynamic_filter_empty_tuple.sql b/tests/queries/0_stateless/04838_top_k_dynamic_filter_empty_tuple.sql new file mode 100644 index 000000000000..3aeedc96367a --- /dev/null +++ b/tests/queries/0_stateless/04838_top_k_dynamic_filter_empty_tuple.sql @@ -0,0 +1,102 @@ +SET enable_nullable_tuple_type = 1; +SET max_threads = 1; +SET max_block_size = 8192; +SET query_plan_max_limit_for_top_k_optimization = 100; +SET use_skip_indexes_for_top_k = 1; +SET use_top_k_dynamic_filtering = 1; +SET use_top_k_dynamic_filtering_for_variable_length_types = 1; + +DROP TABLE IF EXISTS top_k_empty_tuple; + +CREATE TABLE top_k_empty_tuple +( + direct Tuple(), + normal Tuple(UInt64), + nested Tuple(Tuple()), + nullable_nested Tuple(Nullable(Tuple())), + mixed Tuple(UInt64, Tuple()), + array_nested Array(Tuple()), + payload UInt64 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 64; + +INSERT INTO top_k_empty_tuple +SELECT + tuple(), + tuple(number), + tuple(tuple()), + tuple(CAST(if(number >= 99996, NULL, tuple()), 'Nullable(Tuple())')), + tuple(number, tuple()), + if(number >= 99996, [tuple(), tuple()], if(number = 99995, [tuple()], [])), + number +FROM numbers(100000); + +SELECT 'direct_has_filter', count() > 0 +FROM +( + EXPLAIN actions = 1 + SELECT direct FROM top_k_empty_tuple ORDER BY direct DESC LIMIT 5 +) +WHERE explain LIKE '%__topKFilter%'; + +SELECT 'normal_has_filter', count() > 0 +FROM +( + EXPLAIN actions = 1 + SELECT normal FROM top_k_empty_tuple ORDER BY normal DESC LIMIT 5 +) +WHERE explain LIKE '%__topKFilter%'; + +SELECT 'nested_has_filter', count() > 0 +FROM +( + EXPLAIN actions = 1 + SELECT nested FROM top_k_empty_tuple ORDER BY nested DESC LIMIT 5 +) +WHERE explain LIKE '%__topKFilter%'; + +SELECT 'nullable_nested_has_filter', count() > 0 +FROM +( + EXPLAIN actions = 1 + SELECT nullable_nested FROM top_k_empty_tuple ORDER BY nullable_nested DESC NULLS FIRST LIMIT 5 +) +WHERE explain LIKE '%__topKFilter%'; + +SELECT 'mixed_has_filter', count() > 0 +FROM +( + EXPLAIN actions = 1 + SELECT mixed FROM top_k_empty_tuple ORDER BY mixed DESC LIMIT 5 +) +WHERE explain LIKE '%__topKFilter%'; + +SELECT 'array_nested_has_filter', count() > 0 +FROM +( + EXPLAIN actions = 1 + SELECT array_nested FROM top_k_empty_tuple ORDER BY array_nested DESC LIMIT 5 +) +WHERE explain LIKE '%__topKFilter%'; + +SELECT 'direct_results'; +SELECT direct FROM top_k_empty_tuple ORDER BY ALL DESC LIMIT 5; + +SELECT 'normal_results'; +SELECT normal FROM top_k_empty_tuple ORDER BY ALL DESC LIMIT 5; + +SELECT 'nested_results'; +SELECT nested FROM top_k_empty_tuple ORDER BY ALL DESC LIMIT 5; + +SELECT 'nullable_nested_results'; +SELECT nullable_nested FROM top_k_empty_tuple ORDER BY nullable_nested DESC NULLS FIRST LIMIT 5; + +SELECT 'mixed_results'; +SELECT mixed FROM top_k_empty_tuple ORDER BY ALL DESC LIMIT 5; + +SELECT 'array_nested_results'; +SELECT array_nested FROM top_k_empty_tuple ORDER BY ALL DESC LIMIT 5; + +DROP TABLE top_k_empty_tuple; diff --git a/tests/queries/0_stateless/04839_variant_escape_filename_compact_map_buckets.reference b/tests/queries/0_stateless/04839_variant_escape_filename_compact_map_buckets.reference new file mode 100644 index 000000000000..3ce76dc7617d --- /dev/null +++ b/tests/queries/0_stateless/04839_variant_escape_filename_compact_map_buckets.reference @@ -0,0 +1,6 @@ +{'a':1,'b':2,'c':3,'d':4,'e':5} +['a','b','c','d','e'] +[1,2,3,4,5] +{'a':1,'b':2,'c':3,'d':4,'e':5} +['a','b','c','d','e'] +[1,2,3,4,5] diff --git a/tests/queries/0_stateless/04839_variant_escape_filename_compact_map_buckets.sql b/tests/queries/0_stateless/04839_variant_escape_filename_compact_map_buckets.sql new file mode 100644 index 000000000000..cb9c6da90fdb --- /dev/null +++ b/tests/queries/0_stateless/04839_variant_escape_filename_compact_map_buckets.sql @@ -0,0 +1,34 @@ +-- Compact-part regression for the bidirectional escape_variant_subcolumn_filenames fallback. +-- A bucketed Map inside a Variant has an optional MapBucketIndexes substream that preserves the +-- original key order. In compact parts its existence is probed against columns_substreams.txt, whose +-- names use the write-time setting; after the setting is flipped the probe used to miss it and take +-- the "no bucket index" path, reordering the map elements. with_buckets serialization and a constant +-- multi-bucket layout are forced so the substream is actually produced. + +set enable_variant_type=1; + +-- Case 1: written with escaping disabled, then enabled. +drop table if exists test_escape_compact; +create table test_escape_compact (v Variant(Map(String, UInt32))) engine=MergeTree order by tuple() + settings escape_variant_subcolumn_filenames=0, replace_long_file_name_to_hash=0, + map_serialization_version='with_buckets', map_serialization_version_for_zero_level_parts='with_buckets', + map_buckets_strategy='constant', max_buckets_in_map=4, map_buckets_min_avg_size=0; +insert into test_escape_compact select map('a', 1, 'b', 2, 'c', 3, 'd', 4, 'e', 5)::Map(String, UInt32); +alter table test_escape_compact modify setting escape_variant_subcolumn_filenames=1; +select v from test_escape_compact; +select v.`Map(String, UInt32)`.keys from test_escape_compact; +select v.`Map(String, UInt32)`.values from test_escape_compact; +drop table test_escape_compact; + +-- Case 2: written with escaping enabled, then disabled. +drop table if exists test_escape_compact; +create table test_escape_compact (v Variant(Map(String, UInt32))) engine=MergeTree order by tuple() + settings escape_variant_subcolumn_filenames=1, replace_long_file_name_to_hash=0, + map_serialization_version='with_buckets', map_serialization_version_for_zero_level_parts='with_buckets', + map_buckets_strategy='constant', max_buckets_in_map=4, map_buckets_min_avg_size=0; +insert into test_escape_compact select map('a', 1, 'b', 2, 'c', 3, 'd', 4, 'e', 5)::Map(String, UInt32); +alter table test_escape_compact modify setting escape_variant_subcolumn_filenames=0; +select v from test_escape_compact; +select v.`Map(String, UInt32)`.keys from test_escape_compact; +select v.`Map(String, UInt32)`.values from test_escape_compact; +drop table test_escape_compact; diff --git a/tests/queries/0_stateless/04840_trivial_group_by_limit_projection_guards.reference b/tests/queries/0_stateless/04840_trivial_group_by_limit_projection_guards.reference new file mode 100644 index 000000000000..30a75c4405dc --- /dev/null +++ b/tests/queries/0_stateless/04840_trivial_group_by_limit_projection_guards.reference @@ -0,0 +1,4 @@ +3 +5 100000 100000 +10 +10 50000 50000 diff --git a/tests/queries/0_stateless/04840_trivial_group_by_limit_projection_guards.sql b/tests/queries/0_stateless/04840_trivial_group_by_limit_projection_guards.sql new file mode 100644 index 000000000000..9a49d9230c2e --- /dev/null +++ b/tests/queries/0_stateless/04840_trivial_group_by_limit_projection_guards.sql @@ -0,0 +1,51 @@ +-- The trivial `GROUP BY ... LIMIT` optimization (`optimize_trivial_group_by_limit_query`) +-- must not fire when something between the aggregation and the LIMIT consumes or filters +-- the groups: cutting the aggregation at `LIMIT + OFFSET` keys then changes the result +-- instead of merely picking an unspecified subset of the groups. Each of the cases below +-- returned wrong results (fewer rows or wrong values) before the guards were added, as +-- pinned by the expected outputs. +-- +-- The queries run at the top level of an `INSERT ... SELECT` because the pass fires only +-- on the top-level query. The result of the problematic query is captured into a table +-- because its correct output rows are an unspecified subset of the groups; the assertions +-- are on deterministic aggregates of it. + +SET enable_analyzer = 1; +SET optimize_trivial_group_by_limit_query = 1; +SET max_threads = 16; +SET max_block_size = 100; + +DROP TABLE IF EXISTS t_04840; +CREATE TABLE t_04840 (v UInt64) ENGINE = Memory; + +-- DISTINCT collapses the projected groups: 100000 groups yield 4 distinct values of +-- `intDiv(k, 25000)`, and LIMIT 3 must return 3 of them. With aggregation cut at 3 keys +-- the distinct set shrank to 1 value. +INSERT INTO t_04840 SELECT DISTINCT intDiv(k, 25000) FROM (SELECT number AS k FROM numbers_mt(100000)) GROUP BY k LIMIT 3; +SELECT count() FROM t_04840; +TRUNCATE TABLE t_04840; + +-- A window function in the projection is evaluated over all groups: `count() OVER ()` +-- must see all 100000 of them. With aggregation cut near the LIMIT it saw only the kept +-- groups (values like 100). +INSERT INTO t_04840 SELECT count() OVER () FROM (SELECT number AS k FROM numbers_mt(100000)) GROUP BY k LIMIT 5; +SELECT count(), min(v), max(v) FROM t_04840; +TRUNCATE TABLE t_04840; + +-- `arrayJoin` in the projection can drop rows (empty arrays): only 100 of the 100000 +-- groups produce a row, so LIMIT 10 must still find 10 rows. With aggregation cut near +-- the LIMIT almost all kept groups produced nothing (1 row came out). +INSERT INTO t_04840 SELECT arrayJoin(if(k % 1000 = 0, [k], [])) FROM (SELECT number AS k FROM numbers_mt(100000)) GROUP BY k LIMIT 10; +SELECT count() FROM t_04840; +TRUNCATE TABLE t_04840; + +-- QUALIFY filters the groups after the aggregation: 1000 groups pass the filter, so +-- LIMIT 10 must find 10 rows, and the window function must see all 50000 groups of its +-- partition. With aggregation cut near the LIMIT none of the kept groups qualified +-- (0 rows came out). A QUALIFY on plain group keys alone is pushed down below the +-- aggregation and does not need the guard; the window function keeps it in place. +INSERT INTO t_04840 SELECT count() OVER (PARTITION BY k % 2) FROM (SELECT number AS k FROM numbers_mt(100000)) GROUP BY k QUALIFY k >= 99000 LIMIT 10; +SELECT count(), min(v), max(v) FROM t_04840; +TRUNCATE TABLE t_04840; + +DROP TABLE t_04840; diff --git a/tests/queries/0_stateless/04850_parenthesized_definitions_replica_recovery.reference b/tests/queries/0_stateless/04850_parenthesized_definitions_replica_recovery.reference new file mode 100644 index 000000000000..14df008570ec --- /dev/null +++ b/tests/queries/0_stateless/04850_parenthesized_definitions_replica_recovery.reference @@ -0,0 +1,2 @@ +1 1 +1 2 diff --git a/tests/queries/0_stateless/04850_parenthesized_definitions_replica_recovery.sh b/tests/queries/0_stateless/04850_parenthesized_definitions_replica_recovery.sh new file mode 100755 index 000000000000..06e0f5ecb2a8 --- /dev/null +++ b/tests/queries/0_stateless/04850_parenthesized_definitions_replica_recovery.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-shared-merge-tree, no-replicated-database, no-ordinary-database +# Tag no-shared-merge-tree: the test hand-crafts the ZooKeeper nodes of a ReplicatedMergeTree replica. +# Tag no-replicated-database: the test creates an explicit second replica of one table. +# Tag no-ordinary-database: the test creates a table with an explicit UUID. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A replica that failed after creating its ZooKeeper nodes but before saving the local metadata is +# recognized and reused when the table is created again. The nodes may have been written by a server +# that kept the redundant parentheses the user wrote (26.5..26.7), so `createReplicaAttempt` must +# compare its `metadata` and `columns` structurally, not as raw strings. + +ZK_PATH="/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_parens_recovery" +UUID=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_parens_recovery_r1 SYNC" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_parens_recovery_r1 (x UInt64, y UInt64 DEFAULT x + 1) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_parens_recovery', 'r1') ORDER BY (x)" + +# Simulate the leftover nodes of a replica r2 that a 26.5..26.7 server created in ZooKeeper before +# failing to save the local metadata: the same definitions, spelled with the redundant parentheses. +# First prove that the parenthesized spelling actually differs from the stored one. +$CLICKHOUSE_CLIENT -q " + WITH (SELECT value FROM system.zookeeper WHERE path = '$ZK_PATH' AND name = 'metadata') AS m, + (SELECT value FROM system.zookeeper WHERE path = '$ZK_PATH' AND name = 'columns') AS c + SELECT replaceOne(m, 'primary key: x\n', 'primary key: (x)\n') != m, + replaceOne(c, 'DEFAULT\tx + 1', 'DEFAULT\t(x + 1)') != c" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO system.zookeeper (path, name, value) + WITH (SELECT value FROM system.zookeeper WHERE path = '$ZK_PATH' AND name = 'metadata') AS m, + (SELECT value FROM system.zookeeper WHERE path = '$ZK_PATH' AND name = 'columns') AS c + SELECT '$ZK_PATH/replicas/r2', name, value + FROM values('name String, value String', + ('host', ''), + ('log_pointer', ''), + ('queue', ''), + ('parts', ''), + ('flags', ''), + ('is_lost', '1'), + ('metadata_version', '0'), + ('min_unprocessed_insert_time', ''), + ('max_processed_insert_time', ''), + ('mutation_pointer', '')) + UNION ALL + SELECT '$ZK_PATH/replicas/r2', 'metadata', replaceOne(m, 'primary key: x\n', 'primary key: (x)\n') + UNION ALL + SELECT '$ZK_PATH/replicas/r2', 'columns', replaceOne(c, 'DEFAULT\tx + 1', 'DEFAULT\t(x + 1)') + UNION ALL + SELECT '$ZK_PATH/replicas/r2', 'creator_info', concat('$UUID', '|', toString(serverUUID()))" + +# Retrying the creation of r2 with the same definitions (written with the redundant parentheses, +# as the failed server would have them) reuses the existing empty replica instead of throwing +# REPLICA_ALREADY_EXISTS. +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_parens_recovery_r2 UUID '$UUID' (x UInt64, y UInt64 DEFAULT (x + 1)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_parens_recovery', 'r2') ORDER BY ((x))" + +$CLICKHOUSE_CLIENT -q "INSERT INTO t_parens_recovery_r1 (x) VALUES (1)" +$CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA t_parens_recovery_r2" +$CLICKHOUSE_CLIENT -q "SELECT x, y FROM t_parens_recovery_r2" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_parens_recovery_r2 SYNC" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_parens_recovery_r1 SYNC" diff --git a/tests/queries/0_stateless/04881_low_cardinality_default_value_needle.reference b/tests/queries/0_stateless/04881_low_cardinality_default_value_needle.reference new file mode 100644 index 000000000000..215a8ab9dcc3 --- /dev/null +++ b/tests/queries/0_stateless/04881_low_cardinality_default_value_needle.reference @@ -0,0 +1,87 @@ +-- { echo } +SET allow_suspicious_low_cardinality_types = 1; +-- A needle equal to the element type's default value must be found. Every row prints the +-- LowCardinality answer beside the same query over a plain array. +SELECT has(CAST(['', 'a'], 'Array(LowCardinality(String))'), '') AS lc, has(CAST(['', 'a'], 'Array(String)'), '') AS oracle; +1 1 +SELECT indexOf(materialize(CAST(['a', ''], 'Array(LowCardinality(String))')), '') AS lc, indexOf(materialize(CAST(['a', ''], 'Array(String)')), '') AS oracle; +2 2 +SELECT countEqual(materialize(CAST(['', 'a', ''], 'Array(LowCardinality(String))')), '') AS lc, countEqual(materialize(CAST(['', 'a', ''], 'Array(String)')), '') AS oracle; +2 2 +SELECT has(materialize(CAST([0, 5], 'Array(LowCardinality(UInt8))')), 0) AS lc, has(materialize(CAST([0, 5], 'Array(UInt8)')), 0) AS oracle; +1 1 +SELECT has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(LowCardinality(FixedString(3)))')), CAST('', 'FixedString(3)')) AS lc, has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(FixedString(3))')), CAST('', 'FixedString(3)')) AS oracle; +1 1 +-- An Enum needle compares to a FixedString element as a string, where the element type's own padding +-- is not a difference. +SELECT has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(LowCardinality(FixedString(3)))')), CAST('', 'Enum8('''' = 0, ''o'' = 1)')) AS lc, has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(FixedString(3))')), CAST('', 'Enum8('''' = 0, ''o'' = 1)')) AS oracle; +1 1 +-- The Map key and value dictionaries are the second call site. +SELECT mapContainsKey(materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(LowCardinality(String), String)')), '') AS lc, mapContainsKey(materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(String, String)')), '') AS oracle; +1 1 +SELECT materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(LowCardinality(String), String)'))[''] AS lc, materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(String, String)'))[''] AS oracle; +v_empty v_empty +SELECT mapContainsValue(materialize(CAST(map('k', '', 'j', 'v'), 'Map(String, LowCardinality(String))')), '') AS lc, mapContainsValue(materialize(CAST(map('k', '', 'j', 'v'), 'Map(String, String)')), '') AS oracle; +1 1 +-- A constant the element type cannot represent equals no element, while one that survives the cast +-- still finds the default element. The timezone is pinned because the cast to Date drops the +-- needle's time of day and which day that lands on is offset-dependent. +SELECT has(materialize(CAST([0, 5], 'Array(LowCardinality(UInt8))')), 256) AS lc, has(materialize(CAST([0, 5], 'Array(UInt8)')), 256) AS oracle; +0 0 +SELECT mapContainsKey(materialize(CAST(map(0, 'a', 5, 'b'), 'Map(LowCardinality(UInt8), String)')), 256) AS lc, mapContainsKey(materialize(CAST(map(0, 'a', 5, 'b'), 'Map(UInt8, String)')), 256) AS oracle; +0 0 +SELECT has(materialize(CAST([toDate('1970-01-01'), toDate('2020-01-01')], 'Array(LowCardinality(Date))')), toDateTime('1970-01-01 00:00:05')) AS lc, has(materialize(CAST([toDate('1970-01-01'), toDate('2020-01-01')], 'Array(Date)')), toDateTime('1970-01-01 00:00:05')) AS oracle SETTINGS session_timezone = 'UTC'; +0 0 +SELECT has(materialize(CAST([0, 5], 'Array(LowCardinality(UInt8))')), toUInt64(0)) AS widened_needle, has(materialize(CAST([0, 1.5], 'Array(LowCardinality(Float64))')), toUInt8(0)) AS integral_needle; +1 1 +-- An IPv4 element represents a UInt32 needle exactly, and the two have no accurate cast between them, +-- so representability is decided by comparing the needle against its own cast image. +SELECT has(materialize(CAST([toIPv4('0.0.0.0'), toIPv4('1.2.3.4')], 'Array(LowCardinality(IPv4))')), toUInt32(0)) AS lc, has(materialize(CAST([toIPv4('0.0.0.0'), toIPv4('1.2.3.4')], 'Array(IPv4)')), toUInt32(0)) AS oracle; +1 1 +-- A FixedString needle is padded to its own width and equality ignores that padding. +SELECT has(materialize(CAST(['', 'xy'], 'Array(LowCardinality(String))')), CAST('', 'FixedString(4)')) AS lc, length(arrayFilter(x -> x = CAST('', 'FixedString(4)'), materialize(CAST(['', 'xy'], 'Array(String)')))) AS oracle; +1 1 +-- -0.0 and 0.0 are equal but a text format stores them apart, so either zero as a needle must match +-- either spelling, and a count must see both. stored_bits is asserted in the same row: if it ever +-- reads 0 the array no longer holds -0.0 and the arm is void. +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, has(a, toFloat64(0)) AS positive_zero_needle, has(a, reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle, length(arrayFilter(x -> x = toFloat64(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[-0.0,1.5]}'); +[9223372036854775808,4609434218613702656] 1 1 1 +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, indexOf(a, toFloat64(0)) AS positive_zero_needle, indexOf(a, reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle, indexOf(arrayMap(x -> toFloat64(x), a), toFloat64(0)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[1.5,-0.0]}'); +[4609434218613702656,9223372036854775808] 2 2 2 +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, countEqual(a, toFloat64(0)) AS positive_zero_needle, countEqual(a, reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle, length(arrayFilter(x -> x = toFloat64(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[0.0,-0.0,1.5]}'); +[0,9223372036854775808,4609434218613702656] 2 2 2 +SELECT arrayMap(x -> reinterpretAsUInt32(x), a) AS stored_bits, has(a, toFloat32(0)) AS positive_zero_needle, has(a, reinterpretAsFloat32(toUInt32(2147483648))) AS negative_zero_needle, length(arrayFilter(x -> x = toFloat32(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float32))', '{"a":[-0.0,1.5]}'); +[2147483648,1069547520] 1 1 1 +-- A CAST array is folded onto the default slot, so its -0.0 element is stored as +0.0. +SELECT arrayMap(x -> reinterpretAsUInt64(x), materialize(CAST([reinterpretAsFloat64(toUInt64(9223372036854775808)), 1.5], 'Array(LowCardinality(Float64))'))) AS stored_bits, has(materialize(CAST([reinterpretAsFloat64(toUInt64(9223372036854775808)), 1.5], 'Array(LowCardinality(Float64))')), toFloat64(0)) AS positive_zero_needle, has(materialize(CAST([reinterpretAsFloat64(toUInt64(9223372036854775808)), 1.5], 'Array(LowCardinality(Float64))')), reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle; +[0,4609434218613702656] 1 1 +SELECT has(materialize(CAST([0, NULL, 1.5], 'Array(LowCardinality(Nullable(Float64)))')), reinterpretAsFloat64(toUInt64(9223372036854775808))) AS lc, length(arrayFilter(x -> x = reinterpretAsFloat64(toUInt64(9223372036854775808)), materialize(CAST([0, NULL, 1.5], 'Array(LowCardinality(Nullable(Float64)))')))) AS oracle; +1 1 +-- An Enum needle reaches a float element through its underlying number, so it must be declined too. +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, has(a, CAST('z', 'Enum8(\'z\' = 0, \'o\' = 1)')) AS enum8_needle, indexOf(a, CAST('z', 'Enum8(\'z\' = 0, \'o\' = 1)')) AS enum8_index, countEqual(a, CAST('z', 'Enum16(\'z\' = 0, \'o\' = 1)')) AS enum16_count, length(arrayFilter(x -> x = toFloat64(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[-0.0,1.5]}'); +[9223372036854775808,4609434218613702656] 1 1 1 1 +-- Controls that must not move. +SELECT has(materialize(CAST(['', 'a'], 'Array(LowCardinality(String))')), 'zzz') AS absent_needle, has(materialize(CAST(['a', 'b'], 'Array(LowCardinality(String))')), '') AS default_absent; +0 0 +SELECT has(materialize(CAST(['', 'a'], 'Array(LowCardinality(String))')), materialize('')) AS non_const_needle, indexOfAssumeSorted(materialize(CAST(['', 'a'], 'Array(LowCardinality(String))')), '') AS assume_sorted; +1 1 +-- Two answers that only the dictionary shortcut produces, so a build that stopped taking it would +-- move them. Both disagree with the plain-array oracle printed beside them, and both are known +-- defects of the value comparison tracked elsewhere, not of the lookup this test covers: a NaN is +-- one dictionary entry but never equal to itself, and a negative needle is compared to an unsigned +-- element as a raw number. +SELECT has(materialize(CAST([nan, 1.5], 'Array(LowCardinality(Float64))')), nan) AS nan_needle, has(materialize(CAST([nan, 1.5], 'Array(Float64)')), nan) AS oracle; +1 0 +SELECT has(materialize(CAST([0, 255], 'Array(LowCardinality(UInt8))')), toInt8(-1)) AS negative_needle, has(materialize(CAST([0, 255], 'Array(UInt8)')), toInt8(-1)) AS oracle; +1 0 +-- LowCardinality(Nullable(T)): a NULL needle finds the NULL element and a default needle finds the default one. +SELECT indexOf(materialize(CAST(['a', NULL, ''], 'Array(LowCardinality(Nullable(String)))')), NULL) AS null_needle, indexOf(materialize(CAST(['a', NULL, ''], 'Array(LowCardinality(Nullable(String)))')), '') AS default_needle; +2 3 +-- Reached through a real table read rather than a constant-folded literal. +DROP TABLE IF EXISTS t_04881; +CREATE TABLE t_04881 (a Array(LowCardinality(String)), m Map(LowCardinality(String), String)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_04881 VALUES (['', 'a'], map('', 'v_empty', 'k', 'v_k')), (['x', 'y'], map('x', 'v_x', 'y', 'v_y')); +SELECT a, has(a, '') AS has_empty, indexOf(a, '') AS idx_empty, mapContainsKey(m, '') AS key_empty, m[''] AS subscript_empty FROM t_04881 ORDER BY a; +['','a'] 1 1 1 v_empty +['x','y'] 0 0 0 +DROP TABLE t_04881; diff --git a/tests/queries/0_stateless/04881_low_cardinality_default_value_needle.sql b/tests/queries/0_stateless/04881_low_cardinality_default_value_needle.sql new file mode 100644 index 000000000000..dd42b7bea04c --- /dev/null +++ b/tests/queries/0_stateless/04881_low_cardinality_default_value_needle.sql @@ -0,0 +1,69 @@ +-- { echo } +SET allow_suspicious_low_cardinality_types = 1; + +-- A needle equal to the element type's default value must be found. Every row prints the +-- LowCardinality answer beside the same query over a plain array. +SELECT has(CAST(['', 'a'], 'Array(LowCardinality(String))'), '') AS lc, has(CAST(['', 'a'], 'Array(String)'), '') AS oracle; +SELECT indexOf(materialize(CAST(['a', ''], 'Array(LowCardinality(String))')), '') AS lc, indexOf(materialize(CAST(['a', ''], 'Array(String)')), '') AS oracle; +SELECT countEqual(materialize(CAST(['', 'a', ''], 'Array(LowCardinality(String))')), '') AS lc, countEqual(materialize(CAST(['', 'a', ''], 'Array(String)')), '') AS oracle; +SELECT has(materialize(CAST([0, 5], 'Array(LowCardinality(UInt8))')), 0) AS lc, has(materialize(CAST([0, 5], 'Array(UInt8)')), 0) AS oracle; +SELECT has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(LowCardinality(FixedString(3)))')), CAST('', 'FixedString(3)')) AS lc, has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(FixedString(3))')), CAST('', 'FixedString(3)')) AS oracle; +-- An Enum needle compares to a FixedString element as a string, where the element type's own padding +-- is not a difference. +SELECT has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(LowCardinality(FixedString(3)))')), CAST('', 'Enum8('''' = 0, ''o'' = 1)')) AS lc, has(materialize(CAST([CAST('', 'FixedString(3)'), CAST('abc', 'FixedString(3)')], 'Array(FixedString(3))')), CAST('', 'Enum8('''' = 0, ''o'' = 1)')) AS oracle; + +-- The Map key and value dictionaries are the second call site. +SELECT mapContainsKey(materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(LowCardinality(String), String)')), '') AS lc, mapContainsKey(materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(String, String)')), '') AS oracle; +SELECT materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(LowCardinality(String), String)'))[''] AS lc, materialize(CAST(map('', 'v_empty', 'k', 'v_k'), 'Map(String, String)'))[''] AS oracle; +SELECT mapContainsValue(materialize(CAST(map('k', '', 'j', 'v'), 'Map(String, LowCardinality(String))')), '') AS lc, mapContainsValue(materialize(CAST(map('k', '', 'j', 'v'), 'Map(String, String)')), '') AS oracle; + +-- A constant the element type cannot represent equals no element, while one that survives the cast +-- still finds the default element. The timezone is pinned because the cast to Date drops the +-- needle's time of day and which day that lands on is offset-dependent. +SELECT has(materialize(CAST([0, 5], 'Array(LowCardinality(UInt8))')), 256) AS lc, has(materialize(CAST([0, 5], 'Array(UInt8)')), 256) AS oracle; +SELECT mapContainsKey(materialize(CAST(map(0, 'a', 5, 'b'), 'Map(LowCardinality(UInt8), String)')), 256) AS lc, mapContainsKey(materialize(CAST(map(0, 'a', 5, 'b'), 'Map(UInt8, String)')), 256) AS oracle; +SELECT has(materialize(CAST([toDate('1970-01-01'), toDate('2020-01-01')], 'Array(LowCardinality(Date))')), toDateTime('1970-01-01 00:00:05')) AS lc, has(materialize(CAST([toDate('1970-01-01'), toDate('2020-01-01')], 'Array(Date)')), toDateTime('1970-01-01 00:00:05')) AS oracle SETTINGS session_timezone = 'UTC'; +SELECT has(materialize(CAST([0, 5], 'Array(LowCardinality(UInt8))')), toUInt64(0)) AS widened_needle, has(materialize(CAST([0, 1.5], 'Array(LowCardinality(Float64))')), toUInt8(0)) AS integral_needle; +-- An IPv4 element represents a UInt32 needle exactly, and the two have no accurate cast between them, +-- so representability is decided by comparing the needle against its own cast image. +SELECT has(materialize(CAST([toIPv4('0.0.0.0'), toIPv4('1.2.3.4')], 'Array(LowCardinality(IPv4))')), toUInt32(0)) AS lc, has(materialize(CAST([toIPv4('0.0.0.0'), toIPv4('1.2.3.4')], 'Array(IPv4)')), toUInt32(0)) AS oracle; + +-- A FixedString needle is padded to its own width and equality ignores that padding. +SELECT has(materialize(CAST(['', 'xy'], 'Array(LowCardinality(String))')), CAST('', 'FixedString(4)')) AS lc, length(arrayFilter(x -> x = CAST('', 'FixedString(4)'), materialize(CAST(['', 'xy'], 'Array(String)')))) AS oracle; + +-- -0.0 and 0.0 are equal but a text format stores them apart, so either zero as a needle must match +-- either spelling, and a count must see both. stored_bits is asserted in the same row: if it ever +-- reads 0 the array no longer holds -0.0 and the arm is void. +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, has(a, toFloat64(0)) AS positive_zero_needle, has(a, reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle, length(arrayFilter(x -> x = toFloat64(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[-0.0,1.5]}'); +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, indexOf(a, toFloat64(0)) AS positive_zero_needle, indexOf(a, reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle, indexOf(arrayMap(x -> toFloat64(x), a), toFloat64(0)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[1.5,-0.0]}'); +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, countEqual(a, toFloat64(0)) AS positive_zero_needle, countEqual(a, reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle, length(arrayFilter(x -> x = toFloat64(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[0.0,-0.0,1.5]}'); +SELECT arrayMap(x -> reinterpretAsUInt32(x), a) AS stored_bits, has(a, toFloat32(0)) AS positive_zero_needle, has(a, reinterpretAsFloat32(toUInt32(2147483648))) AS negative_zero_needle, length(arrayFilter(x -> x = toFloat32(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float32))', '{"a":[-0.0,1.5]}'); + +-- A CAST array is folded onto the default slot, so its -0.0 element is stored as +0.0. +SELECT arrayMap(x -> reinterpretAsUInt64(x), materialize(CAST([reinterpretAsFloat64(toUInt64(9223372036854775808)), 1.5], 'Array(LowCardinality(Float64))'))) AS stored_bits, has(materialize(CAST([reinterpretAsFloat64(toUInt64(9223372036854775808)), 1.5], 'Array(LowCardinality(Float64))')), toFloat64(0)) AS positive_zero_needle, has(materialize(CAST([reinterpretAsFloat64(toUInt64(9223372036854775808)), 1.5], 'Array(LowCardinality(Float64))')), reinterpretAsFloat64(toUInt64(9223372036854775808))) AS negative_zero_needle; +SELECT has(materialize(CAST([0, NULL, 1.5], 'Array(LowCardinality(Nullable(Float64)))')), reinterpretAsFloat64(toUInt64(9223372036854775808))) AS lc, length(arrayFilter(x -> x = reinterpretAsFloat64(toUInt64(9223372036854775808)), materialize(CAST([0, NULL, 1.5], 'Array(LowCardinality(Nullable(Float64)))')))) AS oracle; + +-- An Enum needle reaches a float element through its underlying number, so it must be declined too. +SELECT arrayMap(x -> reinterpretAsUInt64(x), a) AS stored_bits, has(a, CAST('z', 'Enum8(\'z\' = 0, \'o\' = 1)')) AS enum8_needle, indexOf(a, CAST('z', 'Enum8(\'z\' = 0, \'o\' = 1)')) AS enum8_index, countEqual(a, CAST('z', 'Enum16(\'z\' = 0, \'o\' = 1)')) AS enum16_count, length(arrayFilter(x -> x = toFloat64(0), a)) AS oracle FROM format(JSONEachRow, 'a Array(LowCardinality(Float64))', '{"a":[-0.0,1.5]}'); + +-- Controls that must not move. +SELECT has(materialize(CAST(['', 'a'], 'Array(LowCardinality(String))')), 'zzz') AS absent_needle, has(materialize(CAST(['a', 'b'], 'Array(LowCardinality(String))')), '') AS default_absent; +SELECT has(materialize(CAST(['', 'a'], 'Array(LowCardinality(String))')), materialize('')) AS non_const_needle, indexOfAssumeSorted(materialize(CAST(['', 'a'], 'Array(LowCardinality(String))')), '') AS assume_sorted; + +-- Two answers that only the dictionary shortcut produces, so a build that stopped taking it would +-- move them. Both disagree with the plain-array oracle printed beside them, and both are known +-- defects of the value comparison tracked elsewhere, not of the lookup this test covers: a NaN is +-- one dictionary entry but never equal to itself, and a negative needle is compared to an unsigned +-- element as a raw number. +SELECT has(materialize(CAST([nan, 1.5], 'Array(LowCardinality(Float64))')), nan) AS nan_needle, has(materialize(CAST([nan, 1.5], 'Array(Float64)')), nan) AS oracle; +SELECT has(materialize(CAST([0, 255], 'Array(LowCardinality(UInt8))')), toInt8(-1)) AS negative_needle, has(materialize(CAST([0, 255], 'Array(UInt8)')), toInt8(-1)) AS oracle; + +-- LowCardinality(Nullable(T)): a NULL needle finds the NULL element and a default needle finds the default one. +SELECT indexOf(materialize(CAST(['a', NULL, ''], 'Array(LowCardinality(Nullable(String)))')), NULL) AS null_needle, indexOf(materialize(CAST(['a', NULL, ''], 'Array(LowCardinality(Nullable(String)))')), '') AS default_needle; + +-- Reached through a real table read rather than a constant-folded literal. +DROP TABLE IF EXISTS t_04881; +CREATE TABLE t_04881 (a Array(LowCardinality(String)), m Map(LowCardinality(String), String)) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_04881 VALUES (['', 'a'], map('', 'v_empty', 'k', 'v_k')), (['x', 'y'], map('x', 'v_x', 'y', 'v_y')); +SELECT a, has(a, '') AS has_empty, indexOf(a, '') AS idx_empty, mapContainsKey(m, '') AS key_empty, m[''] AS subscript_empty FROM t_04881 ORDER BY a; +DROP TABLE t_04881; diff --git a/tests/queries/0_stateless/04891_distributed_index_analysis_projections.reference b/tests/queries/0_stateless/04891_distributed_index_analysis_projections.reference new file mode 100644 index 000000000000..3a7ccaf0217d --- /dev/null +++ b/tests/queries/0_stateless/04891_distributed_index_analysis_projections.reference @@ -0,0 +1,15 @@ +-- { echo } +-- The filter matches all rows in all parts (the projection is analyzed, but rejected as not better) +select count(), sum(key) from test_dia_proj where value >= 0 settings distributed_index_analysis=0; +3000000 4499998500000 +select count(), sum(key) from test_dia_proj where value >= 0 settings cluster_for_parallel_replicas='parallel_replicas', distributed_index_analysis=1; +3000000 4499998500000 +-- The filter matches one row in the first part and one row in the last part (the projection is selected for reading) +select count(), sum(key) from test_dia_proj where value in (500000, 2500000) settings distributed_index_analysis=0; +2 3000000 +select count(), sum(key) from test_dia_proj where value in (500000, 2500000) settings cluster_for_parallel_replicas='parallel_replicas', distributed_index_analysis=1; +2 3000000 +distributed_index_analysis=0, DistributedIndexAnalysisMicroseconds>0=0, DistributedIndexAnalysisMissingParts=0, DistributedIndexAnalysisScheduledReplicas>0=0 +distributed_index_analysis=1, DistributedIndexAnalysisMicroseconds>0=1, DistributedIndexAnalysisMissingParts=0, DistributedIndexAnalysisScheduledReplicas>0=1 +distributed_index_analysis=0, DistributedIndexAnalysisMicroseconds>0=0, DistributedIndexAnalysisMissingParts=0, DistributedIndexAnalysisScheduledReplicas>0=0 +distributed_index_analysis=1, DistributedIndexAnalysisMicroseconds>0=1, DistributedIndexAnalysisMissingParts=0, DistributedIndexAnalysisScheduledReplicas>0=1 diff --git a/tests/queries/0_stateless/04891_distributed_index_analysis_projections.sql b/tests/queries/0_stateless/04891_distributed_index_analysis_projections.sql new file mode 100644 index 000000000000..92f3d3d7a3a7 --- /dev/null +++ b/tests/queries/0_stateless/04891_distributed_index_analysis_projections.sql @@ -0,0 +1,62 @@ +-- Tags: no-random-merge-tree-settings, no-random-settings +-- - no-random-merge-tree-settings -- may change number of parts + +drop table if exists test_dia_proj; +create table test_dia_proj +( + key Int, + value Int, + projection prj_value + ( + select key, value order by value + ) +) +engine=MergeTree() +order by key +settings distributed_index_analysis_min_parts_to_activate=0, distributed_index_analysis_min_indexes_bytes_to_activate=0; + +system stop merges test_dia_proj; +insert into test_dia_proj select number, number from numbers(0, 1000000); +insert into test_dia_proj select number, number from numbers(1000000, 1000000); +insert into test_dia_proj select number, number from numbers(2000000, 1000000); + +set allow_experimental_parallel_reading_from_replicas=0; +set cluster_for_parallel_replicas=''; +set max_parallel_replicas=100; +set distributed_index_analysis_for_non_shared_merge_tree=1; +-- Ranges cached by the first (correct) run would narrow the analysis of the second run +set use_query_condition_cache=0; + +--- Ignore warnings when replica does not respond, and analysis is done on initiator +set send_logs_level='error'; + +-- { echo } +-- The filter matches all rows in all parts (the projection is analyzed, but rejected as not better) +select count(), sum(key) from test_dia_proj where value >= 0 settings distributed_index_analysis=0; +select count(), sum(key) from test_dia_proj where value >= 0 settings cluster_for_parallel_replicas='parallel_replicas', distributed_index_analysis=1; + +-- The filter matches one row in the first part and one row in the last part (the projection is selected for reading) +select count(), sum(key) from test_dia_proj where value in (500000, 2500000) settings distributed_index_analysis=0; +select count(), sum(key) from test_dia_proj where value in (500000, 2500000) settings cluster_for_parallel_replicas='parallel_replicas', distributed_index_analysis=1; + +-- { echoOff } +system flush logs query_log; +select format( + 'distributed_index_analysis={}, DistributedIndexAnalysisMicroseconds>0={}, DistributedIndexAnalysisMissingParts={}, DistributedIndexAnalysisScheduledReplicas>0={}', + Settings['distributed_index_analysis'], + ProfileEvents['DistributedIndexAnalysisMicroseconds'] > 0, + ProfileEvents['DistributedIndexAnalysisMissingParts'], + ProfileEvents['DistributedIndexAnalysisScheduledReplicas'] > 0 +) +from system.query_log +where + current_database = currentDatabase() + and event_date >= yesterday() AND event_time >= now() - 600 + and type = 'QueryFinish' + and query_kind = 'Select' + and is_initial_query + and has(Settings, 'distributed_index_analysis') + and endsWith(log_comment, '-' || currentDatabase()) +order by event_time_microseconds; + +drop table test_dia_proj; diff --git a/tests/queries/0_stateless/04891_mergetree_index_respects_row_policy.reference b/tests/queries/0_stateless/04891_mergetree_index_respects_row_policy.reference new file mode 100644 index 000000000000..a86cb1d96d36 --- /dev/null +++ b/tests/queries/0_stateless/04891_mergetree_index_respects_row_policy.reference @@ -0,0 +1,10 @@ +-- without a row policy the index is readable +engineering +finance +hr +-- base table honours the policy +1 engineering +3 engineering +-- mergeTreeIndex must not expose primary key values of policy-hidden rows +-- mergeTreeAnalyzeIndexes returns mark ranges, not values, so the policy does not apply +1 diff --git a/tests/queries/0_stateless/04891_mergetree_index_respects_row_policy.sql b/tests/queries/0_stateless/04891_mergetree_index_respects_row_policy.sql new file mode 100644 index 000000000000..73c5d4cac43e --- /dev/null +++ b/tests/queries/0_stateless/04891_mergetree_index_respects_row_policy.sql @@ -0,0 +1,27 @@ +-- A row policy on the source table must not be bypassed by the `mergeTreeIndex` table function. + +DROP TABLE IF EXISTS t_mt_index_rp; +DROP ROW POLICY IF EXISTS rp_mt_index ON t_mt_index_rp; + +CREATE TABLE t_mt_index_rp (id UInt64, department String) +ENGINE = MergeTree ORDER BY (department, id) SETTINGS index_granularity = 1; + +INSERT INTO t_mt_index_rp VALUES (1, 'engineering'), (2, 'finance'), (3, 'engineering'), (4, 'hr'); + +SELECT '-- without a row policy the index is readable'; +SELECT DISTINCT department FROM mergeTreeIndex(currentDatabase(), 't_mt_index_rp') ORDER BY department; + +CREATE ROW POLICY rp_mt_index ON t_mt_index_rp FOR SELECT USING department = 'engineering' TO ALL; + +SELECT '-- base table honours the policy'; +SELECT id, department FROM t_mt_index_rp ORDER BY id; + +SELECT '-- mergeTreeIndex must not expose primary key values of policy-hidden rows'; +SELECT DISTINCT department FROM mergeTreeIndex(currentDatabase(), 't_mt_index_rp') ORDER BY department; -- { serverError ACCESS_DENIED } +SELECT count() FROM mergeTreeIndex(currentDatabase(), 't_mt_index_rp', with_minmax = true); -- { serverError ACCESS_DENIED } + +SELECT '-- mergeTreeAnalyzeIndexes returns mark ranges, not values, so the policy does not apply'; +SELECT count() >= 0 FROM mergeTreeAnalyzeIndexes(currentDatabase(), t_mt_index_rp, department = 'finance'); + +DROP ROW POLICY rp_mt_index ON t_mt_index_rp; +DROP TABLE t_mt_index_rp; diff --git a/tests/queries/0_stateless/04895_parquet_dictionary_source_prefetch_lifetime.reference b/tests/queries/0_stateless/04895_parquet_dictionary_source_prefetch_lifetime.reference new file mode 100644 index 000000000000..bf116cc7c89e --- /dev/null +++ b/tests/queries/0_stateless/04895_parquet_dictionary_source_prefetch_lifetime.reference @@ -0,0 +1,7 @@ +1 +1 +1 +1 +1 +reloads_that_read 5 +alive 1 diff --git a/tests/queries/0_stateless/04895_parquet_dictionary_source_prefetch_lifetime.sh b/tests/queries/0_stateless/04895_parquet_dictionary_source_prefetch_lifetime.sh new file mode 100755 index 000000000000..eb9b2ceccfe0 --- /dev/null +++ b/tests/queries/0_stateless/04895_parquet_dictionary_source_prefetch_lifetime.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs the Parquet format which is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A dictionary reading Parquet from a local file hands the ReadBuffer to the input format, which owns +# it. When the read throws, the pipeline releases that buffer on the way out while the format's +# background prefetch and decode tasks may still be reading through it. The error must surface every +# time and the server must stay alive; under a sanitizer build the buffer must not be read after +# release. + +DICT="d_${CLICKHOUSE_DATABASE}" +# The dictionary FILE source needs an absolute path, and it must be the path this server actually +# serves -- ask the server rather than assuming a layout. +USER_FILES=$(${CLICKHOUSE_CLIENT} --query "select value from system.server_settings where name = 'user_files_path'") +REL="${CLICKHOUSE_DATABASE}/prefetch_lifetime.parquet" +ABS="${USER_FILES%/}/${REL}" + +# Small row groups so there are many read ranges, hence many queued tasks at throw time. +${CLICKHOUSE_CLIENT} --query=" + insert into function file('${REL}', Parquet, 'key UInt64, val String') + select number, repeat('y', 400) from numbers(2000000) + settings engine_file_truncate_on_insert = 1, output_format_parquet_row_group_size = 5000, + output_format_parquet_compression_method = 'none'; +" + +# `val` is Int64 in the dictionary but holds strings in the file, so the Parquet read throws +# mid-flight, which is what makes the pipeline tear down while tasks are still running. +# +# The settings have to be on the dictionary, not on the query: a dictionary loads in the global +# context, and the `file` source only picks up this SETTINGS clause. min_bytes_for_seek = 1 stops +# range coalescing, so each range becomes its own task, and the prefetch pool has to exist for those +# tasks to run in the background rather than inline on the decoding thread. +${CLICKHOUSE_CLIENT} --query=" + create dictionary ${DICT} (key UInt64, val Int64) primary key key + source(file(path '${ABS}' format 'Parquet')) + layout(flat(max_array_size 5000000)) lifetime(0) + settings(max_download_threads = 32, max_parsing_threads = 32, + input_format_parquet_local_file_min_bytes_for_seek = 1, + input_format_parquet_enable_row_group_prefetch = 1); +" + +# A forced reload, because a plain dictGet would replay the first load's cached exception instead of +# reading the file again. +for _ in 1 2 3 4 5; do + ${CLICKHOUSE_CLIENT} --log_comment="${DICT}_reload" --query="system reload dictionary ${DICT}" 2>&1 \ + | grep -c -m1 -F 'CANNOT_PARSE_TEXT' +done + +# Every iteration has to have reached row group reading, otherwise the loop proves nothing: an +# already-FAILED dictionary replays its stored exception without reading, and a file rejected while +# its footer is parsed reads only the footer, both of which are indistinguishable from a real read by +# the error message alone. ParquetReadRowGroups is counted only once row groups are being read. +${CLICKHOUSE_CLIENT} --query="system flush logs query_log" +${CLICKHOUSE_CLIENT} --query=" + select 'reloads_that_read', countIf(ProfileEvents['ParquetReadRowGroups'] > 0) + from system.query_log + where log_comment = '${DICT}_reload' and current_database = currentDatabase() + and type != 'QueryStart'; +" + +# The server survived every attempt and still answers. +${CLICKHOUSE_CLIENT} --query="select 'alive', count() from system.dictionaries where database = currentDatabase() and name = '${DICT}'" + +${CLICKHOUSE_CLIENT} --query="drop dictionary ${DICT}" +${CLICKHOUSE_CLIENT} --query="select * from file('${REL}', Parquet) where 0 format Null" 2>/dev/null +rm -f "${ABS}" diff --git a/tests/queries/0_stateless/04902_access_entity_map_setting_round_trip.reference b/tests/queries/0_stateless/04902_access_entity_map_setting_round_trip.reference new file mode 100644 index 000000000000..7342020d8e8a --- /dev/null +++ b/tests/queries/0_stateless/04902_access_entity_map_setting_round_trip.reference @@ -0,0 +1,19 @@ +arm01 profile map round trip OK +arm01 profile map reparsed {'Content-Type':'application/json'} +arm02 profile empty map round trip OK +arm03 additional_table_filters round trip OK +arm04 user map round trip OK +arm05 role map round trip OK +arm06 min max round trip OK +arm07 multi key map round trip OK +arm07 multi key map reparsed {'Content-Type':'application/json','X-A':'*','X-B':'1'} +arm08 hostile value round trip OK +arm08 hostile value reparsed {'k,1:{}[]()':'va,l:ue{}[]()\'x','k2':'back\\slash'} +arm09 scalar control round trip OK +arm10 string valued builtin control round trip OK +arm11 custom setting control round trip OK +arm12 alter add settings round trip OK +arm13 system table value {'Content-Type':'application/json'} +arm14 emitted a string literal not a collection literal +arm16 scalar literal stays bare = 5000000 MIN 4000000 MAX 6000000 CONST +arm15 on disk {'X-A':'1','X-B':'2'} {} {'X-A':'1','X-B':'2'} diff --git a/tests/queries/0_stateless/04902_access_entity_map_setting_round_trip.sh b/tests/queries/0_stateless/04902_access_entity_map_setting_round_trip.sh new file mode 100755 index 000000000000..82b745875d3c --- /dev/null +++ b/tests/queries/0_stateless/04902_access_entity_map_setting_round_trip.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +U="${CLICKHOUSE_TEST_UNIQUE_NAME}" +CLEANUP="" + +# Creates an entity, then creates a second one from the statement the server emitted for the first, +# and compares the two emitted statements. This is the display route, via SHOW CREATE; arm15 covers +# the persistence route, where serializeAccessEntity and deserializeAccessEntity run. +# +# Each half is one client invocation: statements that produce no output share it with the single +# SHOW CREATE whose one line is the result, so an arm costs two processes rather than seven. +round_trip() { + local n="$1" kind="$2" label="$3" clause="$4" pin_setting="$5" + local a="a${n}_${U}" b="b${n}_${U}" + local create_kw owner_col + + case "$kind" in + profile) create_kw="SETTINGS PROFILE"; owner_col="profile_name" ;; + user) create_kw="USER"; owner_col="user_name" ;; + role) create_kw="ROLE"; owner_col="role_name" ;; + esac + CLEANUP="${CLEANUP} DROP ${create_kw} IF EXISTS ${a}, ${b};" + + local emitted_a emitted_b reparsed pin_query="" + emitted_a=$(${CLICKHOUSE_CLIENT} -q " + DROP ${create_kw} IF EXISTS ${a}; + CREATE ${create_kw} ${a} SETTINGS ${clause}; + SHOW CREATE ${create_kw} ${a} FORMAT TSVRaw") + + # Comparing the two emitted statements only proves the serializer agrees with itself, so for the + # arms whose value carries structure the surviving value is pinned in .reference as well. It is + # the second line of the same reply. + if [[ -n "$pin_setting" ]]; then + pin_query="SELECT '${label} reparsed', value FROM system.settings_profile_elements WHERE ${owner_col} = '${b}' AND setting_name = '${pin_setting}' FORMAT TSVRaw;" + fi + + # Feed the server's own output back, under the second name. A server that cannot reparse what it + # emitted fails here, which is what the arm is for, so these errors are expected. + reparsed=$(${CLICKHOUSE_CLIENT} --ignore-error -q " + DROP ${create_kw} IF EXISTS ${b}; + ${emitted_a//$a/$b}; + SHOW CREATE ${create_kw} ${b} FORMAT TSVRaw; + ${pin_query}" 2>/dev/null) + emitted_b=$(sed -n 1p <<< "$reparsed") + + if [[ -n "$emitted_a" && "$emitted_a" == "${emitted_b//$b/$a}" ]]; then + echo "${label} round trip OK" + else + echo "${label} round trip FAILED" + echo " emitted: ${emitted_a}" + echo " reparsed: ${emitted_b}" + fi + + [[ -n "$pin_setting" ]] && sed -n 2p <<< "$reparsed" +} + +# kind|label|settings clause|setting whose reparsed value is pinned (empty = shape check only). +# Quoted heredoc: no shell expansion, SQL escapes pass through verbatim. +N=0 +while IFS='|' read -r kind label clause pin; do + [[ -z "$kind" ]] && continue + N=$((N + 1)) + round_trip "$(printf '%02d' "$N")" "$kind" "$label" "$clause" "$pin" +done <<'ARMS' +profile|arm01 profile map|http_response_headers = '{\'Content-Type\':\'application/json\'}' CONST|http_response_headers +profile|arm02 profile empty map|http_response_headers = '{}' CONST| +profile|arm03 additional_table_filters|additional_table_filters = '{\'default.t\':\'x > 0\'}'| +user|arm04 user map|http_response_headers = '{\'a\':\'b\'}'| +role|arm05 role map|additional_table_filters = '{\'default.t\':\'x > 0\'}'| +profile|arm06 min max|http_response_headers = '{\'a\':\'b\'}' MIN '{}' MAX '{\'a\':\'b\'}'| +profile|arm07 multi key map|http_response_headers = '{\'Content-Type\':\'application/json\',\'X-A\':\'*\',\'X-B\':\'1\'}' CONST|http_response_headers +profile|arm08 hostile value|http_response_headers = '{\'k,1:{}[]()\':\'va,l:ue{}[]()\\\'x\',\'k2\':\'back\\\\slash\'}'|http_response_headers +profile|arm09 scalar control|max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST| +profile|arm10 string valued builtin control|log_comment = '{\'not\':\'a map\'}'| +profile|arm11 custom setting control|custom_04902_a = 'plain string'| +ARMS + +# arm12: ALTER then round trip. +P="alter_${U}" +Q="alter2_${U}" +EMITTED=$(${CLICKHOUSE_CLIENT} -q " + DROP SETTINGS PROFILE IF EXISTS ${P}; + CREATE SETTINGS PROFILE ${P} SETTINGS max_memory_usage = 5000000; + ALTER SETTINGS PROFILE ${P} ADD SETTINGS http_response_headers = '{\'a\':\'b\'}' CONST; + SHOW CREATE SETTINGS PROFILE ${P} FORMAT TSVRaw") +REPARSED=$(${CLICKHOUSE_CLIENT} --ignore-error -q " + DROP SETTINGS PROFILE IF EXISTS ${Q}; + ${EMITTED//$P/$Q}; + SHOW CREATE SETTINGS PROFILE ${Q} FORMAT TSVRaw" 2>/dev/null) +if [[ -n "$EMITTED" && "$EMITTED" == "${REPARSED//$Q/$P}" ]]; then + echo "arm12 alter add settings round trip OK" +else + echo "arm12 alter add settings round trip FAILED" + echo " emitted: ${EMITTED}" + echo " reparsed: ${REPARSED}" +fi + +# arm13: the system table shows the canonical text, and SHOW CREATE now emits that same text. +S="sys_${U}" +${CLICKHOUSE_CLIENT} -q " + DROP SETTINGS PROFILE IF EXISTS ${S}; + CREATE SETTINGS PROFILE ${S} SETTINGS http_response_headers = '{\'Content-Type\':\'application/json\'}' CONST; + SELECT 'arm13 system table value', value FROM system.settings_profile_elements WHERE profile_name = '${S}' FORMAT TSVRaw" + +# arm14: the emitted statement of a map setting is a string literal, not a collection literal. +T="lit_${U}" +LIT=$(${CLICKHOUSE_CLIENT} -q " + DROP SETTINGS PROFILE IF EXISTS ${T}; + CREATE SETTINGS PROFILE ${T} SETTINGS http_response_headers = '{\'a\':\'b\'}' CONST; + SHOW CREATE SETTINGS PROFILE ${T} FORMAT TSVRaw") +if [[ "$LIT" == *"= '"* && "$LIT" != *"= ["* ]]; then + echo "arm14 emitted a string literal not a collection literal" +else + echo "arm14 FAILED: ${LIT}" +fi + +# arm16: only a Map is rewritten. A scalar keeps its bare literal, so widening the type check to +# every builtin setting would emit max_memory_usage = '5000000' and change the stored type. +V="num_${U}" +NUM=$(${CLICKHOUSE_CLIENT} -q " + DROP SETTINGS PROFILE IF EXISTS ${V}; + CREATE SETTINGS PROFILE ${V} SETTINGS max_memory_usage = 5000000 MIN 4000000 MAX 6000000 CONST; + SHOW CREATE SETTINGS PROFILE ${V} FORMAT TSVRaw") +echo "arm16 scalar literal stays bare ${NUM#*SETTINGS max_memory_usage}" + +${CLICKHOUSE_CLIENT} -q "${CLEANUP} DROP SETTINGS PROFILE IF EXISTS ${P}, ${Q}, ${S}, ${T}, ${V};" + +# arm15: the on-disk form. The arms above use SHOW CREATE, which takes the display route; the +# stored .sql file is written by the attach route, and a second process has to parse it back +# through deserializeAccessEntity before any query can see the entity. +D="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.ondisk" +rm -rf "$D" +${CLICKHOUSE_LOCAL} --path "$D" -q "CREATE SETTINGS PROFILE ondisk SETTINGS http_response_headers = '{\'X-A\':\'1\',\'X-B\':\'2\'}' MIN '{}' MAX '{\'X-A\':\'1\',\'X-B\':\'2\'}'" -- --max_server_memory_usage=8G --memory_worker_use_cgroup=0 +${CLICKHOUSE_LOCAL} --path "$D" -q "SELECT 'arm15 on disk', value, min, max FROM system.settings_profile_elements WHERE profile_name = 'ondisk' FORMAT TSVRaw" -- --max_server_memory_usage=8G --memory_worker_use_cgroup=0 +rm -rf "$D" diff --git a/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.reference b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.reference new file mode 100644 index 000000000000..31745adb0b13 --- /dev/null +++ b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.reference @@ -0,0 +1,3 @@ +update: update=ok drop=ok +alter: update=ok drop=ok +alias: update=ok drop=ok diff --git a/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.sh b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.sh new file mode 100755 index 000000000000..dca044d4eb87 --- /dev/null +++ b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-replicated-database, no-shared-merge-tree +# Tag no-fasttest: relies on a failpoint (libfiu). +# Tag no-parallel: the test waits on a server-global pauseable failpoint, so a concurrent copy's +# sink could satisfy this copy's wait and this copy's resume could release that sink. +# FailPointInjection::{enable,disable}FailPoint take only a name, so it cannot be scoped. +# Tag no-replicated-database: the test needs its own Memory database, which a Replicated database +# run cannot host, and DROP must reach the synchronous exclusive-lock path. +# Tag no-shared-merge-tree: the failpoint this test parks on is in the ReplicatedMergeTree sink, +# which a SharedMergeTree table does not go through. + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +FP=rmt_pause_before_commit_local_part + +# The failpoint is server-global, so leaving it enabled would park the sink of every later test. +# This has to survive the test failing or being killed part-way through. +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true +} +trap cleanup EXIT + +# A Memory database has no UUID, so DROP takes the table's exclusive lock and removes the data +# inline instead of deferring it to the background cleanup an Atomic database uses. +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE_1}" < /dev/null +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${CLICKHOUSE_DATABASE_1} ENGINE = Memory" < /dev/null + +${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true + +# Every DROP the test's logic depends on pins ignore_drop_queries_probability: the stress runner +# injects 0.2 and clickhouse-client --fake-drop (upgrade check) injects 1, and for a storage that +# keeps data on disk the injection returns success without dropping anything. +function setup_table() +{ + ${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE_1}.t SYNC SETTINGS ignore_drop_queries_probability = 0; + + CREATE TABLE ${CLICKHOUSE_DATABASE_1}.t (id UInt64, c2 String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/04905_$1/', '1') + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; + + INSERT INTO ${CLICKHOUSE_DATABASE_1}.t SELECT number, 'a' FROM numbers(2); + " < /dev/null +} + +# The DROP has to land between the patch-part rename and the commit. The sink parks there and stays +# parked until this function resumes it, so the window is held open instead of being a timed one the +# drop could miss. +function race_with_drop() +{ + local arm=$1 + shift + local qid="${CLICKHOUSE_DATABASE_1}_${arm}_${RANDOM}${RANDOM}" + local drop_qid="${qid}_drop" + + ${CLICKHOUSE_CLIENT} --query "SYSTEM ENABLE FAILPOINT ${FP}" < /dev/null + + # Everything below reads the armed state as evidence, so an arm that is not actually armed would + # make the rest of the checks vacuous rather than failing. + if [ "$(${CLICKHOUSE_CLIENT} --query " + SELECT enabled FROM system.fail_points WHERE name = '${FP}' + SETTINGS enable_parallel_replicas = 0" < /dev/null 2>/dev/null)" != 1 ]; then + echo "$arm: the commit hook was not armed" + return + fi + + ${CLICKHOUSE_CLIENT} --query_id "$qid" "$@" < /dev/null > /dev/null 2>&1 & + local updater=$! + + # Returns once the sink has parked, so the update still holds the lock it took for the pipeline. + # An update that never reaches the hook would otherwise wait here forever. + # shellcheck disable=SC2086 # CLICKHOUSE_CLIENT carries arguments and must word-split + if ! timeout 60 ${CLICKHOUSE_CLIENT} --query "SYSTEM WAIT FAILPOINT ${FP} PAUSE" < /dev/null; then + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true + wait "$updater" 2>/dev/null || true + echo "$arm: the sink never reached the commit hook" + return + fi + + # The wait above returns at once when nothing is parked, so it does not by itself establish that + # the sink reached the hook. The failpoint is one-shot, so it reports itself disabled only after + # something fired it. + if [ "$(${CLICKHOUSE_CLIENT} --query " + SELECT enabled FROM system.fail_points WHERE name = '${FP}' + SETTINGS enable_parallel_replicas = 0" < /dev/null 2>/dev/null)" != 0 ]; then + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true + wait "$updater" 2>/dev/null || true + echo "$arm: the sink never parked at the commit hook" + return + fi + + ${CLICKHOUSE_CLIENT} --query_id "$drop_qid" --query " + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE_1}.t SYNC + SETTINGS ignore_drop_queries_probability = 0" < /dev/null > /dev/null 2>&1 & + local dropper=$! + + # The sink is still parked, so the drop reaches the table lock and blocks on it while the + # window is held open. + sleep 2 + + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null + + local update=ok + wait "$updater" 2>/dev/null || update=failed + + local drop=ok + wait "$dropper" 2>/dev/null || drop=failed + + # The lock wait is charged to the statement that blocked, so a non-zero value here is this + # drop's own wait on this table and no other writer can supply it. A drop that reached the lock + # only after the commit released it reports zero and never covered the race. + local waited + waited=$(${CLICKHOUSE_CLIENT} --query " + SYSTEM FLUSH LOGS query_log; + SELECT ProfileEvents['RWLockWritersWaitMilliseconds'] FROM system.query_log + WHERE query_id = '${drop_qid}' AND type = 'QueryFinish' AND event_date >= yesterday() + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC LIMIT 1 + SETTINGS max_rows_to_read = 0, enable_parallel_replicas = 0" < /dev/null 2>/dev/null) || waited="" + case "$waited" in + '' | *[!0-9]*) waited=0 ;; + esac + + if [ "$waited" -eq 0 ]; then + echo "$arm: the drop did not wait for the table lock" + return + fi + + # A drop that reported success without removing the table took no lock and proved nothing. + if [ "$drop" = ok ] && [ "$(${CLICKHOUSE_CLIENT} --query " + EXISTS ${CLICKHOUSE_DATABASE_1}.t + SETTINGS enable_parallel_replicas = 0" < /dev/null 2>/dev/null)" != 0 ]; then + drop=ignored + fi + + echo "$arm: update=$update drop=$drop" +} + +setup_table update +race_with_drop update --enable_lightweight_update 1 \ + --query "UPDATE ${CLICKHOUSE_DATABASE_1}.t SET c2 = 'xx' WHERE id = 1" + +setup_table alter +race_with_drop alter --enable_lightweight_update 1 --alter_update_mode 'lightweight_force' \ + --query "ALTER TABLE ${CLICKHOUSE_DATABASE_1}.t UPDATE c2 = 'xx' WHERE id = 1" + +# An Alias table resolves a different storage, so the update has to hold the target's lock as well. +setup_table alias +${CLICKHOUSE_CLIENT} --allow_experimental_alias_table_engine 1 --query " + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE_1}.a SYNC SETTINGS ignore_drop_queries_probability = 0; + CREATE TABLE ${CLICKHOUSE_DATABASE_1}.a ENGINE = Alias('${CLICKHOUSE_DATABASE_1}', 't'); +" < /dev/null +race_with_drop alias --allow_experimental_alias_table_engine 1 --enable_lightweight_update 1 \ + --query "UPDATE ${CLICKHOUSE_DATABASE_1}.a SET c2 = 'xx' WHERE id = 1" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE_1}" < /dev/null diff --git a/tests/queries/0_stateless/04920_timeseries_drop_low_sorting_name.reference b/tests/queries/0_stateless/04920_timeseries_drop_low_sorting_name.reference new file mode 100644 index 000000000000..7caea6098b97 --- /dev/null +++ b/tests/queries/0_stateless/04920_timeseries_drop_low_sorting_name.reference @@ -0,0 +1,4 @@ +low_sorting_name_dropped +mv_inner_timeseries_dropped +control_high_sorting_name_dropped +0 diff --git a/tests/queries/0_stateless/04920_timeseries_drop_low_sorting_name.sql b/tests/queries/0_stateless/04920_timeseries_drop_low_sorting_name.sql new file mode 100644 index 000000000000..df245305047e --- /dev/null +++ b/tests/queries/0_stateless/04920_timeseries_drop_low_sorting_name.sql @@ -0,0 +1,28 @@ +SET allow_experimental_time_series_table = 1; +-- The stress runner sets ignore_drop_queries_probability=0.2, which rewrites a DROP of a table +-- that keeps no data on disk into a TRUNCATE; TRUNCATE does not drop inner tables. +SET ignore_drop_queries_probability = 0; + +-- A TimeSeries whose own name sorts below its inner tables' names used to self-deadlock on the +-- DDL guard, so this hung instead of returning. +DROP TABLE IF EXISTS `-ts`; +CREATE TABLE `-ts` ENGINE = TimeSeries; +DROP TABLE `-ts`; +SELECT 'low_sorting_name_dropped'; + +-- Same state reached without choosing a name: the view's own inner name is `.inner_id.`, +-- which always sorts below `.inner_id.metrics.`. +DROP TABLE IF EXISTS mv; +CREATE MATERIALIZED VIEW mv ENGINE = TimeSeries AS SELECT 1 AS a; +DROP TABLE mv; +SELECT 'mv_inner_timeseries_dropped'; + +-- Control: a name that sorts above the inner tables' names takes the other branch of the +-- ordering predicate and always worked. +DROP TABLE IF EXISTS prom; +CREATE TABLE prom ENGINE = TimeSeries; +DROP TABLE prom; +SELECT 'control_high_sorting_name_dropped'; + +-- No inner tables may be left behind by any of the drops above. +SELECT count() FROM system.tables WHERE database = currentDatabase(); diff --git a/tests/queries/0_stateless/04927_uniq_injective_tuple_nullable.reference b/tests/queries/0_stateless/04927_uniq_injective_tuple_nullable.reference new file mode 100644 index 000000000000..fef4388f841b --- /dev/null +++ b/tests/queries/0_stateless/04927_uniq_injective_tuple_nullable.reference @@ -0,0 +1,31 @@ +tuple +3 3 3 3 3 +3 3 3 3 3 +bitmaskToArray and bitPositionsToArray +3 3 +3 3 +nested, LowCardinality and multiple arguments +3 3 +3 3 +2 +2 +constant folding must not change the result +1 +1 +1 +the optimization still applies to a not Nullable argument +QUERY id: 0 + PROJECTION COLUMNS + uniqExact((number)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: uniqExact, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 1 + COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5 + JOIN TREE + TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers + ARGUMENTS + LIST id: 6, nodes: 1 + CONSTANT id: 7, constant_value: UInt64_3, constant_value_type: UInt8 + SETTINGS optimize_injective_functions_inside_uniq=1 diff --git a/tests/queries/0_stateless/04927_uniq_injective_tuple_nullable.sql b/tests/queries/0_stateless/04927_uniq_injective_tuple_nullable.sql new file mode 100644 index 000000000000..199d171cf5a5 --- /dev/null +++ b/tests/queries/0_stateless/04927_uniq_injective_tuple_nullable.sql @@ -0,0 +1,47 @@ +-- Tags: no-old-analyzer +-- The old analyzer rewrites the query before types are resolved and keeps the wrong results. +-- https://github.com/ClickHouse/ClickHouse/issues/114784 +-- `uniq*` skips the rows where an argument is NULL. An injective function whose result cannot be Nullable +-- hides that nullability - `tuple(NULL)` and `bitmaskToArray(NULL)` are values that get counted - so +-- `optimize_injective_functions_inside_uniq` must not remove it. + +SELECT 'tuple'; +SELECT uniq(tuple(x)), uniqExact(tuple(x)), uniqHLL12(tuple(x)), uniqCombined(tuple(x)), uniqCombined64(tuple(x)) +FROM values('x Nullable(Int32)', 1, 2, NULL) +SETTINGS optimize_injective_functions_inside_uniq = 1; + +SELECT uniq(tuple(x)), uniqExact(tuple(x)), uniqHLL12(tuple(x)), uniqCombined(tuple(x)), uniqCombined64(tuple(x)) +FROM values('x Nullable(Int32)', 1, 2, NULL) +SETTINGS optimize_injective_functions_inside_uniq = 0; + +SELECT 'bitmaskToArray and bitPositionsToArray'; +SELECT uniqExact(bitmaskToArray(x)), uniqExact(bitPositionsToArray(x)) +FROM values('x Nullable(Int32)', 1, 2, NULL) +SETTINGS optimize_injective_functions_inside_uniq = 1; + +SELECT uniqExact(bitmaskToArray(x)), uniqExact(bitPositionsToArray(x)) +FROM values('x Nullable(Int32)', 1, 2, NULL) +SETTINGS optimize_injective_functions_inside_uniq = 0; + +SELECT 'nested, LowCardinality and multiple arguments'; +SELECT uniqExact(tuple(tuple(x))), uniqExact(tuple(toLowCardinality(x))) +FROM values('x Nullable(Int32)', 1, 2, NULL) +SETTINGS optimize_injective_functions_inside_uniq = 1; + +SELECT uniqExact(tuple(tuple(x))), uniqExact(tuple(toLowCardinality(x))) +FROM values('x Nullable(Int32)', 1, 2, NULL) +SETTINGS optimize_injective_functions_inside_uniq = 0; + +SELECT uniqExact(tuple(x), y) FROM values('x Nullable(Int32), y Nullable(Int32)', (1, 1), (2, NULL), (NULL, 3)) +SETTINGS optimize_injective_functions_inside_uniq = 1; + +SELECT uniqExact(tuple(x), y) FROM values('x Nullable(Int32), y Nullable(Int32)', (1, 1), (2, NULL), (NULL, 3)) +SETTINGS optimize_injective_functions_inside_uniq = 0; + +SELECT 'constant folding must not change the result'; +SELECT countDistinct(tuple(NULL)); +SELECT countDistinct(tuple(arrayJoin([NULL]))); +SELECT countDistinct(tuple(arrayJoin(emptyArrayToSingle([]::Array(Nullable(Int32)))))); + +SELECT 'the optimization still applies to a not Nullable argument'; +EXPLAIN QUERY TREE SELECT uniqExact(tuple(number)) FROM numbers(3) SETTINGS optimize_injective_functions_inside_uniq = 1; diff --git a/tests/queries/0_stateless/05019_settings_constraints_nested_settings_clamp.reference b/tests/queries/0_stateless/05019_settings_constraints_nested_settings_clamp.reference new file mode 100644 index 000000000000..fe030451bffd --- /dev/null +++ b/tests/queries/0_stateless/05019_settings_constraints_nested_settings_clamp.reference @@ -0,0 +1,25 @@ +-- the row filter installed by the profile is in force +1 +-- a nested clause cannot override a CONST setting: the row filter stays, in a subquery and in a CTE +1 +1 +-- every member of a nested compound query is clamped +2 +2 +1 +-- a nested value above MAX is clamped into the bound, not applied as written +-- a nested value inside the bounds still applies +-- clamping still lets the subquery read up to the bound +1 +-- an unconstrained nested clause still works +1 +-- the session settings are untouched by a nested clause +10 +-- a nested SETTINGS name = DEFAULT is still ignored +1 +10 +-- a SQL SECURITY INVOKER view with an inner clause the invoker may not set reads with the constraints enforced +1 +-- a SQL SECURITY DEFINER view still reads +3 +-- a top-level SETTINGS clause overriding a CONST setting still throws diff --git a/tests/queries/0_stateless/05019_settings_constraints_nested_settings_clamp.sql b/tests/queries/0_stateless/05019_settings_constraints_nested_settings_clamp.sql new file mode 100644 index 000000000000..711b1e040f69 --- /dev/null +++ b/tests/queries/0_stateless/05019_settings_constraints_nested_settings_clamp.sql @@ -0,0 +1,89 @@ +-- Tags: no-parallel, no-old-analyzer +-- no-parallel: a settings profile is server-global rather than per-database, and its name cannot be +-- made unique per run: query parameters are not accepted in access-entity DDL. So this test is not +-- safe against a concurrent copy of itself - which is how the flaky check runs it. +-- no-old-analyzer: the clamp is analyzer-path behaviour; the legacy interpreter has always thrown +-- on a nested clause that violates the constraints, so these expectations do not hold there. + +-- A `SETTINGS` clause nested inside a subquery, a CTE, or a view's inner query must not override the +-- session's settings constraints. The nested form is clamped rather than rejected: a change that +-- violates a `CONST` or `readonly` constraint is dropped, a value outside its `MIN`/`MAX` bounds is +-- clamped into them, and the rest of the clause applies. A clause on the outer query still throws. + +DROP SETTINGS PROFILE IF EXISTS profile_05019; +DROP VIEW IF EXISTS v_invoker_05019; +DROP VIEW IF EXISTS v_definer_05019; +DROP TABLE IF EXISTS t_05019; + +CREATE TABLE t_05019 (tenant_id UInt32, secret String) ENGINE = MergeTree ORDER BY tenant_id; +INSERT INTO t_05019 VALUES (1, 'tenant1-own'), (2, 'tenant2-secret'), (3, 'tenant3-secret'); + +-- Views created by an unconstrained user, carrying an inner SETTINGS clause the constrained session +-- below is not allowed to set. Under SQL SECURITY INVOKER the inner clause is clamped against the +-- invoking session's constraints, so the view keeps reading with the constraints enforced; under +-- DEFINER the definer's own (unconstrained) settings apply, as before. +CREATE VIEW v_invoker_05019 SQL SECURITY INVOKER + AS SELECT count() AS c FROM t_05019 SETTINGS max_execution_time = 5; +CREATE VIEW v_definer_05019 DEFINER = CURRENT_USER SQL SECURITY DEFINER + AS SELECT count() AS c FROM numbers(3) SETTINGS max_execution_time = 5; + +CREATE SETTINGS PROFILE profile_05019 SETTINGS + max_execution_time = 10 CONST, + additional_table_filters = '{''t_05019'':''tenant_id = 1''}' CONST, + max_rows_to_read MAX 2; + +SET profile = 'profile_05019'; + +SELECT '-- the row filter installed by the profile is in force'; +SELECT count() FROM t_05019; + +SELECT '-- a nested clause cannot override a CONST setting: the row filter stays, in a subquery and in a CTE'; +SELECT count() FROM (SELECT * FROM t_05019 SETTINGS additional_table_filters = {'t_05019':'1'}); +WITH cte AS (SELECT * FROM t_05019 SETTINGS additional_table_filters = {'t_05019':'1'}) SELECT count() FROM cte; + +-- A nested compound query carries the clause on one of its member SELECTs, whichever one the parser +-- attached it to, so every member has to be clamped and not just the query as a whole. +SELECT '-- every member of a nested compound query is clamped'; +SELECT count() FROM (SELECT tenant_id FROM t_05019 UNION ALL SELECT tenant_id FROM t_05019 SETTINGS additional_table_filters = {'t_05019':'1'}); +SELECT count() FROM ((SELECT tenant_id FROM t_05019 SETTINGS additional_table_filters = {'t_05019':'1'}) UNION ALL SELECT tenant_id FROM t_05019); +SELECT count() FROM (SELECT tenant_id FROM t_05019 INTERSECT SELECT tenant_id FROM t_05019 SETTINGS additional_table_filters = {'t_05019':'1'}); + +-- Reading 3 rows under `max_rows_to_read MAX 2`: clamped into the bound the read fails, so the error +-- proves the value was clamped - applied as written (100) or dropped (unlimited) it would succeed. +SELECT '-- a nested value above MAX is clamped into the bound, not applied as written'; +SELECT sum(number) FROM (SELECT number FROM numbers(3) SETTINGS max_rows_to_read = 100); -- { serverError TOO_MANY_ROWS } + +SELECT '-- a nested value inside the bounds still applies'; +SELECT sum(number) FROM (SELECT number FROM numbers(3) SETTINGS max_rows_to_read = 1); -- { serverError TOO_MANY_ROWS } + +SELECT '-- clamping still lets the subquery read up to the bound'; +SELECT sum(number) FROM (SELECT number FROM numbers(2) SETTINGS max_rows_to_read = 100); + +SELECT '-- an unconstrained nested clause still works'; +SELECT count() FROM (SELECT * FROM t_05019 SETTINGS max_block_size = 100); + +SELECT '-- the session settings are untouched by a nested clause'; +SELECT getSetting('max_execution_time'); + +-- Applying a nested reset needs a `default_settings` carrier on `QueryNode`; until then it stays +-- ignored, as it always was on this path. Tracked in issue #115415. +SELECT '-- a nested SETTINGS name = DEFAULT is still ignored'; +SELECT count() FROM (SELECT * FROM t_05019 SETTINGS max_execution_time = DEFAULT); +SELECT getSetting('max_execution_time'); + +SELECT '-- a SQL SECURITY INVOKER view with an inner clause the invoker may not set reads with the constraints enforced'; +SELECT c FROM v_invoker_05019; + +SELECT '-- a SQL SECURITY DEFINER view still reads'; +SELECT c FROM v_definer_05019; + +-- Kept last on purpose. A constraint violation on a top-level SETTINGS clause is raised while the +-- server receives the settings packet, so the connection is dropped and the client silently +-- reconnects into a fresh session, losing the `SET profile` above for anything that follows. +SELECT '-- a top-level SETTINGS clause overriding a CONST setting still throws'; +SELECT count() FROM t_05019 SETTINGS additional_table_filters = {'t_05019':'1'}; -- { serverError SETTING_CONSTRAINT_VIOLATION } + +DROP TABLE t_05019; +DROP VIEW v_invoker_05019; +DROP VIEW v_definer_05019; +DROP SETTINGS PROFILE profile_05019; diff --git a/tests/queries/0_stateless/05020_settings_constraints_readonly_user.reference b/tests/queries/0_stateless/05020_settings_constraints_readonly_user.reference new file mode 100644 index 000000000000..b415d21f2d78 --- /dev/null +++ b/tests/queries/0_stateless/05020_settings_constraints_readonly_user.reference @@ -0,0 +1,8 @@ +-- the row filter is in force +tenant1-own +-- a nested SETTINGS clause cannot lift the row filter: it is dropped and the query still works +tenant1-own +-- a readonly user can read a view whose inner query carries a SETTINGS clause +1 +-- the session is still readonly +1 diff --git a/tests/queries/0_stateless/05020_settings_constraints_readonly_user.sh b/tests/queries/0_stateless/05020_settings_constraints_readonly_user.sh new file mode 100755 index 000000000000..7eae12fe67a3 --- /dev/null +++ b/tests/queries/0_stateless/05020_settings_constraints_readonly_user.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# (the clamp is analyzer-path behaviour; the legacy interpreter throws on such a nested clause) +# A readonly user must not be able to override a locked setting through a SETTINGS clause nested in +# a subquery or in a view's inner query. The nested clause is clamped against the constraints: the +# violating changes are dropped and the query keeps working with the constraints enforced. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# A user and a settings profile are server-global, not per-database, so their names carry the test +# database to keep this test safe against a concurrent copy of itself - which is how the flaky check +# runs it. +USER="user_05020_${CLICKHOUSE_DATABASE}" +PROFILE="profile_05020_${CLICKHOUSE_DATABASE}" +TABLE="t_05020" +VIEW="v_05020" + +${CLICKHOUSE_CLIENT} --multiquery --query " +DROP USER IF EXISTS ${USER}; +DROP SETTINGS PROFILE IF EXISTS ${PROFILE}; +DROP VIEW IF EXISTS ${CLICKHOUSE_DATABASE}.${VIEW}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${TABLE}; + +CREATE TABLE ${CLICKHOUSE_DATABASE}.${TABLE} (tenant_id UInt32, secret String) ENGINE = MergeTree ORDER BY tenant_id; +INSERT INTO ${CLICKHOUSE_DATABASE}.${TABLE} VALUES (1, 'tenant1-own'), (2, 'tenant2-secret'), (3, 'tenant3-secret'); + +-- An administrator's view whose inner query carries a SETTINGS clause a readonly user cannot set. +-- Before the nested clause was clamped, reading it as a readonly user was impossible: the clause +-- either threw or, worse, was applied unchecked. +CREATE VIEW ${CLICKHOUSE_DATABASE}.${VIEW} SQL SECURITY INVOKER + AS SELECT count() AS c FROM ${CLICKHOUSE_DATABASE}.${TABLE} SETTINGS max_execution_time = 5; + +CREATE SETTINGS PROFILE ${PROFILE} SETTINGS + readonly = 1 CONST, + additional_table_filters = '{''${CLICKHOUSE_DATABASE}.${TABLE}'':''tenant_id = 1''}' CONST; +CREATE USER ${USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${PROFILE}; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${USER}; +GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${USER}; +" + +RESTRICTED="${CLICKHOUSE_CLIENT_BINARY} --database=${CLICKHOUSE_DATABASE} --user=${USER}" + +echo "-- the row filter is in force" +${RESTRICTED} --query "SELECT secret FROM ${TABLE} ORDER BY tenant_id" + +echo "-- a nested SETTINGS clause cannot lift the row filter: it is dropped and the query still works" +${RESTRICTED} --query "SELECT secret FROM (SELECT * FROM ${TABLE} SETTINGS additional_table_filters = {'${CLICKHOUSE_DATABASE}.${TABLE}':'1'}) ORDER BY secret" + +echo "-- a readonly user can read a view whose inner query carries a SETTINGS clause" +${RESTRICTED} --query "SELECT c FROM ${VIEW}" + +echo "-- the session is still readonly" +${RESTRICTED} --query "CREATE TABLE ${CLICKHOUSE_DATABASE}.should_not_exist_05020 (x UInt64) ENGINE = MergeTree ORDER BY x" 2>&1 | grep -c -F "Cannot execute query in readonly mode" + +${CLICKHOUSE_CLIENT} --multiquery --query " +DROP TABLE ${CLICKHOUSE_DATABASE}.${TABLE}; +DROP VIEW ${CLICKHOUSE_DATABASE}.${VIEW}; +DROP USER ${USER}; +DROP SETTINGS PROFILE ${PROFILE}; +" diff --git a/tests/queries/0_stateless/05023_join_equi_key_pushdown_session_timezone.reference b/tests/queries/0_stateless/05023_join_equi_key_pushdown_session_timezone.reference new file mode 100644 index 000000000000..e543bfe88d3c --- /dev/null +++ b/tests/queries/0_stateless/05023_join_equi_key_pushdown_session_timezone.reference @@ -0,0 +1,6 @@ +RIGHT JOIN USING, Date / DateTime keys, session_timezone: left MergeTree prunes granules +1 +RIGHT JOIN USING, Date / DateTime keys, session_timezone: the row stays matched +2023-06-01 00:00:00 2023-06-01 00:00:00 +RIGHT JOIN USING, Date / DateTime keys, session_timezone: unmatched right row preserved +2023-06-01 12:34:56 diff --git a/tests/queries/0_stateless/05023_join_equi_key_pushdown_session_timezone.sql b/tests/queries/0_stateless/05023_join_equi_key_pushdown_session_timezone.sql new file mode 100644 index 000000000000..9cc6e5da0b48 --- /dev/null +++ b/tests/queries/0_stateless/05023_join_equi_key_pushdown_session_timezone.sql @@ -0,0 +1,48 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: the granule assertions describe the local `MergeTree` read, which parallel +-- replicas replace, and the `RIGHT JOIN` shapes below hit the unrelated logical error of +-- https://github.com/ClickHouse/ClickHouse/issues/113292 there. + +-- The cross-type equi-key substitution of the `RIGHT JOIN` pushdown replaces the `USING` key with +-- `CAST(, supertype)`. For a `Date` key joined to a `DateTime` key that conversion is +-- time-zone-dependent: a `Date` becomes midnight of the effective time zone. The pushed-down +-- predicate must evaluate it exactly like the `JOIN` output column does, or the left input would be +-- pruned on the wrong range and the matched row below would come back with a defaulted left side. +-- `Pacific/Apia` is 13-14 hours away from UTC, so a predicate cast that disagreed with the `JOIN` +-- cast could not land on the same midnights. +-- +-- The `Date` range stays below 2106: past the `DateTime` overflow the statistics-based part pruning +-- discards the part wrongly, which is the unrelated defect of +-- https://github.com/ClickHouse/ClickHouse/issues/111759. + +SET enable_analyzer = 1; +SET query_plan_filter_push_down = 1; +SET query_plan_join_swap_table = 'false'; +SET enable_join_runtime_filters = 0; +SET session_timezone = 'Pacific/Apia'; + +DROP TABLE IF EXISTS mt_date; +CREATE TABLE mt_date (k Date) ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 256, index_granularity_bytes = '10Mi'; +INSERT INTO mt_date SELECT toDate('2020-01-01') + number FROM numbers(2000); + +SELECT 'RIGHT JOIN USING, Date / DateTime keys, session_timezone: left MergeTree prunes granules'; +SELECT count() > 0 FROM ( + EXPLAIN PLAN indexes = 1 + SELECT k FROM mt_date AS l RIGHT JOIN (SELECT toDateTime('2023-06-01 00:00:00') AS k) AS r USING (k) + WHERE k = toDateTime('2023-06-01 00:00:00') +) WHERE explain LIKE '%Granules: 1/%'; + +-- `l.k` pins that the row is truly matched: a wrongly pruned left input would still return the right +-- row, but with the defaulted left side. +SELECT 'RIGHT JOIN USING, Date / DateTime keys, session_timezone: the row stays matched'; +SELECT k, l.k FROM mt_date AS l RIGHT JOIN (SELECT toDateTime('2023-06-01 00:00:00') AS k) AS r USING (k) +WHERE k = toDateTime('2023-06-01 00:00:00'); + +-- A right row at a non-midnight instant matches no left `Date`: it must survive as an unmatched row, +-- however the left input was pruned. +SELECT 'RIGHT JOIN USING, Date / DateTime keys, session_timezone: unmatched right row preserved'; +SELECT k FROM mt_date AS l RIGHT JOIN (SELECT toDateTime('2023-06-01 12:34:56') AS k) AS r USING (k) +WHERE k >= toDateTime('2023-06-01 00:00:00'); + +DROP TABLE mt_date; diff --git a/tests/queries/0_stateless/05023_query_thread_log_progress_per_query.reference b/tests/queries/0_stateless/05023_query_thread_log_progress_per_query.reference new file mode 100644 index 000000000000..ed1a7dbe2913 --- /dev/null +++ b/tests/queries/0_stateless/05023_query_thread_log_progress_per_query.reference @@ -0,0 +1,8 @@ +SELECT arm: queries finished, distinct threads, distinct read_rows values, rows per query +4 1 1 4 +INSERT arm: queries finished, distinct threads, distinct written_rows values, rows per query +4 1 1 4 +SELECT arm: read_rows vs ProfileEvents, and vs query_log +0 0 0 +INSERT arm: written_rows and written_bytes vs ProfileEvents, and vs query_log +0 0 0 diff --git a/tests/queries/0_stateless/05023_query_thread_log_progress_per_query.sh b/tests/queries/0_stateless/05023_query_thread_log_progress_per_query.sh new file mode 100755 index 000000000000..33857ce66fb4 --- /dev/null +++ b/tests/queries/0_stateless/05023_query_thread_log_progress_per_query.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel-replicas + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The queries go over HTTP because the value is only observable on a thread that both initiates the +# query and performs the reads. +# Parallel replicas move the reads off the initiating thread the same way, hence the no-parallel-replicas +# tag. The four queries of an arm are sent as one curl invocation with --next, so they share a +# single keep-alive connection and therefore a single handler thread. +URL="${CLICKHOUSE_URL}&log_queries=1&log_query_threads=1&log_profile_events=1" + +${CLICKHOUSE_CLIENT} -q " + DROP TABLE IF EXISTS t_progress_src; + DROP TABLE IF EXISTS t_progress_dst; + CREATE TABLE t_progress_src (id UInt64) ENGINE = MergeTree ORDER BY id; + CREATE TABLE t_progress_dst (id UInt64) ENGINE = MergeTree ORDER BY id; + INSERT INTO t_progress_src SELECT number FROM numbers(200000); +" + +select_args=() +for i in 1 2 3 4; do + [ "$i" -gt 1 ] && select_args+=(--next) + select_args+=("${URL}&query_id=${CLICKHOUSE_DATABASE}_sel_$i" + --data-binary "SELECT sum(id) FROM t_progress_src SETTINGS max_threads = 1") +done +${CLICKHOUSE_CURL} -sSg "${select_args[@]}" > /dev/null + +insert_args=() +for i in 1 2 3 4; do + [ "$i" -gt 1 ] && insert_args+=(--next) + insert_args+=("${URL}&query_id=${CLICKHOUSE_DATABASE}_ins_$i" + --data-binary "INSERT INTO t_progress_dst SELECT number FROM numbers(50000) SETTINGS max_insert_threads = 1, max_threads = 1") +done +${CLICKHOUSE_CURL} -sSg "${insert_args[@]}" > /dev/null + +# The query_log entry is written after the HTTP response is sent, so flushing once can race the +# last query. Retry until all eight queries have landed in both tables. +for _ in $(seq 1 60); do + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS query_log, query_thread_log" + landed=$(${CLICKHOUSE_CLIENT} -q " + SELECT uniqExactIf(query_id, t = 'ql') = 8 AND uniqExactIf(query_id, t = 'qtl') = 8 + FROM ( + SELECT 'ql' AS t, query_id FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' + AND query_id LIKE '${CLICKHOUSE_DATABASE}\_%' + UNION ALL + SELECT 'qtl' AS t, query_id FROM system.query_thread_log + WHERE current_database = currentDatabase() AND thread_id = master_thread_id + AND query_id LIKE '${CLICKHOUSE_DATABASE}\_%' + ) + ") + [ "$landed" = "1" ] && break + sleep 1 +done + +# Precondition, checked rather than assumed: four successful queries, one reused handler thread. +# Without it the equalities below could hold trivially and the test would assert nothing. The last +# column is a positivity floor: it pins the thread-level value to the per-query amount, so a row +# whose initiating thread read nothing cannot satisfy the equalities. The exact value is safe to +# pin only alongside uniqExact, which stays 1 for a per-query value and grows for a running total. +echo 'SELECT arm: queries finished, distinct threads, distinct read_rows values, rows per query' +${CLICKHOUSE_CLIENT} -q " + SELECT + countIf(ql.type = 'QueryFinish' AND ql.read_rows = 200000), + uniqExact(qtl.thread_id), + uniqExact(qtl.read_rows), + countIf(ql.type = 'QueryFinish' AND qtl.read_rows = 200000 AND qtl.read_bytes > 0) + FROM system.query_thread_log qtl + JOIN system.query_log ql ON ql.query_id = qtl.query_id + WHERE qtl.current_database = currentDatabase() + AND qtl.query_id LIKE '${CLICKHOUSE_DATABASE}_sel_%' AND qtl.thread_id = qtl.master_thread_id +" + +echo 'INSERT arm: queries finished, distinct threads, distinct written_rows values, rows per query' +${CLICKHOUSE_CLIENT} -q " + SELECT + countIf(ql.type = 'QueryFinish' AND ql.written_rows = 50000), + uniqExact(qtl.thread_id), + uniqExact(qtl.written_rows), + countIf(ql.type = 'QueryFinish' AND qtl.written_rows = 50000 AND qtl.written_bytes > 0) + FROM system.query_thread_log qtl + JOIN system.query_log ql ON ql.query_id = qtl.query_id + WHERE qtl.current_database = currentDatabase() + AND qtl.query_id LIKE '${CLICKHOUSE_DATABASE}_ins_%' AND qtl.thread_id = qtl.master_thread_id +" + +# The same row's ProfileEvents come from the performance counters, which are reset per attach, so +# they are the reference the progress columns of that row must agree with. +echo 'SELECT arm: read_rows vs ProfileEvents, and vs query_log' +${CLICKHOUSE_CLIENT} -q " + SELECT + countIf(qtl.read_rows != CAST(qtl.ProfileEvents, 'Map(String, UInt64)')['SelectedRows']), + countIf(qtl.read_bytes != CAST(qtl.ProfileEvents, 'Map(String, UInt64)')['SelectedBytes']), + countIf(qtl.read_rows > ql.read_rows) + FROM system.query_thread_log qtl + JOIN system.query_log ql ON ql.query_id = qtl.query_id + WHERE qtl.current_database = currentDatabase() + AND qtl.query_id LIKE '${CLICKHOUSE_DATABASE}_sel_%' AND qtl.thread_id = qtl.master_thread_id + AND ql.type = 'QueryFinish' +" + +echo 'INSERT arm: written_rows and written_bytes vs ProfileEvents, and vs query_log' +${CLICKHOUSE_CLIENT} -q " + SELECT + countIf(qtl.written_rows != CAST(qtl.ProfileEvents, 'Map(String, UInt64)')['InsertedRows']), + countIf(qtl.written_bytes != CAST(qtl.ProfileEvents, 'Map(String, UInt64)')['InsertedBytes']), + countIf(qtl.written_rows > ql.written_rows) + FROM system.query_thread_log qtl + JOIN system.query_log ql ON ql.query_id = qtl.query_id + WHERE qtl.current_database = currentDatabase() + AND qtl.query_id LIKE '${CLICKHOUSE_DATABASE}_ins_%' AND qtl.thread_id = qtl.master_thread_id + AND ql.type = 'QueryFinish' +" + +${CLICKHOUSE_CLIENT} -q " + DROP TABLE t_progress_src; + DROP TABLE t_progress_dst; +" diff --git a/tests/queries/0_stateless/05023_set_default_bypasses_settings_constraints.reference b/tests/queries/0_stateless/05023_set_default_bypasses_settings_constraints.reference new file mode 100644 index 000000000000..a3abad305446 --- /dev/null +++ b/tests/queries/0_stateless/05023_set_default_bypasses_settings_constraints.reference @@ -0,0 +1,18 @@ +MAX constraint +SETTING_CONSTRAINT_VIOLATION +SETTING_CONSTRAINT_VIOLATION +SETTING_CONSTRAINT_VIOLATION +1000 +CONST constraint +SETTING_CONSTRAINT_VIOLATION +SETTING_CONSTRAINT_VIOLATION +readonly mode +Cannot modify 'readonly' setting in readonly mode +Cannot modify 'readonly' setting in readonly mode +Cannot modify 'readonly' setting in readonly mode +a reset that changes nothing is still allowed +allowed +settings without a declared default are unaffected +custom setting reset +Cannot modify 'SQL_probe_05023' setting in readonly mode +unknown setting ignored diff --git a/tests/queries/0_stateless/05023_set_default_bypasses_settings_constraints.sh b/tests/queries/0_stateless/05023_set_default_bypasses_settings_constraints.sh new file mode 100755 index 000000000000..a4aa146e8a77 --- /dev/null +++ b/tests/queries/0_stateless/05023_set_default_bypasses_settings_constraints.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# `SET = DEFAULT` used to bypass the settings constraints completely: a reset is not a setting +# change, so it never reached `SettingsConstraints`. With `max_query_size = 1000 MAX 1000` in the +# profile, `SET max_query_size = 2000` was rejected while `SET max_query_size = DEFAULT` silently +# restored the much larger built-in default. A `CONST` constraint was escapable the same way, and so +# was readonly mode: `SET readonly = 0` was rejected but `SET readonly = DEFAULT` left readonly mode. +# +# A reset that does not change the value stays allowed, so that resetting an untouched setting keeps +# working under a `CONST` constraint or in readonly mode. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_MAX="u_max_05023_${CLICKHOUSE_DATABASE}" +USER_CONST="u_const_05023_${CLICKHOUSE_DATABASE}" +USER_NOOP="u_noop_05023_${CLICKHOUSE_DATABASE}" +PROFILE_MAX="p_max_05023_${CLICKHOUSE_DATABASE}" +PROFILE_CONST="p_const_05023_${CLICKHOUSE_DATABASE}" +PROFILE_NOOP="p_noop_05023_${CLICKHOUSE_DATABASE}" + +# `max_query_size` is used throughout because its declared default (262144) is far above the maximum +# the profiles below allow, and because the test runner never randomizes it. +DEFAULT_MAX_QUERY_SIZE=$(${CLICKHOUSE_CLIENT} -q "SELECT default FROM system.settings WHERE name = 'max_query_size'") + +${CLICKHOUSE_CLIENT} -q "DROP USER IF EXISTS ${USER_MAX}, ${USER_CONST}, ${USER_NOOP}" +${CLICKHOUSE_CLIENT} -q "DROP PROFILE IF EXISTS ${PROFILE_MAX}, ${PROFILE_CONST}, ${PROFILE_NOOP}" +${CLICKHOUSE_CLIENT} -q "CREATE SETTINGS PROFILE ${PROFILE_MAX} SETTINGS max_query_size = 1000 MAX 1000" +${CLICKHOUSE_CLIENT} -q "CREATE SETTINGS PROFILE ${PROFILE_CONST} SETTINGS max_query_size = 1000 CONST" +${CLICKHOUSE_CLIENT} -q "CREATE SETTINGS PROFILE ${PROFILE_NOOP} SETTINGS max_query_size = ${DEFAULT_MAX_QUERY_SIZE} CONST" +${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER_MAX} SETTINGS PROFILE '${PROFILE_MAX}'" +${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER_CONST} SETTINGS PROFILE '${PROFILE_CONST}'" +${CLICKHOUSE_CLIENT} -q "CREATE USER ${USER_NOOP} SETTINGS PROFILE '${PROFILE_NOOP}'" + +echo 'MAX constraint' +# Assigning a value above the maximum was always rejected +${CLICKHOUSE_CLIENT} --user="${USER_MAX}" -q "SET max_query_size = 2000" 2>&1 | grep -o 'SETTING_CONSTRAINT_VIOLATION' | head -1 +# Resetting to the (larger) default has to be rejected as well, in both spellings +${CLICKHOUSE_CLIENT} --user="${USER_MAX}" -q "SET max_query_size = DEFAULT" 2>&1 | grep -o 'SETTING_CONSTRAINT_VIOLATION' | head -1 +${CLICKHOUSE_CLIENT} --user="${USER_MAX}" -q "SELECT 1 SETTINGS max_query_size = DEFAULT" 2>&1 | grep -o 'SETTING_CONSTRAINT_VIOLATION' | head -1 +# and it must leave the value alone +${CLICKHOUSE_CLIENT} --user="${USER_MAX}" -q "SELECT getSetting('max_query_size')" + +echo 'CONST constraint' +${CLICKHOUSE_CLIENT} --user="${USER_CONST}" -q "SET max_query_size = DEFAULT" 2>&1 | grep -o 'SETTING_CONSTRAINT_VIOLATION' | head -1 +${CLICKHOUSE_CLIENT} --user="${USER_CONST}" -q "SELECT 1 SETTINGS max_query_size = DEFAULT" 2>&1 | grep -o 'SETTING_CONSTRAINT_VIOLATION' | head -1 + +echo 'readonly mode' +# `readonly = 2` keeps every other setting changeable, so the settings the test runner randomizes do +# not run into the readonly check themselves and the error below can only come from `readonly`. +# Assigning to `readonly` was always rejected... +${CLICKHOUSE_CLIENT} -q "SET readonly = 2; SET readonly = 0" 2>&1 | grep -o "Cannot modify 'readonly' setting in readonly mode" | head -1 +# ...and so must resetting it, which used to be a way out of readonly mode +${CLICKHOUSE_CLIENT} -q "SET readonly = 2; SET readonly = DEFAULT" 2>&1 | grep -o "Cannot modify 'readonly' setting in readonly mode" | head -1 +${CLICKHOUSE_CLIENT} -q "SET readonly = 2; SELECT 1 SETTINGS readonly = DEFAULT" 2>&1 | grep -o "Cannot modify 'readonly' setting in readonly mode" | head -1 + +echo 'a reset that changes nothing is still allowed' +# The profile pins `max_query_size` to its declared default and makes it CONST, so the reset is a +# no-op and must not be reported +${CLICKHOUSE_CLIENT} --user="${USER_NOOP}" -q "SET max_query_size = DEFAULT; SELECT 'allowed'" + +echo 'settings without a declared default are unaffected' +# A custom setting is dropped rather than reset, and cannot carry a value constraint. +${CLICKHOUSE_CLIENT} -q "SET SQL_probe_05023 = 1; SET SQL_probe_05023 = DEFAULT; SELECT 'custom setting reset'" +# It is still a session-state mutation, so readonly mode must reject the reset. Unlike `readonly = 2` +# above, `readonly = 1` rejects every setting change, including the ones the client and the test +# runner send along with each statement (`send_logs_level`, `log_comment`, the randomized settings), +# which would be rejected before the reset under test. The check therefore goes over HTTP with a +# session and a bare URL that carries nothing but the session id. +SESSION="s_05023_${CLICKHOUSE_DATABASE}_$$" +SESSION_URL="${CLICKHOUSE_URL%%\?*}?session_id=${SESSION}" +${CLICKHOUSE_CURL} -sS "${SESSION_URL}" -d "SET SQL_probe_05023 = 1" +${CLICKHOUSE_CURL} -sS "${SESSION_URL}" -d "SET readonly = 1" +${CLICKHOUSE_CURL} -sS "${SESSION_URL}" -d "SET SQL_probe_05023 = DEFAULT" 2>&1 | grep -o "Cannot modify 'SQL_probe_05023' setting in readonly mode" | head -1 +# An unknown setting is still silently ignored rather than reported +${CLICKHOUSE_CLIENT} -q "SET nonexistent_setting_05023 = DEFAULT; SELECT 'unknown setting ignored'" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${USER_MAX}, ${USER_CONST}, ${USER_NOOP}" +${CLICKHOUSE_CLIENT} -q "DROP PROFILE ${PROFILE_MAX}, ${PROFILE_CONST}, ${PROFILE_NOOP}" diff --git a/tests/queries/0_stateless/05023_variant_arena_deserialize_bad_discriminator.reference b/tests/queries/0_stateless/05023_variant_arena_deserialize_bad_discriminator.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/05023_variant_arena_deserialize_bad_discriminator.sql b/tests/queries/0_stateless/05023_variant_arena_deserialize_bad_discriminator.sql new file mode 100644 index 000000000000..5553e828a890 --- /dev/null +++ b/tests/queries/0_stateless/05023_variant_arena_deserialize_bad_discriminator.sql @@ -0,0 +1,3 @@ +SELECT groupArrayDistinctMerge( + CAST(unhex('010110') AS AggregateFunction(groupArrayDistinct, Variant(UInt8, String))) +); -- { serverError INCORRECT_DATA } diff --git a/tests/queries/0_stateless/05024_binary_string_literal_uninitialized_byte.reference b/tests/queries/0_stateless/05024_binary_string_literal_uninitialized_byte.reference new file mode 100644 index 000000000000..5388d4505bca --- /dev/null +++ b/tests/queries/0_stateless/05024_binary_string_literal_uninitialized_byte.reference @@ -0,0 +1,6 @@ +01 +1 +01 +0101 +01 +0001 diff --git a/tests/queries/0_stateless/05024_binary_string_literal_uninitialized_byte.sql b/tests/queries/0_stateless/05024_binary_string_literal_uninitialized_byte.sql new file mode 100644 index 000000000000..7fe76d3ef33f --- /dev/null +++ b/tests/queries/0_stateless/05024_binary_string_literal_uninitialized_byte.sql @@ -0,0 +1,10 @@ +-- The buffer of a binary string literal is allocated for the worst case: ceil(bits / 8) bytes. +-- When the number of bits is not a multiple of eight, fewer bytes are written, and the rest of +-- the buffer must not end up in the result. + +SELECT hex(b'000000001'); +SELECT length(b'000000001'); +SELECT hex(b'1'); +SELECT hex(b'100000001'); +SELECT hex(b'00000001'); +SELECT hex(x'0001'); diff --git a/tests/queries/0_stateless/05024_keeper_map_parenthesized_metadata.reference b/tests/queries/0_stateless/05024_keeper_map_parenthesized_metadata.reference new file mode 100644 index 000000000000..d066b3c9e536 --- /dev/null +++ b/tests/queries/0_stateless/05024_keeper_map_parenthesized_metadata.reference @@ -0,0 +1,8 @@ +1 +1 +1 +1 +1 +1 value +1 +1 diff --git a/tests/queries/0_stateless/05024_keeper_map_parenthesized_metadata.sql b/tests/queries/0_stateless/05024_keeper_map_parenthesized_metadata.sql new file mode 100644 index 000000000000..5ad11c78a0f7 --- /dev/null +++ b/tests/queries/0_stateless/05024_keeper_map_parenthesized_metadata.sql @@ -0,0 +1,133 @@ +-- Tags: zookeeper, no-ordinary-database, no-fasttest + +-- The test rewrites a shared `metadata` znode with `replaceOne`. The stress test profile enables the +-- server-side AST fuzzer for every query type, and a permuted argument list makes the replacement +-- literal the haystack, so the znode is left holding that bare literal instead of a full definition. +SET ast_fuzzer_runs = 0; +SET ast_fuzzer_any_query = 0; + +-- Every assertion below reads the state a preceding statement left in Keeper, so a DROP has to drop. +SET ignore_drop_queries_probability = 0; + +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_bad SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_malformed SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_terminated SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_second SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_reverse SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_delimiter_second SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_delimiter SYNC; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata (key UInt64, value String) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata') +PRIMARY KEY key; + +SELECT endsWith(value, 'primary key: key\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_reverse (key UInt64, value String) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata') +PRIMARY KEY(key); + +SELECT endsWith(value, 'primary key: key\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +INSERT INTO system.zookeeper (path, name, value) +SELECT path, name, replaceOne(value, 'primary key: key\n', 'primary key: (key)\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +SELECT endsWith(value, 'primary key: (key)\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_second +( + key UInt64 COMMENT 'comment added later', + value String +) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata') +PRIMARY KEY key; + +SELECT endsWith(value, 'primary key: (key)\n') AND position(value, 'comment added later') = 0 +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +DETACH TABLE 05024_keeper_map_parenthesized_metadata; +ATTACH TABLE 05024_keeper_map_parenthesized_metadata; + +SELECT endsWith(value, 'primary key: (key)\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +INSERT INTO 05024_keeper_map_parenthesized_metadata VALUES (1, 'value'); +SELECT * FROM 05024_keeper_map_parenthesized_metadata_second; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_bad (key UInt64, value String) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata') +PRIMARY KEY value; -- { serverError BAD_ARGUMENTS } + +INSERT INTO system.zookeeper (path, name, value) +SELECT path, name, replaceOne(value, 'primary key: (key)\n', 'primary key: (key\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +SELECT endsWith(value, 'primary key: (key\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_malformed (key UInt64, value String) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata') +PRIMARY KEY key; -- { serverError BAD_ARGUMENTS } + +INSERT INTO system.zookeeper (path, name, value) +SELECT path, name, replaceOne(value, 'primary key: (key\n', 'primary key: key;\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +SELECT endsWith(value, 'primary key: key;\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_terminated (key UInt64, value String) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata') +PRIMARY KEY key; -- { serverError BAD_ARGUMENTS } + +INSERT INTO system.zookeeper (path, name, value) +SELECT path, name, replaceOne(value, 'primary key: key;\n', 'primary key: (key)\n') +FROM system.zookeeper +WHERE path = '/test_keeper_map/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata' + AND name = 'metadata'; + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_delimiter (key UInt64, value String) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata_delimiter') +PRIMARY KEY sipHash64(concat(toString(key), 'primary key: ')); + +CREATE TABLE 05024_keeper_map_parenthesized_metadata_delimiter_second +( + key UInt64 COMMENT 'comment added later', + value String +) +ENGINE = KeeperMap('/' || currentDatabase() || '/05024_keeper_map_parenthesized_metadata_delimiter') +PRIMARY KEY sipHash64(concat(toString(key), 'primary key: ')); + +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_bad SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_malformed SYNC; +DROP TABLE IF EXISTS 05024_keeper_map_parenthesized_metadata_terminated SYNC; +DROP TABLE 05024_keeper_map_parenthesized_metadata_second SYNC; +DROP TABLE 05024_keeper_map_parenthesized_metadata_reverse SYNC; +DROP TABLE 05024_keeper_map_parenthesized_metadata SYNC; +DROP TABLE 05024_keeper_map_parenthesized_metadata_delimiter_second SYNC; +DROP TABLE 05024_keeper_map_parenthesized_metadata_delimiter SYNC; diff --git a/tests/queries/0_stateless/05027_read_wkb_deep_nesting.reference b/tests/queries/0_stateless/05027_read_wkb_deep_nesting.reference new file mode 100644 index 000000000000..330322843671 --- /dev/null +++ b/tests/queries/0_stateless/05027_read_wkb_deep_nesting.reference @@ -0,0 +1 @@ +(0,0) diff --git a/tests/queries/0_stateless/05027_read_wkb_deep_nesting.sql b/tests/queries/0_stateless/05027_read_wkb_deep_nesting.sql new file mode 100644 index 000000000000..3a5004bf62ca --- /dev/null +++ b/tests/queries/0_stateless/05027_read_wkb_deep_nesting.sql @@ -0,0 +1,10 @@ +-- Multi* geometries nest into each other, and the element count limit does not bound the nesting +-- depth, because every level may hold a single element. + +-- Unlike current master, 26.6 does not support the `MultiPoint` geometry (type 4) in `readWKB`, +-- so only `MultiLineString` (type 5) and `MultiPolygon` (type 6) are checked here. + +SELECT readWKB(unhex(repeat('010500000001000000', 100000))); -- { serverError TOO_DEEP_RECURSION } +SELECT readWKB(unhex(repeat('010600000001000000', 100000))); -- { serverError TOO_DEEP_RECURSION } + +SELECT readWKB(unhex('010100000000000000000000000000000000000000')); diff --git a/tests/queries/0_stateless/05028_json_merge_patch_deep_nesting.reference b/tests/queries/0_stateless/05028_json_merge_patch_deep_nesting.reference new file mode 100644 index 000000000000..49bf00e08c1c --- /dev/null +++ b/tests/queries/0_stateless/05028_json_merge_patch_deep_nesting.reference @@ -0,0 +1 @@ +{"a":{"b":1,"c":2}} diff --git a/tests/queries/0_stateless/05028_json_merge_patch_deep_nesting.sql b/tests/queries/0_stateless/05028_json_merge_patch_deep_nesting.sql new file mode 100644 index 000000000000..15b1c11dc926 --- /dev/null +++ b/tests/queries/0_stateless/05028_json_merge_patch_deep_nesting.sql @@ -0,0 +1,9 @@ +-- Tags: no-fasttest +-- Reason: needs RapidJSON, which is not enabled in the fast test build. + +-- The JSON parser is iterative, but merging and serializing the documents recurse over the tree. + +SELECT length(JSONMergePatch('{}', concat(repeat('{"a":', 100000), '1', repeat('}', 100000)))); -- { serverError TOO_DEEP_RECURSION } +SELECT length(JSONMergePatch(concat(repeat('{"a":', 100000), '1', repeat('}', 100000)), '{}')); -- { serverError TOO_DEEP_RECURSION } + +SELECT JSONMergePatch('{"a":{"b":1}}', '{"a":{"c":2}}'); diff --git a/tests/queries/0_stateless/05028_lz4_empty_compressed_body.reference b/tests/queries/0_stateless/05028_lz4_empty_compressed_body.reference new file mode 100644 index 000000000000..c34302cbc7ef --- /dev/null +++ b/tests/queries/0_stateless/05028_lz4_empty_compressed_body.reference @@ -0,0 +1,2 @@ +Cannot decompress LZ4-encoded data +Ok. diff --git a/tests/queries/0_stateless/05028_lz4_empty_compressed_body.sh b/tests/queries/0_stateless/05028_lz4_empty_compressed_body.sh new file mode 100755 index 000000000000..20ef52e589d5 --- /dev/null +++ b/tests/queries/0_stateless/05028_lz4_empty_compressed_body.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A compressed block with an empty body: the compressed size is exactly the size of the header, while +# the uncompressed size is not zero. The decompressor has nothing to read from, so it must fail +# instead of reporting success and handing out the previous contents of the destination buffer. +# +# The layout of the block is: 16 bytes of the checksum (not verified here), 1 byte of the method +# (0x82 is LZ4), 4 bytes of the compressed size including the 9 bytes of the header, and 4 bytes of +# the uncompressed size. The numbers are little endian. + +echo -ne '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x82\x09\x00\x00\x00\x10\x00\x00\x00' | + ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&decompress=1&http_native_compression_disable_checksumming_on_decompress=1" --data-binary @- 2>&1 | + grep -oF 'Cannot decompress LZ4-encoded data' + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary "SELECT 'Ok.'" diff --git a/tests/queries/0_stateless/05029_remote_description_deep_braces.reference b/tests/queries/0_stateless/05029_remote_description_deep_braces.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/05029_remote_description_deep_braces.sql b/tests/queries/0_stateless/05029_remote_description_deep_braces.sql new file mode 100644 index 000000000000..e46fb4bf750f --- /dev/null +++ b/tests/queries/0_stateless/05029_remote_description_deep_braces.sql @@ -0,0 +1,4 @@ +-- Nested braces in the address of a table function are expanded recursively, and the limit on the +-- number of generated addresses does not bound the nesting depth. + +SELECT * FROM url(concat(repeat('{', 100000), ',', repeat('}', 100000))); -- { serverError TOO_DEEP_RECURSION } diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference new file mode 100644 index 000000000000..dff69852207d --- /dev/null +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference @@ -0,0 +1,13 @@ +--- default: aws_external_id hidden, role identifiers visible +aws_external_id = '[HIDDEN]' +aws_access_key_id = '[HIDDEN]' +aws_secret_access_key = '[HIDDEN]' +aws_role_arn = 'arn:aws:iam::1:role/r' +aws_role_session_name = 'sess' +OK: no secret in formatted query +--- show_secrets: aws_external_id visible +aws_external_id = 'SECRET_THAT_MUST_NOT_LEAK' +aws_access_key_id = 'SECRET_THAT_MUST_NOT_LEAK' +aws_secret_access_key = 'SECRET_THAT_MUST_NOT_LEAK' +aws_role_arn = 'arn:aws:iam::1:role/r' +aws_role_session_name = 'sess' diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh new file mode 100755 index 000000000000..f33348fb92d4 --- /dev/null +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, so it must be +# redacted as [HIDDEN] when a DataLakeCatalog CREATE query is formatted (system.databases.engine_full, +# SHOW CREATE DATABASE), while aws_role_arn and aws_role_session_name are non-secret identifiers that +# stay visible. Uses clickhouse-format so it needs no live catalog and is safe to run in parallel. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SECRET="SECRET_THAT_MUST_NOT_LEAK" + +query="CREATE DATABASE d ENGINE = DataLakeCatalog('http://example.invalid/catalog') SETTINGS +catalog_type = 'glue', +region = 'us-east-1', +aws_access_key_id = '${SECRET}', +aws_secret_access_key = '${SECRET}', +aws_external_id = '${SECRET}', +aws_role_arn = 'arn:aws:iam::1:role/r', +aws_role_session_name = 'sess'" + +show_settings() { + local formatted="$1" + for setting in \ + aws_external_id \ + aws_access_key_id \ + aws_secret_access_key \ + aws_role_arn \ + aws_role_session_name + do + echo "$formatted" | grep -oE "(^|[, ])${setting} = '[^']*'" | sed -E "s/^[, ]//" + done +} + +# Arm A: at the default the secret is redacted; arms C (non-secret identifiers stay visible) and +# D (sibling AWS keys still redacted) are asserted by the same output. +echo "--- default: aws_external_id hidden, role identifiers visible" +formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline) +show_settings "$formatted" +if echo "$formatted" | grep -q "$SECRET"; then + echo "FAIL: secret leaked in formatted query" +else + echo "OK: no secret in formatted query" +fi + +# Arm B: an authorized caller can still retrieve the value, so the fix redacts rather than destroys. +echo "--- show_secrets: aws_external_id visible" +formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline --show_secrets) +show_settings "$formatted" diff --git a/tests/queries/0_stateless/05030_ml_method_malformed_state.reference b/tests/queries/0_stateless/05030_ml_method_malformed_state.reference new file mode 100644 index 000000000000..7a0beed21a5e --- /dev/null +++ b/tests/queries/0_stateless/05030_ml_method_malformed_state.reference @@ -0,0 +1 @@ +[0,0] diff --git a/tests/queries/0_stateless/05030_ml_method_malformed_state.sql b/tests/queries/0_stateless/05030_ml_method_malformed_state.sql new file mode 100644 index 000000000000..df796507d64a --- /dev/null +++ b/tests/queries/0_stateless/05030_ml_method_malformed_state.sql @@ -0,0 +1,28 @@ +-- A state of a machine learning aggregate function is deserialized from the data, so the number of +-- weights it holds must agree both with the gradient it holds and with the number of features of +-- the type; otherwise the prediction reads past the end of the weights. + +-- 8 bytes of bias, one weight, iteration number, an empty gradient and the batch size. +SELECT CAST(unhex('00000000000000000100000000000000000000000000000000000000000000000000') + AS AggregateFunction(stochasticLogisticRegression(0.1, 0, 1, 'SGD'), Float64, Float64, Float64, Float64)); -- { serverError INCORRECT_DATA } + +-- The same, with a gradient of two values, so the state itself is consistent, but it declares one +-- weight while the type declares three features. +WITH CAST(unhex('0000000000000000010000000000000000000000000000000002000000000000000000000000000000000000000000000000') + AS AggregateFunction(stochasticLogisticRegression(0.1, 0, 1, 'SGD'), Float64, Float64, Float64, Float64)) AS state +SELECT evalMLMethod(state, toFloat64(1), toFloat64(1), toFloat64(1)); -- { serverError INCORRECT_DATA } + +-- The weights updaters keep their own vectors of the gradient size, so they are validated in the +-- same way. Here the state itself is consistent (one weight, a gradient of two values), but the +-- `Momentum` updater holds a single accumulated gradient value instead of two. +SELECT CAST(unhex('0000000000000000010000000000000000000000000000000002000000000000000000000000000000000000000000000000010000000000000000') + AS AggregateFunction(stochasticLinearRegression(0.1, 0, 1, 'Momentum'), Float64, Float64)); -- { serverError INCORRECT_DATA } + +-- The same for `Adam`, which holds two vectors: the average squared gradient is too short. +SELECT CAST(unhex('00000000000000000100000000000000000000000000000000020000000000000000000000000000000000000000000000000200000000000000000000000000000000010000000000000000') + AS AggregateFunction(stochasticLinearRegression(0.1, 0, 1, 'Adam'), Float64, Float64)); -- { serverError INCORRECT_DATA } + +-- An empty updater vector is valid: versions before 23.2 serialized the vectors empty until the +-- first update. +SELECT finalizeAggregation(CAST(unhex('000000000000000001000000000000000000000000000000000200000000000000000000000000000000000000000000000000') + AS AggregateFunction(stochasticLinearRegression(0.1, 0, 1, 'Momentum'), Float64, Float64))); diff --git a/tests/queries/0_stateless/05031_resample_combinator_overflow.reference b/tests/queries/0_stateless/05031_resample_combinator_overflow.reference new file mode 100644 index 000000000000..f7bfd212974d --- /dev/null +++ b/tests/queries/0_stateless/05031_resample_combinator_overflow.reference @@ -0,0 +1,2 @@ +[1,1,1,1] +[[[[1,0],[0,0]],[[0,0],[0,0]]],[[[0,0],[0,0]],[[0,0],[0,1]]]] diff --git a/tests/queries/0_stateless/05031_resample_combinator_overflow.sql b/tests/queries/0_stateless/05031_resample_combinator_overflow.sql new file mode 100644 index 000000000000..9d5b1a251ea6 --- /dev/null +++ b/tests/queries/0_stateless/05031_resample_combinator_overflow.sql @@ -0,0 +1,14 @@ +-- Every layer of the `Resample` combinator is checked against the limit on the number of elements +-- on its own, but the sizes of the states multiply, so nested combinators can overflow it. + +SELECT countResampleIfResampleIfResampleIfResample(0, 1048576, 1, 0, 1048576, 1, 0, 1048576, 1, 0, 1048576, 1)(number, 1, number, 1, number, 1, number) +FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } + +SELECT countResample(0, 4, 1)(number) FROM numbers(4); +SELECT countResampleIfResampleIfResampleIfResample(0, 2, 1, 0, 2, 1, 0, 2, 1, 0, 2, 1)(number, 1, number, 1, number, 1, number) FROM numbers(2); + +-- Found by the AST fuzzer: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=115701&sha=7ba562b7636f1a8219850caaaed32d51ba64519f&name_0=PR&name_1=AST%20fuzzer%20%28amd_debug%2C%20targeted%29 +-- Here the product of the sizes does not overflow (it is exactly 2^63 bytes), but such a size +-- is treated as a logical error by the allocator, so it must be cut off by the sanity threshold. +SELECT countResampleIfResampleIfResampleIfResample(0, 1048576, 1, 0, 1, 1, 0, 1048576, 1, 0, 1048576, 1)(number, 1, number, 1, number, 1, number) +FROM numbers(1); -- { serverError ARGUMENT_OUT_OF_BOUND } diff --git a/tests/queries/0_stateless/05032_group_uniq_array_malformed_state.reference b/tests/queries/0_stateless/05032_group_uniq_array_malformed_state.reference new file mode 100644 index 000000000000..8e4773407422 --- /dev/null +++ b/tests/queries/0_stateless/05032_group_uniq_array_malformed_state.reference @@ -0,0 +1,2 @@ +[0,1,2] +[0,1,2] diff --git a/tests/queries/0_stateless/05032_group_uniq_array_malformed_state.sql b/tests/queries/0_stateless/05032_group_uniq_array_malformed_state.sql new file mode 100644 index 000000000000..64879b21aebd --- /dev/null +++ b/tests/queries/0_stateless/05032_group_uniq_array_malformed_state.sql @@ -0,0 +1,9 @@ +-- The elements of a state of `groupUniqArray` over a fixed-size type are inserted with +-- `insertData`, which reads the whole width of the value and ignores the length it is given, so a +-- shorter element of a crafted state would read past the end of the buffer. + +SELECT groupUniqArrayMerge(CAST(unhex('010141') AS AggregateFunction(groupUniqArray, Decimal256(0)))); -- { serverError INCORRECT_DATA } +SELECT groupUniqArrayMerge(CAST(unhex('02014120' || repeat('AA', 32)) AS AggregateFunction(groupUniqArray, Decimal256(0)))); -- { serverError INCORRECT_DATA } + +SELECT arraySort(groupUniqArrayMerge(state)) FROM (SELECT groupUniqArrayState(number % 3) AS state FROM numbers(10)); +SELECT arraySort(groupUniqArrayMerge(state)) FROM (SELECT groupUniqArrayState(toDecimal256(number % 3, 0)) AS state FROM numbers(10)); diff --git a/tests/queries/0_stateless/05034_capnproto_deep_schema.reference b/tests/queries/0_stateless/05034_capnproto_deep_schema.reference new file mode 100644 index 000000000000..43b86d38bcb6 --- /dev/null +++ b/tests/queries/0_stateless/05034_capnproto_deep_schema.reference @@ -0,0 +1,4 @@ +1 +data Array(Array(Int32)) +1 +data Tuple(\n data Array(Array(Int32))) diff --git a/tests/queries/0_stateless/05034_capnproto_deep_schema.sh b/tests/queries/0_stateless/05034_capnproto_deep_schema.sh new file mode 100755 index 000000000000..4c02f5b84c21 --- /dev/null +++ b/tests/queries/0_stateless/05034_capnproto_deep_schema.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The Cap'n Proto schema parser is recursive, so a deeply nested type expression used to exhaust +# the thread stack while the schema was being parsed. + +DEPTH=20000 +NESTED="$(python3 -c "print('List(' * $DEPTH + 'Int32' + ')' * $DEPTH)")" + +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') +SETTINGS + format_schema_source = 'string', + format_schema = '@0x844f048b15c12dab;\nstruct M { data @0 :${NESTED}; }', + format_schema_message_name = 'M' +" 2>&1 | grep -c -F 'nested too deeply' + +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') +SETTINGS + format_schema_source = 'string', + format_schema = '@0x844f048b15c12dab;\nstruct M { data @0 :List(List(Int32)); }', + format_schema_message_name = 'M' +" + +# The schema files that a schema `import`s are parsed by the same recursive parser, so a shallow +# entry schema must not be able to smuggle a deeply nested one in through an import. +# `format_schema` with a file must be given an absolute path, and the directory has to be unique +# per test run, because the flaky check runs this test many times concurrently. +SCHEMA_DIR="$(mktemp -d "${CLICKHOUSE_TMP}/05034_capnproto_schemas_XXXXXX")" +SCHEMA_DIR="$(cd "${SCHEMA_DIR}" && pwd)" + +python3 -c " +import sys +directory = sys.argv[1] +depth = int(sys.argv[2]) +with open(directory + '/deep.capnp', 'w') as f: + f.write('@0x844f048b15c12dac;\nstruct D { data @0 :' + 'List(' * depth + 'Int32' + ')' * depth + '; }\n') +with open(directory + '/imports_deep.capnp', 'w') as f: + f.write('@0x844f048b15c12dab;\nusing D = import \"deep.capnp\";\nstruct M { data @0 :D.D; }\n') +with open(directory + '/shallow.capnp', 'w') as f: + f.write('@0x844f048b15c12dad;\nstruct S { data @0 :List(List(Int32)); }\n') +with open(directory + '/imports_shallow.capnp', 'w') as f: + f.write('@0x844f048b15c12dae;\nusing S = import \"shallow.capnp\";\nstruct M { data @0 :S.S; }\n') +" "${SCHEMA_DIR}" "${DEPTH}" + +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/imports_deep.capnp:M' +" 2>&1 | grep -c -F 'nested too deeply' + +${CLICKHOUSE_LOCAL} --logger.console=0 --query " +DESC format(CapnProto, '') SETTINGS format_schema = '${SCHEMA_DIR}/imports_shallow.capnp:M' +" + +rm -rf "${SCHEMA_DIR}" diff --git a/tests/queries/0_stateless/05035_stream_cursor_deep_nesting.reference b/tests/queries/0_stateless/05035_stream_cursor_deep_nesting.reference new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/tests/queries/0_stateless/05035_stream_cursor_deep_nesting.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/05035_stream_cursor_deep_nesting.sql b/tests/queries/0_stateless/05035_stream_cursor_deep_nesting.sql new file mode 100644 index 000000000000..de12230ed09f --- /dev/null +++ b/tests/queries/0_stateless/05035_stream_cursor_deep_nesting.sql @@ -0,0 +1,8 @@ +-- The `CURSOR` clause of `STREAM` is parsed by a helper that recurses directly instead of going +-- through `IParserBase::parse`, so `max_parser_depth` was not in effect and a deeply nested cursor +-- exhausted the thread stack. 26.6 has no `parseQueryToJSON`, so `formatQuery` drives the parser instead. + +SELECT formatQuery(concat('SELECT * FROM t STREAM CURSOR ', repeat('{''a'': ', 100000), '10', repeat('}', 100000))) +SETTINGS max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +SELECT formatQuery('SELECT * FROM t STREAM CURSOR {''a'': {''b'': 10}}') IS NOT NULL; diff --git a/tests/queries/0_stateless/05036_postgresql_execute_non_literal_argument.reference b/tests/queries/0_stateless/05036_postgresql_execute_non_literal_argument.reference new file mode 100644 index 000000000000..61d4a87045c6 --- /dev/null +++ b/tests/queries/0_stateless/05036_postgresql_execute_non_literal_argument.reference @@ -0,0 +1,2 @@ +1 +alive diff --git a/tests/queries/0_stateless/05036_postgresql_execute_non_literal_argument.sh b/tests/queries/0_stateless/05036_postgresql_execute_non_literal_argument.sh new file mode 100755 index 000000000000..4a7c5d5b9a40 --- /dev/null +++ b/tests/queries/0_stateless/05036_postgresql_execute_non_literal_argument.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Requires postgresql-client + +# `ParserExecute` dereferenced the result of a cast to `ASTLiteral` without checking it, so an +# `EXECUTE` statement with an argument that is not a literal dereferenced a null pointer. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The user name must be unique per test run: the flaky check runs this test many times +# concurrently, and a global name would collide with `ACCESS_ENTITY_ALREADY_EXISTS`. +PG_USER="postgresql_user_05036_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -q " +DROP USER IF EXISTS ${PG_USER}; +CREATE USER ${PG_USER} HOST IP '127.0.0.1' IDENTIFIED WITH no_password; +" + +psql --host localhost --port "${CLICKHOUSE_PORT_POSTGRESQL}" "${CLICKHOUSE_DATABASE}" --user "${PG_USER}" --no-align 2>&1 <<'EOF' | grep -c -F 'Syntax error' +PREPARE p AS SELECT 1; +EXECUTE p(1 + 1); +EOF + +# The server must have survived the malformed statement. +${CLICKHOUSE_CLIENT} -q "SELECT 'alive'" + +${CLICKHOUSE_CLIENT} -q "DROP USER ${PG_USER}" diff --git a/tests/queries/0_stateless/05037_postgresql_startup_message_size.reference b/tests/queries/0_stateless/05037_postgresql_startup_message_size.reference new file mode 100644 index 000000000000..897232067187 --- /dev/null +++ b/tests/queries/0_stateless/05037_postgresql_startup_message_size.reference @@ -0,0 +1,3 @@ +huge declared size: ERROR: Can't correctly handle Startup message +unterminated parameter: ERROR: Can't correctly handle Startup message +well-formed message: authentication request diff --git a/tests/queries/0_stateless/05037_postgresql_startup_message_size.sh b/tests/queries/0_stateless/05037_postgresql_startup_message_size.sh new file mode 100755 index 000000000000..f6d2d315b9b1 --- /dev/null +++ b/tests/queries/0_stateless/05037_postgresql_startup_message_size.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the PostgreSQL compatibility port is not enabled in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The startup message is processed before authentication, so its size must be bounded, +# and parsing must not read past the declared size of the message. + +CLICKHOUSE_PORT_POSTGRESQL="$CLICKHOUSE_PORT_POSTGRESQL" python3 - <<'PYTHON' +import os +import socket +import struct + +port = int(os.environ["CLICKHOUSE_PORT_POSTGRESQL"]) + +def connect(): + sock = socket.create_connection(("127.0.0.1", port), timeout=30) + sock.settimeout(30) + return sock + +def read_error(sock): + """Read the reply and return the human-readable error message, if any.""" + data = b"" + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + return "TIMEOUT" + except ConnectionResetError: + break + if not chunk: + break + data += chunk + if not data: + return "NO REPLY" + if data[0:1] != b"E": + return "UNEXPECTED REPLY" + if b"Can't correctly handle Startup message" in data: + return "ERROR: Can't correctly handle Startup message" + return "ERROR: " + data.decode("utf-8", "replace") + +# A startup message with an absurdly large declared size must be rejected without allocating it. +sock = connect() +sock.sendall(struct.pack(">ii", 1000000000, 196608)) +print("huge declared size:", read_error(sock)) +sock.close() + +# A startup message that declares a small size, but then streams an unterminated parameter name, +# must be rejected as well: the parser must not keep reading past the declared size. +sock = connect() +sock.sendall(struct.pack(">ii", 30, 196608)) +try: + sock.sendall(b"x" * 65536) +except (BrokenPipeError, ConnectionResetError, socket.timeout): + pass +print("unterminated parameter:", read_error(sock)) +sock.close() + +# A well-formed startup message is still accepted (the reply is an authentication request). +sock = connect() +payload = b"user\x00default\x00\x00" +sock.sendall(struct.pack(">ii", 8 + len(payload), 196608) + payload) +reply = sock.recv(1) +print("well-formed message:", "authentication request" if reply == b"R" else "unexpected reply " + repr(reply)) +sock.close() +PYTHON diff --git a/tests/queries/0_stateless/05043_merge_prefilter_delegating_children.reference b/tests/queries/0_stateless/05043_merge_prefilter_delegating_children.reference new file mode 100644 index 000000000000..90520bf749bf --- /dev/null +++ b/tests/queries/0_stateless/05043_merge_prefilter_delegating_children.reference @@ -0,0 +1,17 @@ +Merge over Distributed +3 +0 +t05043_leaf +3 +Merge over Merge +2 +0 +Merge over Buffer +2 +1 +Merge over Alias +1 +0 +Pruning still works +1 +1 diff --git a/tests/queries/0_stateless/05043_merge_prefilter_delegating_children.sql b/tests/queries/0_stateless/05043_merge_prefilter_delegating_children.sql new file mode 100644 index 000000000000..c57335c2f711 --- /dev/null +++ b/tests/queries/0_stateless/05043_merge_prefilter_delegating_children.sql @@ -0,0 +1,76 @@ +-- Tags: shard + +-- A `WHERE _table = ...` or `WHERE _database = ...` filter over a `Merge` table silently returned +-- zero rows when the matching child reads its data from other tables (`Distributed`, `Merge`, +-- `Buffer`, `Alias`): the rows of such a child carry the name of the table that actually produced +-- them, while the pruning in `ReadFromMerge::getSelectedTables` matched the predicate against the +-- child's own name. Such children are now always read, and the predicate filters their rows. + +DROP TABLE IF EXISTS t05043_leaf; +DROP TABLE IF EXISTS t05043_dist; +DROP TABLE IF EXISTS t05043_inner_leaf; +DROP TABLE IF EXISTS t05043_inner_merge; +DROP TABLE IF EXISTS t05043_buf_dst; +DROP TABLE IF EXISTS t05043_buf; +DROP TABLE IF EXISTS t05043_alias_target; +DROP TABLE IF EXISTS t05043_alias; +DROP TABLE IF EXISTS t05043_plain; +DROP VIEW IF EXISTS t05043_throwing; + +SELECT 'Merge over Distributed'; +CREATE TABLE t05043_leaf (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t05043_leaf VALUES (1), (2), (3); +CREATE TABLE t05043_dist (x UInt64) ENGINE = Distributed(test_shard_localhost, currentDatabase(), t05043_leaf); + +SELECT count() FROM merge(currentDatabase(), '^t05043_dist$') WHERE _table = 't05043_leaf'; +SELECT count() FROM merge(currentDatabase(), '^t05043_dist$') WHERE _table = 't05043_dist'; +SELECT DISTINCT _table FROM merge(currentDatabase(), '^t05043_dist$'); +-- The same at the `FetchColumns` stage (`ARRAY JOIN` prevents forwarding the query to the child): +SELECT count() FROM merge(currentDatabase(), '^t05043_dist$') ARRAY JOIN [1] AS one WHERE _table = 't05043_leaf'; + +SELECT 'Merge over Merge'; +CREATE TABLE t05043_inner_leaf (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t05043_inner_leaf VALUES (10), (20); +CREATE TABLE t05043_inner_merge (x UInt64) ENGINE = Merge(currentDatabase(), '^t05043_inner_leaf$'); + +SELECT count() FROM merge(currentDatabase(), '^t05043_inner_merge$') WHERE _table = 't05043_inner_leaf'; +SELECT count() FROM merge(currentDatabase(), '^t05043_inner_merge$') WHERE _table = 't05043_inner_merge'; + +SELECT 'Merge over Buffer'; +CREATE TABLE t05043_buf_dst (x UInt64) ENGINE = MergeTree ORDER BY x; +-- The time/rows/bytes thresholds are high enough that nothing is flushed during the test. +CREATE TABLE t05043_buf (x UInt64) ENGINE = Buffer(currentDatabase(), t05043_buf_dst, 1, 1000, 1000, 1000000, 1000000, 100000000, 100000000); +INSERT INTO t05043_buf_dst VALUES (100), (200); +INSERT INTO t05043_buf VALUES (300); + +SELECT count() FROM merge(currentDatabase(), '^t05043_buf$') WHERE _table = 't05043_buf_dst'; +SELECT count() FROM merge(currentDatabase(), '^t05043_buf$') WHERE _table = 't05043_buf'; + +SELECT 'Merge over Alias'; +CREATE TABLE t05043_alias_target (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t05043_alias_target VALUES (1000); +CREATE TABLE t05043_alias ENGINE = Alias('t05043_alias_target'); + +SELECT count() FROM merge(currentDatabase(), '^t05043_alias$') WHERE _table = 't05043_alias_target'; +SELECT count() FROM merge(currentDatabase(), '^t05043_alias$') WHERE _table = 't05043_alias'; + +SELECT 'Pruning still works'; +-- A child that does not read from other tables is still pruned by its own name: +-- the view throws on read, so the query only succeeds if the view is never read. +CREATE TABLE t05043_plain (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t05043_plain VALUES (1); +CREATE VIEW t05043_throwing AS SELECT throwIf(number >= 0, 'must not be read') + number AS x FROM system.numbers LIMIT 1; + +SELECT count() FROM merge(currentDatabase(), '^t05043_(plain|throwing)$') WHERE _table = 't05043_plain'; +SELECT count() FROM merge(currentDatabase(), '^t05043_(plain|throwing)$') WHERE _database = currentDatabase() AND _table = 't05043_plain'; + +DROP TABLE t05043_dist; +DROP TABLE t05043_leaf; +DROP TABLE t05043_inner_merge; +DROP TABLE t05043_inner_leaf; +DROP TABLE t05043_buf; +DROP TABLE t05043_buf_dst; +DROP TABLE t05043_alias; +DROP TABLE t05043_alias_target; +DROP VIEW t05043_throwing; +DROP TABLE t05043_plain; diff --git a/tests/queries/0_stateless/05045_empty_in_set_short_circuit.reference b/tests/queries/0_stateless/05045_empty_in_set_short_circuit.reference new file mode 100644 index 000000000000..e06afd16bdf9 --- /dev/null +++ b/tests/queries/0_stateless/05045_empty_in_set_short_circuit.reference @@ -0,0 +1,8 @@ +0 +0 +0 +0 +100000 +0 +0 +0 diff --git a/tests/queries/0_stateless/05045_empty_in_set_short_circuit.sql b/tests/queries/0_stateless/05045_empty_in_set_short_circuit.sql new file mode 100644 index 000000000000..b72e31f4d7d7 --- /dev/null +++ b/tests/queries/0_stateless/05045_empty_in_set_short_circuit.sql @@ -0,0 +1,35 @@ +-- an empty set behind IN (subquery) must skip the read, Nullable column included +-- prewhere is pinned off so the condition lands in a filter above the read, which is what folds it +DROP TABLE IF EXISTS t_short_circuit; +DROP TABLE IF EXISTS t_short_circuit_final; +DROP TABLE IF EXISTS t_short_circuit_set; + +CREATE TABLE t_short_circuit (a UInt64, b Nullable(UInt64)) ENGINE = MergeTree ORDER BY a; +CREATE TABLE t_short_circuit_final (a UInt64, b Nullable(UInt64)) ENGINE = ReplacingMergeTree ORDER BY a; +CREATE TABLE t_short_circuit_set (b UInt64) ENGINE = MergeTree ORDER BY b; + +INSERT INTO t_short_circuit SELECT number, number FROM numbers(100000); +INSERT INTO t_short_circuit_final SELECT number, number FROM numbers(100000); + +SET optimize_move_to_prewhere = 0, query_plan_optimize_prewhere = 0; + +SELECT count() /* assert_no_read */ FROM t_short_circuit WHERE b IN (SELECT b FROM t_short_circuit_set); +SELECT count() /* assert_no_read */ FROM t_short_circuit_final FINAL WHERE b IN (SELECT b FROM t_short_circuit_set); +-- transform_null_in rewrites IN to nullIn, which must short-circuit as well +SELECT count() /* assert_no_read */ FROM t_short_circuit WHERE b IN (SELECT b FROM t_short_circuit_set) SETTINGS transform_null_in = 1; +-- a conjunct is enough to make the whole filter false; read_rows is not asserted here because +-- whether this lands in a filter or inside the read step depends on the plan +SELECT count() FROM t_short_circuit WHERE b IN (SELECT b FROM t_short_circuit_set) AND a > 10; +-- NOT IN over an empty set matches everything, so this one reads the whole table +SELECT count() FROM t_short_circuit WHERE b NOT IN (SELECT b FROM t_short_circuit_set); + +SYSTEM FLUSH LOGS query_log; + +SELECT read_rows FROM system.query_log +WHERE current_database = currentDatabase() AND type = 'QueryFinish' + AND query LIKE '%assert_no_read%' AND query NOT LIKE '%query_log%' +ORDER BY event_time_microseconds; + +DROP TABLE t_short_circuit; +DROP TABLE t_short_circuit_final; +DROP TABLE t_short_circuit_set; diff --git a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference new file mode 100644 index 000000000000..9f607859a6e8 --- /dev/null +++ b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference @@ -0,0 +1,7 @@ +TableProxy +3 +t05045_leaf +3 +0 +3 +0 diff --git a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.sql b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.sql new file mode 100644 index 000000000000..967b85a5436b --- /dev/null +++ b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.sql @@ -0,0 +1,46 @@ +-- Tags: shard, no-replicated-database +-- no-replicated-database: `DETACH DATABASE` / `ATTACH DATABASE` of an `Atomic` database +-- with the `lazy_load_tables` setting. + +-- In a database with `lazy_load_tables = 1`, an unloaded table is a `StorageTableProxy`. +-- The proxy must forward `readsFromOtherTables` to the nested storage: otherwise a +-- `WHERE _table = ...` filter over a `Merge` table would prune a lazily loaded `Distributed` +-- (or `Merge`, `Buffer`, `Alias`) child by the proxy's own name and silently return no rows +-- after a restart or `ATTACH DATABASE` (found by review in +-- https://github.com/ClickHouse/ClickHouse/pull/116371). + +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Atomic SETTINGS lazy_load_tables = 1; + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t05045_leaf (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t05045_leaf VALUES (1), (2), (3); +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t05045_dist (x UInt64) ENGINE = Distributed(test_shard_localhost, {CLICKHOUSE_DATABASE_1:String}, t05045_leaf); + +-- Re-attach the database so the tables become unloaded lazy proxies. +DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; + +-- Prove the child is still an unloaded proxy at the time of the query below. +-- The `system.tables` filter is spelled with `currentDatabase()` rather than the equivalent +-- `{CLICKHOUSE_DATABASE_1:String}` because the style check only recognizes the former; `USE` +-- does not load the lazy tables, so the engine reported below is still the proxy. +USE {CLICKHOUSE_DATABASE_1:Identifier}; +SELECT engine FROM system.tables WHERE database = currentDatabase() AND name = 't05045_dist'; +USE {CLICKHOUSE_DATABASE:Identifier}; + +-- The rows of the `Distributed` child carry the leaf's name; the proxy must not be pruned. +SELECT count() FROM merge({CLICKHOUSE_DATABASE_1:String}, '^t05045_dist$') WHERE _table = 't05045_leaf'; +SELECT DISTINCT _table FROM merge({CLICKHOUSE_DATABASE_1:String}, '^t05045_dist$'); +-- The same at the `FetchColumns` stage (`ARRAY JOIN` prevents forwarding the query to the child): +SELECT count() FROM merge({CLICKHOUSE_DATABASE_1:String}, '^t05045_dist$') ARRAY JOIN [1] AS one WHERE _table = 't05045_leaf'; +-- No rows carry the child's own name: +SELECT count() FROM merge({CLICKHOUSE_DATABASE_1:String}, '^t05045_dist$') WHERE _table = 't05045_dist'; + +-- A lazily loaded `MergeTree` child does not delegate its reads and stays prunable: +-- filtering on another name reads nothing from it. +DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; +SELECT count() FROM merge({CLICKHOUSE_DATABASE_1:String}, '^t05045_leaf$') WHERE _table = 't05045_leaf'; +SELECT count() FROM merge({CLICKHOUSE_DATABASE_1:String}, '^t05045_leaf$') WHERE _table = 'no_such_table'; + +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/05052_client_untrusted_server_allocations.reference b/tests/queries/0_stateless/05052_client_untrusted_server_allocations.reference new file mode 100644 index 000000000000..a381dfc73e4d --- /dev/null +++ b/tests/queries/0_stateless/05052_client_untrusted_server_allocations.reference @@ -0,0 +1,4 @@ +block declaring a 2 GiB decompressed size: Too large size_decompressed +block declaring 1e12 rows: TOO_LARGE_ARRAY_SIZE +exception from the server: reported +exception with a 1 GiB message that is not sent: reported diff --git a/tests/queries/0_stateless/05052_client_untrusted_server_allocations.sh b/tests/queries/0_stateless/05052_client_untrusted_server_allocations.sh new file mode 100755 index 000000000000..89b61391af82 --- /dev/null +++ b/tests/queries/0_stateless/05052_client_untrusted_server_allocations.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A client allocates from sizes that the server puts on the wire, so a size on its own must not turn +# into an allocation: the strings of an exception received during the handshake are read as they +# arrive, the decompressed size of a block is bounded, and the row count of a block is not used to +# preallocate the columns. + +# The decompressed size of a block is taken from its header, before the payload is read. +# `[checksum: 16][method: 1][size_compressed: 4][size_decompressed: 4]`, method `0x82` is LZ4. +printf '%s' "00000000000000000000000000000000821300000000000080" | xxd -r -p > "${CLICKHOUSE_TMP}/05052_huge_block.compressed" +echo -n 'block declaring a 2 GiB decompressed size: ' +$CLICKHOUSE_COMPRESSOR --decompress --no-checksum-validation --input "${CLICKHOUSE_TMP}/05052_huge_block.compressed" --output "${CLICKHOUSE_TMP}/05052_huge_block.decompressed" 2>&1 | grep -o -m1 'Too large size_decompressed' + +# A `Native` block with one `UInt64` column that declares a thousand billion rows and carries none. +printf '%s' "0180a094a58d1d01780655496e74363400" | xxd -r -p > "${CLICKHOUSE_TMP}/05052_huge_rows.native" +echo -n 'block declaring 1e12 rows: ' +$CLICKHOUSE_LOCAL --query "SELECT count() FROM file('${CLICKHOUSE_TMP}/05052_huge_rows.native', 'Native', 'x UInt64')" 2>&1 | grep -o -m1 -e 'TOO_LARGE_ARRAY_SIZE' -e 'MEMORY_LIMIT_EXCEEDED' + +CLICKHOUSE_CLIENT_BINARY="$CLICKHOUSE_CLIENT_BINARY" python3 - <<'PYTHON' +import os +import shlex +import socket +import struct +import subprocess +import threading + +client = shlex.split(os.environ["CLICKHOUSE_CLIENT_BINARY"]) + + +def varint(value): + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + out.append(byte | 0x80 if value else byte) + if not value: + return bytes(out) + + +def string(value): + return varint(len(value)) + value + + +def exception_packet(message, declared_message_size=None): + """An `Exception` packet: type, code, name, message, stack trace, `has_nested`.""" + body = string(b"DB::Exception") + if declared_message_size is None: + body += string(message) + else: + body += varint(declared_message_size) + message + body += string(b"") + b"\x00" + return varint(2) + struct.pack(", m MAP) + TBLPROPERTIES ('file.format'='parquet'); +INSERT INTO paimon.default.t VALUES (1, array(1, 2), map('k', 1)), (2, NULL, NULL); +``` diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/bucket-0/data-c05044f3-6565-4dae-99d7-67acae444025-0.parquet b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/bucket-0/data-c05044f3-6565-4dae-99d7-67acae444025-0.parquet new file mode 100644 index 000000000000..288989e38eac Binary files /dev/null and b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/bucket-0/data-c05044f3-6565-4dae-99d7-67acae444025-0.parquet differ diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/bucket-0/data-c4476bf5-7447-452e-842a-8b07ff1cdaec-0.parquet b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/bucket-0/data-c4476bf5-7447-452e-842a-8b07ff1cdaec-0.parquet new file mode 100644 index 000000000000..2ac61c8bab2b Binary files /dev/null and b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/bucket-0/data-c4476bf5-7447-452e-842a-8b07ff1cdaec-0.parquet differ diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-9040de1d-709b-4a72-bf08-0e42eb834d61-0 b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-9040de1d-709b-4a72-bf08-0e42eb834d61-0 new file mode 100644 index 000000000000..fe9c87a17ed8 Binary files /dev/null and b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-9040de1d-709b-4a72-bf08-0e42eb834d61-0 differ diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-0 b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-0 new file mode 100644 index 000000000000..a7a20ba42390 Binary files /dev/null and b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-0 differ diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-1 b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-1 new file mode 100644 index 000000000000..d48fc8b9d3c7 Binary files /dev/null and b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/manifest/manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-1 differ diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/schema/schema-0 b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/schema/schema-0 new file mode 100644 index 000000000000..abfa23b738d4 --- /dev/null +++ b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/schema/schema-0 @@ -0,0 +1,32 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "arr", + "type" : { + "type" : "ARRAY", + "element" : "INT" + } + }, { + "id" : 2, + "name" : "m", + "type" : { + "type" : "MAP", + "key" : "STRING NOT NULL", + "value" : "INT" + } + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "owner" : "root", + "file.format" : "parquet" + }, + "timeMillis" : 1785911615531 +} \ No newline at end of file diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/EARLIEST b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/EARLIEST new file mode 100644 index 000000000000..56a6051ca2b0 --- /dev/null +++ b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/LATEST b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/LATEST new file mode 100644 index 000000000000..56a6051ca2b0 --- /dev/null +++ b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/snapshot-1 b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/snapshot-1 new file mode 100644 index 000000000000..9778e14c2a09 --- /dev/null +++ b/tests/queries/0_stateless/data_minio/paimon_nullable_composites/snapshot/snapshot-1 @@ -0,0 +1,18 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-0", + "baseManifestListSize" : 884, + "deltaManifestList" : "manifest-list-a8943d65-5c43-4f19-af88-f2e5f215952c-1", + "deltaManifestListSize" : 989, + "changelogManifestList" : null, + "commitUser" : "a7034c2b-895c-4f19-89b2-45df7be00511", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1785911617611, + "logOffsets" : { }, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "changelogRecordCount" : 0 +} \ No newline at end of file diff --git a/tests/queries/0_stateless/data_native/variant_corrupted_compact_discriminator.native b/tests/queries/0_stateless/data_native/variant_corrupted_compact_discriminator.native new file mode 100644 index 000000000000..51d4a7e9a43a Binary files /dev/null and b/tests/queries/0_stateless/data_native/variant_corrupted_compact_discriminator.native differ diff --git a/tests/queries/0_stateless/data_native/variant_corrupted_discriminators.native b/tests/queries/0_stateless/data_native/variant_corrupted_discriminators.native new file mode 100644 index 000000000000..cdf97e71133c Binary files /dev/null and b/tests/queries/0_stateless/data_native/variant_corrupted_discriminators.native differ diff --git a/tests/queries/0_stateless/data_parquet/04654_bloom_filter_bitset_out_of_bounds.parquet b/tests/queries/0_stateless/data_parquet/04654_bloom_filter_bitset_out_of_bounds.parquet new file mode 100644 index 000000000000..cbf35d8a12f6 Binary files /dev/null and b/tests/queries/0_stateless/data_parquet/04654_bloom_filter_bitset_out_of_bounds.parquet differ