From a498939153ce439f4f7e5062ecea8955bc1ebb5d Mon Sep 17 00:00:00 2001 From: Darick Tong Date: Tue, 17 Nov 2026 22:59:45 -0800 Subject: [PATCH] chore: vendor latest bedrock (52082d4b80) Move the bundled SQLite from bedrock 2137918d6a (2026-08-10) to 52082d4b80 (2026-11-09, reports 3.55.0). This picks up the wal2 fix for the writer/checkpointer race (4bc8fdd581): when a COMMIT switches wal files, a checkpointer taking a lock in the gap could make the writer return a spurious SQLITE_BUSY. See https://sqlite.org/forum/forumpost/63ba0208a6 Also included: - SQLite 3.54.0 and 3.54.1 - most fixes queued for 3.54.2, including two WAL/shm recovery race fixes, a b-tree cursor cell-cache fix, ALTER TABLE DROP COLUMN leaving a stale index, and several wrong-result query fixes Only CHECKIN in deps/download.sh and the regenerated deps/sqlite3/* change. --- deps/download.sh | 2 +- deps/sqlite3/shell.c | 2534 ++++++++++----- deps/sqlite3/sqlite3.c | 6236 +++++++++++++++++++++++++------------ deps/sqlite3/sqlite3.h | 55 +- deps/sqlite3/sqlite3ext.h | 16 +- 5 files changed, 6154 insertions(+), 2689 deletions(-) diff --git a/deps/download.sh b/deps/download.sh index d10dd08..fa03e22 100755 --- a/deps/download.sh +++ b/deps/download.sh @@ -18,7 +18,7 @@ # 4. node-gyp links the two resulting binaries to generate better_sqlite3.node. # === -CHECKIN="2137918d6aa8cc59" +CHECKIN="52082d4b80537268" # Defines below are sorted alphabetically. DEFINES=" diff --git a/deps/sqlite3/shell.c b/deps/sqlite3/shell.c index cbed801..12ef142 100644 --- a/deps/sqlite3/shell.c +++ b/deps/sqlite3/shell.c @@ -16,6 +16,7 @@ ** ext/misc/fileio.c ** ext/misc/ieee754.c ** ext/misc/memtrace.c +** ext/misc/optname.c ** ext/misc/pcachetrace.c ** ext/misc/regexp.c ** ext/misc/series.c @@ -636,6 +637,7 @@ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); + if( z==0 ) return -1; sqlite3_fputs(z, out); rc = (int)strlen(z); sqlite3_free(z); @@ -657,6 +659,7 @@ int sqlite3_vfprintf(FILE *out, const char *zFormat, va_list ap){ */ char *z; z = sqlite3_vmprintf(zFormat, ap); + if( z==0 ) return -1; sqlite3_fputs(z, out); rc = (int)strlen(z); sqlite3_free(z); @@ -748,6 +751,8 @@ struct sqlite3_qrf_spec { char **pzOutput; /* Storage location for output string */ /* The following are available in iVersion 2 and later */ unsigned char bRowCount; /* Show the number of rows at end of each query */ + char *zIFmt; /* Format string for integers */ + char *zFpFmt; /* Format string for floating point */ /* Additional fields may be added in the future */ }; @@ -889,6 +894,18 @@ int sqlite3_qrf_wcwidth(int c); */ size_t sqlite3_qrf_wcswidth(const char*); +/* +** The argument is a proposed format string for the zIFmt or zFpFmt +** parameters. Return value indicates: +** +** 0 The input is not a valid format string. If this string +** appears in either zIFmt or zFpFmt, it will be ignored. +** +** 1 The input is a valid format string for integers. +** +** 2 The input is a valid format string for floating point. +*/ +int sqlite3_qrf_ckformat(const char*); #ifdef __cplusplus } @@ -971,6 +988,7 @@ struct Qrf { } u; sqlite3_int64 nRow; /* Number of rows handled so far */ int *actualWidth; /* Actual width of each column */ + char zFmt[24]; /* Space to hold the true integer rendering fmt */ sqlite3_qrf_spec spec; /* Copy of the original spec */ }; @@ -1015,6 +1033,44 @@ static const char qrfCType[] = { # endif #endif +/* +** The argument is a proposed format string for the zIFmt or zFpFmt +** parameters. Return value indicates: +** +** 0 The input is not a valid format string. If this string +** appears in either zIFmt or zFpFmt, it will be ignored. +** +** 1 The input is a valid format string for integers. +** +** 2 The input is a valid format string for floating point. +*/ +int sqlite3_qrf_ckformat(const char *z){ + size_t n; + if( z==0 ) return 0; + if( z[0]!='%' ) return 0; + z++; + n = strspn(z,"+-,#0"); + if( n>5 ) return 0; + z += n; + if( z[0]>='1' && z[0]<='9' ){ + n = strspn(z,"0123456789"); + if( n>3 ) return 0; + z += n; + } + if( z[0]=='.' ){ + z++; + n = strspn(z,"0123456789"); + if( n>3 ) return 0; + z += n; + } + n = strspn(z,"eEgGf"); + if( n==1 && z[1]==0 ) return 2; + n = strspn(z,"dxXo"); + if( n==1 && z[1]==0 ) return 1; + return 0; +} + + /* ** Set an error code and error message. */ @@ -1277,7 +1333,7 @@ static void qrfEqpStats(Qrf *p){ if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ break; } - n = (int)strlen(z) + qrfStatsHeight(pS,i)*3; + n = (z ? (int)strlen(z) : 0) + qrfStatsHeight(pS,i)*3; if( n>nWidth ) nWidth = n; } nWidth += 2; @@ -1349,9 +1405,9 @@ static void qrfEqpStats(Qrf *p){ } if( p->u.pGraph ) p->u.pGraph->nWidth = nWidth; qrfStrErr(p, pLine); - sqlite3_free(sqlite3_str_finish(pLine)); + sqlite3_str_free(pLine); qrfStrErr(p, pStats); - sqlite3_free(sqlite3_str_finish(pStats)); + sqlite3_str_free(pStats); #endif } @@ -1721,15 +1777,23 @@ static int qrfRelaxable(Qrf *p, const char *z){ if( p->spec.zNull && strcmp(p->spec.zNull,z)==0 ) return 0; i = (z[0]=='-' || z[0]=='+'); if( strcmp(z+i,"Inf")==0 ) return 0; - if( !qrfDigit(z[i]) ) return 1; - i++; - while( qrfDigit(z[i]) ){ i++; } - if( z[i]==0 ) return 0; - if( z[i]=='.' ){ + if( z[i]=='0' && (z[i+1]=='x' || z[i+1]=='X') ){ + /* Hexadecimal integer literal: 0x1f */ + i += 2; + n = strspn(&z[i], "0123456789abcdefABCDEF"); + return n==0 || z[i+n]!=0; + } + if( z[i]=='.' && qrfDigit(z[i+1]) ){ + /* Real literal with no digits before the ".", such as ".5" */ i++; + }else if( !qrfDigit(z[i]) ){ + return 1; + }else{ while( qrfDigit(z[i]) ){ i++; } - if( z[i]==0 ) return 0; + if( z[i]=='.' ) i++; } + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; if( z[i]=='e' || z[i]=='E' ){ i++; if( z[i]=='+' || z[i]=='-' ){ i++; } @@ -2002,12 +2066,18 @@ static void qrfRenderValue(Qrf *p, sqlite3_str *pOut, int iCol){ } switch( sqlite3_column_type(p->pStmt,iCol) ){ case SQLITE_INTEGER: { - sqlite3_str_appendf(pOut, "%lld", sqlite3_column_int64(p->pStmt,iCol)); + sqlite3_str_appendf(pOut, p->spec.zIFmt, + sqlite3_column_int64(p->pStmt,iCol)); break; } case SQLITE_FLOAT: { - const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); - sqlite3_str_appendall(pOut, zTxt); + if( p->spec.zFpFmt ){ + double r = sqlite3_column_double(p->pStmt,iCol); + sqlite3_str_appendf(pOut,p->spec.zFpFmt,r); + }else{ + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + sqlite3_str_appendall(pOut, zTxt); + } break; } case SQLITE_BLOB: { @@ -2862,6 +2932,21 @@ static void qrfRestrictScreenWidth(qrfColData *pData, Qrf *p){ } } +/* +** Invoke sqlite3_str_finish(). But if it returns NULL, try to allocate +** an empty string to replace the result. +** +** The result might still be an empty string, but only after an OOM and +** in that case qrfOom() has been called already and so it is safe to +** return immediately. +*/ +static char *qrfStrFinish(Qrf *p, sqlite3_str *pStr){ + char *z = sqlite3_str_finish(pStr); + if( z==0 ) z = sqlite3_mprintf(""); + if( z==0 ) qrfOom(p); + return z; +} + /* ** Columnar modes require that the entire query be evaluated first, with ** results written into memory, so that we can compute appropriate column @@ -2925,7 +3010,8 @@ static void qrfColumnar(Qrf *p){ qrfEncodeText(p, pStr, z ? z : ""); n = sqlite3_str_length(pStr); qrfStrErr(p, pStr); - z = data.az[data.n] = sqlite3_str_finish(pStr); + z = data.az[data.n] = qrfStrFinish(p, pStr); + if( z==0 ) goto columnar_done; if( p->spec.nTitleLimit ){ nNL = 0; data.aiWth[data.n] = w = qrfTitleLimit(data.az[data.n], @@ -2954,7 +3040,8 @@ static void qrfColumnar(Qrf *p){ qrfRenderValue(p, pStr, i); n = sqlite3_str_length(pStr); qrfStrErr(p, pStr); - z = data.az[data.n] = sqlite3_str_finish(pStr); + z = data.az[data.n] = qrfStrFinish(p,pStr); + if( z==0 ) goto columnar_done; data.abNum[data.n] = eType==SQLITE_INTEGER || eType==SQLITE_FLOAT; data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); data.n++; @@ -3242,6 +3329,7 @@ static void qrfColumnar(Qrf *p){ } qrfWrite(p); +columnar_done: qrfColDataFree(&data); return; } @@ -3646,6 +3734,12 @@ static void qrfOneSimpleRow(Qrf *p){ pVal = sqlite3_str_new(p->db); nSep = (int)strlen(p->spec.zColumnSep); mxW = p->mxWidth - (nSep + p->u.sLine.mxColWth); + if( mxW<2 ){ + /* The column names leave little or no room for the value. Use a + ** width of at least 2, enough for any one character, so that + ** qrfWrapLine() always makes progress. */ + mxW = 2; + } bWW = p->spec.bWordWrap==QRF_Yes; for(i=0; inCol; i++){ const char *zVal; @@ -3673,7 +3767,7 @@ static void qrfOneSimpleRow(Qrf *p){ sqlite3_str_reset(pVal); } qrfStrErr(p, pVal); - sqlite3_free(sqlite3_str_finish(pVal)); + sqlite3_str_free(pVal); qrfWrite(p); break; } @@ -3730,7 +3824,11 @@ static void qrfInitialize( p->iErr = SQLITE_OK; p->nCol = sqlite3_column_count(p->pStmt); p->nRow = 0; - sz = sizeof(sqlite3_qrf_spec); + switch( pSpec->iVersion ){ + case 0: sz = offsetof(sqlite3_qrf_spec, bRowCount); break; + case 1: sz = offsetof(sqlite3_qrf_spec, bRowCount); break; + default: sz = sizeof(sqlite3_qrf_spec); break; + } memcpy(&p->spec, pSpec, sz); if( p->spec.zNull==0 ) p->spec.zNull = ""; p->mxWidth = p->spec.nScreenWidth; @@ -3742,10 +3840,10 @@ static void qrfInitialize( if( p->spec.eText>QRF_TEXT_Relaxed ) p->spec.eText = QRF_Auto; if( p->spec.eTitle>QRF_TEXT_Relaxed ) p->spec.eTitle = QRF_Auto; if( p->spec.eBlob>QRF_BLOB_Size ) p->spec.eBlob = QRF_Auto; - if( pSpec->iVersion<=1 ){ - p->spec.bRowCount = 0; - } qrf_reinit: + if( p->spec.zFpFmt && sqlite3_qrf_ckformat(p->spec.zFpFmt)!=2 ){ + p->spec.zFpFmt = 0; + } switch( p->spec.eStyle ){ case QRF_Auto: { switch( sqlite3_stmt_isexplain(pStmt) ){ @@ -3764,6 +3862,7 @@ static void qrfInitialize( case QRF_STYLE_Json: { p->spec.eText = QRF_TEXT_Json; p->spec.zNull = "null"; + if( p->spec.zFpFmt==0 ) p->spec.zFpFmt = "%!0.17g"; break; } case QRF_STYLE_Html: { @@ -3777,6 +3876,7 @@ static void qrfInitialize( if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ p->spec.zTableName = "tab"; } + if( p->spec.zFpFmt==0 ) p->spec.zFpFmt = "%!0.17g"; p->u.nIns = 0; break; } @@ -3792,6 +3892,7 @@ static void qrfInitialize( p->spec.zColumnSep = ","; p->spec.zRowSep = "\r\n"; p->spec.zNull = ""; + if( p->spec.eEsc==QRF_Auto ) p->spec.eEsc = QRF_ESC_Off; break; } case QRF_STYLE_Quote: { @@ -3821,7 +3922,11 @@ static void qrfInitialize( } } if( p->spec.eEsc==QRF_Auto ){ - p->spec.eEsc = QRF_ESC_Ascii; + if( p->spec.eText==QRF_TEXT_Csv ){ + p->spec.eEsc = QRF_ESC_Off; + }else{ + p->spec.eEsc = QRF_ESC_Ascii; + } } if( p->spec.eText==QRF_Auto ){ p->spec.eText = QRF_TEXT_Plain; @@ -3869,6 +3974,17 @@ static void qrfInitialize( } if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = ","; if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; + if( p->spec.zIFmt==0 || sqlite3_qrf_ckformat(p->spec.zIFmt)!=1 ){ + p->spec.zIFmt = "%lld"; + }else{ + size_t n = strlen(p->spec.zIFmt); + memcpy(p->zFmt, p->spec.zIFmt, n-1); + p->zFmt[n-1] = 'l'; + p->zFmt[n] = 'l'; + p->zFmt[n+1] = p->spec.zIFmt[n-1]; + p->zFmt[n+2] = 0; + p->spec.zIFmt = p->zFmt; + } } /* @@ -3936,14 +4052,15 @@ static void qrfFinalize(Qrf *p){ qrfOom(p); }else{ p->spec.pzOutput[0] = zCombined; - memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n+1); + if( n ) memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n); + zCombined[sz+n] = 0; } - sqlite3_free(sqlite3_str_finish(p->pOut)); + sqlite3_str_free(p->pOut); }else{ - p->spec.pzOutput[0] = sqlite3_str_finish(p->pOut); + p->spec.pzOutput[0] = qrfStrFinish(p, p->pOut); } }else if( p->pOut ){ - sqlite3_free(sqlite3_str_finish(p->pOut)); + sqlite3_str_free(p->pOut); } if( p->expMode>0 ){ sqlite3_stmt_explain(p->pStmt, p->expMode-1); @@ -6341,7 +6458,7 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ ** pB->isNull==0 */ static int decimal_cmp(Decimal *pA, Decimal *pB){ - int nASig, nBSig, rc, n; + int nASig, nBSig, rc, n, iA, iB; while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ pA->nDigit--; pA->nFrac--; @@ -6358,16 +6475,18 @@ static int decimal_cmp(Decimal *pA, Decimal *pB){ pA = pB; pB = pTemp; } - nASig = pA->nDigit - pA->nFrac; - nBSig = pB->nDigit - pB->nFrac; + for(iA=0; iAnDigit-pA->nFrac && pA->a[iA]==0; iA++){} + for(iB=0; iBnDigit-pB->nFrac && pB->a[iB]==0; iB++){} + nASig = pA->nDigit - pA->nFrac - iA; + nBSig = pB->nDigit - pB->nFrac - iB; if( nASig!=nBSig ){ return nASig - nBSig; } - n = pA->nDigit; - if( n>pB->nDigit ) n = pB->nDigit; - rc = memcmp(pA->a, pB->a, n); + n = pA->nDigit - iA; + if( n>pB->nDigit - iB ) n = pB->nDigit - iB; + rc = memcmp(pA->a+iA, pB->a+iB, n); if( rc==0 ){ - rc = pA->nDigit - pB->nDigit; + rc = (pA->nDigit - iA) - (pB->nDigit - iB); } return rc; } @@ -6643,6 +6762,31 @@ static Decimal *decimalFromDouble(double r){ return pA; } +/* +** Extract a 32-bit signed integer from argument pVal and write the +** results into *pResult. Return 0 on success and 1 if the value is +** too larger to fit in 32-bits. +*/ +static int decimalSafeInt( + sqlite3_context *context, + sqlite3_value *pVal, + int *pResult +){ + sqlite3_int64 ii = sqlite3_value_int64(pVal); + if( ii<-2147483648LL || ii>2147483647LL ){ + char *zMsg = sqlite3_mprintf( + "value %lld is too large: range limit -2147483648..+2147483647", + ii); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + *pResult = 0; + return 1; + }else{ + *pResult = (int)ii; + return 0; + } +} + /* ** SQL Function: decimal(X) ** OR: decimal_exp(X) @@ -6664,8 +6808,12 @@ static void decimalFunc( Decimal *p = decimal_new(context, argv[0], 0); int N; if( argc==2 ){ - N = sqlite3_value_int(argv[1]); - if( N>0 ) decimal_round(p, N); + if( decimalSafeInt(context, argv[1], &N) ){ + decimal_free(p); + p = 0; + }else if( N>0 ){ + decimal_round(p, N); + } }else{ N = 0; } @@ -6718,8 +6866,10 @@ static void decimalAddFunc( Decimal *pA = decimal_new(context, argv[0], 1); Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); - decimal_add(pA, pB); - decimal_result(context, pA); + if( pA && pB ){ + decimal_add(pA, pB); + decimal_result(context, pA); + } decimal_free(pA); decimal_free(pB); } @@ -6731,7 +6881,7 @@ static void decimalSubFunc( Decimal *pA = decimal_new(context, argv[0], 1); Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); - if( pB ){ + if( pA && pB ){ pB->sign = !pB->sign; decimal_add(pA, pB); decimal_result(context, pA); @@ -6840,9 +6990,14 @@ static void decimalPow2Func( ){ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ - Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); - decimal_result_sci(context, pA, 0); - decimal_free(pA); + int N; + Decimal *pA; + if( decimalSafeInt(context, argv[0], &N) ) return; + pA = decimalPow2(N); + if( pA ){ + decimal_result_sci(context, pA, 0); + decimal_free(pA); + } } } @@ -7652,6 +7807,121 @@ int main(int na, char *av[]){ #endif /************************* End ext/misc/base85.c ********************/ +/************************* Begin ext/misc/optname.c ******************/ +/* +** 2026-10-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** The sqlite3_test_control() interface with SQLITE_TESTCTRL_OPTIMIZATIONS +** and SQLITE_TESTCTRL_GETOPT are used to enable and/or disable various +** optimizations within SQLite. Optimizations are identified by a bitmask. +** This file contains external code that will map symbolic names into +** bitmask values, and bitmask values into symbolic names. +** +** This code is not part of the SQLite core because we don't want to use +** code space to store all the symbolic names. +*/ +/* #include "sqlite3.h" */ + +/* +** Convert an optimization number (as recorded in sqliteInt.h) into +** a mask for that optimization. +*/ +#define OPT_MASK(N) ((1ull)<<((N)-1)) + + +/* +** This is the mapping between bits and names. +** +** Some bit values have multiple names. The preferred name always +** comes first. +*/ +static const struct OptName { + sqlite3_uint64 uMask; + const char *zName; +} aOpt[] = { + { OPT_MASK(1), "QueryFlattener" }, + { OPT_MASK(1), "Flatten" }, /* Alias for prior */ + { OPT_MASK(2), "WindowFunc" }, + { OPT_MASK(3), "GroupByOrder" }, + { OPT_MASK(4), "JoinMiss" }, + { OPT_MASK(5), "DistinctOpt" }, + { OPT_MASK(6), "CoverIdxScan" }, + { OPT_MASK(7), "OrderByIdxJoin" }, + { OPT_MASK(8), "Transitive" }, + { OPT_MASK(9), "OmitNoopJoin" }, + { OPT_MASK(10), "CountOfView" }, + { OPT_MASK(11), "CursorHints" }, + { OPT_MASK(12), "Stat4" }, + { OPT_MASK(13), "PushDown" }, + { OPT_MASK(14), "SimplifyJoin" }, + { OPT_MASK(15), "SkipScan" }, + { OPT_MASK(16), "PropagateConst" }, + { OPT_MASK(17), "MinMaxOpt" }, + { OPT_MASK(18), "SeekScan" }, + { OPT_MASK(19), "OmitOrderBy" }, + { OPT_MASK(20), "BloomFilter" }, + /* 21 currently unused */ + { OPT_MASK(22), "BalancedMerge" }, + { OPT_MASK(23), "ReleaseReg" }, + { OPT_MASK(24), "FlttnUnionAll" }, + { OPT_MASK(25), "IndexedExpr" }, + { OPT_MASK(26), "Coroutines" }, + { OPT_MASK(27), "NullUnusedCols" }, + { OPT_MASK(28), "OnePass" }, + { OPT_MASK(29), "OrderBySubq" }, + { OPT_MASK(29), "OrderBySubquery" }, /* Alias for prior */ + { OPT_MASK(30), "StarQuery" }, + { OPT_MASK(31), "ExistsToJoin" }, + { OPT_MASK(32), "UnionLimit" }, + /* Special names */ + { 0xffffffffffffffffULL, "All" }, + { 0x0000000000000000ULL, "None" }, +}; + +/* +** Given a symbolic name, try to compute the corresponding mask. +** +** The return value is: +** +** SQLITE_OK Successful match +** SQLITE_NOTFOUND No match +** SQLITE_MISUSE Bad arguments +*/ +int sqlite3_optimization_mask(const char *zName, sqlite3_uint64 *pMask){ + unsigned int i; + if( zName==0 || pMask==0 ){ return SQLITE_MISUSE; } + for(i=0; i-1000 && e<1000 ){ + }else if( m==0 ){ sqlite3_result_double(context, 0.0); return; } @@ -8194,6 +8464,7 @@ struct series_cursor { sqlite3_int64 iValue; /* Current value */ u8 bDesc; /* iStep is really negative */ u8 bDone; /* True if stepped past last element */ + u8 bLimit; /* There exists a limit clause */ }; /* @@ -8305,10 +8576,10 @@ static int seriesNext(sqlite3_vtab_cursor *cur){ pCur->bDone = 1; }else if( pCur->bDesc ){ pCur->iValue = sub64(pCur->iValue, pCur->iStep); - assert( pCur->iValue>=pCur->iTerm ); + assert( pCur->iValue>=pCur->iTerm || pCur->bLimit ); }else{ pCur->iValue = add64(pCur->iValue, pCur->iStep); - assert( pCur->iValue<=pCur->iTerm ); + assert( pCur->iValue<=pCur->iTerm || pCur->bLimit ); } return SQLITE_OK; } @@ -8429,6 +8700,13 @@ static sqlite3_int64 seriesRealToI64(double r){ return (sqlite3_int64)r; } +/* Return true if the type of the value is TEXT or BLOB. +*/ +static int seriesIsTextOrBlob(sqlite3_value *pVal){ + int eType = sqlite3_value_type(pVal); + return eType==SQLITE_TEXT || eType==SQLITE_BLOB; +} + /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least @@ -8538,14 +8816,24 @@ static int seriesFilter( if( idxNum & 0x40 ){ iOffset = sqlite3_value_int64(argv[iArg++]); } + if( iLimit>0 ) pCur->bLimit = 1; } /* Narrow the range of iMin and iMax (the minimum and maximum outputs) ** based on equality and inequality constraints on the "value" column. + ** + ** The "value" column has no affinity, so a TEXT or BLOB operand X is + ** not converted into a number. Every INTEGER is less than any TEXT + ** or BLOB, so "value=X", "value>X" and "value>=X" are never true and + ** "value=(double)SMALLEST_INT64 @@ -8560,7 +8848,9 @@ static int seriesFilter( } }else{ if( idxNum & 0x0300 ){ /* value>X (0x200) or value>=X (0x100) */ - if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + if( seriesIsTextOrBlob(argv[iArg]) ){ + goto series_no_rows; + }else if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ double r = sqlite3_value_double(argv[iArg++]); if( r<=(double)SMALLEST_INT64 ){ iMin = SMALLEST_INT64; @@ -8585,7 +8875,9 @@ static int seriesFilter( } } if( idxNum & 0x3000 ){ /* value=(double)LARGEST_INT64 ){ iMax = LARGEST_INT64; @@ -9398,7 +9690,10 @@ static unsigned re_esc_char(ReCompiled *p){ static const char zTrans[] = "\a\f\n\r\t\v"; int i, v = 0; char c; - if( p->sIn.i>=p->sIn.mx ) return 0; + if( p->sIn.i>=p->sIn.mx ){ + p->zErr = "unknown \\ escape"; + return 0; + } c = p->sIn.z[p->sIn.i]; if( c=='u' && p->sIn.i+4sIn.mx ){ const unsigned char *zIn = p->sIn.z + p->sIn.i; @@ -10129,8 +10424,10 @@ static int fileio_mkdir(const char *zPath){ */ static void readFileContents(sqlite3_context *ctx, const char *zName){ FILE *in; - sqlite3_int64 nIn; - void *pBuf; + sqlite3_int64 nIn; /* Size of the file according to ftell() */ + sqlite3_int64 nAlloc; /* Bytes allocated for pBuf */ + sqlite3_int64 nRead = 0; /* Bytes read so far */ + unsigned char *pBuf; sqlite3 *db; int mxBlob; @@ -10139,9 +10436,12 @@ static void readFileContents(sqlite3_context *ctx, const char *zName){ /* File does not exist or is unreadable. Leave the result set to NULL. */ return; } + /* The size reported by ftell() is only a hint. It is -1 for pipes and + ** FIFOs and 0 for many files under /proc. Keep reading until EOF. */ fseek(in, 0, SEEK_END); nIn = ftell(in); rewind(in); + if( nIn<0 ) nIn = 0; db = sqlite3_context_db_handle(ctx); mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); if( nIn>mxBlob ){ @@ -10149,17 +10449,41 @@ static void readFileContents(sqlite3_context *ctx, const char *zName){ fclose(in); return; } - pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); + /* One extra byte so that EOF on a regular file is seen without a + ** reallocation. */ + nAlloc = nIn+1; + pBuf = sqlite3_malloc64( nAlloc ); if( pBuf==0 ){ sqlite3_result_error_nomem(ctx); fclose(in); return; } - if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ - sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); - }else{ + while( 1 ){ + unsigned char *pNew; + nRead += fread(pBuf+nRead, 1, (size_t)(nAlloc-nRead), in); + if( nRead>mxBlob ){ + sqlite3_result_error_code(ctx, SQLITE_TOOBIG); + sqlite3_free(pBuf); + fclose(in); + return; + } + if( nRead(sqlite3_int64)mxBlob+1 ) nAlloc = (sqlite3_int64)mxBlob+1; + pNew = sqlite3_realloc64(pBuf, nAlloc); + if( pNew==0 ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(pBuf); + fclose(in); + return; + } + pBuf = pNew; + } + if( ferror(in) ){ sqlite3_result_error_code(ctx, SQLITE_IOERR); sqlite3_free(pBuf); + }else{ + sqlite3_result_blob64(ctx, pBuf, nRead, sqlite3_free); } fclose(in); } @@ -10841,13 +11165,38 @@ static int fsdirFilter( assert( argc>i ); pCur->zBase = (const char*)sqlite3_value_text(argv[i++]); } + pCur->mxLvl = 1000000000; if( (idxNum & 0x0c)!=0 ){ + /* The LEVEL constraint is only used to limit the depth of the + ** search. SQLite still checks the constraint against every row, so + ** mxLvl may admit rows that do not match, but must never exclude + ** a row that does. Only entries with leveli ); - pCur->mxLvl = sqlite3_value_int(argv[i++]); - if( idxNum & 0x08 ) pCur->mxLvl++; - if( pCur->mxLvl<=0 ) pCur->mxLvl = 1000000000; - }else{ - pCur->mxLvl = 1000000000; + pMx = argv[i++]; + switch( sqlite3_value_type(pMx) ){ + case SQLITE_NULL: { + /* No row can match a NULL constraint */ + pCur->mxLvl = 1; + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pMx); + if( r<1.0 ){ + pCur->mxLvl = 1; + }else if( r<1.0e9 ){ + int iMx = (int)r; + if( (idxNum & 0x08)!=0 || (double)iMxmxLvl = iMx; + } + break; + } + default: { + /* TEXT or BLOB. Do not limit the search. */ + break; + } + } } if( pCur->zBase ){ pCur->nBase = (int)strlen(pCur->zBase)+1; @@ -10880,7 +11229,11 @@ static int fsdirFilter( ** ** 0x01 The path value is supplied by argv[0] ** 0x02 dir is in argv[1] -** 0x04 maxdepth is in argv[1] or [2] +** 0x04 LEVEL< bound is in argv[1] or [2] +** 0x08 LEVEL<= or LEVEL= bound is in argv[1] or [2] +** +** The LEVEL constraint is never omitted. It is used only to limit the +** depth of the directory search, and SQLite rechecks it for each row. */ static int fsdirBestIndex( sqlite3_vtab *tab, @@ -10891,7 +11244,6 @@ static int fsdirBestIndex( int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ int idxLevel = -1; /* Index in pIdxInfo->aConstraint of LEVEL< or <= */ int idxLevelEQ = 0; /* 0x08 for LEVEL<= or LEVEL=. 0x04 for LEVEL< */ - int omitLevel = 0; /* omit the LEVEL constraint */ int seenPath = 0; /* True if an unusable PATH= constraint is seen */ int seenDir = 0; /* True if an unusable DIR= constraint is seen */ const struct sqlite3_index_constraint *pConstraint; @@ -10923,7 +11275,6 @@ static int fsdirBestIndex( if( pConstraint->usable && idxLevel<0 ){ idxLevel = i; idxLevelEQ = 0x08; - omitLevel = 0; } break; } @@ -10936,11 +11287,9 @@ static int fsdirBestIndex( if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE ){ idxLevel = i; idxLevelEQ = 0x08; - omitLevel = 1; }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ idxLevel = i; idxLevelEQ = 0x04; - omitLevel = 1; } } } @@ -10967,7 +11316,6 @@ static int fsdirBestIndex( pIdxInfo->estimatedCost /= 1.0e4; } if( idxLevel>=0 ){ - pIdxInfo->aConstraintUsage[idxLevel].omit = omitLevel; pIdxInfo->aConstraintUsage[idxLevel].argvIndex = i++; pIdxInfo->idxNum |= idxLevelEQ; pIdxInfo->estimatedCost /= 1.0e4; @@ -12528,6 +12876,7 @@ static const char ZIPFILE_SCHEMA[] = #define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "file" in the above */ #define ZIPFILE_MX_NAME (250) /* Windows limitation on filename size */ +#define ZIPFILE_MX_ENTRY (65535) /* Max entries without zip64 extensions */ /* ** The buffer should be large enough to contain 3 65536 byte strings - the @@ -14066,10 +14415,10 @@ static int zipfileUpdate( } } for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ + if( pOld==0 ) return SQLITE_OK; if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ break; } - assert( pOld->pNext ); } } @@ -14156,7 +14505,9 @@ static int zipfileUpdate( ** entry with a path, thereby making it into a duplicate. */ if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){ ZipfileEntry *p; + int nEntry = 0; for(p=pTab->pFirstEntry; p; p=p->pNext){ + nEntry++; if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ switch( sqlite3_vtab_on_conflict(pTab->db) ){ case SQLITE_IGNORE: { @@ -14175,6 +14526,13 @@ static int zipfileUpdate( break; } } + + /* The EOCD record stores the number of entries as a 16-bit value. + ** Refuse to add an entry that would cause it to overflow. */ + if( p==0 && pOld==0 && nEntry>=ZIPFILE_MX_ENTRY ){ + zipfileTableErr(pTab, "too many entries; max: %d", ZIPFILE_MX_ENTRY); + rc = SQLITE_CONSTRAINT; + } } if( rc==SQLITE_OK ){ @@ -14308,6 +14666,7 @@ static int zipfileCommit(sqlite3_vtab *pVtab){ /* Write out the EOCD record */ eocd.iDisk = 0; eocd.iFirstDisk = 0; + assert( nEntry<=ZIPFILE_MX_ENTRY ); eocd.nEntry = (u16)nEntry; eocd.nEntryTotal = (u16)nEntry; eocd.nSize = (u32)(pTab->szCurrent - iOffset); @@ -14500,6 +14859,12 @@ static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ rc = SQLITE_ERROR; goto zipfile_step_out; } + if( p->nEntry>=ZIPFILE_MX_ENTRY ){ + zErr = sqlite3_mprintf("too many entries for zipfile(); max: %d", + ZIPFILE_MX_ENTRY); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use ** deflate compression) or NULL (choose automatically). */ @@ -14627,6 +14992,7 @@ static void zipfileFinal(sqlite3_context *pCtx){ if( p==0 ) return; if( p->nEntry>0 ){ memset(&eocd, 0, sizeof(eocd)); + assert( p->nEntry<=ZIPFILE_MX_ENTRY ); eocd.nEntry = (u16)p->nEntry; eocd.nEntryTotal = (u16)p->nEntry; eocd.nSize = p->cds.n; @@ -15316,7 +15682,7 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ int nColl = STRLEN(zColl); assert( *pRc==SQLITE_OK ); - pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1); + pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) + nColl + 1); if( pNew ){ pNew->zColl = (char*)&pNew[1]; memcpy(pNew->zColl, zColl, nColl+1); @@ -16009,12 +16375,21 @@ static int idxCreateFromCons( do{ int i; char *zFind; - for(i=0; zCols[i]; i++){ - h += ((h<<3) + zCols[i]); - } + sqlite3_free(zName); - zName = sqlite3_mprintf("%s_idx_%08x", zTable, h); + if( collisions<10 ){ + for(i=0; zCols[i]; i++){ + h += ((h<<3) + zCols[i]); + } + zName = sqlite3_mprintf("%s_idx_%08x", zTable, h); + }else{ + /* After several hash attempts, generate a random name */ + sqlite3_uint64 u; + sqlite3_randomness(sizeof(u), &u); + zName = sqlite3_mprintf("%s_idx_%llx", zTable, u); + } if( zName==0 ) break; + /* Is is unique among table, view and index names? */ zFmt = "SELECT count(*) FROM sqlite_schema WHERE name=%Q" " AND type in ('index','table','view')"; @@ -16028,8 +16403,8 @@ static int idxCreateFromCons( break; } ++collisions; - }while( collisions<50 && zName!=0 ); - if( collisions ){ + }while( ALWAYS(collisions<500) && zName!=0 ); + if( NEVER(collisions) ){ /* This return means "Gave up trying to find a unique index name." */ rc = SQLITE_BUSY_TIMEOUT; }else if( zName==0 ){ @@ -16051,9 +16426,9 @@ static int idxCreateFromCons( idxHashAdd(&rc, &p->hIdx, zName, zIdx); } } - sqlite3_free(zName); sqlite3_free(zIdx); } + sqlite3_free(zName); } sqlite3_free(zCols); @@ -16599,7 +16974,10 @@ static void idxRemFunc( assert( argc==2 ); iSlot = sqlite3_value_int(argv[0]); - assert( iSlotnSlot ); + if( iSlot<0 || iSlot>=p->nSlot ){ + sqlite3_result_error(pCtx, "index out of range.", -1); + return; + } pSlot = &p->aSlot[iSlot]; switch( pSlot->eType ){ @@ -18374,15 +18752,11 @@ int sqlite3_intck_unlock(sqlite3_intck *p){ */ const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ sqlite3_free(p->zTestSql); + p->zTestSql = 0; if( zObj ){ p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); - }else{ - if( p->zObj ){ - p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); - }else{ - sqlite3_free(p->zTestSql); - p->zTestSql = 0; - } + }else if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); } return p->zTestSql; } @@ -19980,7 +20354,12 @@ static void diskusedPercent( }else{ r = num*100.0/(double)denom; } - sqlite3_snprintf(sizeof(zNum)-5, zNum, r>=10.0 ? "%.3g" :"%.2g", r); + if( r>0.0 && r<0.0001 ){ + /* Avoid exponential notation for very small percentages */ + sqlite3_snprintf(sizeof(zNum)-5, zNum, "<0.0001"); + }else{ + sqlite3_snprintf(sizeof(zNum)-5, zNum, r>=10.0 ? "%.3g" :"%.2g", r); + } sz = (int)strlen(zNum); zDP = strchr(zNum, '.'); if( zDP==0 ){ @@ -20146,6 +20525,7 @@ static void diskusedFunc( int rc; sqlite3_stmt *pStmt; int n; + char *z; sqlite3_int64 ii; sqlite3_int64 pgsz; sqlite3_int64 nPage; @@ -20469,9 +20849,11 @@ static void diskusedFunc( sqlite3_str_appendf(s.pOut, "The following SQL will create a table named \"space_used\" which\n" "contains most of the information used to generate the report above.\n" - "*/\n" ); + z = sqlite3_str_value(s.pOut); + while( z && (z = strstr(z,"*/"))!=0 ){ z[0] = '+'; } sqlite3_str_appendf(s.pOut, + "*/\n" "BEGIN;\n" "CREATE TABLE space_used(\n" " name text, -- A table or index\n" /* 0 */ @@ -20620,6 +21002,14 @@ SQLITE_EXTENSION_INIT1 #include #include +/* +** No-op wrapper around free() to work around problems with non-CDECL +** calling conventions on 32-bit Windows builds. +*/ +static void strdupFree(void *p){ + free(p); +} + /* ** Make a copy of a string or BLOB in memory obtained from malloc(). */ @@ -20643,7 +21033,7 @@ static void strdupfunc( return; } if( nIn>0 ) memcpy(zOut, zIn, nIn); - sqlite3_result_blob(context, zOut, nIn, free); + sqlite3_result_blob(context, zOut, nIn, strdupFree); }else{ zIn = (const unsigned char*)sqlite3_value_text(argv[0]); if( zIn==0 ) return; @@ -20655,7 +21045,7 @@ static void strdupfunc( } memcpy(zOut, zIn, nIn); zOut[nIn] = 0; - sqlite3_result_text64(context, (char*)zOut, nIn, free, + sqlite3_result_text64(context, (char*)zOut, nIn, strdupFree, SQLITE_UTF8_ZT); } } @@ -21451,11 +21841,39 @@ static void dbdataValue( switch( enc ){ #ifndef SQLITE_OMIT_UTF16 case SQLITE_UTF16BE: - sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; - case SQLITE_UTF16LE: - sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + case SQLITE_UTF16LE: { + /* sqlite3_result_text16be/le() remove a leading BOM. If + ** the stored text begins with U+FEFF or U+FFFE, prepend + ** a BOM in the database byte order, so that only the + ** added BOM is removed. */ + u8 *aBuf = 0; + if( n>=2 && ((pData[0]==0xFE && pData[1]==0xFF) + || (pData[0]==0xFF && pData[1]==0xFE)) + ){ + aBuf = (u8*)sqlite3_malloc64(n+2); + if( aBuf==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + aBuf[0] = (enc==SQLITE_UTF16BE) ? 0xFE : 0xFF; + aBuf[1] = (enc==SQLITE_UTF16BE) ? 0xFF : 0xFE; + memcpy(&aBuf[2], pData, n); + } + if( enc==SQLITE_UTF16BE ){ + if( aBuf ){ + sqlite3_result_text16be(pCtx, aBuf, n+2, sqlite3_free); + }else{ + sqlite3_result_text16be(pCtx, pData, n, SQLITE_TRANSIENT); + } + }else{ + if( aBuf ){ + sqlite3_result_text16le(pCtx, aBuf, n+2, sqlite3_free); + }else{ + sqlite3_result_text16le(pCtx, pData, n, SQLITE_TRANSIENT); + } + } break; + } #endif default: sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); @@ -22028,6 +22446,12 @@ typedef struct RecoverColumn RecoverColumn; ** aCol[] is an array of nCol columns. In the order in which they appear ** in the table. ** +** nField: +** The number of fields in each record of the table's b-tree. This may +** be less than nCol (VIRTUAL generated columns have no field) or, for +** a WITHOUT ROWID table with a column that appears more than once in +** the PRIMARY KEY, greater than nCol. +** ** bIntkey: ** Set to true for intkey tables, false for WITHOUT ROWID. ** @@ -22052,6 +22476,7 @@ struct RecoverTable { u32 iRoot; /* Root page in original database */ char *zTab; /* Name of table */ int nCol; /* Number of columns in table */ + int nField; /* Number of fields in each record */ RecoverColumn *aCol; /* Array of columns */ int bIntkey; /* True for intkey, false for without rowid */ int iRowidBind; /* If >0, bind rowid to INSERT here */ @@ -22135,7 +22560,7 @@ struct RecoverStateW1 { int nInsert; RecoverTable *pTab; /* Table currently being written */ - int nMax; /* Max column count in any schema table */ + int nMax; /* Max field count in any schema table */ sqlite3_value **apVal; /* Array of nMax values */ int nVal; /* Number of valid entries in apVal[] */ int bHaveRowid; @@ -22381,7 +22806,7 @@ static void recoverBitmapFree(RecoverBitmap *pMap){ ** Set the bit associated with page iPg in bitvec pMap. */ static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ - if( iPg<=pMap->nPg ){ + if( iPg<=pMap->nPg && iPg>0 ){ int iElem = (iPg / 32); int iBit = (iPg % 32); pMap->aElem[iElem] |= (((u32)1) << iBit); @@ -23041,7 +23466,7 @@ static void recoverAddTable( i64 iRoot /* Root page of same table in INPUT db */ ){ sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, - "PRAGMA table_xinfo(%Q)", zName + "PRAGMA %Q.table_xinfo(%Q)", p->zDb, zName ); if( pStmt ){ @@ -23095,6 +23520,7 @@ static void recoverAddTable( csr += (n+1); } + pNew->nField = iField; pNew->pNext = p->pTblList; p->pTblList = pNew; pNew->bIntkey = 1; @@ -23102,15 +23528,25 @@ static void recoverAddTable( recoverFinalize(p, pStmt); - pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + /* If this is a WITHOUT ROWID table, the fields of each record are + ** ordered as reported by PRAGMA index_xinfo on the table. A column may + ** appear more than once if it is listed more than once in the PRIMARY + ** KEY (with different collation sequences). In that case use the first + ** field - the others are redundant copies. */ + pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA %Q.index_xinfo(%Q)", p->zDb, zName); while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ int iField = sqlite3_column_int(pStmt, 0); int iCol = sqlite3_column_int(pStmt, 1); - assert( iColnCol ); - pNew->aCol[iCol].iField = iField; - - pNew->bIntkey = 0; + assert( iCol>=0 && iColnCol ); + if( pNew->bIntkey ){ + int ii; + for(ii=0; iinCol; ii++) pNew->aCol[ii].iField = -1; + pNew->bIntkey = 0; + } + if( pNew->aCol[iCol].iField<0 ) pNew->aCol[iCol].iField = iField; + pNew->nField = iField+1; iPk = -2; } recoverFinalize(p, pStmt); @@ -23300,7 +23736,7 @@ static sqlite3_stmt *recoverInsertStmt( if( nField<=0 ) return 0; - assert( nField<=pTab->nCol ); + assert( nField<=pTab->nField ); zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); @@ -23316,12 +23752,18 @@ static sqlite3_stmt *recoverInsertStmt( zSep = ", "; } - for(ii=0; iiaCol[ii].eHidden; - if( eHidden!=RECOVER_EHIDDEN_VIRTUAL - && eHidden!=RECOVER_EHIDDEN_STORED - ){ - assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); + /* Include each column that has a bind parameter and either has a field + ** in the nField-field record or is the IPK. This must match the set of + ** parameters bound by recoverWriteDataStep(). Note that nField counts + ** record fields, not table columns - VIRTUAL generated columns have no + ** field (iField<0), and the order of fields need not match the order of + ** columns (WITHOUT ROWID tables). */ + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + if( pCol->iBind>0 && (pCol->iFieldbIPK) ){ + assert( pCol->iField>=0 ); + assert( pCol->eHidden!=RECOVER_EHIDDEN_VIRTUAL ); + assert( pCol->eHidden!=RECOVER_EHIDDEN_STORED ); zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); if( bSql ){ @@ -23661,10 +24103,10 @@ static int recoverWriteDataInit(sqlite3_recover *p){ RecoverTable *pTbl = 0; int nByte = 0; - /* Figure out the maximum number of columns for any table in the schema */ + /* Figure out the maximum number of fields for any table in the schema */ assert( p1->nMax==0 ); for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ - if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; + if( pTbl->nField>p1->nMax ) p1->nMax = pTbl->nField; } /* Allocate an array of (sqlite3_value*) in which to accumulate the values @@ -23771,7 +24213,7 @@ static int recoverWriteDataStep(sqlite3_recover *p){ int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); assert( bNewCell==0 || (iField==-1 || iField==0) ); - assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nField ); if( bNewCell ){ int ii = 0; @@ -23821,15 +24263,15 @@ static int recoverWriteDataStep(sqlite3_recover *p){ assert( p1->nVal==-1 ); p1->nVal = 0; p1->bHaveRowid = 1; - }else if( iFieldnCol ){ + }else if( iFieldnField ){ assert( apVal[iField]==0 ); apVal[iField] = sqlite3_value_dup( pVal ); if( apVal[iField]==0 ){ recoverError(p, SQLITE_NOMEM, 0); } p1->nVal = iField+1; - }else if( pTab->nCol==0 ){ - p1->nVal = pTab->nCol; + }else if( pTab->nField==0 ){ + p1->nVal = pTab->nField; } p1->iPrevCell = iCell; p1->iPrevPage = iPage; @@ -24938,6 +25380,8 @@ typedef struct Mode { u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ u8 eMode; /* One of the MODE_ values */ + char zIFmt[24]; /* Space to hold the --ifmt value */ + char zFpFmt[24]; /* Space to hold the --fpfmt value */ sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ } Mode; @@ -24946,6 +25390,20 @@ typedef struct Mode { #define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ #define MFLG_HDR 0x04 /* .header used to change headers on/off */ +/* Bit values for ShellState.mStats +*/ +#define CLISTAT_VMSTEP 0x01 /* Show vmsteps */ +#define CLISTAT_STMT 0x02 /* Stats for each statement */ +#define CLISTAT_MEM 0x04 /* Memory stats */ +#define CLISTAT_XSTMT 0x08 /* Extra statement stats */ +#define CLISTAT_XMEM 0x10 /* Extra memory stats */ +#define CLISTAT_IO 0x20 /* I/O stats (Linux only) */ +#define CLISTAT_ZIPVFS 0x40 /* ZIPVFS stats */ +#define CLISTAT_ON 0x17 /* Usual stats */ +#define CLISTAT_ALL 0x7f /* All stats */ +#define CLISTAT_RESET 0x80 /* Reset stat counters each time */ + + /* A file that needs to be deleted, but only after a delay. */ typedef struct Unlink { @@ -24953,6 +25411,11 @@ typedef struct Unlink { char *zFN; /* Name of file. Space from sqlite3_malloc() */ } Unlink; +/* +** Maximum nested depth for .if statements. +*/ +#define MX_IF 10 + /* ** State information about the database connection is contained in an ** instance of the following structure. @@ -24967,12 +25430,13 @@ struct ShellState { u8 bSafeMode; /* True to prohibit unsafe operations */ u8 bSafeModePersist; /* The long-term value of bSafeMode */ u8 eRestoreState; /* See comments above doAutoDetectRestore() */ - unsigned statsOn; /* True to display memory stats before each finalize */ + u8 mStats; /* Which stats to display (if any) */ unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ u8 nPopOutput; /* Revert .output settings when reaching zero */ u8 nPopMode; /* Revert .mode settings when reaching zero */ u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ u8 bDelimitNonprint; /* Add \001...\002 around non-printing in prompts */ + u8 bTestParse; /* Undocumented --test-parse flags */ int inputNesting; /* Track nesting level of .read and other redirects */ double prevTimer; /* Last reported timer value */ double tmProgress; /* --timeout option for .progress */ @@ -24992,9 +25456,13 @@ struct ShellState { unsigned nTestRun; /* Number of test cases run */ unsigned nTestErr; /* Number of test cases that failed */ sqlite3_int64 szMax; /* --maxsize argument to .open */ + char *zInput; /* Simulated inputs */ + i64 nInputUsed; /* Simulated input read so far */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ char *zErrPrefix; /* Alternative error message prefix */ + char *zAuthIgnore; /* Authorizer returns SQLITE_IGNORE for glob matches */ + char *zAuthDeny; /* Authorizer returns SQLITE_DENY for glob matches */ sqlite3_stmt *pStmt; /* Current statement if any. */ FILE *pLog; /* Write log output here */ char *azPrompt[2]; /* Main and continuation prompt strings */ @@ -25032,6 +25500,13 @@ struct ShellState { int *aiOfst; /* Offset into zOrig[] for start of each arg */ char *abQuot; /* True if the argment was originally quoted */ } dot; + u32 mIf; /* Mask of false if conditions */ + u32 mElse; /* An else seen at this level */ + u32 mIfHasBeenTrue; /* True branch seen in if-elseif-elseif-else chain */ + int nIfLevel; /* Current number of if levels */ + int nIfAtStart; /* If level at start of current script */ + i64 lnIf[MX_IF]; /* Line number starting an if */ + i64 lnElse[MX_IF]; /* Line number of the else */ #ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ @@ -25082,7 +25557,7 @@ static ShellState shellState; */ static const char *qrfEscNames[] = { "auto", "off", "ascii", "symbol" }; static const char *qrfQuoteNames[] = - { "off","off","sql","hex","csv","tcl","json","relaxed"}; + { "off","off","sql","csv","html","tcl","json","relaxed"}; /* ** These are the allowed shellFlgs values @@ -25344,7 +25819,7 @@ static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ static void cli_exit(int rc){ if( cli_output_capture ){ char *z = sqlite3_str_finish(cli_output_capture); - sqlite3_fputs(z, stdout); + sqlite3_fputs(z ? z : "", stdout); fflush(stdout); } exit(rc); @@ -25475,6 +25950,25 @@ static FILE * openChrSource(const char *zFile){ #undef STAT_CHR_SRC } +/* +** Works like fgets except that if "in" really points to p->zInput +** then text is read out of the p->zInput buffer. +*/ +static char *shell_fgets(ShellState *p, char *zOut, int nOut, FILE *in){ + int i, j; + char *zIn; + if( p==0 || in!=(FILE*)&p->zInput ){ + return sqlite3_fgets(zOut, nOut, in); + } + zIn = p->zInput; + for(i=0, j=p->nInputUsed; inInputUsed], i); + zOut[i] = 0; + p->nInputUsed += i; + return i>0 ? zOut : 0; +} + /* ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer @@ -25484,7 +25978,7 @@ static FILE * openChrSource(const char *zFile){ ** If zLine is not NULL then it is a malloced buffer returned from ** a previous call to this routine that may be reused. */ -static char *local_getline(char *zLine, FILE *in){ +static char *local_getline(ShellState *p, char *zLine, FILE *in){ int nLine = zLine==0 ? 0 : 100; int n = 0; @@ -25498,7 +25992,7 @@ static char *local_getline(char *zLine, FILE *in){ zLine = realloc(zLine, nLine); shell_check_oom(zLine); } - if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ + if( shell_fgets(p, &zLine[n], nLine - n, in)==0 ){ if( n==0 ){ free(zLine); return 0; @@ -25611,7 +26105,24 @@ static const char *shellPromptAppDef(int c){ } #endif /* !defined(SQLITE_PS_APPDEF) */ - +#ifdef SQLITE_DEBUG +/* +** SQL function: sqlite3_incomplete(TEXT) +** +** For use internally and for debugging and testing only. This function +** is subject to change or removal in future versions. +** +** Invoke the sqlite3_incomplete() interface and return its result. +*/ +static void shellIncompleteFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + sqlite3_result_int64(pCtx, zSql?sqlite3_incomplete(zSql):SQLITE_MISUSE); +} +#endif /* SQLITE_DEBUG */ /* ** Return the raw (unexpanded) prompt string. This will be the @@ -25713,6 +26224,89 @@ static int nAnsiEscape(const char *z){ return i; } +/* +** Append text on pOut that is sufficient to complete the statement zBase +** (according to sqlite3_complete()). eFlags: +** +** 0x01 Include the final ";" if needed +** 0x04 Assume zBase ends with '\n' +*/ +static void appendCompletion( + sqlite3_str *pOut, /* Append the completion here */ + const char *zBase, /* Existing SQL text */ + int eFlags /* 0x01: Include ";", 0x04: Assume '\n' */ +){ + sqlite3_int64 R = zBase ? sqlite3_incomplete(zBase) : 0; + int cc = (R>>16)&0xff; + int nParen = R>>32; + int eSemi = (R>>8)&0xff; + int eStat = R&0xff; + int bSemi = (eFlags & 0x01)!=0; + if( cc==0 ){ + /* no-op */ + }else if( cc=='-' ){ + if( (eFlags & 0x04)==0 ) sqlite3_str_appendchar(pOut, 1, '\n'); + }else if( cc=='/' ){ + sqlite3_str_append(pOut,"*/",2); + }else{ + sqlite3_str_appendchar(pOut, 1, cc); + } + if( nParen>0 ){ + sqlite3_str_appendf(pOut, "%.*c",nParen,')'); + } + if( eStat==SQLITE_OK || eStat==SQLITE_EMPTY ){ + /* Nothing to add */ + }else if( eSemi==1 ){ + if( bSemi ) sqlite3_str_append(pOut, ";", 1); + }else if( eSemi==2 ){ + sqlite3_str_append(pOut, "END;", 3+(bSemi!=0)); + }else if( eSemi==3 ){ + sqlite3_str_append(pOut, ";END;", 4+(bSemi!=0)); + } +} + +/* +** SQL function: shell_complete_sql(SQL) +** shell_complete_sql(SQL, FLAGS) +** +** For use internally and for debugging and testing only. This function +** is subject to change or removal in future versions. +** +** If the SQL input is a string, return a new string which is the same +** text with all pending quotes and comments finish, so that the +** SQL is ready to receive its final semicolon. The optional FLAGS +** parameter can be an integer where bits mean: +** +** 0x01 Include the final semicolon +** 0x02 Only return the completion text. Omit the original SQL. +** 0x04 Assume that SQL has a \n at the end. +** +** The /C substitution in the CLI prompt is computed using the +** equivalent of shell_complete_sql(SQL,3); +*/ +static void shell_complete_sql( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql; /* Input SQL text */ + sqlite3_str *pOut; /* Completed SQL */ + int flags; + + if( nVal!=1 && nVal!=2 ) return; + if( nVal==2 ){ + flags = 0x7 & sqlite3_value_int64(apVal[1]); + }else{ + flags = 0; + } + zSql = (const char*)sqlite3_value_text(apVal[0]); + if( zSql==0 ) return; + pOut = sqlite3_str_new(sqlite3_context_db_handle(pCtx)); + if( (flags & 0x02)==0 ) sqlite3_str_appendall(pOut, zSql); + appendCompletion(pOut, zSql, flags & 0x05); + sqlite3_result_str(pCtx, pOut, SQLITE_FINISH); +} + /* ** Expand escapes in the given input prompt string. Return the ** expanded prompt in memory obtained from sqlite3_malloc(). The @@ -25889,29 +26483,7 @@ static char *expand_prompt( if( c=='C' ){ /* /C becomes text needed to terminate current input */ if( !mOff ){ - sqlite3_int64 R = zPrior ? sqlite3_incomplete(zPrior) : 0; - int cc = (R>>16)&0xff; - int nParen = R>>32; - int eSemi = (R>>8)&0xff; - if( cc==0 ){ - /* no-op */ - }else if( cc=='-' ){ - sqlite3_str_append(pOut,"\\n",3); - }else if( cc=='/' ){ - sqlite3_str_append(pOut,"*/",2); - }else{ - sqlite3_str_appendchar(pOut, 1, cc); - } - if( nParen>0 ){ - sqlite3_str_appendf(pOut, "%.*c",nParen,')'); - } - if( eSemi==1 ){ - sqlite3_str_append(pOut, ";", 1); - }else if( eSemi==2 ){ - sqlite3_str_append(pOut, "END;", 4); - }else if( eSemi==3 ){ - sqlite3_str_append(pOut, ";END;", 5); - } + appendCompletion(pOut, zPrior, 0x5); } zPrompt += 2; i = -1; @@ -26034,9 +26606,8 @@ static char *one_input_line( int bContinue /* True for a continuation prompt */ ){ char *zResult; - FILE *in = p->in; - if( in!=0 ){ - zResult = local_getline(zPrior, in); + if( p->in!=0 ){ + zResult = local_getline(p, zPrior, p->in); }else{ const char *zBase = prompt_string(p, bContinue!=0); char *zPrompt = expand_prompt(p, zAll, zBase); @@ -26058,7 +26629,7 @@ static char *one_input_line( sputz(stdout, zPrompt); fflush(stdout); do{ - zResult = local_getline(zPrior, stdin); + zResult = local_getline(p, zPrior, stdin); zPrior = 0; /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ if( zResult==0 ) sqlite3_sleep(50); @@ -26235,7 +26806,7 @@ static char *shellFakeSchema( const char *zName /* The name of the virtual table */ ){ sqlite3_stmt *pStmt = 0; - char *zSql; + char *zSql, *zCom; ShellText s; char cQuote; char *zDiv = "("; @@ -26269,13 +26840,84 @@ static char *shellFakeSchema( if( nRow==0 ){ freeText(&s); s.zTxt = 0; + }else{ + while( s.zTxt && (zCom = strstr(s.zTxt,"*/"))!=0 ){ zCom[1] = '*'; } } return s.zTxt; } +/* +** SQL function: capable(X) +** +** Return true if the current build has capability X. Return 0 +** if X is not a capability or NULL if X is NULL. +** +** This function is intended to be used with the ".if" command +** to bypass test cases that use features that are not part of +** the build. Examples: +** +** .if capable('fts5') +** # do tests that require the FTS5 extension +** .endif +** +** .if capable('generate_series') +** # do tests that require the generate_series() table-valued functions +** .endif +*/ +static void shellCapable( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + const char *z = (const char*)sqlite3_value_text(apVal[0]); + static const char *azSql[] = { + "PRAGMA compile_options", + "PRAGMA module_list", + "PRAGMA function_list" + }; + static const char *azInversions[] = { + "vtab", "OMIT_VIRTUALTABLE", + "fkey", "OMIT_FOREIGN_KEY", + "load_ext", "OMIT_LOAD_EXTENSION", + }; + size_t i; + int invert = 0; + int res = 0; + + UNUSED_PARAMETER(nVal); + if( z==0 ) return; + for(i=0; idb, azSql[i], -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zRow = (const char*)sqlite3_column_text(pStmt, 0); + if( sqlite3_stricmp(zRow, z)==0 ){ + res = 1; + break; + } + } + sqlite3_finalize(pStmt); + if( res ) break; + } + if( invert ) res = !res; + sqlite3_result_int(pCtx, res); +} + + /* ** SQL function: strtod(X) ** +** For use internally and for debugging and testing only. This function +** is subject to change or removal in future versions. +** ** Use the C-library strtod() function to convert string X into a double. ** Used for comparing the accuracy of SQLite's internal text-to-float conversion ** routines against the C-library. @@ -26294,6 +26936,9 @@ static void shellStrtod( /* ** SQL function: dtostr(X) ** +** For use internally and for debugging and testing only. This function +** is subject to change or removal in future versions. +** ** Use the C-library printf() function to convert real value X into a string. ** Used for comparing the accuracy of SQLite's internal float-to-text conversion ** routines against the C-library. @@ -26304,19 +26949,30 @@ static void shellDtostr( sqlite3_value **apVal ){ double r = sqlite3_value_double(apVal[0]); - int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + i64 nn = nVal>=2 ? sqlite3_value_int64(apVal[1]) : 26; + int n; char z[400]; - if( n<1 ) n = 1; - if( n>350 ) n = 350; + if( nn<1 ){ + n = 1; + }else if( nn>350 ){ + n = 350; + }else{ + n = (int)nn; + } z[sizeof(z)-1] = 0; snprintf(z, sizeof(z)-1, "%#+.*e", n, r); sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } /* -** SQL function: shell_add_schema(S,X) +** SQL function: shell_add_schema(S,X,N) +** +** For use internally and for debugging and testing only. This function +** is subject to change or removal in future versions. +** +** Add the schema X to the CREATE statement in S for table N and return +** the result. ** -** Add the schema name X to the CREATE statement in S and return the result. ** Examples: ** ** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); @@ -26355,31 +27011,28 @@ static void shellAddSchemaName( for(i=0; ispec.zNull ){ pDest->spec.zNull = strdup(pSrc->spec.zNull); } + if( pDest->spec.zFpFmt ){ + assert( pDest->spec.zFpFmt==pSrc->zFpFmt ); + pDest->spec.zFpFmt = pDest->zFpFmt; + } + if( pDest->spec.zIFmt ){ + assert( pDest->spec.zIFmt==pSrc->zIFmt ); + pDest->spec.zIFmt = pDest->zIFmt; + } } /* @@ -26867,6 +27531,9 @@ static void shellLog(void *pArg, int iErrCode, const char *zMsg){ /* ** SQL function: shell_putsnl(X) ** +** For use internally and for debugging and testing only. This function +** is subject to change or removal in future versions. +** ** Write the text X to the screen (or whatever output is being directed) ** adding a newline at the end, and then return X. */ @@ -27395,24 +28062,20 @@ static int shellAuth( "ANALYZE", "CREATE_VTABLE", "DROP_VTABLE", "FUNCTION", "SAVEPOINT", "RECURSIVE" }; - int i; - const char *az[4]; - az[0] = zA1; - az[1] = zA2; - az[2] = zA3; - az[3] = zA4; - cli_printf(p->out, "authorizer: %s", azAction[op]); - for(i=0; i<4; i++){ - cli_puts(" ", p->out); - if( az[i] ){ - output_c_string(p->out, az[i]); - }else{ - cli_puts("NULL", p->out); - } - } - cli_puts("\n", p->out); + char *zResult = "OK"; + int rc = SQLITE_OK; + char *zMsg = sqlite3_mprintf("%s %Q %Q %Q %Q", azAction[op],zA1,zA2,zA3,zA4); + if( p->zAuthDeny && sqlite3_strglob(p->zAuthDeny, zMsg)==0 ){ + zResult = "DENY"; + rc = SQLITE_DENY; + }else if( p->zAuthIgnore && sqlite3_strglob(p->zAuthIgnore, zMsg)==0 ){ + zResult = "IGNORE"; + rc = SQLITE_IGNORE; + } + cli_printf(p->out, "authorizer: %s -> %s\n", zMsg, zResult); + sqlite3_free(zMsg); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); - return SQLITE_OK; + return rc; } #endif @@ -27427,32 +28090,13 @@ static int shellAuth( ** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ - char *zToFree = 0; if( z==0 ) return; if( zTail==0 ) return; - if( zTail[0]==';' && (strstr(z, "/*")!=0 || strstr(z,"--")!=0) ){ - const char *zOrig = z; - static const char *azTerm[] = { "", "*/", "\n" }; - int i; - for(i=0; iout==0 ) return 0; out = pArg->out; + bReset = (pArg->mStats & CLISTAT_RESET)!=0; - if( pArg->pStmt && pArg->statsOn==2 ){ + if( pArg->pStmt && (pArg->mStats & CLISTAT_XSTMT)!=0 ){ int nCol, i, x; sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; @@ -27930,91 +28575,92 @@ static int display_stats( } } - if( pArg->statsOn==3 ){ - if( pArg->pStmt ){ - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - cli_printf(out, "VM-steps: %d\n", iCur); - } - return 0; + if( pArg->pStmt && (pArg->mStats & CLISTAT_VMSTEP)!=0 ){ + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); + cli_printf(out, "VM-steps: %d\n", iCur); } - displayStatLine(out, "Memory Used:", - "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); - displayStatLine(out, "Number of Outstanding Allocations:", - "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); - if( pArg->shellFlgs & SHFLG_Pagecache ){ - displayStatLine(out, "Number of Pcache Pages Used:", - "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); - } - displayStatLine(out, "Number of Pcache Overflow Bytes:", - "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - displayStatLine(out, "Largest Allocation:", - "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); - displayStatLine(out, "Largest Pcache Allocation:", - "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); + if( (pArg->mStats & CLISTAT_MEM)!=0 ){ + displayStatLine(out, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(out, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); + } + if( (pArg->mStats & CLISTAT_XMEM)!=0 ){ + if( pArg->shellFlgs & SHFLG_Pagecache ){ + displayStatLine(out, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); + } + displayStatLine(out, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); + displayStatLine(out, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(out, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - displayStatLine(out, "Deepest Parser Stack:", - "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); + displayStatLine(out, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif - - if( db ){ - if( pArg->shellFlgs & SHFLG_Lookaside ){ + + if( db ){ + if( pArg->shellFlgs & SHFLG_Lookaside ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + cli_printf(out, + "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); + cli_printf(out, + "Successful lookaside attempts: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); + cli_printf(out, + "Lookaside failures due to size: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); + cli_printf(out, + "Lookaside failures due to OOM: %d\n", iHiwtr); + } iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, - &iCur, &iHiwtr, bReset); - cli_printf(out, - "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, - &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); cli_printf(out, - "Successful lookaside attempts: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, - &iCur, &iHiwtr, bReset); + "Pager Heap Usage: %d bytes\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); cli_printf(out, - "Lookaside failures due to size: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, - &iCur, &iHiwtr, bReset); + "Page cache hits: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); cli_printf(out, - "Lookaside failures due to OOM: %d\n", iHiwtr); + "Page cache misses: %d\n", iCur); + iHiwtr64 = iCur64 = -1; + sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, + 0); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); + cli_printf(out, + "Page cache writes: %d\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); + cli_printf(out, + "Page cache spills: %d\n", iCur); + cli_printf(out, + "Temporary data spilled to disk: %lld\n", iCur64); + sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, + 1); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); + cli_printf(out, + "Schema Heap Usage: %d bytes\n", iCur); + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); + cli_printf(out, + "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, - "Pager Heap Usage: %d bytes\n", iCur); - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - cli_printf(out, - "Page cache hits: %d\n", iCur); - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - cli_printf(out, - "Page cache misses: %d\n", iCur); - iHiwtr64 = iCur64 = -1; - sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, - 0); - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - cli_printf(out, - "Page cache writes: %d\n", iCur); - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - cli_printf(out, - "Page cache spills: %d\n", iCur); - cli_printf(out, - "Temporary data spilled to disk: %lld\n", iCur64); - sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, - 1); - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, - "Schema Heap Usage: %d bytes\n", iCur); - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, - "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } - - if( pArg->pStmt ){ + + if( pArg->pStmt && (pArg->mStats & CLISTAT_STMT)!=0 ){ int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); @@ -28034,9 +28680,6 @@ static int display_stats( cli_printf(out, "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - cli_printf(out, - "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); cli_printf(out, "Reprepare operations: %d\n", iCur); @@ -28049,10 +28692,16 @@ static int display_stats( } #ifdef __linux__ - displayLinuxIoStats(pArg->out); + if( (pArg->mStats & CLISTAT_IO)!=0 ){ + displayLinuxIoStats(pArg->out); + } #endif - /* Do not remove this machine readable comment: extra-stats-output-here */ +#ifdef SQLITE_ENABLE_ZIPVFS_VTAB + if( (pArg->mStats & CLISTAT_ZIPVFS)!=0 ){ + /* Do not remove this machine readable comment: extra-stats-output-here */ + } +#endif return 0; } @@ -28442,7 +29091,7 @@ static int shell_exec( bind_prepared_stmt(pArg, pStmt); if( isExplain && pArg->mode.autoExplain ){ spec.eStyle = isExplain==1 ? QRF_STYLE_Explain : QRF_STYLE_Eqp; - sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + rc = sqlite3_format_query_result(pStmt, &spec, pzErrMsg); }else if( pArg->mode.eMode==MODE_Www ){ cli_printf(pArg->out, "\n" @@ -28454,12 +29103,12 @@ static int shell_exec( "
");
       }else{
         spec.eStyle = eStyle;
-        sqlite3_format_query_result(pStmt, &spec, pzErrMsg);
+        rc = sqlite3_format_query_result(pStmt, &spec, pzErrMsg);
       }
 
       /* print usage stats if stats on */
-      if( pArg && pArg->statsOn ){
-        display_stats(db, pArg, 0);
+      if( pArg && pArg->mStats!=0 ){
+        display_stats(db, pArg);
       }
 
       /* print loop-counters if required */
@@ -28482,11 +29131,11 @@ static int shell_exec(
       ** copy of the error message. Otherwise, set zSql to point to the
       ** next statement to execute. */
       rc2 = sqlite3_finalize(pStmt);
-      if( rc!=SQLITE_NOMEM ) rc = rc2;
+      if( rc2 && rc!=SQLITE_NOMEM ) rc = rc2;
       if( rc==SQLITE_OK ){
         zSql = zLeftover;
         while( IsSpace(zSql[0]) ) zSql++;
-      }else if( pzErrMsg ){
+      }else if( pzErrMsg && pzErrMsg[0]==0 ){
         *pzErrMsg = save_err_msg(db, "stepping", rc, 0);
       }
 
@@ -28736,6 +29385,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
     initText(&sSelect);
     appendText(&sSelect, "SELECT ", 0);
     if( azCol[0] ){
+      appendText(&sSelect, azCol[0], 0);
+      appendText(&sSelect, " AS ", 0);
       appendText(&sSelect, azCol[0], 0);
       appendText(&sSelect, ",", 0);
     }
@@ -28757,6 +29408,9 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
     p->mode.spec.eBlob = QRF_BLOB_Sql;
     p->mode.spec.bTitles = QRF_No;
     p->mode.spec.nCharLimit = 0;
+    p->mode.spec.bTextJsonb = QRF_No;
+    p->mode.spec.zIFmt = 0;
+    p->mode.spec.zFpFmt = 0;
     rc = shell_exec(p, sSelect.zTxt, 0);
     if( (rc&0xff)==SQLITE_CORRUPT ){
       cli_puts("/****** CORRUPTION ERROR *******/\n", p->out);
@@ -28846,6 +29500,9 @@ static const char *(azHelp[]) = {
 #endif
 #ifndef SQLITE_OMIT_AUTHORIZATION
   ".auth ON|OFF             Show authorizer callbacks",
+  "   Options:",
+  "     --deny GLOB           Return SQLITE_DENY if pattern matches",
+  "     --ignore GLOB         Return SQLITE_IGNORE if pattern matches",
 #endif
 #ifndef SQLITE_SHELL_FIDDLE
   ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
@@ -29092,6 +29749,17 @@ static const struct {
   const char *zCmd;   /* Name of the dot-command */
   const char *zUsage; /* Documentation */
 } aUsage[] = {
+  { ".clone",
+"USAGE: .clone FILE\n"
+"\n"
+"The original purpose of the \".clone\" command (when it was first introduced\n"
+"in 2014) was to try to recover content for a corrupt database file and\n"
+"write the recovered content into a new database named FILE.  However,\n"
+"the command has always been experimental and undocumented.  Today, it\n"
+"does nothing more than make a copy of the main database into a new\n"
+"database named by its only argument, FILE.  No repair attempts are\n"
+"made.  The command is retained for legacy only.\n"
+  },
   { ".import",
 "USAGE: .import [OPTIONS] FILE TABLE\n"
 "\n"
@@ -29130,7 +29798,9 @@ static const struct {
 "\n"
 "Change the output mode to MODE and/or apply OPTIONS to the output mode.\n"
 "Arguments are processed from left to right.  If no arguments, show the\n"
-"current output mode and relevant options.\n"
+"current output mode and relevant options.  Use the --list option to get\n"
+"a list of available MODE values.  Use -v to show all the current output\n"
+"mode settings.\n"
 "\n"
 "Options:\n"
 "  --align STRING           Set the alignment of text in columnar modes\n"
@@ -29146,8 +29816,12 @@ static const struct {
 "                           truncated. Zero means \"no limit\".\n"
 "  --colsep STRING          Use STRING as the column separator\n"
 "  --escape ESC             Enable/disable escaping of control characters\n"
-"                           found in the output. ESC can be \"off\", \"ascii\",\n"
-"                           or \"symbol\".\n"
+"                           found in the output. ESC can be \"auto\", \"off\",\n"
+"                           \"ascii\", or \"symbol\".  Default is \"auto\"\n"
+"  --fpfmt STRING           String is a printf-style format string used to\n"
+"                           render floating-point values.\n"
+"  --ifmt STRING            String is a printf-style format string used to\n"
+"                           render integer values.\n"
 "  --linelimit N            Set the maximum number of output lines to show for\n"
 "                           any single SQL value to N. Longer values are\n"
 "                           truncated. Zero means \"no limit\". Only works\n"
@@ -29176,7 +29850,7 @@ static const struct {
 "                           means \"no limit\".  Or N can be \"auto\" to set the\n"
 "                           width automatically.\n"
 "  --tablename NAME         Set the name of the table for \"insert\" mode.\n"
-"  --tag NAME               Save mode to the left as NAME.\n"
+"  --tag NAME               Save current settings as a new mode called NAME.\n"
 "  --textjsonb BOOLEAN      If enabled, JSONB text is displayed as text JSON.\n"
 "  --title ARG              Whether or not to show column headers, and if so\n"
 "                           how to encode them.  ARG can be \"off\", \"on\",\n"
@@ -29261,6 +29935,27 @@ static const struct {
 "                         will follow.\n"
 "  --notglob              Output should not match PATTERN\n"
 "  --show                 Write testcase output to the screen, for debugging.\n"
+  },
+  { ".stats",
+"USAGE: .stats [OPTIONS] ...\n"
+"\n"
+"Show status information about the database and database connection.\n"
+"The stats setting is persistent, unless the --once option is used.\n"
+"Options are cumulative and are processed from left to right.\n"
+"If no options are given, that is the same as \".stats all --once\".\n"
+"\n"
+"Options:\n"
+"  all          Show all available stats.\n"
+"  io           Show I/O stats (Linux only).\n"
+"  mem          Show basic memory usage.\n"
+"  off          Show nothing.  Turn off stat display.\n"
+"  on           Shorthand for \"stmt xmem\".\n"
+"  --once       Show the stats once. Do not change the default stats display.\n"
+"  reset        Reset \"max\" values after next display of each that value.\n"
+"  stmt         Show information about the last SQL statement.\n"
+"  vmstep       Show the number of virtual machine steps taken.\n"
+"  xmem         Show detailed memory status information\n"
+"  xstmt        Show details about each SQL statement\n"
   },
   { ".testcase",
 "USAGE: .testcase [OPTIONS] NAME\n"
@@ -29272,6 +29967,37 @@ static const struct {
 "Options:\n"
 "  --error-prefix TEXT       Change error message prefix text to TEXT\n"
   },
+  { ".if",
+"USAGE: .if SQL-EXPRESSION\n"
+"\n"
+"The argument can be any SQL expression, including a subquery in\n"
+"parentheses.  If the expression evalutes to true (if it is non-zero\n"
+"when converted into an number) then subsequent statements are\n"
+"evaluated.  If SQL-EXPRESSION evalutes to false or NULL or raises\n"
+"an error, then subsequent statements are skipped, up until the first\n"
+"matching .elseif, .else, or .endif.\n"
+"\n"
+"Maximum nested depth is limited.  An error is raised if .if statements\n"
+"are nested to deep.\n"
+"\n"
+"Every .if must be followed by a matching .endif, or an error is raised.\n"
+  },
+  { ".elseif",
+"USAGE: .elseif SQL-EXPRESSION\n"
+"\n"
+"An \"else-if\".  The prior .if is finished and a new condition is applied.\n"
+  },
+  { ".else",
+"USAGE: .else\n"
+"\n"
+"Invert the sense of the inner-most .if.  Start processing if the prior\n"
+".if was false.  Stop processing if the prior .if was true.\n"
+  },
+  { ".endif",
+"USAGE: .endif\n"
+"\n"
+"Terminate an .if statement.\n"
+  },
 };
 
 /*
@@ -29429,6 +30155,11 @@ static char *readFile(const char *zName, int *pnByte){
     return 0;
   }
   nIn = ftell(in);
+  if( nIn<0 ){
+    cli_printf(stderr,"Error: cannot read '%s'\n", zName);
+    fclose(in);
+    return 0;
+  }
   rewind(in);
   pBuf = sqlite3_malloc64( nIn+1 );
   if( pBuf==0 ){
@@ -29436,9 +30167,9 @@ static char *readFile(const char *zName, int *pnByte){
     fclose(in);
     return 0;
   }
-  nRead = fread(pBuf, nIn, 1, in);
+  nRead = fread(pBuf, 1, nIn, in);
   fclose(in);
-  if( nRead!=1 ){
+  if( nRead!=(size_t)nIn ){
     sqlite3_free(pBuf);
     cli_printf(stderr,"Error: cannot read '%s'\n", zName);
     return 0;
@@ -29655,7 +30386,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
   }
   *pnData = 0;
   nLine++;
-  if( sqlite3_fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
+  if( shell_fgets(p, zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
   rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz);
   if( rc!=2 ) goto readHexDb_error;
   if( n<0 ) goto readHexDb_error;
@@ -29667,7 +30398,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
   a = sqlite3_malloc64( sz ? sz : 1 );
   shell_check_oom(a);
   memset(a, 0, sz);
-  for(nLine++; sqlite3_fgets(zLine, sizeof(zLine), in)!=0; nLine++){
+  for(nLine++; shell_fgets(p, zLine, sizeof(zLine), in)!=0; nLine++){
     int j = 0;                    /* Page number from "| page" line */
     int k = 0;                    /* Offset from "| page" line */
     if( nLine>=2000000000 ){
@@ -29705,7 +30436,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
   if( in!=p->in ){
     fclose(in);
   }else{
-    while( sqlite3_fgets(zLine, sizeof(zLine), p->in)!=0 ){
+    while( shell_fgets(p, zLine, sizeof(zLine), p->in)!=0 ){
       nLine++;
       if(cli_strncmp(zLine, "| end ", 6)==0 ) break;
     }
@@ -29734,6 +30465,9 @@ static void shellUSleepFunc(
 /*
 ** SQL function:  shell_module_schema(X)
 **
+** For use internally and for debugging and testing only.  This function
+** is subject to change or removal in future versions.
+**
 ** Return a fake schema for the table-valued function or eponymous virtual
 ** table X.
 */
@@ -29758,6 +30492,7 @@ static void shellModuleSchema(
   p->pLog = pSavedLog;
 
   if( zFake ){
+    assert( strstr(zFake,"*/")==0 );
     sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake),
                         -1, sqlite3_free);
     sqlite3_free(zFake);
@@ -29888,7 +30623,9 @@ static void open_db(ShellState *p, int openFlags){
     sqlite3_diskused_init(p->db, 0, 0);
     sqlite3_strdup_init(p->db, 0, 0);
 #ifndef SQLITE_SHELL_FIDDLE
-    sqlite3_fileio_init(p->db, 0, 0);
+    if( !p->bSafeMode ){
+      sqlite3_fileio_init(p->db, 0, 0);
+    }
     sqlite3_completion_init(p->db, 0, 0);
 #endif
 #ifdef SQLITE_HAVE_ZLIB
@@ -29925,6 +30662,8 @@ static void open_db(ShellState *p, int openFlags){
     }
 #endif
 
+    sqlite3_create_function(p->db, "capable", 1, SQLITE_UTF8, p,
+                            shellCapable, 0, 0);
     sqlite3_create_function(p->db, "strtod", 1, SQLITE_UTF8, 0,
                             shellStrtod, 0, 0);
     sqlite3_create_function(p->db, "dtostr", 1, SQLITE_UTF8, 0,
@@ -29939,6 +30678,10 @@ static void open_db(ShellState *p, int openFlags){
                             shellPutsFunc, 0, 0);
     sqlite3_create_function(p->db, "shell_format_schema", 2, SQLITE_UTF8, p,
                             shellFormatSchema, 0, 0);
+    sqlite3_create_function(p->db, "shell_complete_sql", 1, SQLITE_UTF8, p,
+                            shell_complete_sql, 0, 0);
+    sqlite3_create_function(p->db, "shell_complete_sql", 2, SQLITE_UTF8, p,
+                            shell_complete_sql, 0, 0);
     sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0,
                             shellUSleepFunc, 0, 0);
 #ifndef SQLITE_NOHAVE_SYSTEM
@@ -29951,6 +30694,10 @@ static void open_db(ShellState *p, int openFlags){
                             p, shellExpandPrompt, 0, 0);
     sqlite3_create_function(p->db, "shell_temp_filename", 1, SQLITE_UTF8,
                             p, shellTempFilenameFunc, 0, 0);
+#ifdef SQLITE_DEBUG
+    sqlite3_create_function(p->db, "sqlite3_incomplete", 1, SQLITE_UTF8,
+                            p, shellIncompleteFunc, 0, 0);
+#endif
 
 
     if( p->openMode==SHELL_OPEN_ZIPFILE ){
@@ -30493,223 +31240,44 @@ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){
   return p->z;
 }
 
+#ifndef SQLITE_SHELL_FIDDLE
 /*
-** Try to transfer data for table zTable.  If an error is seen while
-** moving forward, try to go backwards.  The backwards movement won't
-** work for WITHOUT ROWID tables.
-*/
-static void tryToCloneData(
-  ShellState *p,
-  sqlite3 *newDb,
-  const char *zTable
-){
-  sqlite3_stmt *pQuery = 0;
-  sqlite3_stmt *pInsert = 0;
-  char *zQuery = 0;
-  char *zInsert = 0;
-  int rc;
-  int i, j, n;
-  int nTable = strlen30(zTable);
-  int k = 0;
-  int cnt = 0;
-  const int spinRate = 10000;
-
-  zQuery = sqlite3_mprintf("SELECT * FROM \"%w\"", zTable);
-  shell_check_oom(zQuery);
-  rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-  if( rc ){
-    cli_printf(stderr,"Error %d: %s on [%s]\n",
-          sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
-    goto end_data_xfer;
-  }
-  n = sqlite3_column_count(pQuery);
-  zInsert = sqlite3_malloc64(200 + nTable + n*3);
-  shell_check_oom(zInsert);
-  sqlite3_snprintf(200+nTable,zInsert,
-                   "INSERT OR IGNORE INTO \"%s\" VALUES(?", zTable);
-  i = strlen30(zInsert);
-  for(j=1; jdb, zQuery, -1, &pQuery, 0);
-    if( rc ){
-      cli_printf(stderr,"Warning: cannot step \"%s\" backwards", zTable);
-      break;
-    }
-  } /* End for(k=0...) */
-
-end_data_xfer:
-  sqlite3_finalize(pQuery);
-  sqlite3_finalize(pInsert);
-  sqlite3_free(zQuery);
-  sqlite3_free(zInsert);
-}
-
-
-/*
-** Try to transfer all rows of the schema that match zWhere.  For
-** each row, invoke xForEach() on the object defined by that row.
-** If an error is encountered while moving forward through the
-** sqlite_schema table, try again moving backwards.
+** DOT-COMMAND: .clone
+**
+** USAGE: .clone FILE
+**
+** The original purpose of the ".clone" command (when it was first introduced
+** in 2014) was to try to recover content for a corrupt database file and
+** write the recovered content into a new database named FILE.  However,
+** the command has always been experimental and undocumented.  Today, it
+** does nothing more than make a copy of the main database into a new
+** database named by its only argument, FILE.  No repair attempts are
+** made.  The command is retained for legacy only.
 */
-static void tryToCloneSchema(
-  ShellState *p,
-  sqlite3 *newDb,
-  const char *zWhere,
-  void (*xForEach)(ShellState*,sqlite3*,const char*)
-){
-  sqlite3_stmt *pQuery = 0;
-  char *zQuery = 0;
+static int dotCmdClone(ShellState *p){
+  int nArg = p->dot.nArg;     /* Number of arguments */
+  char **azArg = p->dot.azArg;/* Argument list */
+  char *zSql;
+  sqlite3_stmt *pStmt = 0;
   int rc;
-  const unsigned char *zName;
-  const unsigned char *zSql;
-  char *zErrMsg = 0;
 
-  zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema"
-                           " WHERE %s ORDER BY rowid ASC", zWhere);
-  shell_check_oom(zQuery);
-  rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-  if( rc ){
-    cli_printf(stderr,
-          "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db),
-          sqlite3_errmsg(p->db), zQuery);
-    goto end_schema_xfer;
-  }
-  while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){
-    zName = sqlite3_column_text(pQuery, 0);
-    zSql = sqlite3_column_text(pQuery, 1);
-    if( zName==0 || zSql==0 ) continue;
-    if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){
-      cli_printf(stdout, "%s... ", zName); fflush(stdout);
-      sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
-      if( zErrMsg ){
-        cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
-        sqlite3_free(zErrMsg);
-        zErrMsg = 0;
-      }
-    }
-    if( xForEach ){
-      xForEach(p, newDb, (const char*)zName);
-    }
-    sputz(stdout, "done\n");
-  }
-  if( rc!=SQLITE_DONE ){
-    sqlite3_finalize(pQuery);
-    sqlite3_free(zQuery);
-    zQuery = sqlite3_mprintf("SELECT name, sql FROM sqlite_schema"
-                             " WHERE %s ORDER BY rowid DESC", zWhere);
-    shell_check_oom(zQuery);
-    rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
-    if( rc ){
-      cli_printf(stderr,"Error: (%d) %s on [%s]\n",
-            sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
-      goto end_schema_xfer;
-    }
-    while( sqlite3_step(pQuery)==SQLITE_ROW ){
-      zName = sqlite3_column_text(pQuery, 0);
-      zSql = sqlite3_column_text(pQuery, 1);
-      if( zName==0 || zSql==0 ) continue;
-      if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue;
-      cli_printf(stdout, "%s... ", zName); fflush(stdout);
-      sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
-      if( zErrMsg ){
-        cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
-        sqlite3_free(zErrMsg);
-        zErrMsg = 0;
-      }
-      if( xForEach ){
-        xForEach(p, newDb, (const char*)zName);
-      }
-      sputz(stdout, "done\n");
-    }
-  }
-end_schema_xfer:
-  sqlite3_finalize(pQuery);
-  sqlite3_free(zQuery);
-}
-
-/*
-** Open a new database file named "zNewDb".  Try to recover as much information
-** as possible out of the main database (which might be corrupt) and write it
-** into zNewDb.
-*/
-static void tryToClone(ShellState *p, const char *zNewDb){
-  int rc;
-  sqlite3 *newDb = 0;
-  if( access(zNewDb,0)==0 ){
-    cli_printf(stderr,"File \"%s\" already exists.\n", zNewDb);
-    return;
+  failIfSafeMode(p, "cannot run .clone in safe mode");
+  if( nArg!=2 ){
+    dotCmdError(p, nArg, 0, "missing FILE argument");
+    return 1;
   }
-  rc = sqlite3_open(zNewDb, &newDb);
-  if( rc ){
-    cli_printf(stderr,
-        "Cannot create output database: %s\n", sqlite3_errmsg(newDb));
-  }else{
-    sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0);
-    sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0);
-    tryToCloneSchema(p, newDb, "type='table'", tryToCloneData);
-    tryToCloneSchema(p, newDb, "type!='table'", 0);
-    sqlite3_exec(newDb, "COMMIT;", 0, 0, 0);
-    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
+  zSql = sqlite3_mprintf("VACUUM INTO %Q", azArg[1]);
+  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc!=SQLITE_OK || SQLITE_DONE!=sqlite3_step(pStmt) ){
+    cli_printf(stderr,"Error: cannot clone into \"%s\"\n", azArg[1]);
+    sqlite3_finalize(pStmt);
+    return 1;
   }
-  close_db(newDb);
+  sqlite3_finalize(pStmt);
+  return 0;
 }
+#endif /* !defined(SQLITE_SHELL_FIDDLE) */
 
 #ifndef SQLITE_SHELL_FIDDLE
 /*
@@ -30759,7 +31327,11 @@ static void output_reset(ShellState *p){
       "xdg-open";
 #endif
       char *zCmd;
+#ifdef __linux__
+      zCmd = sqlite3_mprintf("%s %s 2>/dev/null", zXdgOpenCmd, p->zTempFile);
+#else
       zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile);
+#endif
       if( system(zCmd) ){
         cli_printf(stderr,"Failed: [%s]\n", zCmd);
       }else{
@@ -30768,7 +31340,7 @@ static void output_reset(ShellState *p){
         ** p->zTempFile data file out from under it */
         p->aUnlink = realloc(p->aUnlink, sizeof(Unlink)*(1+p->nUnlink));
         shell_check_oom(p->aUnlink);
-        p->aUnlink[p->nUnlink].tm = timeOfDay()+10000;
+        p->aUnlink[p->nUnlink].tm = timeOfDay()+10000000;
         p->aUnlink[p->nUnlink].zFN = p->zTempFile;
         p->nUnlink++;
         p->zTempFile = 0;
@@ -31209,8 +31781,9 @@ int shellDeleteFile(const char *zFilename, int *pRetry){
       sqlite3_sleep(pRetry[0]);
       pRetry[0] = 0;
       continue;
+    }else{
+      break;
     }
-    break;
   }
   return rc;
 }
@@ -31233,23 +31806,20 @@ int shellDeleteFile(const char *zFilename, int *pRetry){
 ** wait for nDelay milliseconds and try again before giving up.  Only
 ** a single wait occurs, even if problems are encountered with multiple
 ** temp files.  Only the first problem seen takes the delay.  Thus
-** the total delay never exceeds nDelay milliseconds.  When nDelay
-** is non-zero, memory used to hold the filename is reclaimed regardless
-** of whether or not the temporary files were successfully deleted.
+** the total delay never exceeds nDelay milliseconds.
 **
 ** If bForce is true, deletion is attempted on p->aUnlink[] files
 ** even if their time has not expired.  However, there is a pause
-** of up to nDelay milliseconds before doing the deletion.
-**
-** When nDelay is zero and the deletion attempt fails, memory used to
-** store temporary filesnames is not reclaimed.
+** of up to nDelay milliseconds before doing the deletion.  Memory
+** used to hold filenames is reclaimed regardless of whether or not
+** the delete is successful.
 **
 ** The bForce flag is used only when the process is about to exit.  When
 ** bForce is set, that indicates that this is our last opportunity to
 ** clean up temporary files.
 */
 static void clearTempFile(ShellState *p, int nDelay, int bForce){
-  int alwaysFree = (nDelay>0) || bForce;
+  int alwaysFree = bForce;
   int rc = 0;
   if( p->zTempFile && (!p->doXdgOpen || bForce) ){
     rc = shellDeleteFile(p->zTempFile, &nDelay);
@@ -31265,7 +31835,11 @@ static void clearTempFile(ShellState *p, int nDelay, int bForce){
       int doDelete =  tmToGo<=0;
       if( !doDelete && alwaysFree ){
         int tmSleep = nDelay;
-        if( tmSleep > tmToGo ) tmSleep = (int)tmToGo;
+        if( tmSleep > tmToGo/1000 ) tmSleep = (int)(tmToGo/1000);
+        if( tmSleep>=1000 ){
+          cli_printf(p->out,"Waiting about %d.%d seconds to delete temp"
+                            " file(s)\n", tmSleep/1000, (tmSleep/100)%10);
+        }
         sqlite3_sleep(tmSleep);
         nDelay -= tmSleep;
         doDelete = 1;
@@ -31330,6 +31904,9 @@ static void newTempFile(ShellState *p, const char *zSuffix){
 /*
 ** SQL function:  shell_temp_filename(SUFFIX)
 **
+** For use internally and for debugging and testing only.  This function
+** is subject to change or removal in future versions.
+**
 ** Return a randomly generated temporary filename.
 */
 static void shellTempFilenameFunc(
@@ -32065,7 +32642,7 @@ static int arListCommand(ArCommand *pAr){
   shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose],
                      pAr->zSrcTable, zWhere);
   if( pAr->bDryRun ){
-    cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql));
+    if( pSql ) cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql));
   }else{
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       if( pAr->bVerbose ){
@@ -32501,8 +33078,19 @@ static int arDotCommand(
 */
 static int recoverSqlCb(void *pCtx, const char *zSql){
   ShellState *pState = (ShellState*)pCtx;
-  cli_printf(pState->out, "%s;\n", zSql);
-  return SQLITE_OK;
+  sqlite3_str *pOut = sqlite3_str_new(pState->db);
+  char *zOut;
+  int rc = SQLITE_OK;
+  sqlite3_str_appendall(pOut, zSql);
+  appendCompletion(pOut, zSql, 0);
+  zOut = sqlite3_str_value(pOut);
+  if( zOut ){
+    cli_printf(pState->out, "%s;\n", zOut);
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+  sqlite3_str_free(pOut);
+  return rc;
 }
 
 /*
@@ -33008,9 +33596,17 @@ static int dotCmdImport(ShellState *p){
       }
     }else if( cli_strcmp(z,"-v")==0 ){
       eVerbose++;
-    }else if( cli_strcmp(z,"-schema")==0 && izInFile;
     sCtx.nLine = p->lineno+1;
     iLineOffset = p->lineno;
-    while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){
+    while( shell_fgets(p, zLine,sizeof(zLine),p->in) ){
       if( ckEnd && cli_strncmp(zLine,zEndMark,nEndMark)==0 ){
         ckEnd = 2;
         if( strchr(zLine,'\n') ) p->lineno++;
@@ -33157,7 +33761,7 @@ static int dotCmdImport(ShellState *p){
   import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
   if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) 
    && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema"
-                       " WHERE name=%Q AND type='view'",
+                       " WHERE name=%Q COLLATE nocase AND type='view'",
                        zSchema ? zSchema : "main", zTable)
   ){
     /* Table does not exist.  Create it. */
@@ -33409,7 +34013,9 @@ static int modeTitleDsply(ShellState *p, int bAll){
 **
 ** Change the output mode to MODE and/or apply OPTIONS to the output mode.
 ** Arguments are processed from left to right.  If no arguments, show the
-** current output mode and relevant options.
+** current output mode and relevant options.  Use the --list option to get
+** a list of available MODE values.  Use -v to show all the current output
+** mode settings.
 **
 ** Options:
 **   --align STRING           Set the alignment of text in columnar modes
@@ -33425,8 +34031,12 @@ static int modeTitleDsply(ShellState *p, int bAll){
 **                            truncated. Zero means "no limit".
 **   --colsep STRING          Use STRING as the column separator
 **   --escape ESC             Enable/disable escaping of control characters
-**                            found in the output. ESC can be "off", "ascii",
-**                            or "symbol".
+**                            found in the output. ESC can be "auto", "off",
+**                            "ascii", or "symbol".  Default is "auto"
+**   --fpfmt STRING           String is a printf-style format string used to
+**                            render floating-point values.
+**   --ifmt STRING            String is a printf-style format string used to
+**                            render integer values.
 **   --linelimit N            Set the maximum number of output lines to show for
 **                            any single SQL value to N. Longer values are
 **                            truncated. Zero means "no limit". Only works
@@ -33455,7 +34065,7 @@ static int modeTitleDsply(ShellState *p, int bAll){
 **                            means "no limit".  Or N can be "auto" to set the
 **                            width automatically.
 **   --tablename NAME         Set the name of the table for "insert" mode.
-**   --tag NAME               Save mode to the left as NAME.
+**   --tag NAME               Save current settings as a new mode called NAME.
 **   --textjsonb BOOLEAN      If enabled, JSONB text is displayed as text JSON.
 **   --title ARG              Whether or not to show column headers, and if so
 **                            how to encode them.  ARG can be "off", "on",
@@ -33590,14 +34200,14 @@ static int dotCmdMode(ShellState *p){
       if( (++i)>=nArg ){
         dotCmdError(p, i-1, "missing argument", 0);
         return 1;
-      }                       /*  0     1       2  <-- One less than QRF_ESC_ */
-      k = pickStr(azArg[i],&zErr,"off","ascii","symbol","");
+      }                       /* 0        1     2       3 */
+      k = pickStr(azArg[i],&zErr,"auto", "off","ascii","symbol","");
       if( k<0 ){
         dotCmdError(p, i, "unknown escape type", "%s", zErr);
         sqlite3_free(zErr);
         return 1;
       }
-      p->mode.spec.eEsc = k+1;
+      p->mode.spec.eEsc = k;
       chng = 1;
     }else if( optionMatch(z,"limits") ){
       if( (++i)>=nArg ){
@@ -33683,6 +34293,40 @@ static int dotCmdMode(ShellState *p){
           break;
       }
       chng = 1;
+    }else if( optionMatch(z,"ifmt") ){
+      if( i+1>=nArg ){
+        dotCmdError(p, i, "missing argument", 0);
+        return 1;
+      }
+      i++;
+      if( azArg[i][0]==0 || cli_strcmp(azArg[i],"auto")==0 ){
+        p->mode.spec.zIFmt = 0;
+      }else if( sqlite3_qrf_ckformat(azArg[i])!=1 ){
+        dotCmdError(p, i, "not a valid integer format", 0);
+        return 1;
+      }else{
+        assert( strlen(azArg[i])mode.zIFmt)-1 );
+        memcpy(p->mode.zIFmt, azArg[i], strlen(azArg[i])+1);
+        p->mode.spec.zIFmt = p->mode.zIFmt;
+      }
+      chng = 1;
+    }else if( optionMatch(z,"fpfmt") ){
+      if( i+1>=nArg ){
+        dotCmdError(p, i, "missing argument", 0);
+        return 1;
+      }
+      i++;
+      if( azArg[i][0]==0 || cli_strcmp(azArg[i],"auto")==0 ){
+        p->mode.spec.zFpFmt = 0;
+      }else if( sqlite3_qrf_ckformat(azArg[i])!=2 ){
+        dotCmdError(p, i, "not a valid floating-point format", 0);
+        return 1;
+      }else{
+        assert( strlen(azArg[i])mode.zFpFmt)-1 );
+        memcpy(p->mode.zFpFmt, azArg[i], strlen(azArg[i])+1);
+        p->mode.spec.zFpFmt = p->mode.zFpFmt;
+      }
+      chng = 1;
     }else if( optionMatch(z,"reset") ){
       int saved_eMode = p->mode.eMode;
       modeFree(&p->mode);
@@ -33810,10 +34454,10 @@ static int dotCmdMode(ShellState *p){
         return 1;
       }
       zW = azArg[++i];
-      /* Every width value takes at least 2 bytes in the input string to
-      ** specify, so strlen(zW) bytes should be plenty of space to hold the
-      ** result. */
-      aWidth = malloc( strlen(zW) );
+      /* Every width value except a single-digit final value takes at
+      ** least 2 bytes in the input string to specify, so strlen(zW)+1
+      ** entries is plenty of space to hold the result. */
+      aWidth = malloc( strlen(zW)+1 );
       while( IsSpace(zW[0]) ) zW++;
       while( zW[0] ){
         int w = 0;
@@ -33917,6 +34561,14 @@ static int dotCmdMode(ShellState *p){
     if( bAll || p->mode.spec.eEsc!=QRF_Auto ){
       sqlite3_str_appendf(pDesc, " --escape %s",qrfEscNames[p->mode.spec.eEsc]);
     }
+    if( bAll || p->mode.spec.zFpFmt!=0 ){
+      const char *z = p->mode.spec.zFpFmt;
+      sqlite3_str_appendf(pDesc, " --fpfmt %s", z ? z : "auto");
+    }
+    if( bAll || p->mode.spec.zIFmt!=0 ){
+      const char *z = p->mode.spec.zIFmt;
+      sqlite3_str_appendf(pDesc, " --ifmt %s", z ? z : "auto");
+    }
     if( bAll
      || (p->mode.spec.nLineLimit>0 && pI->eCx>0)
      || p->mode.spec.nCharLimit>0
@@ -33952,7 +34604,8 @@ static int dotCmdMode(ShellState *p){
     if( bAll 
      || (pI->eText!=p->mode.spec.eText && (pI->eText>1 || p->mode.spec.eText>1))
     ){
-      sqlite3_str_appendf(pDesc," --quote %s",qrfQuoteNames[p->mode.spec.eText]);
+      sqlite3_str_appendf(pDesc," --quote %s",
+                          qrfQuoteNames[p->mode.spec.eText]);
     }
     if( bAll 
      || ((p->mode.spec.bRowCount==QRF_Yes) != ((pI->mFlg&4)!=0))
@@ -34123,7 +34776,7 @@ static int dotCmdOutput(ShellState *p){
         }
         eMode = z[1];
       }else if( cli_strcmp(z,"-show")==0 ){
-        if( cli_output_capture ){
+        if( sqlite3_str_length(cli_output_capture)>0 ){
           sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture));
         }
       }else if( cli_strcmp(z,"-keep")==0 ){
@@ -34259,6 +34912,30 @@ static int dotCmdOutput(ShellState *p){
   return 1;
 }
 
+/*
+** Return true if the argument is <db);
+  char zLine[2000];
+  while( shell_fgets(p, zLine,sizeof(zLine),p->in) ){
+    if( strchr(zLine,'\n') ) p->lineno++;
+    if( cli_strncmp(zTerminator,zLine,nTerminator)==0 ) break;
+    sqlite3_str_appendall(pHeredoc, zLine);
+  }
+  return sqlite3_str_finish(pHeredoc);
+}
+
 /*
 ** DOT-COMMAND: .check
 ** USAGE: .check [OPTIONS] PATTERN
@@ -34292,6 +34969,15 @@ static int dotCmdCheck(ShellState *p){
   int isOk;                          /* True if results are OK */
   sqlite3_int64 iStart = p->lineno;  /* Line number of .check statement */
 
+  if( p->mIf!=0 ){
+    /* Unlike most other dot-commands, the .check command runs even if
+    ** inside the false branch of a .if.  It needs to do this so that it
+    ** can capture any "<0 && isHereDoc(azArg[nArg-1]) ){
+      sqlite3_free(captureHereDoc(p, azArg[nArg-1]+2));
+    }
+    return 0;
+  }
   if( p->zTestcase[0]==0 ){
     dotCmdError(p, 0, "no .testcase is active", 0);
     return 1;
@@ -34302,7 +34988,7 @@ static int dotCmdCheck(ShellState *p){
     if( cli_strcmp(z,"-keep")==0 ){
       bKeep = 1;
     }else if( cli_strcmp(z,"-show")==0 ){
-      if( cli_output_capture ){
+      if( sqlite3_str_length(cli_output_capture)>0 ){
         sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture));
       }
       bKeep = 1;
@@ -34325,23 +35011,15 @@ static int dotCmdCheck(ShellState *p){
     dotCmdError(p, 0, "no PATTERN specified", 0);
     return 1;
   }
-  if( cli_output_capture && sqlite3_str_length(cli_output_capture) ){
+  if( sqlite3_str_length(cli_output_capture) ){
     zTest = sqlite3_str_value(cli_output_capture);
     shell_check_oom(zTest);
   }else{
     zTest = "";
   }
   p->nTestRun++;
-  if( zCheck[0]=='<' && zCheck[1]=='<' && zCheck[2]!=0 ){
-    int nCheck = strlen30(zCheck);
-    sqlite3_str *pPattern = sqlite3_str_new(p->db);
-    char zLine[2000];
-    while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){
-      if( strchr(zLine,'\n') ) p->lineno++;
-      if( cli_strncmp(&zCheck[2],zLine,nCheck-2)==0 ) break;
-      sqlite3_str_appendall(pPattern, zLine);
-    }
-    zPattern = sqlite3_str_finish(pPattern);
+  if( isHereDoc(zCheck) ){
+    zPattern = captureHereDoc(p, zCheck+2);
     if( zPattern==0 ){
       zPattern = sqlite3_mprintf("");
     }
@@ -34399,6 +35077,97 @@ static int dotCmdCheck(ShellState *p){
   return 0;
 }
 
+/*
+** DOT-COMMAND: .stats
+** USAGE: .stats [OPTIONS] ...
+**
+** Show status information about the database and database connection.
+** The stats setting is persistent, unless the --once option is used.
+** Options are cumulative and are processed from left to right.
+** If no options are given, that is the same as ".stats all --once".
+**
+** Options:
+**   all          Show all available stats.
+**   io           Show I/O stats (Linux only).
+**   mem          Show basic memory usage.
+**   off          Show nothing.  Turn off stat display.
+**   on           Shorthand for "stmt xmem".
+**   --once       Show the stats once. Do not change the default stats display.
+**   reset        Reset "max" values after next display of each that value.
+**   stmt         Show information about the last SQL statement.
+**   vmstep       Show the number of virtual machine steps taken.
+**   xmem         Show detailed memory status information
+**   xstmt        Show details about each SQL statement
+*/
+static int dotCmdStats(ShellState *p){
+  int nArg = p->dot.nArg;        /* Number of arguments */
+  char **azArg = p->dot.azArg;   /* Text of the arguments */
+  int ii;                        /* Loop counter */
+  int bOnce = 0;                 /* --once option seen */
+  u8 mNew = 0;                   /* New status setting */
+  u8 savedStats = p->mStats;     /* Current stats setting */
+
+  if( nArg==1 ){
+    p->mStats = CLISTAT_ALL;
+    display_stats(p->db, p);
+    p->mStats = savedStats;
+    return 0;
+  }
+  for(ii=1; ii=0 ){
+      mNew |= CLISTAT_ON;
+    }else if( pickStr(z, 0, "off", "no", "OFF", "NO", "")>=0 ){
+      mNew = 0;
+    }else{
+      dotCmdError(p, ii, "unknown argument", 0);
+      return 1;
+    }
+  }
+  p->mStats = mNew;
+  if( bOnce ){
+    display_stats(p->db, p);
+    p->mStats = savedStats;
+  }
+  return 0;
+}
+
 /*
 ** DOT-COMMAND: .testcase
 ** USAGE: .testcase [OPTIONS] NAME
@@ -34448,6 +35217,143 @@ static int dotCmdTestcase(ShellState *p){
   return 0;
 }
 
+/*
+** Evaluate the text of zExpr as an SQL expression.  Return true if the
+** expression is true.  Return false if the expression evaluates to
+** false or NULL.  Return false if there is any kind of error, either
+** a run-time or a compile-time error.
+*/
+static int sqlExprIsTrue(ShellState *p, const char *zExpr){
+  char *zSql = sqlite3_mprintf("SELECT %s", zExpr);
+  sqlite3_stmt *pStmt;
+  int rc;
+
+  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc ){
+    rc = 0;
+  }else{
+    bind_prepared_stmt(p, pStmt);
+    rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ){
+      rc = sqlite3_column_double(pStmt,0)!=0.0;
+    }else{
+      rc = 0;
+    }
+  }
+  sqlite3_finalize(pStmt);
+  return rc;
+}
+
+/*
+** DOT-COMMAND: .if
+** USAGE: .if SQL-EXPRESSION
+**
+** The argument can be any SQL expression, including a subquery in
+** parentheses.  If the expression evalutes to true (if it is non-zero
+** when converted into an number) then subsequent statements are
+** evaluated.  If SQL-EXPRESSION evalutes to false or NULL or raises
+** an error, then subsequent statements are skipped, up until the first
+** matching .elseif, .else, or .endif.
+**
+** Maximum nested depth is limited.  An error is raised if .if statements
+** are nested to deep.
+**
+** Every .if must be followed by a matching .endif, or an error is raised.
+*/
+static int dotCmdIf(ShellState *p){
+  int jj;
+  if( p->nIfLevel==MX_IF ){
+    dotCmdError(p, 0, 0, ".if nested too deep");
+    return 1;
+  }
+  jj = p->nIfLevel++;
+  open_db(p, 0);
+  if( sqlExprIsTrue(p, p->dot.zOrig+3) ){
+    p->mIfHasBeenTrue |= 1<mIf |= 1<lnIf[jj] = p->lineno;
+  return 0;
+}
+
+/*
+** DOT-COMMAND: .elseif
+** USAGE: .elseif SQL-EXPRESSION
+**
+** An "else-if".  The prior .if is finished and a new condition is applied.
+*/
+static int dotCmdElseif(ShellState *p){
+  int jj = p->nIfLevel-1;
+  if( jjnIfAtStart ){
+    dotCmdError(p, 0, 0, ".elseif with no .if");
+    return 1;
+  }
+  if( (p->mElse & (1<lnElse[jj]);
+    p->mIf |= 1<mIfHasBeenTrue & (1<mIf |= 1<dot.zOrig+7) ){
+    p->mIf &= ~(1<mIfHasBeenTrue |= (1<mIf |= 1<nIfLevel<=p->nIfAtStart ){
+    dotCmdError(p, 0, 0, ".else without a .if");
+    return 1;
+  }
+  ii = p->nIfLevel-1;
+  if( (p->mElse & (1<lnElse[ii]);
+    return 1;
+  }
+  if( p->mIfHasBeenTrue & (1<mIf |= (1<mIf &= ~(1<mElse |= (1<lnElse[ii] = p->lineno;
+  return 0;
+}
+
+/*
+** DOT-COMMAND: .endif
+** USAGE: .endif
+**
+** Terminate an .if statement.
+*/
+static int dotCmdEndif(ShellState *p){
+  if( p->nIfLevel<=p->nIfAtStart ){
+    dotCmdError(p, 0, 0, ".endif without a .if");
+    return 1;
+  }
+  p->nIfLevel--;
+  p->mIf &= ~(1<nIfLevel);
+  p->mElse &= ~(1<nIfLevel);
+  p->mIfHasBeenTrue &= ~(1<nIfLevel);
+  return 0;
+}
+
+
+
 /*
 ** Enlarge the space allocated in p->dot so that it can hold more
 ** than nArg parsed command-line arguments.
@@ -34550,13 +35456,57 @@ static int do_meta_command(const char *zLine, ShellState *p){
 
 #ifndef SQLITE_OMIT_AUTHORIZATION
   if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){
-    if( nArg!=2 ){
-      cli_printf(stderr, "Usage: .auth ON|OFF\n");
+    int ii;
+    int eOnOff = -1;
+    open_db(p, 0);
+    for(ii=1; ii=nArg ){
+          dotCmdError(p, ii-1, "missing argument", 0);
+          rc = 1;
+          goto meta_command_exit;
+        }else{
+          free(p->zAuthDeny);
+          p->zAuthDeny = 0;
+          if( azArg[ii][0] ){
+            p->zAuthDeny = strdup(azArg[ii]);
+            shell_check_oom(p->zAuthDeny);
+          }
+        }
+      }else
+      if( strcmp(z,"-ignore")==0 ){
+        ii++;
+        if( ii>=nArg ){
+          dotCmdError(p, ii-1, "missing argument", 0);
+          rc = 1;
+          goto meta_command_exit;
+        }else{
+          free(p->zAuthIgnore);
+          p->zAuthIgnore = 0;
+          if( azArg[ii][0] ){
+            p->zAuthIgnore = strdup(azArg[ii]);
+            shell_check_oom(p->zAuthIgnore);
+          }
+        }
+      }else
+      if( eOnOff<0 ){
+        eOnOff = booleanValue(z);
+      }else
+      {
+        dotCmdError(p, ii, "unknown argument", 0);
+        rc = 1;
+        goto meta_command_exit;
+      }
+    }
+    if( eOnOff<0 ){
+      cli_printf(stderr, "Usage: .auth ON|OFF [--deny GLOB] [--ignore]\n");
       rc = 1;
       goto meta_command_exit;
     }
-    open_db(p, 0);
-    if( booleanValue(azArg[1]) ){
+    if( eOnOff ){
       sqlite3_set_authorizer(p->db, shellAuth, p);
     }else if( p->bSafeModePersist ){
       sqlite3_set_authorizer(p->db, safeModeAuth, p);
@@ -34709,13 +35659,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
 
 #ifndef SQLITE_SHELL_FIDDLE
   if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){
-    failIfSafeMode(p, "cannot run .clone in safe mode");
-    if( nArg==2 ){
-      tryToClone(p, azArg[1]);
-    }else{
-      eputz("Usage: .clone FILENAME\n");
-      rc = 1;
-    }
+    rc = dotCmdClone(p);
   }else
 #endif /* !defined(SQLITE_SHELL_FIDDLE) */
 
@@ -34996,7 +35940,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
     p->nErr = 0;
     if( zLike==0 ) zLike = sqlite3_mprintf("true");
     zSql = sqlite3_mprintf(
-      "SELECT name, type, sql FROM sqlite_schema AS o "
+      "SELECT name, type, shell_complete_sql(sql) FROM sqlite_schema AS o "
       "WHERE (%s) AND type=='table'"
       "  AND sql NOT NULL"
       " ORDER BY tbl_name='sqlite_sequence', rowid",
@@ -35006,7 +35950,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
     sqlite3_free(zSql);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
       zSql = sqlite3_mprintf(
-        "SELECT sql FROM sqlite_schema AS o "
+        "SELECT shell_complete_sql(sql) FROM sqlite_schema AS o "
         "WHERE (%s) AND sql NOT NULL"
         "  AND type IN ('index','trigger','view') "
         "ORDER BY type COLLATE NOCASE DESC",
@@ -35023,11 +35967,13 @@ static int do_meta_command(const char *zLine, ShellState *p){
     sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
     sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
-      cli_puts(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out);
+      cli_puts(p->nErr?";\nROLLBACK; -- due to errors\n":"COMMIT;\n", p->out);
     }
     p->shellFlgs = savedShellFlags;
     modeFree(&p->mode);
     p->mode = saved_mode;
+    if( p->mode.spec.zFpFmt ) p->mode.spec.zFpFmt = p->mode.zFpFmt;
+    if( p->mode.spec.zIFmt ) p->mode.spec.zIFmt = p->mode.zIFmt;
     rc = p->nErr>0;
   }else
 
@@ -35044,6 +35990,18 @@ static int do_meta_command(const char *zLine, ShellState *p){
     }
   }else
 
+  if( c=='e' && n==6 && cli_strcmp(azArg[0],"elseif")==0 ){
+    rc = dotCmdElseif(p);
+  }else
+
+  if( c=='e' && n==4 && cli_strcmp(azArg[0],"else")==0 ){
+    rc = dotCmdElse(p);
+  }else
+
+  if( c=='e' && n==5 && cli_strcmp(azArg[0],"endif")==0 ){
+    rc = dotCmdEndif(p);
+  }else
+
   if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){
     if( nArg==2 ){
       if( p->mode.autoEQPtrace ){
@@ -35268,7 +36226,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
     }
     open_db(p, 0);
     zSql = sqlite3_mprintf(
-       "SELECT shell_format_schema(sql,%d) FROM"
+       "SELECT shell_format_schema(shell_complete_sql(sql),%d) FROM"
        "  (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x"
        "     FROM sqlite_schema UNION ALL"
        "   SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) "
@@ -35280,7 +36238,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
     data.mode.eMode = MODE_List;
     data.mode.spec.eText = QRF_TEXT_Plain;
     data.mode.spec.nCharLimit = 0;
-    data.mode.spec.zRowSep = "\n";
+    data.mode.spec.zRowSep = ";\n";
     rc = shell_exec(&data,zSql,0);
     sqlite3_free(zSql);
     if( rc==SQLITE_OK ){
@@ -35343,6 +36301,10 @@ static int do_meta_command(const char *zLine, ShellState *p){
     }
   }else
 
+  if( c=='i' && n==2 && cli_strcmp(azArg[0],"if")==0 ){
+    rc = dotCmdIf(p);
+  }else
+
 #ifndef SQLITE_SHELL_FIDDLE
   if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){
     rc = dotCmdImport(p);
@@ -35598,6 +36560,8 @@ static int do_meta_command(const char *zLine, ShellState *p){
       { "trigger_depth",         SQLITE_LIMIT_TRIGGER_DEPTH             },
       { "worker_threads",        SQLITE_LIMIT_WORKER_THREADS            },
       { "schema",                SQLITE_LIMIT_SCHEMA                    },
+      { "trigger_steps",         SQLITE_LIMIT_TRIGGER_STEPS             },
+      { "ctes",                  SQLITE_LIMIT_CTES                      },
     };
     int i, n2;
     open_db(p, 0);
@@ -35632,8 +36596,8 @@ static int do_meta_command(const char *zLine, ShellState *p){
         goto meta_command_exit;
       }
       if( nArg==3 ){
-        sqlite3_limit(p->db, aLimit[iLimit].limitCode,
-                      (int)integerValue(azArg[2]));
+        int v = (int)integerValue(azArg[2]);
+        sqlite3_limit(p->db, aLimit[iLimit].limitCode, v);
       }else{
         cli_printf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName,
               sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1));
@@ -36089,15 +37053,23 @@ static int do_meta_command(const char *zLine, ShellState *p){
 #endif
 
 #ifndef SQLITE_SHELL_FIDDLE
-  if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){
+  if( c=='r' && n==4 && cli_strcmp(azArg[0], "read")==0 ){
     FILE *inSaved = p->in;
     i64 savedLineno = p->lineno;
-    failIfSafeMode(p, "cannot run .read in safe mode");
+    if( p->mIf ){
+      if( nArg==2 && isHereDoc(azArg[1]) ){
+        sqlite3_free(captureHereDoc(p, azArg[1]+2));
+      }
+      return 0;
+    }
     if( nArg!=2 ){
       eputz("Usage: .read FILE\n");
       rc = 1;
       goto meta_command_exit;
     }
+    if( !isHereDoc(azArg[1]) ){
+      failIfSafeMode(p, "cannot run .read in safe mode");
+    }
     if( azArg[1][0]=='|' ){
 #ifdef SQLITE_OMIT_POPEN
       eputz("Error: pipes are not supported in this OS\n");
@@ -36112,6 +37084,20 @@ static int do_meta_command(const char *zLine, ShellState *p){
         pclose(p->in);
       }
 #endif
+    }else if( isHereDoc(azArg[1]) ){
+      char *saved_zInput;
+      i64 saved_nInputUsed;
+      char *zScript = captureHereDoc(p, azArg[1]+2);
+      savedLineno = p->lineno;
+      saved_zInput = p->zInput;
+      saved_nInputUsed = p->nInputUsed;
+      p->zInput = zScript;
+      p->nInputUsed = 0;
+      p->in = (FILE*)&p->zInput;
+      rc = process_input(p, "");
+      sqlite3_free(p->zInput);
+      p->zInput = saved_zInput;
+      p->nInputUsed = saved_nInputUsed;
     }else if( (p->in = openChrSource(azArg[1]))==0 ){
       cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
       rc = 1;
@@ -36283,7 +37269,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
       goto meta_command_exit;
     }
     pSql = sqlite3_str_new(p->db);
-    sqlite3_str_appendf(pSql, "SELECT sql FROM", 0);
+    sqlite3_str_appendf(pSql, "SELECT shell_complete_sql(sql,1) FROM", 0);
     iSchema = 0;
     while( sqlite3_step(pStmt)==SQLITE_ROW ){
       const char *zDb = (const char*)sqlite3_column_text(pStmt, 1);
@@ -36911,7 +37897,6 @@ static int do_meta_command(const char *zLine, ShellState *p){
 
   if( c=='s' && cli_strncmp(azArg[0], "show", n)==0 ){
     static const char *azBool[] = { "off", "on", "trigger", "full"};
-    const char *zOut;
     int i;
     if( nArg!=1 ){
       eputz("Usage: .show\n");
@@ -36951,13 +37936,42 @@ static int do_meta_command(const char *zLine, ShellState *p){
     cli_printf(p->out, "%12.12s: ", "rowseparator");
     output_c_string(p->out, p->mode.spec.zRowSep);
     cli_puts("\n", p->out);
-    switch( p->statsOn ){
-      case 0:  zOut = "off";     break;
-      default: zOut = "on";      break;
-      case 2:  zOut = "stmt";    break;
-      case 3:  zOut = "vmstep";  break;
+    cli_printf(p->out, "%12.12s:", "stats");
+    if( p->mStats==0 ){
+      cli_printf(p->out, " off\n");
+    }else{
+      u8 mStats = p->mStats;
+      if( (mStats & CLISTAT_ALL)==CLISTAT_ALL ){
+        cli_printf(p->out, " all");
+        mStats &= ~CLISTAT_ALL;
+      }
+      if( (mStats & CLISTAT_ON)==CLISTAT_ON ){
+        cli_printf(p->out, " on");
+        mStats &= ~CLISTAT_ON;
+      }
+      if( mStats & CLISTAT_XSTMT ){
+        cli_printf(p->out, " xstmt");
+      }else if( mStats & CLISTAT_STMT ){
+        cli_printf(p->out, " stmt");
+      }else if( mStats & CLISTAT_VMSTEP ){
+        cli_printf(p->out, " vmstep");
+      }
+      if( mStats & CLISTAT_XMEM ){
+        cli_printf(p->out, " xmem");
+      }else if( mStats & CLISTAT_MEM ){
+        cli_printf(p->out, " mem");
+      }
+      if( mStats & CLISTAT_IO ){
+        cli_printf(p->out, " io");
+      }
+      if( mStats & CLISTAT_ZIPVFS ){
+        cli_printf(p->out, " zipvfs");
+      }
+      if( mStats & CLISTAT_RESET ){
+        cli_printf(p->out, " reset");
+      }
+      cli_printf(p->out,"\n");
     }
-    cli_printf(p->out, "%12.12s: %s\n","stats", zOut);
     cli_printf(p->out, "%12.12s: ", "width");
     for(i=0; imode.spec.nWidth; i++){
       cli_printf(p->out, "%d ", (int)p->mode.spec.aWidth[i]);
@@ -36968,20 +37982,7 @@ static int do_meta_command(const char *zLine, ShellState *p){
   }else
 
   if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){
-    if( nArg==2 ){
-      if( cli_strcmp(azArg[1],"stmt")==0 ){
-        p->statsOn = 2;
-      }else if( cli_strcmp(azArg[1],"vmstep")==0 ){
-        p->statsOn = 3;
-      }else{
-        p->statsOn = (u8)booleanValue(azArg[1]);
-      }
-    }else if( nArg==1 ){
-      display_stats(p->db, p, 0);
-    }else{
-      eputz("Usage: .stats ?on|off|stmt|vmstep?\n");
-      rc = 1;
-    }
+    rc = dotCmdStats(p);
   }else
 
   if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) ){
@@ -37131,50 +38132,14 @@ static int do_meta_command(const char *zLine, ShellState *p){
         **      INTEGER      Mask of optimizations to disable
         */
         case SQLITE_TESTCTRL_OPTIMIZATIONS: {
-          static const struct {
-             unsigned int mask;    /* Mask for this optimization */
-             unsigned int bDsply;  /* Display this on output */
-             const char *zLabel;   /* Name of optimization */
-          } aLabel[] = {
-            { 0x00000001, 1, "QueryFlattener" },
-            { 0x00000001, 0, "Flatten" },
-            { 0x00000002, 1, "WindowFunc" },
-            { 0x00000004, 1, "GroupByOrder" },
-            { 0x00000008, 1, "FactorOutConst" },
-            { 0x00000010, 1, "DistinctOpt" },
-            { 0x00000020, 1, "CoverIdxScan" },
-            { 0x00000040, 1, "OrderByIdxJoin" },
-            { 0x00000080, 1, "Transitive" },
-            { 0x00000100, 1, "OmitNoopJoin" },
-            { 0x00000200, 1, "CountOfView" },
-            { 0x00000400, 1, "CursorHints" },
-            { 0x00000800, 1, "Stat4" },
-            { 0x00001000, 1, "PushDown" },
-            { 0x00002000, 1, "SimplifyJoin" },
-            { 0x00004000, 1, "SkipScan" },
-            { 0x00008000, 1, "PropagateConst" },
-            { 0x00010000, 1, "MinMaxOpt" },
-            { 0x00020000, 1, "SeekScan" },
-            { 0x00040000, 1, "OmitOrderBy" },
-            { 0x00080000, 1, "BloomFilter" },
-            { 0x00100000, 1, "BloomPulldown" },
-            { 0x00200000, 1, "BalancedMerge" },
-            { 0x00400000, 1, "ReleaseReg" },
-            { 0x00800000, 1, "FlttnUnionAll" },
-            { 0x01000000, 1, "IndexedEXpr" },
-            { 0x02000000, 1, "Coroutines" },
-            { 0x04000000, 1, "NullUnusedCols" },
-            { 0x08000000, 1, "OnePass" },
-            { 0x10000000, 1, "OrderBySubq" },
-            { 0x20000000, 1, "StarQuery" },
-            { 0x40000000, 1, "ExistsToJoin" },
-            { 0xffffffff, 0, "All" },
-          };
-          unsigned int curOpt;
-          unsigned int newOpt;
-          unsigned int m;
-          int ii;
-          int nOff;
+          unsigned int curOpt;       /* Current opt-disabled mask */
+          unsigned int newOpt;       /* New opt-disabled mask */
+          unsigned int m;            /* A working mask value */
+          sqlite3_uint64 mm;         /* 64-bit working mask value */
+          int ii, jj;                /* Loop counters */
+          int nOff;                  /* Number of disabled optimizations */
+          const char *zName;         /* Name of an optimization */
+
           sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, p->db, &curOpt);
           newOpt = curOpt;
           for(ii=2; ii=ArraySize(aLabel) ){
+              if( sqlite3_optimization_mask(zLabel,&mm)==SQLITE_NOTFOUND ){
                 cli_printf(stderr,
                     "Error: no such optimization: \"%s\"\n", zLabel);
-                cli_puts("Should be one of:", stderr);
-                for(jj=0; jjout);
-            for(ii=0; iiout, " -%s", aLabel[ii].zLabel);
+            for(ii=0, m=1; ii<32; ii++, m <<= 1){
+              if( (newOpt & m)!=0 
+               && (zName = sqlite3_optimization_name(m))!=0
+              ){
+                cli_printf(p->out, " -%s", zName);
               }
             }
           }else{
             cli_puts("-All", p->out);
-            for(ii=0; iiout, " +%s", aLabel[ii].zLabel);
+            for(ii=0, m=1; ii<32; ii++, m <<= 1){
+              if( (newOpt & m)==0
+               && (zName = sqlite3_optimization_name(m))!=0
+              ){
+                cli_printf(p->out, " +%s", zName);
               }
             }
           }
@@ -37744,23 +38709,6 @@ static int line_is_all_whitespace(const char *z){
   return 1;
 }
 
-/*
-** Return TRUE if the line typed in is an SQL command terminator other
-** than a semi-colon.  The SQL Server style "go" command is understood
-** as is the Oracle "/".
-*/
-static int line_is_command_terminator(const char *zLine){
-  while( IsSpace(zLine[0]) ){ zLine++; };
-  if( zLine[0]=='/' && line_is_all_whitespace(&zLine[1]) ){
-    return 1;  /* Oracle */
-  }
-  if( ToLower(zLine[0])=='g' && ToLower(zLine[1])=='o'
-         && line_is_all_whitespace(&zLine[2]) ){
-    return 1;  /* SQL Server */
-  }
-  return 0;
-}
-
 /*
 ** The CLI needs a working sqlite3_complete() to work properly.  So error
 ** out of the build if compiling with SQLITE_OMIT_COMPLETE.
@@ -37769,19 +38717,33 @@ static int line_is_command_terminator(const char *zLine){
 # error the CLI application is incompatible with SQLITE_OMIT_COMPLETE.
 #endif
 
+#ifdef SQLITE_SHELL_LEGACY_COMMAND_TERMINATOR
 /*
-** Return true if zSql is a complete SQL statement.  Return false if it
-** ends in the middle of a string literal or C-style comment.
+** Return TRUE if the line typed in is an SQL command terminator other
+** than a semi-colon.  The SQL Server style "go" command is understood
+** as is the Oracle "/".
 */
-static int line_is_complete(char *zSql, int nSql){
-  int rc;
-  if( zSql==0 ) return 1;
-  zSql[nSql] = ';';
-  zSql[nSql+1] = 0;
-  rc = sqlite3_complete(zSql);
-  zSql[nSql] = 0;
-  return rc;
+static int line_is_command_terminator(
+  const char *zLine,         /* Current line of input */
+  char *zSql,                /* Total accumulated input text */
+  int nSql                   /* Bytes of accumulated input text */
+){
+  while( IsSpace(zLine[0]) ){ zLine++; };
+  if( (zLine[0]=='/' && line_is_all_whitespace(&zLine[1]))  /* Oracle */
+   || (sqlite3_strnicmp(zLine,"go",2)==0
+             && line_is_all_whitespace(&zLine[2]))          /* SQL Server */
+  }{
+    int rc;
+    if( zSql==0 ) return 1;
+    zSql[nSql] = ';';
+    zSql[nSql+1] = 0;
+    rc = sqlite3_complete(zSql);
+    zSql[nSql] = 0;
+    return rc;
+  }
+  return 0;
 }
+#endif
 
 /*
 ** This function is called after processing each line of SQL in the
@@ -37868,6 +38830,63 @@ static int doAutoDetectRestore(ShellState *p, const char *zSql){
   return rc;
 }
 
+/*
+** When the (undocumented) command-line option --test-parse is used,
+** this routine is run instead of runOneSqlLine() to process SQL.  Don't
+** actually run any SQL.  Instead just provide a one-line summary of the
+** SQL that would have run.
+*/
+static void runTestParse(
+  ShellState *p,            /* Execution context */
+  char *zSql                /* SQL to be run */
+){
+  int nNL = 0;
+  char *zNL;
+  size_t i, j, n;
+  int nPre = 0;
+  int nSuf = 0;
+  int mxPre = 20;
+  char zPre[50], zSuf[50];
+
+  for(zNL=strchr(zSql,'\n'); zNL!=0; zNL=strchr(zNL+1,'\n')){ nNL++; }
+  n = strlen(zSql);
+  if( n<40 ) mxPre = 40;
+  for(i=0; nPre0 && c<' ' ){
+      zPre[nPre++] = '^';
+      zPre[nPre++] = c+0x40;
+    }else{
+      zPre[nPre++] = c;
+    }
+  }
+  zPre[nPre] = 0;
+  if( n<=i+20 ){
+    j = i;
+  }else{
+    j = n - 20;
+    memcpy(zSuf," ... ", 5);
+    nSuf = 5;
+  }
+  for(i=j; i0 && c<' ' ){
+      zSuf[nSuf++] = '^';
+      zSuf[nSuf++] = c+0x40;
+    }else{
+      zSuf[nSuf++] = c;
+    }
+  }
+  zSuf[nSuf] = 0;
+  cli_printf(p->out, "%d line%s: %s%s\n", nNL+1, (nNL?"s":""), zPre, zSuf);
+}
+
 /*
 ** Run a single line of SQL.  Return the number of errors.
 */
@@ -37883,9 +38902,14 @@ static int runOneSqlLine(
   open_db(p, 0);
   if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql);
   if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0;
-  BEGIN_TIMER(p);
-  rc = shell_exec(p, zSql, &zErrMsg);
-  END_TIMER(p);
+  if( p->bTestParse ){
+    runTestParse(p, zSql);
+    rc = SQLITE_OK;
+  }else{
+    BEGIN_TIMER(p);
+    rc = shell_exec(p, zSql, &zErrMsg);
+    END_TIMER(p);
+  }
   if( rc || zErrMsg ){
     sqlite3_str *pPrefix = sqlite3_str_new(p->db);
     const char *zErrorTail;
@@ -37934,8 +38958,11 @@ static int runOneSqlLine(
 
 static void echo_group_input(ShellState *p, const char *zDo){
   if( p->mode.mFlags & MFLG_ECHO ){
+    sqlite3_str *savedCliCapture = cli_output_capture;
+    cli_output_capture = 0;  /* .echo should not be captured by .testcase */
     cli_printf(p->out, "%s\n", zDo);
     fflush(p->out);
+    cli_output_capture = savedCliCapture;
   }
 }
 
@@ -37978,6 +39005,51 @@ static char *one_input_line(
 }
 #endif /* SQLITE_SHELL_FIDDLE */
 
+/*
+** Return true if the input would is complete SQL, or complete SQL
+** followed by only whitespace.  This is similar to sqlite3_complete()
+** except that if the final semicolon is followed by whitespace, it
+** still returns true.
+*/
+static int line_is_complete(char *zSql, int nSql){
+  sqlite3_int64 x;
+  assert( nSql>=0 && zSql[nSql]==0 );
+  x = sqlite3_incomplete(zSql);
+  if( x==0x00 || x==0x10 ) return 1;
+  if( x==0x2d0001 ){
+    zSql[nSql] = '\n';
+    zSql[nSql+1] = 0;
+    x = sqlite3_incomplete(zSql);
+    zSql[nSql] = 0;
+    if( x==0x00 || x==0x10 ) return 1;
+  }
+  return 0;
+}
+
+/*
+** Return true if zLine is a dot-command that should always be run
+** even if inside of the false leg of an ".if"
+*/
+static int alwaysRun(const char *zLine){
+  const char *az[] = {
+     "3.if",
+     "7.elseif",
+     "5.else",
+     "6.endif",
+     "6.check",    /* For its heredoc */
+     "5.read",     /* For its heredoc */
+     "0"
+  };
+  int jj, n;
+  for(jj=0; (n = az[jj][0] - '0')>0; jj++){
+    if( strncmp(zLine,az[jj]+1,n)==0 && (IsSpace(zLine[n]) || zLine[n]==0) ){
+      return 1;
+    }
+  }
+  return 0;
+}
+  
+
 /*
 ** Read input from *in and process it.  If *in==0 then input
 ** is interactive - the user is typing it it.  Otherwise, input
@@ -37999,6 +39071,7 @@ static int process_input(ShellState *p, const char *zSrc){
   i64 startline = 0;        /* Line number for start of current input */
   const char *saved_zInFile;      /* Prior value of p->zInFile */
   i64 saved_lineno;               /* Prior value of p->lineno */
+  int saved_nIfAtStart;           /* Prior value of p->nIfAtStart */
 
   if( p->inputNesting==MAX_INPUT_NESTING ){
     /* This will be more informative in a later version. */
@@ -38011,6 +39084,8 @@ static int process_input(ShellState *p, const char *zSrc){
   p->zInFile = zSrc;
   saved_lineno = p->lineno;
   p->lineno = 0;
+  saved_nIfAtStart = p->nIfAtStart;
+  p->nIfAtStart = p->nIfLevel;
   while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){
     fflush(p->out);
     zLine = one_input_line(p, zLine, zSql, nSql>0);
@@ -38032,21 +39107,26 @@ static int process_input(ShellState *p, const char *zSrc){
     if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){
       echo_group_input(p, zLine);
       if( zLine[0]=='.' ){
-        rc = do_meta_command(zLine, p);
-        if( rc==2 ){ /* exit requested */
-          break;
-        }else if( rc ){
-          errCnt++;
+        if( p->mIf==0 || alwaysRun(zLine) ){
+          rc = do_meta_command(zLine, p);
+          if( rc==2 ){ /* exit requested */
+            p->nIfLevel = p->nIfAtStart;
+            break;
+          }else if( rc ){
+            errCnt++;
+          }
         }
       }
       continue;
     }
-    if( line_is_command_terminator(zLine) && line_is_complete(zSql, nSql) ){
+#ifdef SQLITE_SHELL_LEGACY_COMMAND_TERMINATOR
+    if( line_is_command_terminator(zLine, zSql, nSql) ){
       memcpy(zLine,";",2);
     }
-    hasSemi = strchr(zLine,';')!=0;
+#endif
+    if( strchr(zLine,';')!=0 ) hasSemi = 1;
     nLine = strlen(zLine);
-    if( nSql+nLine+2>=nAlloc ){
+    if( nSql+nLine+4>=nAlloc ){
       /* Grow buffer by half-again increments when big. */
       nAlloc = nSql+(nSql>>1)+nLine+100;
       zSql = realloc(zSql, nAlloc);
@@ -38073,9 +39153,11 @@ static int process_input(ShellState *p, const char *zSrc){
       errCnt++;
       break;
     }
-    if( nSql && hasSemi && sqlite3_complete(zSql) ){
+    if( nSql && hasSemi && line_is_complete(zSql, nSql) ){
       echo_group_input(p, zSql);
-      errCnt += runOneSqlLine(p, zSql, p->zInFile, startline);
+      if( p->mIf==0 ){
+        errCnt += runOneSqlLine(p, zSql, p->zInFile, startline);
+      }
       nSql = 0;
       if( p->nPopOutput ){
         output_reset(p);
@@ -38096,13 +39178,24 @@ static int process_input(ShellState *p, const char *zSrc){
   if( nSql ){
     /* This may be incomplete. Let the SQL parser deal with that. */
     echo_group_input(p, zSql);
-    errCnt += runOneSqlLine(p, zSql, p->zInFile, startline);
+    if( p->mIf==0 ){
+      errCnt += runOneSqlLine(p, zSql, p->zInFile, startline);
+    }
   }
   free(zSql);
   free(zLine);
   --p->inputNesting;
+  while( p->nIfLevel > p->nIfAtStart ){
+    int ii = --p->nIfLevel;
+    cli_printf(stderr,"%s:%lld: unterminated .if\n", zSrc, p->lnIf[ii]);
+    p->mIf &= ~(1<mElse &= ~(1<mIfHasBeenTrue &= ~(1<zInFile = saved_zInFile;
   p->lineno = saved_lineno;
+  p->nIfAtStart = saved_nIfAtStart;
   return errCnt>0;
 }
 
@@ -38807,10 +39900,11 @@ int SQLITE_CDECL main(int argc, char **argv){
     }else if( cli_strcmp(z,"-nonce")==0 ){
       free(data.zNonce);
       data.zNonce = strdup(cmdline_option_value(argc, argv, ++i));
-    }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
+    }else if( cli_strcmp(z,"-unsafe-testing")==0
+           || cli_strcmp(z,"-unsafe")==0 ){
       ShellSetFlag(&data,SHFLG_TestingMode);
     }else if( cli_strcmp(z,"-safe")==0 ){
-      /* no-op - catch this on the second pass */
+      noInit = 1; /* --safe implies --noinit */
     }else if( cli_strcmp(z,"-escape")==0 && i+1SQLITE_LIMIT_SCHEMA
 ** 
The maximum number of objects (tables, indexes, triggers, and views) ** defined by the database schema.
)^ +** +** [[SQLITE_LIMIT_TRIGGER_STEPS]] ^(
SQLITE_LIMIT_TRIGGER_STEPS
+**
The maximum number of SQL statements that can be contained within +** a single trigger.
)^ +** +** [[SQLITE_LIMIT_CTES]] ^(
SQLITE_LIMIT_CTES
+**
The maximum number of [common table expressions] (CTEs) allowed +** in a single SQL statements. This limit is used to limit possible +** exponential growth in the number of required cursors required for +** nested joins of CTEs. The limit is an absolute upper bound, and +** SQLite may refuse to compile a statement if the number of CTEs simply +** approaches this limit. Set this limit to 0 to disable the restriction. +**
)^ ** */ #define SQLITE_LIMIT_LENGTH 0 @@ -4774,6 +4796,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_WORKER_THREADS 11 #define SQLITE_LIMIT_PARSER_DEPTH 12 #define SQLITE_LIMIT_SCHEMA 13 +#define SQLITE_LIMIT_TRIGGER_STEPS 14 +#define SQLITE_LIMIT_CTES 15 /* ** CAPI3REF: Prepare Flags @@ -6186,6 +6210,10 @@ SQLITE_API int sqlite3_create_window_function( ** These constants define integer codes that represent the various ** text encodings supported by SQLite. ** +** If any value other than the ones defined here are passed into the +** "encoding" argument of an API function, then the results are +** unspecified and subject to change. +** **
** [[SQLITE_UTF8]]
SQLITE_UTF8
Text is encoding as UTF-8
** @@ -7168,6 +7196,10 @@ SQLITE_API void sqlite3_activate_cerod( ** at all. ^In SQLite version 3.42.0 and later, a negative ** argument passed into sqlite3_sleep() is changed to zero before it is relayed ** down into the xSleep method of the VFS. +** +** The maximum sleep time is 2147483 milliseconds (about 35 minutes and +** 47 seconds). If the argument to sqlite3_sleep() is greater than 2147483, +** the sleep time is capped at 2147483 milliseconds. */ SQLITE_API int sqlite3_sleep(int); @@ -9083,7 +9115,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ #define SQLITE_TESTCTRL_ATOF 34 -#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_EXPIRE_STMTS 35 +#define SQLITE_TESTCTRL_LAST 35 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -11587,7 +11620,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_wal_info( ** ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory -** allocation error occurs. +** allocation or I/O error occurs. ** ** This interface is omitted if SQLite is compiled with the ** [SQLITE_OMIT_DESERIALIZE] option. @@ -15217,6 +15250,35 @@ struct fts5_api { # define SQLITE_MAX_SCHEMA 10000000 #endif +/* +** Maximum number of SQL statements allowed in the body of a single +** trigger. +*/ +#ifndef SQLITE_MAX_TRIGGER_STEPS +# define SQLITE_MAX_TRIGGER_STEPS 65000 +#endif + +/* +** Maximum number of CTE cursors (approximately). This limit will be +** exceeded if any CTE uses a cursor number larger than this value, even +** of some of the cursors are used for something other than CTEs. +** +** The purpose of this limit is to prevent resource exhaustion caused +** by hostile SQL inputs such as: +** +** WITH c0 AS (SELECT 1 AS x), +** c1 AS (SELECT a.x AS x FROM c0 a, c0 b), +** c2 AS (SELECT a.x AS x FROM c1 a, c1 b), +** ... +** c17 AS (SELECT a.x AS x FROM c16 a, c16 b) +** SELECT count(*) FROM c17; +** +** See Bug 2026-10-04T05:35:14Z +*/ +#ifndef SQLITE_MAX_CTES +# define SQLITE_MAX_CTES 50000 +#endif + /************** End of sqliteLimit.h *****************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -15293,25 +15355,38 @@ struct fts5_api { ** ** SQLITE_OPT_INLINE In-line this function if building the ** amalgamation. +** +** SQLITE_USES_INLINE Set to 1 or 0 according to whether or not +** inline functions are supported. */ -#if defined(__GNUC__) +#if defined(SQLITE_DISABLE_INLINE) +# define SQLITE_NOINLINE +# define SQLITE_INLINE +# define SQLITE_OPT_INLINE +# define SQLITE_USES_INLINE 0 +#elif defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) # define SQLITE_INLINE __attribute__((always_inline)) inline # define SQLITE_OPT_INLINE __attribute__((always_inline)) inline +# define SQLITE_USES_INLINE 1 #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) # define SQLITE_INLINE __forceinline # define SQLITE_OPT_INLINE __forceinline +# define SQLITE_USES_INLINE 1 #else # define SQLITE_NOINLINE # define SQLITE_INLINE # define SQLITE_OPT_INLINE +# define SQLITE_USES_INLINE 0 #endif #if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__) # undef SQLITE_INLINE # define SQLITE_INLINE # undef SQLITE_OPT_INLINE # define SQLITE_OPT_INLINE +# undef SQLITE_USES_INLINE +# define SQLITE_USES_INLINE 0 #endif #if !defined(SQLITE_AMALGAMATION) # undef SQLITE_OPT_INLINE @@ -17355,6 +17430,10 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager*); #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_WALSTAT) +SQLITE_PRIVATE void sqlite3PagerWalStat(Pager*,sqlite3_str*); +#endif + #endif /* SQLITE_PAGER_H */ /************** End of pager.h ***********************************************/ @@ -17867,8 +17946,6 @@ struct VdbeOp { int i; /* Integer value if p4type==P4_INT32 */ void *p; /* Generic pointer */ char *z; /* Pointer to data for string (char array) types */ - i64 *pI64; /* Used when p4type is P4_INT64 */ - double *pReal; /* Used when p4type is P4_REAL */ FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ @@ -17924,6 +18001,19 @@ struct VdbeOpList { }; typedef struct VdbeOpList VdbeOpList; +/* +** Combine two int32 values into a single int64. The least significant +** term comes first. +*/ +#define INT32_TO_64(a,b) (i64)(((u64)(u32)(b)<<32)|(u32)(a)) + +/* +** Return an int32 that is the lower or upper 32-bits of an int64. +*/ +#define LOWER32(x) (int)(((u64)(x))&0xffffffff) +#define UPPER32(x) (int)(((u64)(x))>>32) + + /* ** Allowed values of VdbeOp.p4type */ @@ -17943,12 +18033,10 @@ typedef struct VdbeOpList VdbeOpList; #define P4_EXPR (-10) /* P4 is a pointer to an Expr tree */ #define P4_MEM (-11) /* P4 is a pointer to a Mem* structure */ #define P4_VTAB (-12) /* P4 is a pointer to an sqlite3_vtab structure */ -#define P4_REAL (-13) /* P4 is a 64-bit floating point value */ -#define P4_INT64 (-14) /* P4 is a 64-bit signed integer */ -#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ -#define P4_FUNCCTX (-16) /* P4 is a pointer to an sqlite3_context object */ -#define P4_TABLEREF (-17) /* Like P4_TABLE, but reference counted */ -#define P4_SUBRTNSIG (-18) /* P4 is a SubrtnSig pointer */ +#define P4_INTARRAY (-13) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-14) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-15) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-16) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -18015,33 +18103,33 @@ typedef struct VdbeOpList VdbeOpList; #define OP_SeekLE 22 /* jump0, synopsis: key=r[P3@P4] */ #define OP_SeekGE 23 /* jump0, synopsis: key=r[P3@P4] */ #define OP_SeekGT 24 /* jump0, synopsis: key=r[P3@P4] */ -#define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ -#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ -#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 30 /* jump0, synopsis: intkey=r[P3] */ -#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 32 /* jump0 */ -#define OP_IfSizeBetween 33 /* jump */ -#define OP_SorterSort 34 /* jump */ -#define OP_Sort 35 /* jump */ -#define OP_Rewind 36 /* jump0 */ -#define OP_IfEmpty 37 /* jump, synopsis: if( empty(P1) ) goto P2 */ -#define OP_SorterNext 38 /* jump */ -#define OP_Prev 39 /* jump */ -#define OP_Next 40 /* jump */ -#define OP_IdxLE 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 42 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 43 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNoHope 25 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 29 /* jump0, synopsis: intkey=r[P3] */ +#define OP_NotExists 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 31 /* jump0 */ +#define OP_IfSizeBetween 32 /* jump */ +#define OP_SorterSort 33 /* jump */ +#define OP_Sort 34 /* jump */ +#define OP_Rewind 35 /* jump0 */ +#define OP_IfEmpty 36 /* jump, synopsis: if( empty(P1) ) goto P2 */ +#define OP_SorterNext 37 /* jump */ +#define OP_Prev 38 /* jump */ +#define OP_Next 39 /* jump */ +#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGE 43 /* jump, synopsis: key=r[P3@P4] */ #define OP_Or 44 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 45 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_IdxGE 46 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IFindKey 47 /* jump */ -#define OP_RowSetRead 48 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 49 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 50 /* jump0 */ -#define OP_FkIfZero 51 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IFindKey 46 /* jump */ +#define OP_RowSetRead 47 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 48 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 49 /* jump0 */ +#define OP_FkIfZero 50 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 51 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ #define OP_IsNull 52 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 53 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 54 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -18051,48 +18139,48 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 58 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ #define OP_ElseEq 60 /* jump, same as TK_ESCAPE */ -#define OP_IfPos 61 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ -#define OP_IfNotZero 62 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 63 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 64 /* jump */ -#define OP_VNext 65 /* jump */ -#define OP_Filter 66 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ -#define OP_PureFunc 67 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Function 68 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Return 69 -#define OP_EndCoroutine 70 -#define OP_HaltIfNull 71 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 72 -#define OP_Integer 73 /* synopsis: r[P2]=P1 */ -#define OP_Int64 74 /* synopsis: r[P2]=P4 */ -#define OP_String 75 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_BeginSubrtn 76 /* synopsis: r[P2]=NULL */ -#define OP_Null 77 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 78 /* synopsis: r[P1]=NULL */ -#define OP_Blob 79 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 80 /* synopsis: r[P2]=parameter(P1) */ -#define OP_Move 81 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 82 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 83 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 84 /* synopsis: r[P2]=r[P1] */ -#define OP_FkCheck 85 -#define OP_ResultRow 86 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 87 -#define OP_AddImm 88 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 89 -#define OP_Cast 90 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 91 -#define OP_Compare 92 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_IsTrue 93 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_ZeroOrNull 94 /* synopsis: r[P2] = 0 OR NULL */ -#define OP_Offset 95 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 96 /* synopsis: r[P3]=PX cursor P1 column P2 */ -#define OP_TypeCheck 97 /* synopsis: typecheck(r[P1@P2]) */ -#define OP_Affinity 98 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 99 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 100 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 101 -#define OP_SetCookie 102 +#define OP_IfNotZero 61 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 62 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 63 /* jump */ +#define OP_VNext 64 /* jump */ +#define OP_Filter 65 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ +#define OP_PureFunc 66 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 67 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 68 +#define OP_EndCoroutine 69 +#define OP_HaltIfNull 70 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 71 +#define OP_Integer 72 /* synopsis: r[P2]=P1 */ +#define OP_Int64 73 /* synopsis: r[P2]=PINT13 */ +#define OP_String 74 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_BeginSubrtn 75 /* synopsis: r[P2]=NULL */ +#define OP_Null 76 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 77 /* synopsis: r[P1]=NULL */ +#define OP_Blob 78 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 79 /* synopsis: r[P2]=parameter(P1) */ +#define OP_Move 80 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 81 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 82 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 83 /* synopsis: r[P2]=r[P1] */ +#define OP_FkCheck 84 +#define OP_ResultRow 85 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 86 +#define OP_AddImm 87 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 88 +#define OP_Cast 89 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 90 +#define OP_Compare 91 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 92 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 93 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 94 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 95 /* synopsis: r[P3]=PX cursor P1 column P2 */ +#define OP_TypeCheck 96 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 97 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 98 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 99 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 100 +#define OP_SetCookie 101 +#define OP_ReopenIdx 102 /* synopsis: root=P2 iDb=P3 */ #define OP_BitAnd 103 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 104 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 105 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 163 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 164 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 165 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 166 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 167 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 168 -#define OP_CursorLock 169 -#define OP_CursorUnlock 170 -#define OP_TableLock 171 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 172 -#define OP_VCreate 173 -#define OP_VDestroy 174 -#define OP_VOpen 175 -#define OP_VCheck 176 -#define OP_VInitIn 177 /* synopsis: r[P2]=ValueList(P1,P3) */ -#define OP_VColumn 178 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 179 -#define OP_Pagecount 180 -#define OP_MaxPgcnt 181 -#define OP_ClrSubtype 182 /* synopsis: r[P1].subtype = 0 */ -#define OP_GetSubtype 183 /* synopsis: r[P2] = r[P1].subtype */ -#define OP_SetSubtype 184 /* synopsis: r[P2].subtype = r[P1] */ -#define OP_FilterAdd 185 /* synopsis: filter(P1) += key(P3@P4) */ -#define OP_Trace 186 -#define OP_CursorHint 187 -#define OP_ReleaseReg 188 /* synopsis: release r[P1@P2] mask P3 */ -#define OP_Noop 189 -#define OP_Explain 190 -#define OP_Abortable 191 +#define OP_OpenEphemeral 119 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 120 +#define OP_SequenceTest 121 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 122 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 123 +#define OP_ColumnsUsed 124 /* synopsis: Cursor P1 uses columns PHEX23 */ +#define OP_SeekScan 125 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 126 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 127 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 128 /* synopsis: r[P2]=rowid */ +#define OP_Insert 129 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 130 +#define OP_Delete 131 +#define OP_ResetCount 132 +#define OP_SorterCompare 133 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 134 /* synopsis: r[P2]=data */ +#define OP_RowData 135 /* synopsis: r[P2]=data */ +#define OP_Rowid 136 /* synopsis: r[P2]=PX rowid of P1 */ +#define OP_NullRow 137 +#define OP_SeekEnd 138 +#define OP_IdxInsert 139 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 140 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 141 /* synopsis: key=r[P2@P5] */ +#define OP_DeferredSeek 142 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 143 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 144 +#define OP_Destroy 145 +#define OP_Clear 146 +#define OP_ResetSorter 147 +#define OP_CreateBtree 148 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 149 +#define OP_ParseSchema 150 +#define OP_LoadAnalysis 151 +#define OP_DropTable 152 +#define OP_DropIndex 153 +#define OP_Real 154 /* same as TK_FLOAT, synopsis: r[P2]=PDBL13 */ +#define OP_DropTrigger 155 +#define OP_IntegrityCk 156 +#define OP_RowSetAdd 157 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 158 +#define OP_FkCounter 159 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 160 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 161 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 162 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 164 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 165 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 166 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 167 +#define OP_CursorLock 168 +#define OP_CursorUnlock 169 +#define OP_TableLock 170 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 171 +#define OP_VCreate 172 +#define OP_VDestroy 173 +#define OP_VOpen 174 +#define OP_VCheck 175 +#define OP_VInitIn 176 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 177 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 178 +#define OP_Pagecount 179 +#define OP_MaxPgcnt 180 +#define OP_ClrSubtype 181 /* synopsis: r[P1].subtype = 0 */ +#define OP_GetSubtype 182 /* synopsis: r[P2] = r[P1].subtype */ +#define OP_SetSubtype 183 /* synopsis: r[P2].subtype = r[P1] */ +#define OP_FilterAdd 184 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 185 +#define OP_CursorHint 186 +#define OP_ReleaseReg 187 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 188 +#define OP_Explain 189 +#define OP_Abortable 190 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -18199,28 +18286,27 @@ typedef struct VdbeOpList VdbeOpList; /* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ /* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ /* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ -/* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ -/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x01, 0x41,\ -/* 40 */ 0x41, 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x09,\ -/* 48 */ 0x23, 0x0b, 0x81, 0x01, 0x03, 0x03, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03,\ -/* 64 */ 0x01, 0x41, 0x01, 0x00, 0x00, 0x02, 0x02, 0x08,\ -/* 72 */ 0x00, 0x10, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10,\ -/* 80 */ 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00,\ -/* 88 */ 0x02, 0x02, 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20,\ -/* 96 */ 0x40, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x26,\ +/* 24 */ 0xc9, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49, 0xc1,\ +/* 32 */ 0x01, 0x41, 0x41, 0xc1, 0x01, 0x01, 0x41, 0x41,\ +/* 40 */ 0x41, 0x41, 0x41, 0x41, 0x26, 0x26, 0x09, 0x23,\ +/* 48 */ 0x0b, 0x81, 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01,\ +/* 64 */ 0x41, 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00,\ +/* 72 */ 0x10, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10,\ +/* 80 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02,\ +/* 88 */ 0x02, 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40,\ +/* 96 */ 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x26,\ /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 112 */ 0x26, 0x40, 0x40, 0x12, 0x00, 0x40, 0x10, 0x40,\ -/* 120 */ 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x40, 0x40,\ -/* 128 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,\ -/* 136 */ 0x00, 0x50, 0x00, 0x40, 0x04, 0x04, 0x00, 0x40,\ -/* 144 */ 0x50, 0x40, 0x10, 0x00, 0x00, 0x10, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x06, 0x10,\ -/* 160 */ 0x00, 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40,\ -/* 176 */ 0x10, 0x50, 0x40, 0x00, 0x10, 0x10, 0x02, 0x12,\ -/* 184 */ 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -} +/* 112 */ 0x26, 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40,\ +/* 120 */ 0x00, 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10,\ +/* 128 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00,\ +/* 136 */ 0x50, 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50,\ +/* 144 */ 0x40, 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00,\ +/* 160 */ 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10,\ +/* 176 */ 0x50, 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12,\ +/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} /* The resolve3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -18228,7 +18314,7 @@ typedef struct VdbeOpList VdbeOpList; ** generated this include file strives to group all JUMP opcodes ** together near the beginning of the list. */ -#define SQLITE_MX_JUMP_OPCODE 66 /* Maximum JUMP opcode */ +#define SQLITE_MX_JUMP_OPCODE 65 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -18252,8 +18338,9 @@ SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddInt64(Vdbe*,int,i64); +SQLITE_PRIVATE int sqlite3VdbeAddDouble(Vdbe*,int,double); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall(Parse*,int,int,int,int,const FuncDef*,int); SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); @@ -18345,7 +18432,7 @@ SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); -SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); +SQLITE_PRIVATE void sqlite3VdbeReprepareOnBind(Vdbe*, int, int); #ifndef SQLITE_OMIT_TRACE SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif @@ -18504,6 +18591,7 @@ SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr) */ #ifndef _PCACHE_H_ +#define _PCACHE_H_ typedef struct PgHdr PgHdr; typedef struct PCache PCache; @@ -18726,7 +18814,7 @@ SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); # define SQLITE_MUTEX_OMIT #endif #if SQLITE_THREADSAFE && !defined(SQLITE_MUTEX_NOOP) -# if SQLITE_OS_UNIX +# if SQLITE_OS_UNIX || defined(__CYGWIN__) # define SQLITE_MUTEX_PTHREADS # elif SQLITE_OS_WIN # define SQLITE_MUTEX_W32 @@ -18873,7 +18961,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_SCHEMA+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_CTES+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -19012,7 +19100,6 @@ struct sqlite3 { int errByteOffset; /* Byte offset of error in SQL statement */ int errMask; /* & result codes with this before returning */ int iSysErrno; /* Errno value from last system error */ - u32 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 eConcurrent; /* CONCURRENT_* value */ @@ -19029,10 +19116,12 @@ struct sqlite3 { u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ u8 eOpenState; /* Current condition of the connection */ u8 nFpDigit; /* Significant digits to keep on double->text */ + u8 bPendingSchemaReset; /* 1 or more schema resets pending */ int nextPagesize; /* Pagesize after VACUUM if >0 */ FastPrng sPrng; /* State of the per-connection PRNG */ i64 nChange; /* Value returned by sqlite3_changes() */ i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ + u32 dbOptFlags; /* Flags to enable/disable optimizations */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ @@ -19157,10 +19246,13 @@ struct sqlite3 { #define CONCURRENT_SCHEMA 2 /* -** A macro to discover the encoding of a database. +** A macro to discover the encoding of a database or for managing +** encoding values. */ #define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) #define ENC(db) ((db)->enc) +#define ENC_MASK 0x1f /* Usable encoding bits */ +#define ENC_KEEPBOM 0x80 /* Do not remove BOM */ /* ** A u64 constant where the lower 32 bits are all zeros. Only the @@ -19246,49 +19338,49 @@ struct sqlite3 { /* ** Bits of the sqlite3.dbOptFlags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to -** selectively disable various optimizations. -*/ -#define SQLITE_QueryFlattener 0x00000001 /* Query flattening */ -#define SQLITE_WindowFunc 0x00000002 /* Use xInverse for window functions */ -#define SQLITE_GroupByOrder 0x00000004 /* GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x00000008 /* Constant factoring */ -#define SQLITE_DistinctOpt 0x00000010 /* DISTINCT using indexes */ -#define SQLITE_CoverIdxScan 0x00000020 /* Covering index scans */ -#define SQLITE_OrderByIdxJoin 0x00000040 /* ORDER BY of joins via index */ -#define SQLITE_Transitive 0x00000080 /* Transitive constraints */ -#define SQLITE_OmitNoopJoin 0x00000100 /* Omit unused tables in joins */ -#define SQLITE_CountOfView 0x00000200 /* The count-of-view optimization */ -#define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ -#define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ - /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ -#define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ -#define SQLITE_SkipScan 0x00004000 /* Skip-scans */ -#define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ -#define SQLITE_MinMaxOpt 0x00010000 /* The min/max optimization */ -#define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */ -#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */ - /* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */ -#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */ -#define SQLITE_BloomPulldown 0x00100000 /* Run Bloom filters early */ -#define SQLITE_BalancedMerge 0x00200000 /* Balance multi-way merges */ -#define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ -#define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ - /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ -#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ -#define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ -#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ -#define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ -#define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ -#define SQLITE_StarQuery 0x20000000 /* Heurists for star queries */ -#define SQLITE_ExistsToJoin 0x40000000 /* The EXISTS-to-JOIN optimization */ -#define SQLITE_AllOpts 0xffffffff /* All optimizations */ +** selectively disable various optimizations. The numbers in each macro +** are the one-based bit position (for improved readability). +** +** Keep these values in sync with the ext/misc/optname.c extension. +*/ +#define SQLITE_QueryFlattener 1 /* Query flattening */ +#define SQLITE_WindowFunc 2 /* Use xInverse for window functions */ +#define SQLITE_GroupByOrder 3 /* GROUPBY cover of ORDERBY */ +#define SQLITE_JoinMiss 4 /* Unwind join loops on a lookup miss*/ +#define SQLITE_DistinctOpt 5 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 6 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 7 /* ORDER BY of joins via index */ +#define SQLITE_Transitive 8 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 9 /* Omit unused tables in joins */ +#define SQLITE_CountOfView 10 /* The count-of-view optimization */ +#define SQLITE_CursorHints 11 /* Add OP_CursorHint opcodes */ +#define SQLITE_Stat4 12 /* Use STAT4 data */ +#define SQLITE_PushDown 13 /* WHERE-clause push-down opt */ +#define SQLITE_SimplifyJoin 14 /* Convert LEFT JOIN to JOIN */ +#define SQLITE_SkipScan 15 /* Skip-scans */ +#define SQLITE_PropagateConst 16 /* The constant propagation opt */ +#define SQLITE_MinMaxOpt 17 /* The min/max optimization */ +#define SQLITE_SeekScan 18 /* The OP_SeekScan optimization */ +#define SQLITE_OmitOrderBy 19 /* Omit pointless ORDER BY */ +#define SQLITE_BloomFilter 20 /* Use a Bloom filters */ + /* 21 -- Available for reuse */ +#define SQLITE_BalancedMerge 22 /* Balance multi-way merges */ +#define SQLITE_ReleaseReg 23 /* Use OP_ReleaseReg for testing */ +#define SQLITE_FlttnUnionAll 24 /* Disable the UNION ALL flattener */ +#define SQLITE_IndexedExpr 25 /* Pull exprs from index when able */ +#define SQLITE_Coroutines 26 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 27 /* NULL unused columns in subqueries */ +#define SQLITE_OnePass 28 /* Single-pass DELETE and UPDATE */ +#define SQLITE_OrderBySubq 29 /* ORDER BY in subquery helps outer */ +#define SQLITE_StarQuery 30 /* Heurists for star queries */ +#define SQLITE_ExistsToJoin 31 /* The EXISTS-to-JOIN optimization */ +#define SQLITE_UnionLimit 32 /* Optimizations for UNION + LIMIT */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ -#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) -#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) +#define OptimizationDisabled(D,M) (((D)->dbOptFlags&(1ul<<((M)-1)))!=0) +#define OptimizationEnabled(D,M) (((D)->dbOptFlags&(1ul<<((M)-1)))==0) /* ** Return true if it OK to factor constant expressions into the initialization @@ -21163,6 +21255,7 @@ struct TriggerPrg { # define DbMaskTest(M,I) (((M)[(I)/8]&(1<<((I)&7)))!=0) # define DbMaskZero(M) memset((M),0,sizeof(M)) # define DbMaskSet(M,I) (M)[(I)/8]|=(1<<((I)&7)) +# define DbMaskUnion(M,X) sqlite3DbMaskUnion(M,X) # define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) # define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) #else @@ -21170,10 +21263,15 @@ struct TriggerPrg { # define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) # define DbMaskZero(M) ((M)=0) # define DbMaskSet(M,I) ((M)|=(((yDbMask)1)<<(I))) +# define DbMaskUnion(M,X) ((M)|=(X)) # define DbMaskAllZero(M) ((M)==0) # define DbMaskNonZero(M) ((M)!=0) #endif +/* Should be in vdbe.h, but we needed yDbMask definition first */ +SQLITE_PRIVATE void sqlite3VdbeSchemaInUse(sqlite3*,Vdbe*,yDbMask*); + + /* ** For each index X that has as one of its arguments either an expression ** or the name of a virtual generated column, and if X is in scope such that @@ -21208,6 +21306,13 @@ struct ParseCleanup { void (*xCleanup)(sqlite3*,void*); /* Deallocation routine */ }; +/* +** Number of 64-bit entried in the variable number bitmap cache (VNBMC). +*/ +#ifndef SQLITE_VNBMC +# define SQLITE_VNBMC 2 +#endif + /* ** An SQL parser context. A copy of this structure is passed through ** the parser and down into all the parser action routine in order to @@ -21262,6 +21367,7 @@ struct Parse { int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ + int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ int iSelfTab; /* Table associated with an index on expr, or negative ** of the base register during check-constraint eval */ int nNestSel; /* Number of nested SELECT statements and/or VIEWs */ @@ -21285,6 +21391,7 @@ struct Parse { Table *pTriggerTab; /* Table triggers are being coded for */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ ParseCleanup *pCleanup; /* List of cleanup operations to run after parse */ + Trigger *pTrigCoded; /* Trigger whose sub-program is being coded */ /************************************************************************** ** Fields above must be initialized to zero. The fields that follow, @@ -21326,6 +21433,7 @@ struct Parse { Token sLastToken; /* The last token parsed */ ynVar nVar; /* Number of '?' variables seen in the SQL so far */ + u64 aVnbmc[SQLITE_VNBMC]; /* Varible Number Bitmap Cache */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ u8 explain; /* True if the EXPLAIN flag is found on the query */ u8 eParseMode; /* PARSE_MODE_XXX constant */ @@ -22365,6 +22473,7 @@ SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); SQLITE_PRIVATE void sqlite3ExprListDeleteGeneric(sqlite3*,void*); +SQLITE_PRIVATE int sqlite3ExprCanReturnSubtype(Parse*,Expr*); SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); @@ -22375,11 +22484,12 @@ SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3*,const char *zName); #endif SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); -SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int,Vdbe*); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); SQLITE_PRIVATE void sqlite3ColumnSetExpr(Parse*,Table*,Column*,Expr*); SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table*,Column*); +SQLITE_PRIVATE Expr *sqlite3ColumnExprAuth(Table*,Column*,Parse*); SQLITE_PRIVATE void sqlite3ColumnSetColl(sqlite3*,Column*,const char*zColl); SQLITE_PRIVATE const char *sqlite3ColumnColl(Column*); SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); @@ -22451,6 +22561,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); #if SQLITE_MAX_ATTACHED>30 SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); +SQLITE_PRIVATE void sqlite3DbMaskUnion(yDbMask,yDbMask); #endif SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); @@ -22590,11 +22701,11 @@ SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse*,Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int,int); SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); -#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE int sqlite3ExprListIsConstant(Parse *pParse, ExprList *pList, int bNoIs); SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); -#endif -SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*, Parse*); +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*, Parse*, int); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3ExprIsLikeOperator(const Expr*); @@ -22746,6 +22857,7 @@ SQLITE_PRIVATE int sqlite3VListNameToNum(VList*,const char*,int); */ SQLITE_PRIVATE int sqlite3PutVarint(unsigned char*, u64); SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *, u64 *); +SQLITE_PRIVATE i64 sqlite3VarintValue(const unsigned char*); SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *, u32 *); SQLITE_PRIVATE int sqlite3VarintLen(u64 v); @@ -22761,9 +22873,6 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); #define putVarint32(A,B) \ (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ sqlite3PutVarint((A),(B))) -#define getVarint sqlite3GetVarint -#define putVarint sqlite3PutVarint - SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3*,const Table*); @@ -22799,14 +22908,23 @@ SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs*); SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); +SQLITE_PRIVATE int sqlite3BinaryCompare(void*,int,const void*,int,const void*); +#if SQLITE_USES_INLINE +static SQLITE_INLINE int sqlite3IsBinary(const CollSeq *p){ + assert( p==0 || p->xCmp!=sqlite3BinaryCompare + || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==sqlite3BinaryCompare; +} +#else SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq*); +#endif SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr); SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr); SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,const Expr*,const Expr*); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(const Parse *pParse, Expr*, const Token*, int); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(const Parse*,Expr*,const char*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); @@ -22880,6 +22998,7 @@ SQLITE_PRIVATE void sqlite3AlterSetNotNull(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE i64 sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); +SQLITE_PRIVATE void sqlite3ExpireStmtsUsingSchema(sqlite3*, int, Vdbe*); SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int, int); SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); @@ -23048,10 +23167,12 @@ SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db); SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName); SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3*,Table*,const char*); SQLITE_PRIVATE void sqlite3MarkAllShadowTablesOf(sqlite3*, Table*); +SQLITE_PRIVATE void sqlite3VtabEponymousTableClearAll(sqlite3*); #else # define sqlite3ShadowTableName(A,B) 0 # define sqlite3IsShadowTableOf(A,B,C) 0 # define sqlite3MarkAllShadowTablesOf(A,B) +# define sqlite3VtabEponymousTableClearAll(X) #endif SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); @@ -25205,7 +25326,9 @@ struct Vdbe { VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ - u32 expmask; /* Binding to these vars invalidates VM */ + u32 expmask; /* Binding to these vars might invalidate VM */ + u32 smimask; /* Only invalid if changing to/from small integer */ + /* Note: smimask is always a subset of expmask */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ AuxData *pAuxData; /* Linked list of auxdata allocations */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -25355,7 +25478,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); SQLITE_PRIVATE int sqlite3MemRealValueRC(Mem*, double*); SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); -SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*,int bFromText); SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem*,u8,u8); @@ -26118,6 +26241,7 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ }else{ s = 0; } + if( h==24 && (m!=0 || s!=0 || ms!=0.0) ) return 1; p->validJD = 0; p->rawS = 0; p->validHMS = 1; @@ -26154,6 +26278,10 @@ static void computeJD(DateTime *p){ Y = 2000; /* If no YMD specified, assume 2000-Jan-01 */ M = 1; D = 1; + if( p->validHMS && p->h==24 ){ + p->h = 0; + D = 2; + } } if( Y<-4713 || Y>9999 || p->rawS ){ datetimeError(p); @@ -26182,6 +26310,21 @@ static void computeJD(DateTime *p){ } } +/* +** Return the number of dates in a specific month. +*/ +static int daysOfMonth(int Y, int M){ + if( (1<validYMD || p->isError ); - assert( p->D>=0 && p->D<=31 ); + assert( p->D>=0 && p->D<=32 ); assert( p->M>=0 && p->M<=12 ); - if( p->D<=28 ){ - p->nFloor = 0; - }else if( (1<M) & 0x15aa ){ - p->nFloor = 0; - }else if( p->M!=2 ){ - p->nFloor = (p->D==31); - }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ - p->nFloor = p->D - 28; - }else{ - p->nFloor = p->D - 29; - } + mxD = daysOfMonth(p->Y, p->M); + p->nFloor = p->D>mxD ? p->D - mxD : 0; } /* @@ -26229,10 +26364,23 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ return 1; } + if( neg ) Y = -Y; zDate += 10; while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; } if( parseHhMmSs(zDate, p)==0 ){ - /* We got the time */ + if( p->h==24 ){ + int mxD = daysOfMonth(Y,M); + p->h = 0; + D++; + if( D>mxD ){ + D -= mxD; + M++; + if( M>12 ){ + M = 1; + Y++; + } + } + } }else if( *zDate==0 ){ p->validHMS = 0; }else{ @@ -26240,7 +26388,7 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ } p->validJD = 0; p->validYMD = 1; - p->Y = neg ? -Y : Y; + p->Y = Y; p->M = M; p->D = D; computeFloor(p); @@ -30729,6 +30877,12 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ pTo->xMutexNotheld = pFrom->xMutexNotheld; sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; + }else{ + /* Another thread may have just installed the default methods above. + ** This barrier pairs with the one prior to setting xMutexAlloc, so + ** that the load of xMutexInit below cannot be satisfied before the + ** load of xMutexAlloc on processors with weak memory ordering. */ + sqlite3MemoryBarrier(); } assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); @@ -31575,8 +31729,11 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ # define ALIGN128 #endif +#ifdef _MSC_VER #pragma warning(push) #pragma warning(disable: 4324) +#endif /* _MSC_VER */ + /* ** Each SQLite mutex is an instance of the following structure. ** @@ -31599,7 +31756,9 @@ struct sqlite3_mutex { volatile LONG trace; /* True to trace changes */ #endif }; +#ifdef _MSC_VER #pragma warning(pop) +#endif /* _MSC_VER */ #ifdef SQLITE_DEBUG /* @@ -33013,9 +33172,16 @@ static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ #define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ /* -** Hard limit on the precision of floating-point conversions. +** Hard limit on the precision of floating-point conversions to +** the minimum of SQLITE_PRINTF_PRECISION_LIMIT and 100 million. */ -#ifndef SQLITE_PRINTF_PRECISION_LIMIT +#ifdef SQLITE_PRINTF_PRECISION_LIMIT +# if SQLITE_PRINTF_PRECISION_LIMIT<100000000 +# define SQLITE_FP_PRECISION_LIMIT SQLITE_PRINTF_PRECISION_LIMIT +# else +# define SQLITE_FP_PRECISION_LIMIT 100000000 +# endif +#else # define SQLITE_FP_PRECISION_LIMIT 100000000 #endif @@ -33375,11 +33541,9 @@ SQLITE_API void sqlite3_str_vappendf( realvalue = va_arg(ap,double); } if( precision<0 ) precision = 6; /* Set default precision */ -#ifdef SQLITE_FP_PRECISION_LIMIT if( precision>SQLITE_FP_PRECISION_LIMIT ){ precision = SQLITE_FP_PRECISION_LIMIT; } -#endif if( xtype==etFLOAT ){ iRound = -precision; }else if( xtype==etGENERIC ){ @@ -33469,7 +33633,7 @@ SQLITE_API void sqlite3_str_vappendf( /* Unable to allocate space in pAccum, perhaps because it ** is coming from sqlite3_snprintf() or similar. We'll have ** to render into temporary space and the memcpy() it over. */ - bufpt = sqlite3_malloc(szBufNeeded); + bufpt = sqlite3_malloc64(szBufNeeded); if( bufpt==0 ){ sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); return; @@ -34007,10 +34171,13 @@ SQLITE_PRIVATE void sqlite3RecordErrorByteOffset(sqlite3 *db, const char *z){ ** as the error offset. */ SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExpr){ - while( pExpr - && (ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) || pExpr->w.iOfst<=0) - ){ - pExpr = pExpr->pLeft; + while( pExpr ){ + if( ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ) return; + if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) || pExpr->w.iOfst<=0 ){ + pExpr = pExpr->pLeft; + }else{ + break; + } } if( pExpr==0 ) return; if( ExprHasProperty(pExpr, EP_FromDDL) ) return; @@ -35169,6 +35336,10 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); break; } + case TK_ASTERISK: { + sqlite3TreeViewLine(pView,"ASKTERISK"); + break; + } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ @@ -36517,9 +36688,7 @@ SQLITE_PRIVATE int sqlite3AppendOneUtf8Character(char *zOut, u32 v){ while( zIn=0xD800 && i<=0xDFFF ) t = 0xFFFD; - if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD; assert( c==t ); assert( (z-zBuf)==n ); } @@ -37792,11 +37958,13 @@ static double sqlite3Fp10Convert2(u64 d, int p){ ** there is no prefix of the input that can be seen as ** a valid floating point number. ** bit 1 => Set if the input contains a decimal point or eNNN -** clause. Zero if the input is an integer. +** clause or other text that disqualifies it as an +** integer value. Zero if the input is an integer. ** bit 2 => The input is exactly 0.0, not an underflow from ** some value near zero. ** bit 3 => Set if there are more than about 19 significant ** digits in the input. +** bit 4 => The input is a special value like "Inf" or "NaN" ** ** If the input contains a syntax error but begins with text that might ** be a valid number of some kind, then the result is negative. The @@ -37881,8 +38049,13 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *zIn, double *pResult){ } mState |= 2; }else if( mState==0 ){ - *pResult = 0.0; - return 0; + if( sqlite3StrICmp((const char*)z,"Inf")==0 ){ + *pResult = neg ? -INFINITY : INFINITY; + return 0x13; + }else{ + *pResult = 0.0; + return 0; + } } /* if exponent is present */ @@ -38331,7 +38504,9 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ ** ** If iRound>0 round to min(iRound,mxRound) significant digits total. ** -** mxRound must be positive. +** mxRound must be positive. iRound must not be less than -100000000 +** (the SQLITE_FP_PRECISION_LIMIT maximum in printf.c) so that the +** computation of the number of significant digits cannot overflow. ** ** The significant digits of the decimal representation are ** stored in p->z[] which is a often (but not always) a pointer @@ -38348,6 +38523,7 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou p->isSpecial = 0; assert( mxRound>0 ); + assert( iRound>=-100000000 ); /* Convert negative numbers to positive. Deal with Infinity, 0.0, and ** NaN. */ @@ -38496,7 +38672,7 @@ SQLITE_PRIVATE int sqlite3GetUInt32(const char *z, u32 *pI){ int i; for(i=0; sqlite3Isdigit(z[i]); i++){ v = v*10 + z[i] - '0'; - if( v>4294967296LL ){ *pI = 0; return 0; } + if( v>4294967295LL ){ *pI = 0; return 0; } } if( i==0 || z[i]!=0 ){ *pI = 0; return 0; } *pI = (u32)v; @@ -38569,171 +38745,83 @@ SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ return putVarint64(p,v); } -/* -** Bitmasks used by sqlite3GetVarint(). These precomputed constants -** are defined here rather than simply putting the constant expressions -** inline in order to work around bugs in the RVT compiler. -** -** SLOT_2_0 A mask for (0x7f<<14) | 0x7f -** -** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 -*/ -#define SLOT_2_0 0x001fc07f -#define SLOT_4_2_0 0xf01fc07f - - /* ** Read a 64-bit variable-length integer from memory starting at p[0]. ** Return the number of bytes read. The value is stored in *v. */ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ - u32 a,b,s; - - if( ((signed char*)p)[0]>=0 ){ - *v = *p; - return 1; - } - if( ((signed char*)p)[1]>=0 ){ - *v = ((u32)(p[0]&0x7f)<<7) | p[1]; - return 2; - } - - /* Verify that constants are precomputed correctly */ - assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); - assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); - - a = ((u32)p[0])<<14; - b = p[1]; - p += 2; - a |= *p; - /* a: p0<<14 | p2 (unmasked) */ - if (!(a&0x80)) - { - a &= SLOT_2_0; - b &= 0x7f; - b = b<<7; - a |= b; - *v = a; - return 3; - } - - /* CSE1 from below */ - a &= SLOT_2_0; - p++; - b = b<<14; - b |= *p; - /* b: p1<<14 | p3 (unmasked) */ - if (!(b&0x80)) - { - b &= SLOT_2_0; - /* moved CSE1 up */ - /* a &= (0x7f<<14)|(0x7f); */ - a = a<<7; - a |= b; - *v = a; - return 4; - } - - /* a: p0<<14 | p2 (masked) */ - /* b: p1<<14 | p3 (unmasked) */ - /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ - /* moved CSE1 up */ - /* a &= (0x7f<<14)|(0x7f); */ - b &= SLOT_2_0; - s = a; - /* s: p0<<14 | p2 (masked) */ - - p++; - a = a<<14; - a |= *p; - /* a: p0<<28 | p2<<14 | p4 (unmasked) */ - if (!(a&0x80)) - { - /* we can skip these cause they were (effectively) done above - ** while calculating s */ - /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ - /* b &= (0x7f<<14)|(0x7f); */ - b = b<<7; - a |= b; - s = s>>18; - *v = ((u64)s)<<32 | a; - return 5; - } - - /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ - s = s<<7; - s |= b; - /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ - - p++; - b = b<<14; - b |= *p; - /* b: p1<<28 | p3<<14 | p5 (unmasked) */ - if (!(b&0x80)) - { - /* we can skip this cause it was (effectively) done above in calc'ing s */ - /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ - a &= SLOT_2_0; - a = a<<7; - a |= b; - s = s>>18; - *v = ((u64)s)<<32 | a; - return 6; - } - - p++; - a = a<<14; - a |= *p; - /* a: p2<<28 | p4<<14 | p6 (unmasked) */ - if (!(a&0x80)) - { - a &= SLOT_4_2_0; - b &= SLOT_2_0; - b = b<<7; - a |= b; - s = s>>11; - *v = ((u64)s)<<32 | a; - return 7; + u64 iKey = p[0]; + const u8 *pStart = p; + if( iKey>=0x80 ){ + u8 x; + iKey = (iKey<<7) ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<8) ^ 0x8000 ^ (*++p); + } + } + } + } + } + }else{ + iKey ^= 0x204000; + } + }else{ + iKey ^= 0x4000; + } } + *v = iKey; + return (u8)(p - pStart) + 1; +} - /* CSE2 from below */ - a &= SLOT_2_0; - p++; - b = b<<14; - b |= *p; - /* b: p3<<28 | p5<<14 | p7 (unmasked) */ - if (!(b&0x80)) - { - b &= SLOT_4_2_0; - /* moved CSE2 up */ - /* a &= (0x7f<<14)|(0x7f); */ - a = a<<7; - a |= b; - s = s>>4; - *v = ((u64)s)<<32 | a; - return 8; +/* +** Return the value of a variable-length integer without computing +** its length. This is an optimization on sqlite3GetVarint() for the +** cases when the return value of sqlite3GetVarint() is not needed. +*/ +SQLITE_PRIVATE i64 sqlite3VarintValue(const unsigned char *p){ + u64 iKey = p[0]; + if( iKey>=0x80 ){ + u8 x; + iKey = (iKey<<7) ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++p); + if( x>=0x80 ){ + iKey = (iKey<<8) ^ 0x8000 ^ (*++p); + } + } + } + } + } + }else{ + iKey ^= 0x204000; + } + }else{ + iKey ^= 0x4000; + } } - - p++; - a = a<<15; - a |= *p; - /* a: p4<<29 | p6<<15 | p8 (unmasked) */ - - /* moved CSE2 up */ - /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ - b &= SLOT_2_0; - b = b<<8; - a |= b; - - s = s<<4; - b = p[-4]; - b &= 0x7f; - b = b>>3; - s |= b; - - *v = ((u64)s)<<32 | a; - - return 9; + return *(i64*)&iKey; } /* @@ -39541,7 +39629,6 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ return 0; } - /************** End of hash.c ************************************************/ /************** Begin file opcodes.c *****************************************/ /* Automatically generated. Do not edit */ @@ -39581,33 +39668,33 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 22 */ "SeekLE" OpHelp("key=r[P3@P4]"), /* 23 */ "SeekGE" OpHelp("key=r[P3@P4]"), /* 24 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 25 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), - /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), - /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 29 */ "Found" OpHelp("key=r[P3@P4]"), - /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), - /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 32 */ "Last" OpHelp(""), - /* 33 */ "IfSizeBetween" OpHelp(""), - /* 34 */ "SorterSort" OpHelp(""), - /* 35 */ "Sort" OpHelp(""), - /* 36 */ "Rewind" OpHelp(""), - /* 37 */ "IfEmpty" OpHelp("if( empty(P1) ) goto P2"), - /* 38 */ "SorterNext" OpHelp(""), - /* 39 */ "Prev" OpHelp(""), - /* 40 */ "Next" OpHelp(""), - /* 41 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 42 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 43 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 25 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 26 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 27 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 28 */ "Found" OpHelp("key=r[P3@P4]"), + /* 29 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 30 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 31 */ "Last" OpHelp(""), + /* 32 */ "IfSizeBetween" OpHelp(""), + /* 33 */ "SorterSort" OpHelp(""), + /* 34 */ "Sort" OpHelp(""), + /* 35 */ "Rewind" OpHelp(""), + /* 36 */ "IfEmpty" OpHelp("if( empty(P1) ) goto P2"), + /* 37 */ "SorterNext" OpHelp(""), + /* 38 */ "Prev" OpHelp(""), + /* 39 */ "Next" OpHelp(""), + /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 43 */ "IdxGE" OpHelp("key=r[P3@P4]"), /* 44 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 45 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 46 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 47 */ "IFindKey" OpHelp(""), - /* 48 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 49 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 50 */ "Program" OpHelp(""), - /* 51 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 46 */ "IFindKey" OpHelp(""), + /* 47 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 48 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 49 */ "Program" OpHelp(""), + /* 50 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 51 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), /* 52 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 53 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 54 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), @@ -39617,48 +39704,48 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 58 */ "Lt" OpHelp("IF r[P3]=r[P1]"), /* 60 */ "ElseEq" OpHelp(""), - /* 61 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), - /* 62 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 63 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 64 */ "IncrVacuum" OpHelp(""), - /* 65 */ "VNext" OpHelp(""), - /* 66 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), - /* 67 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), - /* 68 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), - /* 69 */ "Return" OpHelp(""), - /* 70 */ "EndCoroutine" OpHelp(""), - /* 71 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 72 */ "Halt" OpHelp(""), - /* 73 */ "Integer" OpHelp("r[P2]=P1"), - /* 74 */ "Int64" OpHelp("r[P2]=P4"), - /* 75 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 76 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), - /* 77 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 78 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 79 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 80 */ "Variable" OpHelp("r[P2]=parameter(P1)"), - /* 81 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 82 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 83 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 84 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 85 */ "FkCheck" OpHelp(""), - /* 86 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 87 */ "CollSeq" OpHelp(""), - /* 88 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 89 */ "RealAffinity" OpHelp(""), - /* 90 */ "Cast" OpHelp("affinity(r[P1])"), - /* 91 */ "Permutation" OpHelp(""), - /* 92 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 93 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 94 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), - /* 95 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 96 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), - /* 97 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), - /* 98 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 99 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 100 */ "Count" OpHelp("r[P2]=count()"), - /* 101 */ "ReadCookie" OpHelp(""), - /* 102 */ "SetCookie" OpHelp(""), + /* 61 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 62 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 63 */ "IncrVacuum" OpHelp(""), + /* 64 */ "VNext" OpHelp(""), + /* 65 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), + /* 66 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 67 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 68 */ "Return" OpHelp(""), + /* 69 */ "EndCoroutine" OpHelp(""), + /* 70 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 71 */ "Halt" OpHelp(""), + /* 72 */ "Integer" OpHelp("r[P2]=P1"), + /* 73 */ "Int64" OpHelp("r[P2]=PINT13"), + /* 74 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 75 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), + /* 76 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 77 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 78 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 79 */ "Variable" OpHelp("r[P2]=parameter(P1)"), + /* 80 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 81 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 82 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 83 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 84 */ "FkCheck" OpHelp(""), + /* 85 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 86 */ "CollSeq" OpHelp(""), + /* 87 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 88 */ "RealAffinity" OpHelp(""), + /* 89 */ "Cast" OpHelp("affinity(r[P1])"), + /* 90 */ "Permutation" OpHelp(""), + /* 91 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 92 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 93 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 94 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 95 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), + /* 96 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 97 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 98 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 99 */ "Count" OpHelp("r[P2]=count()"), + /* 100 */ "ReadCookie" OpHelp(""), + /* 101 */ "SetCookie" OpHelp(""), + /* 102 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), /* 103 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 104 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 105 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 163 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 164 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 165 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 166 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 167 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 168 */ "Expire" OpHelp(""), - /* 169 */ "CursorLock" OpHelp(""), - /* 170 */ "CursorUnlock" OpHelp(""), - /* 171 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 172 */ "VBegin" OpHelp(""), - /* 173 */ "VCreate" OpHelp(""), - /* 174 */ "VDestroy" OpHelp(""), - /* 175 */ "VOpen" OpHelp(""), - /* 176 */ "VCheck" OpHelp(""), - /* 177 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), - /* 178 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 179 */ "VRename" OpHelp(""), - /* 180 */ "Pagecount" OpHelp(""), - /* 181 */ "MaxPgcnt" OpHelp(""), - /* 182 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), - /* 183 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), - /* 184 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), - /* 185 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), - /* 186 */ "Trace" OpHelp(""), - /* 187 */ "CursorHint" OpHelp(""), - /* 188 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), - /* 189 */ "Noop" OpHelp(""), - /* 190 */ "Explain" OpHelp(""), - /* 191 */ "Abortable" OpHelp(""), + /* 119 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 120 */ "SorterOpen" OpHelp(""), + /* 121 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 122 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 123 */ "Close" OpHelp(""), + /* 124 */ "ColumnsUsed" OpHelp("Cursor P1 uses columns PHEX23"), + /* 125 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 126 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 127 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 128 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 129 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 130 */ "RowCell" OpHelp(""), + /* 131 */ "Delete" OpHelp(""), + /* 132 */ "ResetCount" OpHelp(""), + /* 133 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 134 */ "SorterData" OpHelp("r[P2]=data"), + /* 135 */ "RowData" OpHelp("r[P2]=data"), + /* 136 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), + /* 137 */ "NullRow" OpHelp(""), + /* 138 */ "SeekEnd" OpHelp(""), + /* 139 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 140 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 141 */ "IdxDelete" OpHelp("key=r[P2@P5]"), + /* 142 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 143 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 144 */ "FinishSeek" OpHelp(""), + /* 145 */ "Destroy" OpHelp(""), + /* 146 */ "Clear" OpHelp(""), + /* 147 */ "ResetSorter" OpHelp(""), + /* 148 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 149 */ "SqlExec" OpHelp(""), + /* 150 */ "ParseSchema" OpHelp(""), + /* 151 */ "LoadAnalysis" OpHelp(""), + /* 152 */ "DropTable" OpHelp(""), + /* 153 */ "DropIndex" OpHelp(""), + /* 154 */ "Real" OpHelp("r[P2]=PDBL13"), + /* 155 */ "DropTrigger" OpHelp(""), + /* 156 */ "IntegrityCk" OpHelp(""), + /* 157 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 158 */ "Param" OpHelp(""), + /* 159 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 160 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 161 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 162 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 163 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 164 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 165 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 166 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 167 */ "Expire" OpHelp(""), + /* 168 */ "CursorLock" OpHelp(""), + /* 169 */ "CursorUnlock" OpHelp(""), + /* 170 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 171 */ "VBegin" OpHelp(""), + /* 172 */ "VCreate" OpHelp(""), + /* 173 */ "VDestroy" OpHelp(""), + /* 174 */ "VOpen" OpHelp(""), + /* 175 */ "VCheck" OpHelp(""), + /* 176 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 177 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 178 */ "VRename" OpHelp(""), + /* 179 */ "Pagecount" OpHelp(""), + /* 180 */ "MaxPgcnt" OpHelp(""), + /* 181 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 182 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), + /* 183 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), + /* 184 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 185 */ "Trace" OpHelp(""), + /* 186 */ "CursorHint" OpHelp(""), + /* 187 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 188 */ "Noop" OpHelp(""), + /* 189 */ "Explain" OpHelp(""), + /* 190 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -41997,7 +42083,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ int n; /* Length of zAbsoluteName string */ - assert( zAbsoluteName[0]=='/' ); + if( zAbsoluteName==0 || zAbsoluteName[0]=='/' ); n = (int)strlen(zAbsoluteName); pNew = sqlite3_malloc64( sizeof(*pNew) + (n+1) ); if( pNew==0 ) return 0; @@ -45448,19 +45534,19 @@ struct unixShmNode { char *zFilename; /* Name of the mmapped file */ int hShm; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ - u16 nRegion; /* Size of array apRegion */ + int nRegion; /* Size of array apRegion */ + int nRef; /* Number of unixShm objects pointing to this */ u8 isReadonly; /* True if read-only */ u8 isUnlocked; /* True if no DMS lock held */ +#ifdef SQLITE_DEBUG + u8 nextShmId; /* Next available unixShm.id value */ +#endif char **apRegion; /* Array of mapped shared-memory regions */ - int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK]; #endif int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ -#ifdef SQLITE_DEBUG - u8 nextShmId; /* Next available unixShm.id value */ -#endif #ifdef SQLITE_MUTEXFREE_SHMLOCK /* In unix-excl mode, if SQLITE_MUTEXFREE_SHMLOCK is defined, all locks @@ -46622,28 +46708,35 @@ static void unixRemapfile( #endif if( pOrig ){ + i64 nReuse = pFd->mmapSize & ~((int)osGetpagesize()-1); + + /* Unmap any part of the current mapping that will not be reused */ + if( nReusemmapSizeActual ){ + osMunmap(&pOrig[nReuse], pFd->mmapSizeActual - nReuse); + } + + /* If nReuse is now zero, then the entire mapping was unmapped by the + ** call above. In this case do not attempt to extend pOrig. */ + if( nReuse>0 ){ #if HAVE_MREMAP - i64 nReuse = pFd->mmapSize; - pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); - zErr = "mremap"; + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; #else - const int szSyspage = osGetpagesize(); - i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); - u8 *pReq = &pOrig[nReuse]; - pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); - if( pNew!=MAP_FAILED ){ - if( pNew!=pReq ){ - osMunmap(pNew, nNew - nReuse); - pNew = 0; - }else{ - pNew = pOrig; + pNew = osMmap(&pOrig[nReuse], nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=&pOrig[nReuse] ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } } - } #endif - /* The attempt to extend the existing mapping failed. Free it. */ - if( pNew==MAP_FAILED || pNew==0 ){ - osMunmap(pOrig, nReuse); + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } } } @@ -47140,10 +47233,14 @@ static int fillInUnixFile( } #if OS_VXWORKS - pNew->pId = vxworksFindFileId(zFilename); - if( pNew->pId==0 ){ - ctrlFlags |= UNIXFILE_NOLOCK; - rc = SQLITE_NOMEM_BKPT; + if( zFilename ){ + pNew->pId = vxworksFindFileId(zFilename); + if( pNew->pId==0 ){ + ctrlFlags |= UNIXFILE_NOLOCK; + rc = SQLITE_NOMEM_BKPT; + } + }else{ + pNew->pId = 0; } #endif @@ -47251,12 +47348,15 @@ static int fillInUnixFile( int n; sqlite3_snprintf(MAX_PATHNAME, zSemName, "/%s.sem", pNew->pId->zCanonicalName); - for( n=1; zSemName[n]; n++ ) - if( zSemName[n]=='/' ) zSemName[n] = '_'; + for(n=1; zSemName[n]; n++){ + if( zSemName[n]=='/' && !sqlite3FaultSim(801) ) zSemName[n] = '_'; + } pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); if( pNew->pInode->pSem == SEM_FAILED ){ rc = SQLITE_NOMEM_BKPT; pNew->pInode->aSemName[0] = '\0'; + releaseInodeInfo(pNew); + pNew->pInode = 0; } } unixLeaveMutex(); @@ -47680,9 +47780,12 @@ static int unixOpen( */ zName = unixTempFileDir(); if( zName - && (fd = robust_open(zName, O_RDWR|O_CREAT|O_EXCL|O_TMPFILE, 0600))>=0 + && (fd = robust_open(zName, O_RDWR|O_EXCL|O_TMPFILE, 0600))>=0 ){ - rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + if( pOutFlags ){ + *pOutFlags = flags; + } + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags|UNIXFILE_NOLOCK); goto open_finished; } #endif @@ -48203,7 +48306,12 @@ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ UNUSED_PARAMETER(NotUsed); return microseconds; #else - int seconds = (microseconds+999999)/1000000; + int seconds; + if( microseconds>2146000000 ){ + seconds = 2147; + }else{ + seconds = (microseconds+999999)/1000000; + } sleep(seconds); UNUSED_PARAMETER(NotUsed); return seconds*1000000; @@ -49652,7 +49760,6 @@ SQLITE_API int sqlite3_os_end(void){ ** Include the header file for the Windows VFS. */ /* #include "os_win.h" */ -#define SQLITE_WIN32_HAS_WIDE 1 /* ** This constant should already be defined (in the "WinDef.h" SDK file). @@ -49967,24 +50074,6 @@ static void winMemShutdown(void *pAppData); SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetWin32(void); #endif /* SQLITE_WIN32_MALLOC */ -/* -** The following variable is (normally) set once and never changes -** thereafter. It records whether the operating system is Win9x -** or WinNT. -** -** 0: Operating system unknown. -** 1: Operating system is Win9x. -** 2: Operating system is WinNT. -** -** In order to facilitate testing on a WinNT system, the test fixture -** can manually set this value to 1 to emulate Win98 behavior. -*/ -#ifdef SQLITE_TEST -SQLITE_API LONG volatile sqlite3_os_type = 0; -#else -static LONG volatile sqlite3_os_type = 0; -#endif - #ifndef SYSCALL # define SYSCALL sqlite3_syscall_ptr #endif @@ -50336,6 +50425,9 @@ static struct win_syscall { #define osCygwin_conv_path ((size_t(*)(unsigned int, \ const void *, void *, size_t))aSyscall[58].pCurrent) + { "GetDriveTypeW", (SYSCALL)GetDriveTypeW, 0 }, +#define osGetDriveTypeW ((UINT(WINAPI*)(LPCWSTR))aSyscall[59].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -50506,6 +50598,7 @@ SQLITE_API int sqlite3_win32_reset_heap(){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** This function outputs the specified (ANSI) string to the Win32 debugger ** (if available). Undocumented. Might go away at any moment. @@ -50542,6 +50635,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ return rc; } #endif +#endif /* _WIN32 */ #ifdef SQLITE_WIN32_MALLOC /* @@ -50802,6 +50896,7 @@ static char *winUnicodeToUtf8(LPCWSTR zWideText){ return zText; } +#ifdef _WIN32 /* ** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM ** code page. @@ -50831,7 +50926,6 @@ static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ return zMbcsText; } -#ifdef _WIN32 /* ** Convert a Microsoft Unicode string to a multi-byte character string, ** using the ANSI or OEM code page. @@ -50859,7 +50953,6 @@ static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ } return zText; } -#endif /* _WIN32 */ /* ** Convert a multi-byte character string to UTF-8. @@ -50879,7 +50972,6 @@ static char *winMbcsToUtf8(const char *zText, int useAnsi){ return zTextUtf8; } -#ifdef _WIN32 /* ** Convert a UTF-8 string to a multi-byte character string. ** @@ -50929,7 +51021,6 @@ SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ #endif return winUnicodeToUtf8(zWideText); } -#endif /* _WIN32 */ /* ** This is a public wrapper for the winMbcsToUtf8() function. @@ -50947,7 +51038,6 @@ SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){ return winMbcsToUtf8(zText, osAreFileApisANSI()); } -#ifdef _WIN32 /* ** This is a public wrapper for the winMbcsToUtf8() function. */ @@ -51210,13 +51300,13 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ return 0; } if( winIoerrCanRetry1(e) ){ - sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + osSleep(winIoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } #if defined(winIoerrCanRetry2) else if( winIoerrCanRetry2(e) ){ - sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + osSleep(winIoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } @@ -51446,7 +51536,7 @@ static int winClose(sqlite3_file *id){ do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ - }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (osSleep(100), 1) ); if( rc ){ pFile->h = NULL; } @@ -52060,7 +52150,7 @@ static int winLock(sqlite3_file *id, int locktype){ } cnt--; - if( cnt>0 ) sqlite3_win32_sleep(1); + if( cnt>0 ) osSleep(1); } gotPendingLock = res; } @@ -52269,7 +52359,7 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ /* Forward references to VFS helper methods used for temporary files */ static int winGetTempname(sqlite3_vfs *, char **); -static int winIsDir(const void *); +static int winIsDir(LPCWSTR); static BOOL winIsLongPathPrefix(const char *); static BOOL winIsDriveLetterAndColon(const char *); @@ -52501,11 +52591,10 @@ static SYSTEM_INFO winSysInfo; ** happen when the file path >32k, in which case winUtf8ToUnicode() ** will fail too. */ -static void *winConvertFromUtf8Filename(const char *zFilename){ - void *zConverted = 0; +static LPWSTR winConvertFromUtf8Filename(const char *zFilename){ + LPWSTR zConverted = 0; #ifdef __CYGWIN__ int nChar; - LPWSTR zWideFilename; if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) && winIsDirSep(zFilename[2])) ){ @@ -52519,20 +52608,19 @@ static void *winConvertFromUtf8Filename(const char *zFilename){ if ( zConverted==0 ){ return zConverted; } - zWideFilename = zConverted; /* Filenames should be prefixed, except when converted * full path already starts with "\\?\". */ if( osCygwin_conv_path(convertflag, zFilename, - zWideFilename+4, nByte)==0 ){ + zConverted+4, nByte)==0 ){ if( (convertflag&CCP_RELATIVE) ){ - memmove(zWideFilename, zWideFilename+4, nByte); - }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ - memcpy(zWideFilename, L"\\\\?\\", 8); - }else if( zWideFilename[6]!='?' ){ - memmove(zWideFilename+6, zWideFilename+4, nByte); - memcpy(zWideFilename, L"\\\\?\\UNC", 14); + memmove(zConverted, zConverted+4, nByte); + }else if( memcmp(zConverted+4, L"\\\\", 4) ){ + memcpy(zConverted, L"\\\\?\\", 8); + }else if( zConverted[6]!='?' ){ + memmove(zConverted+6, zConverted+4, nByte); + memcpy(zConverted, L"\\\\?\\UNC", 14); }else{ - memmove(zWideFilename, zWideFilename+4, nByte); + memmove(zConverted, zConverted+4, nByte); } return zConverted; } @@ -52543,28 +52631,27 @@ static void *winConvertFromUtf8Filename(const char *zFilename){ if( nChar==0 ){ return 0; } - zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); - if( zWideFilename==0 ){ + zConverted = sqlite3MallocZero( ((u64)nChar+6)*sizeof(WCHAR) ); + if( zConverted==0 ){ return 0; } nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, - zWideFilename, nChar); + zConverted, nChar); if( nChar==0 ){ - sqlite3_free(zWideFilename); - zWideFilename = 0; + sqlite3_free(zConverted); + zConverted = 0; }else if( nChar>MAX_PATH && winIsDriveLetterAndColon(zFilename) && winIsDirSep(zFilename[2]) ){ - memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); - zWideFilename[2] = '\\'; - memcpy(zWideFilename, L"\\\\?\\", 8); + memmove(zConverted+4, zConverted, nChar*sizeof(WCHAR)); + zConverted[2] = '\\'; + memcpy(zConverted, L"\\\\?\\", 8); }else if( nChar>MAX_PATH && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) && zFilename[2] != '?' ){ - memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); - memcpy(zWideFilename, L"\\\\?\\UNC", 14); + memmove(zConverted+6, zConverted, nChar*sizeof(WCHAR)); + memcpy(zConverted, L"\\\\?\\UNC", 14); } - zConverted = zWideFilename; #else /* if !defined(__CYGWIN__) */ zConverted = winUtf8ToUnicode(zFilename); #endif /* __CYGWIN__ */ @@ -52735,6 +52822,28 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } } +/* +** Handle h is open on the *-shm file. This function zeroes the first +** 8 bytes of the file. If SQLITE_ENABLE_SETLK_TIMEOUT is defined, h may +** have been opened with FILE_FLAG_OVERLAPPED. +*/ +static int winZeroSharedMemory(HANDLE h){ + u8 aZero[8] = {0,0,0,0,0,0,0,0}; + DWORD nWrite = 0; + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + if( !osWriteFile(h, aZero, sizeof(aZero), &nWrite, &ovlp) ){ + /* If the file was opened with FILE_FLAG_OVERLAPPED, the write may still + ** be pending. Wait for it with GetOverlappedResult(). */ + if( osGetLastError()!=ERROR_IO_PENDING + || !GetOverlappedResult(h, &ovlp, &nWrite, TRUE) + ){ + nWrite = 0; + } + } + return (nWrite==sizeof(aZero) ? SQLITE_OK : SQLITE_IOERR_WRITE); +} + /* ** The DMS lock has not yet been taken on the shm file associated with ** pShmNode. Take the lock. Truncate the *-shm file if required. @@ -52753,7 +52862,12 @@ static int winLockSharedMemory(winShmNode *pShmNode, DWORD nMs){ if( pShmNode->isReadonly ){ rc = SQLITE_READONLY_CANTINIT; }else{ - rc = winHandleTruncate(h, 0); + if( SQLITE_OK!=winHandleTruncate(h, 0) ){ + /* Sometimes the truncate operation may fail because some other + ** process is still holding an open mapping. So try to write + ** zeroes into the start of the file instead. */ + rc = winZeroSharedMemory(h); + } } /* Release the EXCLUSIVE lock acquired above. */ @@ -52786,7 +52900,7 @@ static int winHandleOpen( HANDLE *ph /* OUT: New HANDLE for file */ ){ int rc = SQLITE_OK; - void *zConverted = 0; + LPWSTR zConverted = 0; int bReadonly = *pbReadonly; HANDLE h = INVALID_HANDLE_VALUE; @@ -52814,7 +52928,7 @@ static int winHandleOpen( /* TODO: platforms. ** TODO: retry-on-ioerr. */ - h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ + h = osCreateFileW(zConverted, /* lpFileName */ (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ NULL, /* lpSecurityAttributes */ @@ -52912,6 +53026,23 @@ static int winIsUNCPath(const char *zFile){ return sqlite3_win_test_unc_locking; } +/* +** Return true if the string passed in is the name of a file on a +** remote (network-mounted) volume. +*/ +static int winIsRemoteVolume(const char *zFile){ + WCHAR zRoot[4]; + if( strncmp(zFile, "\\\\?\\",4)==0 ) zFile += 4; + if( !sqlite3Isalpha(zFile[0]) ) return 0; + if( zFile[1]!=':' ) return 0; + if( !winIsDirSep(zFile[2]) ) return 0; + zRoot[0] = (WCHAR)zFile[0]; + zRoot[1] = ':'; + zRoot[2] = '\\'; + zRoot[3] = 0; + return osGetDriveTypeW(zRoot)==DRIVE_REMOTE; +} + /* ** Open the shared-memory area associated with database file pDbFd. */ @@ -52937,7 +53068,11 @@ static int winOpenSharedMemory(winFile *pDbFd){ pNew->zFilename = (char*)&pNew[1]; pNew->hSharedShm = INVALID_HANDLE_VALUE; pNew->isUnlocked = 1; - pNew->bUseSharedLockHandle = winIsUNCPath(pDbFd->zPath); + if( winIsUNCPath(pDbFd->zPath) || winIsRemoteVolume(pDbFd->zPath) ){ + pNew->bUseSharedLockHandle = 1; + }else{ + pNew->bUseSharedLockHandle = 0; + } sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); @@ -53777,7 +53912,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !azDirs[3] ) azDirs[3] = osGetenv("TEMP"); if( !azDirs[4] ) azDirs[4] = osGetenv("USERPROFILE"); for(i=0; ipage, only on paths where pPage is +** known to be non-NULL. PAGE_TO_PGHDR1() is likewise only ever given a +** non-NULL page that this module previously handed out. +*/ +#define PAGE_TO_PGHDR1(pPg) \ + ((PgHdr1*)(((char*)(pPg)) - offsetof(PgHdr1,page))) + /* ** A page is pinned if it is not on the LRU list. To be "pinned" means ** that the page is in active use and must not be deallocated. @@ -57903,6 +58070,7 @@ static void pcache1ResizeHash(PCache1 *p){ if( nNew<256 ){ nNew = 256; } + assert( (nNew & (nNew-1))==0 ); /* nNew is always a power of two */ pcache1LeaveMutex(p->pGroup); if( p->nHash ){ sqlite3BeginBenignMalloc(); } @@ -57914,7 +58082,7 @@ static void pcache1ResizeHash(PCache1 *p){ PgHdr1 *pPage; PgHdr1 *pNext = p->apHash[i]; while( (pPage = pNext)!=0 ){ - unsigned int h = pPage->iKey % nNew; + unsigned int h = (unsigned int)(pPage->iKey & (nNew-1)); pNext = pPage->pNext; pPage->pNext = apNew[h]; apNew[h] = pPage; @@ -57933,7 +58101,7 @@ static void pcache1ResizeHash(PCache1 *p){ ** ** The PGroup mutex must be held when this function is called. */ -static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ +static sqlite3_pcache_page *pcache1PinPage(PgHdr1 *pPage){ assert( pPage!=0 ); assert( PAGE_IS_UNPINNED(pPage) ); assert( pPage->pLruNext ); @@ -57947,7 +58115,7 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ assert( pPage->isAnchor==0 ); assert( pPage->pCache->pGroup->lru.isAnchor==1 ); pPage->pCache->nRecyclable--; - return pPage; + return &pPage->page; } @@ -57964,7 +58132,8 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ PgHdr1 **pp; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - h = pPage->iKey % pCache->nHash; + assert( pCache->nHash>0 && (pCache->nHash & (pCache->nHash-1))==0 ); + h = pPage->iKey & (pCache->nHash-1); for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); *pp = (*pp)->pNext; @@ -58017,14 +58186,14 @@ static void pcache1TruncateUnsafe( unsigned int h, iStop; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); assert( pCache->iMaxKey >= iLimit ); - assert( pCache->nHash > 0 ); + assert( pCache->nHash>0 && (pCache->nHash & (pCache->nHash-1))==0 ); if( pCache->iMaxKey - iLimit < pCache->nHash ){ /* If we are just shaving the last few pages off the end of the ** cache, then there is no point in scanning the entire hash table. ** Only scan those hash slots that might contain pages that need to ** be removed. */ - h = iLimit % pCache->nHash; - iStop = pCache->iMaxKey % pCache->nHash; + h = iLimit & (pCache->nHash-1); + iStop = pCache->iMaxKey & (pCache->nHash-1); TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ }else{ /* This is the general case where many pages are being removed. @@ -58055,7 +58224,7 @@ static void pcache1TruncateUnsafe( } } if( h==iStop ) break; - h = (h+1) % pCache->nHash; + h = (h+1) & (pCache->nHash-1); } assert( nPage<0 || pCache->nPage==(unsigned)nPage ); } @@ -58243,7 +58412,7 @@ static int pcache1Pagecount(sqlite3_pcache *p){ ** usually not needed, and by avoiding the stack initialization required ** for these steps, the main pcache1Fetch() procedure can run faster. */ -static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( +static SQLITE_NOINLINE sqlite3_pcache_page *pcache1FetchStage2( PCache1 *pCache, unsigned int iKey, int createFlag @@ -58267,6 +58436,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( if( pCache->nPage>=pCache->nHash ) pcache1ResizeHash(pCache); assert( pCache->nHash>0 && pCache->apHash ); + assert( (pCache->nHash & (pCache->nHash-1))==0 ); /* Step 4. Try to recycle a page. */ if( pCache->bPurgeable @@ -58295,7 +58465,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( } if( pPage ){ - unsigned int h = iKey % pCache->nHash; + unsigned int h = iKey & (pCache->nHash-1); pCache->nPage++; pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; @@ -58308,8 +58478,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( if( iKey>pCache->iMaxKey ){ pCache->iMaxKey = iKey; } + return &pPage->page; } - return pPage; + return 0; } /* @@ -58371,7 +58542,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper ** invokes the appropriate routine. */ -static PgHdr1 *pcache1FetchNoMutex( +static sqlite3_pcache_page *pcache1FetchNoMutex( sqlite3_pcache *p, unsigned int iKey, int createFlag @@ -58394,7 +58565,7 @@ static PgHdr1 *pcache1FetchNoMutex( if( PAGE_IS_UNPINNED(pPage) ){ return pcache1PinPage(pPage); }else{ - return pPage; + return &pPage->page; } }else if( createFlag ){ /* Steps 3, 4, and 5 implemented by this subroutine */ @@ -58404,13 +58575,13 @@ static PgHdr1 *pcache1FetchNoMutex( } } #if PCACHE1_MIGHT_USE_GROUP_MUTEX -static PgHdr1 *pcache1FetchWithMutex( +static sqlite3_pcache_page *pcache1FetchWithMutex( sqlite3_pcache *p, unsigned int iKey, int createFlag ){ PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage; + sqlite3_pcache_page *pPage; pcache1EnterMutex(pCache->pGroup); pPage = pcache1FetchNoMutex(p, iKey, createFlag); @@ -58428,7 +58599,6 @@ static sqlite3_pcache_page *pcache1Fetch( PCache1 *pCache = (PCache1 *)p; #endif - assert( offsetof(PgHdr1,page)==0 ); assert( pCache->bPurgeable || createFlag!=1 ); assert( pCache->bPurgeable || pCache->nMin==0 ); assert( pCache->bPurgeable==0 || pCache->nMin==10 ); @@ -58436,11 +58606,11 @@ static sqlite3_pcache_page *pcache1Fetch( assert( pCache->nHash>0 ); #if PCACHE1_MIGHT_USE_GROUP_MUTEX if( pCache->pGroup->mutex ){ - return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); + return pcache1FetchWithMutex(p, iKey, createFlag); }else #endif { - return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); + return pcache1FetchNoMutex(p, iKey, createFlag); } } @@ -58456,7 +58626,7 @@ static void pcache1Unpin( int reuseUnlikely ){ PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage = (PgHdr1 *)pPg; + PgHdr1 *pPage = PAGE_TO_PGHDR1(pPg); PGroup *pGroup = pCache->pGroup; assert( pPage->pCache==pCache ); @@ -58492,7 +58662,7 @@ static void pcache1Rekey( unsigned int iNew ){ PCache1 *pCache = (PCache1 *)p; - PgHdr1 *pPage = (PgHdr1 *)pPg; + PgHdr1 *pPage = PAGE_TO_PGHDR1(pPg); PgHdr1 **pp; unsigned int hOld, hNew; assert( pPage->iKey==iOld ); @@ -58501,8 +58671,9 @@ static void pcache1Rekey( pcache1EnterMutex(pCache->pGroup); - assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ - hOld = iOld%pCache->nHash; + assert( pcache1FetchNoMutex(p, iOld, 0)==&pPage->page ); /* pPg really iOld */ + assert( pCache->nHash>0 && (pCache->nHash & (pCache->nHash-1))==0 ); + hOld = iOld & (pCache->nHash-1); pp = &pCache->apHash[hOld]; while( (*pp)!=pPage ){ pp = &(*pp)->pNext; @@ -58510,7 +58681,7 @@ static void pcache1Rekey( *pp = pPage->pNext; assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ - hNew = iNew%pCache->nHash; + hNew = iNew & (pCache->nHash-1); pPage->iKey = iNew; pPage->pNext = pCache->apHash[hNew]; pCache->apHash[hNew] = pPage; @@ -59368,6 +59539,10 @@ SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal*); #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_WALSTAT) +SQLITE_PRIVATE void sqlite3WalStat(Wal*,sqlite3_str*); +#endif + /* sqlite3_wal_info() data */ SQLITE_PRIVATE int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame); @@ -64442,7 +64617,15 @@ SQLITE_PRIVATE int sqlite3PagerOpen( */ if( zFilename && zFilename[0] ){ int fout = 0; /* VFS flags returned by xOpen() */ + int bImmutable = sqlite3_uri_boolean(pPager->zFilename, "immutable", 0); + if( bImmutable ){ + vfsFlags |= SQLITE_OPEN_READONLY; + vfsFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + } rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + if( bImmutable ){ + goto act_like_temp_file; + } assert( !memDb ); pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0; readOnly = (fout&SQLITE_OPEN_READONLY)!=0; @@ -64482,10 +64665,9 @@ SQLITE_PRIVATE int sqlite3PagerOpen( #endif } pPager->noLock = sqlite3_uri_boolean(pPager->zFilename, "nolock", 0); - if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 - || sqlite3_uri_boolean(pPager->zFilename, "immutable", 0) ){ - vfsFlags |= SQLITE_OPEN_READONLY; - goto act_like_temp_file; + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; } } }else{ @@ -64505,6 +64687,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); + assert( readOnly==0 || readOnly==1 ); } /* The following call to PagerSetPagesize() serves to set the value of @@ -66425,6 +66608,20 @@ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ } #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_WALSTAT) +/* +** Append JSON structure elements describing the current state of +** the write-ahead log to pStr. +*/ +SQLITE_PRIVATE void sqlite3PagerWalStat(Pager *pPager, sqlite3_str *pStr){ +#ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + sqlite3WalStat(pPager->pWal, pStr); + } +#endif +} +#endif + /* ** Return the approximate number of bytes of memory currently ** used by the pager and its associated cache. @@ -68321,10 +68518,41 @@ struct Wal { ** walIteratorFree() - Free an iterator. ** ** This functionality is used by the checkpoint code (see walCheckpoint()). +** +** There are two possible algorithms: +** +** WALITER-1 Load all pages mentioned in the WAL file and sort them, +** then hand them back out in sorted order as requested by +** walIteratorNext(). nPagemap==0 for this algorithm. +** +** WALITER-2 Create a map of all database file pages and record the +** last WAL file frame that updates that page. Then walk +** through the map in acsending order and return all +** non-zero entries. nPagemap is positive for this algorithm. +** +** Either algorithm should work for any database and WAL file, however +** algorithm WALITER-1 is faster and uses less memory when the database is +** larger than the WAL file and algorithm WALITER-2 is faster and uses +** less memory when the WAL file is larger than the database, modulo a +** constant of proportionality. The walIteratorInit() routine selects +** the best algorithm for each situation. WALITER-2 was added for +** SQLite 3.54.0. Prior to that, WALITER-1 was used always. +** +** For algorithm WALITER-2, nPagemap stores the number of pages in the +** database file, nSegment==1, and aSegment[0].aPgno[] is the page map array +** that is nPagemap+1 entries in length. The value of aSegment[0].aPgno[PGNO] +** is the frame number of the last (newest) entry for page PGNO in the WAL +** file, or 0 if PGNO is not in the WAL file. (Aside: There is no page +** number 0 in a well-formed database, so the memory allocation is arranged +** such that aSegment[0].aPgno[0] and aSegment[0].iZero are aliases for the +** same four bits of memory. This saves us from having to use -1 and +1 +** offsets to translate between 1-based SQLite page numbers and 0-based +** C-language index numbers. tag-20260903-1) */ struct WalIterator { u32 iPrior; /* Last result returned from the iterator */ int nSegment; /* Number of entries in aSegment[] */ + u32 nPagemap; /* Non-zero in "page-map" mode */ struct WalSegment { int iNext; /* Next slot in aIndex[] not yet returned */ ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ @@ -68338,6 +68566,7 @@ struct WalIterator { #define SZ_WALITERATOR(N) \ (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment)) + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -69268,6 +69497,7 @@ static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){ if( !aFrame ){ return SQLITE_NOMEM_BKPT; } + memset(aFrame, 0, szFrame + WALINDEX_PGSZ); aData = &aFrame[WAL_FRAME_HDRSIZE]; aPrivate = (u32*)&aData[szPage]; @@ -69474,7 +69704,6 @@ static int walIndexRecover(Wal *pWal){ } } } - if( rc==SQLITE_OK ){ volatile WalCkptInfo *pInfo; @@ -69736,24 +69965,38 @@ static int walIteratorNext( u32 *piPage, /* OUT: The page number of the next page */ u32 *piFrame /* OUT: Wal frame index of next page */ ){ - u32 iMin; /* Result pgno must be greater than iMin */ u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ - int i; /* For looping through segments */ - - iMin = p->iPrior; - assert( iMin<0xffffffff ); - for(i=p->nSegment-1; i>=0; i--){ - struct WalSegment *pSegment = &p->aSegment[i]; - while( pSegment->iNextnEntry ){ - u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; - if( iPg>iMin ){ - if( iPgiZero + pSegment->aIndex[pSegment->iNext]; - } + + if( p->nPagemap ){ + /* Algorithm WALITER-2 */ + while( p->iPriornPagemap ){ + p->iPrior++; + if( p->aSegment[0].aPgno[p->iPrior] ){ + iRet = p->iPrior; + *piFrame = p->aSegment[0].aPgno[p->iPrior]; break; } - pSegment->iNext++; + } + }else{ + /* Algorithm WALITER-1 */ + u32 iMin; /* Result pgno must be greater than iMin */ + int i; /* For looping through segments */ + + iMin = p->iPrior; + assert( iMin<0xffffffff ); + for(i=p->nSegment-1; i>=0; i--){ + struct WalSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextnEntry ){ + u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgiZero + pSegment->aIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } } } @@ -69927,47 +70170,71 @@ static int walIteratorInit( u32 nBackfill, WalIterator **pp ){ + int iFirstPage; /* First 32KB *-shm page to visit */ + int iLastPage; /* Last 32KB *-shm page to visit */ + u32 iZero; /* Frame before first frame to visit */ WalIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ sqlite3_int64 nByte; /* Number of bytes to allocate */ + sqlite3_int64 nExtra; /* Number of bytes to allocate */ int i; /* Iterator variable */ - int iLastSeg; /* Last hash table to iterate though */ - ht_slot *aTmp; /* Temp space used by merge-sort */ + ht_slot *aTmp = 0; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ + int eAlg; /* Which algorithm to use. 1 or 2 */ + int iMode = isWalMode2(pWal) ? 2 : 1; assert( isWalMode2(pWal) || iWal==0 ); assert( 0==isWalMode2(pWal) || nBackfill==0 ); /* This routine only runs while holding the checkpoint lock. And - ** it only runs if there is actually content in the log (mxFrame>0). - */ + ** it only runs if there is actually content in the log (mxFrame>0). */ iLast = walidxGetMxFrame(&pWal->hdr, iWal); assert( pWal->ckptLock && iLast>0 ); - if( iMode==2 ){ - iLastSeg = walFramePage2(iWal, iLast); + iLastPage = walFramePage2(iWal, iLast); + iFirstPage = iWal; + iZero = 0; }else{ - iLastSeg = walFramePage(iLast); + iLastPage = walFramePage(iLast); + iFirstPage = walFramePage(nBackfill+1); + iZero = iFirstPage ? (HASHTABLE_NPAGE_ONE+(iFirstPage-1)*HASHTABLE_NPAGE):0; + } + + if( iMode==1 && (iLast-iZero)*sizeof(ht_slot)>pWal->hdr.nPage*sizeof(u32) ){ + /* Algorithm WALITER-2 */ + nSegment = 1; + eAlg = 2; + nByte = SZ_WALITERATOR(nSegment) + pWal->hdr.nPage * sizeof(u32); + nExtra = 0; + }else{ + /* Algorithm WALITER-1 */ + assert( ((iLastPage - iFirstPage) % iMode)==0 ); + nSegment = ((iLastPage - iFirstPage)/iMode) + 1; + eAlg = 1; + nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot); + nExtra = sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast); } - nSegment = 1 + (iLastSeg/iMode); /* Allocate space for the WalIterator object. */ - nByte = SZ_WALITERATOR(nSegment) - + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc64(nByte - + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); + p = (WalIterator*)sqlite3_malloc64(nByte + nExtra); + if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; - aTmp = (ht_slot*)&(((u8*)p)[nByte]); + if( eAlg==2 ){ + p->nPagemap = pWal->hdr.nPage; + p->aSegment[0].aPgno = ((u32*)&p->aSegment[1]) - 1; + assert( (u8*)&(p->aSegment[0].aPgno[0]) == (u8*)&(p->aSegment[0].iZero) ); + /* ^-- verifies tag-20260903-1 */ + }else{ + aTmp = (ht_slot*)&(((u8*)p)[nByte]); + } SEH_FREE_ON_ERROR(0, p); - i = iMode==2 ? iWal : walFramePage(nBackfill+1); - for(; rc==SQLITE_OK && i<=iLastSeg; i+=iMode){ + for(i=iFirstPage; rc==SQLITE_OK && i<=iLastPage; i+=iMode){ WalHashLoc sLoc; rc = walHashGet(pWal, i, &sLoc); @@ -69975,32 +70242,45 @@ static int walIteratorInit( int j; /* Counter variable */ int nEntry; /* Number of entries in this segment */ ht_slot *aIndex; /* Sorted index for this segment */ - u32 iZero; + u32 iHashZero; if( iMode==2 ){ - walExternalDecode(sLoc.iZero+1, &iZero); - iZero--; - assert( iZero==0 || i>=2 ); + walExternalDecode(sLoc.iZero+1, &iHashZero); + iHashZero--; + assert( iHashZero==0 || i>=2 ); }else{ - iZero = sLoc.iZero; + iHashZero = sLoc.iZero; } - if( i==iLastSeg ){ - nEntry = (int)(iLast - iZero); + if( i==iLastPage ){ + nEntry = (int)(iLast - iHashZero); }else{ nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno); } - aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[iZero]; - iZero++; - for(j=0; jaSegment[0].aPgno; + assert( iMode==1 ); + for(j=0; jnPagemap ){ + aPagemap[pgno] = sLoc.iZero + j + 1; + } + } + }else{ + assert( i!=iFirstPage || iMode==2 || iHashZero==sLoc.iZero ); + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[iHashZero - iZero]; + iHashZero++; + + for(j=0; jaSegment[(i-iFirstPage)/iMode].iZero = iHashZero; + p->aSegment[(i-iFirstPage)/iMode].nEntry = nEntry; + p->aSegment[(i-iFirstPage)/iMode].aIndex = aIndex; + p->aSegment[(i-iFirstPage)/iMode].aPgno = (u32*)sLoc.aPgno; } - walMergesort((u32*)sLoc.aPgno, aTmp, aIndex, &nEntry); - p->aSegment[i/iMode].iZero = iZero; - p->aSegment[i/iMode].nEntry = nEntry; - p->aSegment[i/iMode].aIndex = aIndex; - p->aSegment[i/iMode].aPgno = (u32*)sLoc.aPgno; } } if( rc!=SQLITE_OK ){ @@ -70194,8 +70474,39 @@ static int wal2RestartOk(Wal *pWal, int iApp){ return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3); } -static void wal2RestartFinished(Wal *pWal, int iApp){ - walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3); + +/* +** This function is called by a writer that has just switched the current wal +** file from wal file iApp to wal file !iApp. The caller holds the exclusive +** locks taken by wal2RestartOk(pWal, iApp) and a shared lock on read-lock +** slot pWal->readLock (iApp*3+1). This function releases all of those locks +** and takes a shared lock on the new read-lock slot such that there is no +** opportunity for a concurrent checkpointer to take an exclusive lock that +** interferes with this connection taking the new shared-lock. +*/ +static int wal2RestartFinished(Wal *pWal, int iApp){ + int iNew = !iApp; + int iNewLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; + int iHold = iNew ? WAL_LOCK_PART2 : WAL_LOCK_PART1; + int rc; + + assert( WAL_LOCK_PART1==1 ); + assert( WAL_LOCK_PART1_FULL2==2 ); + assert( WAL_LOCK_PART2_FULL1==3 ); + assert( WAL_LOCK_PART2==4 ); + assert( pWal->readLock==iApp*3+1 ); + + /* wal2RestartOk() locked slots 1, 2 and 3 (iApp==1) or 2, 3 and 4 + ** (iApp==0). In both cases, release slots 2 and 3, and keep slot iHold + ** until the new shared lock has been obtained. The old read-lock slot + ** (slot 4 if iApp==1, or slot 1 if iApp==0) is not part of the + ** checkpointer's pair, so it may be released first. */ + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + walUnlockExclusive(pWal, WAL_READ_LOCK(WAL_LOCK_PART1_FULL2), 2); + pWal->readLock = iNewLock; + rc = walLockShared(pWal, WAL_READ_LOCK(iNewLock)); + walUnlockExclusive(pWal, WAL_READ_LOCK(iHold), 1); + return rc; } /* @@ -70829,8 +71140,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( badHdr ){ if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + /* Retry the header now that the WRITE_LOCK is held. If it still + ** fails, the header really is corrupt and recovery is required. + ** Otherwise, continue with the now valid header. */ + if( page0 ) badHdr = walIndexTryHdr(pWal, pChanged); walUnlockShared(pWal, WAL_WRITE_LOCK); - rc = SQLITE_READONLY_RECOVERY; + if( badHdr ) rc = SQLITE_READONLY_RECOVERY; } }else{ int bWriteLock = pWal->writeLock; @@ -71341,25 +71656,35 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ mxI = i; } } - if( (pWal->readOnly & WAL_SHM_RDONLY)==0 - && (mxReadMarkaReadMark+i,mxFrame); - mxReadMark = mxFrame; - mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; + + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 ){ + if( mxReadMarkaReadMark+i,mxFrame); + mxReadMark = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } } } + }else if( mxI==0 ){ + /* If the loop above could not find a usable slot (mxI==0), then check + ** if the wal-index header has been modified by another connection since + ** it was read. If not, return SQLITE_READONLY_CANTINIT. If it has + ** been modified, fall through and return WAL_RETRY. */ + walShmBarrier(pWal); + if( 0==memcmp((void*)walIndexHdr(pWal),&pWal->hdr,sizeof(WalIndexHdr)) ){ + return SQLITE_READONLY_CANTINIT; + } } + if( mxI==0 ){ - assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; + return WAL_RETRY; } (void)walEnableBlockingMs(pWal, nBlockTmout); @@ -71916,6 +72241,10 @@ SQLITE_PRIVATE int sqlite3WalReadFrame( /* Figure out the page size */ sz = pWal->hdr.szPage; sz = (sz&0xfe00) + ((sz&0x0001)<<16); + if( nOut>sz ){ + memset(pOut+sz, 0, nOut-sz); + nOut = sz; + } testcase( sz<=32768 ); testcase( sz>=65536 ); @@ -71930,7 +72259,7 @@ SQLITE_PRIVATE int sqlite3WalReadFrame( WALTRACE(("WAL%p: reading frame %d wal %d\n", pWal, iRead, iWal)); iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->apWalFd[iWal], pOut, (nOut>sz?sz:nOut), iOffset); + return sqlite3OsRead(pWal->apWalFd[iWal], pOut, nOut, iOffset); } /* @@ -72521,10 +72850,7 @@ static int walRestartLog(Wal *pWal){ sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; - wal2RestartFinished(pWal, iApp); - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; - rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + rc = wal2RestartFinished(pWal, iApp); }else if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; } @@ -73399,6 +73725,64 @@ SQLITE_PRIVATE int sqlite3WalJournalMode(Wal *pWal){ return (isWalMode2(pWal) ? PAGER_JOURNALMODE_WAL2 : PAGER_JOURNALMODE_WAL); } +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_WALSTAT) +/* +** Helper function to sqlite3WalStat(). +** +** Append JSON-5 structure elements onto describing the current state +** of pWal onto the end of pStr. This function is for debugging and +** analysis only and is not included in production builds. +*/ +static void walIndexHdrStat(volatile WalIndexHdr *pHdr, sqlite3_str *pStr){ + sqlite3_str_appendf(pStr, "iVersion:%u",pHdr->iVersion); + sqlite3_str_appendf(pStr, ",iChange:%u",pHdr->iChange); + sqlite3_str_appendf(pStr, ",isInit:%u",pHdr->isInit); + sqlite3_str_appendf(pStr, ",bigEndCksum:%u",pHdr->bigEndCksum); + sqlite3_str_appendf(pStr, ",szPage:%u",pHdr->szPage); + sqlite3_str_appendf(pStr, ",mxFrame:%u",pHdr->mxFrame); + sqlite3_str_appendf(pStr, ",nPage:%u",pHdr->nPage); +#if 0 /* Not often needed. Declutter. */ + sqlite3_str_appendf(pStr, ",aFrameCksum:[0x%08x,0x%08x]", + pHdr->aFrameCksum[0], pHdr->aFrameCksum[1]); + sqlite3_str_appendf(pStr, ",aSalt:[0x%08x,0x%08x]", + pHdr->aSalt[0], pHdr->aSalt[1]); + sqlite3_str_appendf(pStr, ",aCksum:[0x%08x,0x%08x]", + pHdr->aCksum[0], pHdr->aCksum[1]); +#endif +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_WALSTAT */ + + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_WALSTAT) +/* +** Append JSON-5 structure elements onto describing the current state +** of pWal onto the end of pStr. This function is for debugging and +** analysis only and is not included in production builds. +*/ +SQLITE_PRIVATE void sqlite3WalStat(Wal *pWal, sqlite3_str *pStr){ + SEH_TRY { + int i; + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + volatile WalIndexHdr *pHdr = walIndexHdr(pWal); + sqlite3_str_appendf(pStr, "hdr0:{"); + walIndexHdrStat(pHdr,pStr); + sqlite3_str_appendf(pStr, "},ckpt:{"); + sqlite3_str_appendf(pStr, "nBackfill:%u",pInfo->nBackfill); + sqlite3_str_appendf(pStr, ",aReadMark:[%u", pInfo->aReadMark[0]); + for(i=1; iaReadMark[i]); + } + sqlite3_str_appendf(pStr, "],nBackfillAttempted:%u}", + pInfo->nBackfillAttempted); + + sqlite3_str_appendf(pStr, ",nWiData:%d", pWal->nWiData); + sqlite3_str_appendf(pStr, ",readLock:%d", pWal->readLock); + sqlite3_str_appendf(pStr, ",minFrame:%u", pWal->minFrame); + } + SEH_EXCEPT( ; ) +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_WALSTAT */ + #endif /* #ifndef SQLITE_OMIT_WAL */ /************** End of wal.c *************************************************/ @@ -74153,6 +74537,7 @@ struct IntegrityCk { int nErr; /* Number of messages written to zErrMsg so far */ int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ u32 nStep; /* Number of steps into the integrity_check process */ + u8 nAbove; /* Current btree recursion depth */ const char *zPfx; /* Error message prefix */ Pgno v0; /* Value for first %u substitution in zPfx (root page) */ Pgno v1; /* Value for second %u substitution in zPfx (current pg) */ @@ -74454,12 +74839,18 @@ SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ } SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ int i; + u8 skipOk = 1; + /* As in the threadsafe build: set db->noSharedCache when no attached + ** Btree is sharable, so that schema and table-lock checks are skipped */ + if( db->noSharedCache ) return; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ p->pBt->db = p->db; + if( p->sharable ) skipOk = 0; } } + db->noSharedCache = skipOk; } #endif /* if SQLITE_THREADSAFE */ @@ -76004,7 +76395,7 @@ static void btreeParseCellPtrNoPayload( #ifndef SQLITE_DEBUG UNUSED_PARAMETER(pPage); #endif - pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nSize = 4 + sqlite3GetVarint(&pCell[4], (u64*)&pInfo->nKey); pInfo->nPayload = 0; pInfo->nLocal = 0; pInfo->pPayload = 0; @@ -76044,7 +76435,7 @@ static void btreeParseCellPtr( /* The next block of code is equivalent to: ** - ** pIter += getVarint(pIter, (u64*)&pInfo->nKey); + ** pIter += sqlite3GetVarint(pIter, (u64*)&pInfo->nKey); ** ** The code is inlined and the loop is unrolled for performance. ** This routine is a high-runner. @@ -76894,7 +77285,7 @@ static int btreeComputeFreeSpace(MemPage *pPage){ size = get2byte(&data[pc+2]); if( size<4 && sqlite3FaultSim(422)==SQLITE_OK ){ /* Minimum freeblock size is 4. Enable fault-sim 422 to disable this - ** check to reach interesting error stats. However, disabling this + ** check to reach interesting error states. However, disabling this ** check can cause assertion faults due to min-heap overflow. All ** fault-sims are for testing use only, but this one especially so. */ return SQLITE_CORRUPT_PAGE(pPage); @@ -76929,7 +77320,8 @@ static int btreeComputeFreeSpace(MemPage *pPage){ /* ** Do additional sanity check after btreeInitPage() if -** PRAGMA cell_size_check=ON +** PRAGMA cell_size_check=ON. Raise a corruption error and +** leave the page uninitialized if any problems are found. */ static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ int iCellFirst; /* First allowable cell or freeblock offset */ @@ -76952,11 +77344,13 @@ static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ testcase( pc==iCellFirst ); testcase( pc==iCellLast ); if( pciCellLast ){ + pPage->isInit = 0; return SQLITE_CORRUPT_PAGE(pPage); } sz = pPage->xCellSize(pPage, &data[pc]); testcase( pc+sz==usableSize ); if( pc+sz>usableSize ){ + pPage->isInit = 0; return SQLITE_CORRUPT_PAGE(pPage); } } @@ -80913,7 +81307,7 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( } } } - getVarint(pCell, (u64*)&nCellKey); + nCellKey = sqlite3VarintValue(pCell); if( nCellKeyupr ){ c = -1; break; } @@ -82027,10 +82421,8 @@ static SQLITE_NOINLINE int clearCellOverflow( pBt = pPage->pBt; assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; - nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; - assert( nOvfl>0 || - (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)nPayload - pInfo->nLocal + (u64)ovflPageSize-1)/ovflPageSize; + assert( nOvfl>0 ); while( nOvfl-- ){ Pgno iNext = 0; MemPage *pOvfl = 0; @@ -82131,7 +82523,7 @@ static int fillInCell( nSrc = pX->nData; assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ nHeader += putVarint32(&pCell[nHeader], nPayload); - nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); + nHeader += sqlite3PutVarint(&pCell[nHeader], *(u64*)&pX->nKey); }else{ assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); nSrc = nPayload = (int)pX->nKey; @@ -82423,7 +82815,7 @@ static int insertCell( memcpy(&data[idx+4], pCell+4, sz-4); put4byte(&data[idx], iChild); pIns = pPage->aCellIdx + i*2; - memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + if( inCell ) memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); pPage->nCell++; /* increment the cell count */ @@ -82508,7 +82900,7 @@ static int insertCellFast( pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); pIns = pPage->aCellIdx + i*2; - memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + if( inCell ) memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); pPage->nCell++; /* increment the cell count */ @@ -83461,6 +83853,7 @@ static int balance_nonroot( u16 maskPage = pOld->maskPage; u8 *piCell = aData + pOld->cellOffset; u8 *piEnd; + u8 **ppCell; VVA_ONLY( int nCellAtStart = b.nCell; ) /* Verify that all sibling pages are of the same "type" (table-leaf, @@ -83496,10 +83889,10 @@ static int balance_nonroot( } limit = pOld->aiOvfl[0]; for(j=0; jnOverflow; k++){ assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ b.apCell[b.nCell] = pOld->apOvfl[k]; @@ -83507,12 +83900,14 @@ static int balance_nonroot( } } piEnd = aData + pOld->cellOffset + 2*pOld->nCell; + /* Collect cell pointers before updating the cell count. */ + ppCell = &b.apCell[b.nCell]; while( piCellnCell+pOld->nOverflow) ); cntOld[i] = b.nCell; @@ -83881,7 +84276,7 @@ static int balance_nonroot( j--; pNew->xParseCell(pNew, b.apCell[j], &info); pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); + sz = 4 + sqlite3PutVarint(&pCell[4], info.nKey); pTemp = 0; }else{ pCell -= 4; @@ -84486,6 +84881,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( testcase( pCur->eState==CURSOR_FAULT ); rc = moveToRoot(pCur); if( rc && rc!=SQLITE_EMPTY ) return rc; + loc = 0; /* Indicate the the cursor is not in a known position */ } assert( cursorOwnsBtShared(pCur) ); @@ -84509,11 +84905,13 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( invalidateIncrblobCursors(p, pCur->pgnoRoot, pX->nKey, 0); } - /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing - ** to a row with the same key as the new entry being inserted. + /* If BTREE_SAVEPOSITION is set and seekResult is non-zero and + ** if we didn't have to reposition the cursor initially, then + ** the cursor must already be pointing to a row with the same + ** key as the new entry being inserted. Verify that this is so. */ #ifdef SQLITE_DEBUG - if( flags & BTREE_SAVEPOSITION ){ + if( (flags & BTREE_SAVEPOSITION)!=0 && loc!=0 ){ assert( pCur->curFlags & BTCF_ValidNKey ); assert( pX->nKey==pCur->info.nKey ); assert( loc==0 ); @@ -84768,7 +85166,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 }else{ aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload); } - if( pDest->pKeyInfo==0 ) aOut += putVarint(aOut, iKey); + if( pDest->pKeyInfo==0 ) aOut += sqlite3PutVarint(aOut, iKey); nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ @@ -85010,7 +85408,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); if( rc==SQLITE_OK ){ - rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n); + rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4-(pCell[0]==2), pTmp, n); } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; @@ -85062,6 +85460,12 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ }else{ pCur->skipNext = 1; } + /* The pCur->info structure is currently populated with the details + ** of the cell that was just removed. But pCur->ix now points to a + ** different cell in the b-tree. So zero info.nSize here so that if + ** saveCursorPosition() is called on this cursor it does not trust + ** the contents of pCur->info. */ + pCur->info.nSize = 0; }else{ rc = moveToRoot(pCur); if( bPreserve ){ @@ -85241,11 +85645,20 @@ SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, Pgno *piTable, int flags){ /* ** Erase the given database page and all its children. Return ** the page to the freelist. +** +** The freePageFlag parameter serves a double role: +** +** * Bit 0 (freePageFlag&1) means that the page should be freed +** after it is cleared. +** +** * Bits 1-31 (freePageFlag>>1) is the depth of recursion. Use this +** to prevent a corrupt database file from recursing too deeply and +** overflowing the CPU stack. */ static int clearDatabasePage( BtShared *pBt, /* The BTree that contains the table */ Pgno pgno, /* Page number to clear */ - int freePageFlag, /* Deallocate page if true */ + int freePageFlag, /* bit 0: Deallocate page. Bits 1-31: depth */ i64 *pnChange, /* Add number of Cells freed to this counter */ Pgno pgnoRoot ){ @@ -85260,6 +85673,9 @@ static int clearDatabasePage( if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_PGNO(pgno); } + if( (freePageFlag>>1) > BTCURSOR_MAX_DEPTH ){ + return SQLITE_CORRUPT_PGNO(pgno); + } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; setMempageRoot(pPage, pgnoRoot); @@ -85273,7 +85689,8 @@ static int clearDatabasePage( for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot); + rc = clearDatabasePage(pBt, get4byte(pCell), + (freePageFlag+2)|1, pnChange, pgnoRoot); if( rc ) goto cleardatabasepage_out; } BTREE_CLEAR_CELL(rc, pPage, pCell, info); @@ -85281,7 +85698,8 @@ static int clearDatabasePage( } if( !pPage->leaf ){ rc = clearDatabasePage( - pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot + pBt, get4byte(&pPage->aData[hdr+8]), + (freePageFlag+2)|1, pnChange, pgnoRoot ); if( rc ) goto cleardatabasepage_out; if( pPage->intKey ) pnChange = 0; @@ -85290,7 +85708,7 @@ static int clearDatabasePage( testcase( !pPage->intKey ); *pnChange += pPage->nCell; } - if( freePageFlag ){ + if( (freePageFlag&1)!=0 ){ freePage(pPage, &rc); }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); @@ -85420,6 +85838,9 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ } pMove = 0; rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pMove->pDbPage); + } freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -85925,19 +86346,24 @@ static int checkTreePage( /* Check that the page exists */ checkProgress(pCheck); - if( pCheck->mxErr==0 ) goto end_of_check; + if( pCheck->mxErr==0 ) return 0; pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; pCheck->zPfx = "Tree %u page %u: "; pCheck->v1 = iPage; + pCheck->nAbove++; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); if( rc==SQLITE_IOERR_NOMEM ) pCheck->rc = SQLITE_NOMEM; goto end_of_check; } + if( pCheck->nAbove > BTCURSOR_MAX_DEPTH ){ + checkAppendMsg(pCheck,"btree depth exceeds %d",BTCURSOR_MAX_DEPTH); + goto end_of_check; + } /* Clear MemPage.isInit to make sure the corruption detection code in ** btreeInitPage() is executed. */ @@ -86157,6 +86583,7 @@ static int checkTreePage( pCheck->zPfx = saved_zPfx; pCheck->v1 = saved_v1; pCheck->v2 = saved_v2; + pCheck->nAbove--; return depth+1; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -86234,9 +86661,9 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( checkOom(&sCheck); goto integrity_ck_cleanup; } - sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); + sCheck.heap = (u32*)sqlite3Malloc( pBt->pageSize*2 ); #ifdef SQLITE_DEBUG - sCheck.mxHeap = pBt->pageSize/4 - 1; + sCheck.mxHeap = pBt->pageSize/2 - 1; #endif if( sCheck.heap==0 ){ checkOom(&sCheck); @@ -86289,7 +86716,9 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( } #endif sCheck.v0 = aRoot[i]; + assert( sCheck.nAbove==0 ); checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + assert( sCheck.nAbove==0 ); } sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } @@ -86324,7 +86753,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( /* Clean up and report errors. */ integrity_ck_cleanup: - sqlite3PageFree(sCheck.heap); + sqlite3_free(sCheck.heap); sqlite3_free(sCheck.aPgRef); *pnErr = sCheck.nErr; if( sCheck.nErr==0 ){ @@ -88378,8 +88807,28 @@ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ /* ** The MEM structure is already a MEM_Real or MEM_IntReal. Try to ** make it a MEM_Int if we can. -*/ -SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ +** +** This routine only converts REAL values into INTEGER if the REAL +** has no fractional part and if the value can be exactly expressed +** as a signed 64-bit integer. +** +** The bFromText parameter is true if the MEM was originally text. +** If the value was originally text then this routine will not convert +** -9223372036854775808.0 into -9223372036854775808 because if the value +** was originally text, it would have been converted into an integer +** already, and since it was not, that means that the original text +** content had the last four significant digits greater than "5808" +** and the hence the text value was not really -9223372036854775808.0 +** but rather some nearby value that happens to have the same IEEE 754 +** representation. We don't want text '-9223372036854775900.0' to get +** converted into -9223372036854775808. +** +** But if bFromText is false, then the value started out as REAL and +** so it is ok to convert -9223372036854775808.0 into -9223372036854775808 +** in that case, because -9223372036854775808.0 is the exact value of +** the underlying IEEE 754 representation. +*/ +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem, int bFromText){ assert( pMem!=0 ); assert( pMem->flags & (MEM_Real|MEM_IntReal) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); @@ -88401,7 +88850,9 @@ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ ** the second condition under the assumption that addition overflow causes ** values to wrap around. */ - if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.r==ix && ixSMALLEST_INT64) + ){ pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); } @@ -88844,11 +89295,16 @@ SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ ** either case, SQLITE_TOOBIG is returned. ** ** The "enc" parameter is the text encoding for the string, or zero -** to store a blob. +** to store a blob. If enc not 0 or SQLITE_UTF8 and the string +** begins with a BOM, the BOM is removed. To retain the BOM on a UTF16 +** string, use a non-negative "n" and set enc to SQLITE_UTF8, and +** separately set pMem->enc to the correct encoding after this function +** returns. ** -** If n is negative, then the string consists of all bytes up to but -** excluding the first zero character. The n parameter must be -** non-negative for blobs. +** If n is negative and enc is not zero and is set correctly, then the +** string consists of all bytes up to but excluding the first zero +** character. The n parameter must be non-negative for blobs or if +** enc is not set accurately. */ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( Mem *pMem, /* Memory cell to set to string value */ @@ -88860,11 +89316,13 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( i64 nByte = n; /* New value for pMem->n */ int iLimit; /* Maximum allowed string or blob size */ u16 flags; /* New value for pMem->flags */ + u8 xenc; /* Encoding without the ENC_KEEPBOM bit */ assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); - assert( enc!=0 || n>=0 ); + xenc = enc & ENC_MASK; + assert( xenc!=0 || n>=0 ); /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ if( !z ){ @@ -88878,16 +89336,16 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( iLimit = SQLITE_MAX_LENGTH; } if( nByte<0 ){ - assert( enc!=0 ); - if( enc==SQLITE_UTF8 ){ + assert( xenc!=0 ); + if( xenc==SQLITE_UTF8 ){ nByte = strlen(z); }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } flags= MEM_Str|MEM_Term; - }else if( enc==0 ){ + }else if( xenc==0 ){ flags = MEM_Blob; - enc = SQLITE_UTF8; + xenc = SQLITE_UTF8; }else{ flags = MEM_Str; } @@ -88910,7 +89368,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( if( xDel==SQLITE_TRANSIENT ){ i64 nAlloc = nByte; if( flags&MEM_Term ){ - nAlloc += (enc==SQLITE_UTF8?1:2); + nAlloc += (xenc==SQLITE_UTF8?1:2); } testcase( nAlloc==0 ); testcase( nAlloc==31 ); @@ -88934,10 +89392,13 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( pMem->n = (int)(nByte & 0x7fffffff); pMem->flags = flags; - pMem->enc = enc; + pMem->enc = xenc; #ifndef SQLITE_OMIT_UTF16 - if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + if( xenc>SQLITE_UTF8 + && (enc & ENC_KEEPBOM)==0 + && sqlite3VdbeMemHandleBom(pMem) + ){ return SQLITE_NOMEM_BKPT; } #endif @@ -89055,10 +89516,10 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( if( (u64)amt + (u64)offset > (u64)sqlite3BtreeMaxRecordSize(pCur) ){ return SQLITE_CORRUPT_BKPT; } - if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){ + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+8)) ){ rc = sqlite3BtreePayload(pCur, offset, amt, pMem->z); if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; /* Overrun area used when reading malformed records */ + memset(pMem->z+amt,0,8); /* Overrun area for malformed records */ pMem->flags = MEM_Blob; pMem->n = (int)amt; }else{ @@ -89447,14 +89908,15 @@ static int valueFromExpr( zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r); + pVal->flags = MEM_Real; + } } } if( affinity==SQLITE_AFF_BLOB ){ - if( op==TK_FLOAT ){ - assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); - sqlite3AtoF(pVal->z, &pVal->u.r); - pVal->flags = MEM_Real; - }else if( op==TK_INTEGER ){ + if( op==TK_INTEGER ){ /* This case is required by -9223372036854775808 and other strings ** that look like integers but cannot be handled by the ** sqlite3DecOrHexToI64() call above. */ @@ -89510,6 +89972,7 @@ static int valueFromExpr( assert( zVal[nVal]=='\'' ); sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2, 0, SQLITE_DYNAMIC); + pVal->enc = enc; } #endif #ifdef SQLITE_ENABLE_STAT4 @@ -89562,7 +90025,30 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( u8 affinity, /* Affinity to use */ sqlite3_value **ppVal /* Write the new value here */ ){ - return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; + int rc = SQLITE_OK; + sqlite3_value *pVal = 0; + if( pExpr ){ + rc = valueFromExpr(db, pExpr, enc, affinity, &pVal, 0); + if( rc==SQLITE_OK && pVal + && affinity==SQLITE_AFF_REAL + && (pVal->flags & MEM_Int) + && (pVal->u.i<-140737488355328LL || pVal->u.i>140737488355327LL) + ){ + /* If the integer value is too large to fit in a 6-byte integer and + ** the affinity is REAL, convert it to a real value now. In most + ** cases an OP_RealAffinity opcode will be used to convert the + ** value to an actual real, but this opcode is omitted if the values + ** are being read directly from a table in order to create an index + ** key. It is important to get a "real" real value for the larger + ** magnitude integer values so that comparisons work correctly - + ** SQLite by default will convert real values to integers before + ** doing the comparison, which is different from converting to real + ** first. */ + sqlite3VdbeMemRealify(pVal); + } + } + *ppVal = pVal; + return rc; } #ifdef SQLITE_ENABLE_STAT4 @@ -89607,7 +90093,7 @@ static int stat4ValueFromExpr( }else if( pExpr->op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ Vdbe *v; int iBindVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + sqlite3VdbeReprepareOnBind(pParse->pVdbe, iBindVar, 0); if( (v = pParse->pReprepare)!=0 ){ pVal = valueNew(db, pAlloc); if( pVal ){ @@ -89881,6 +90367,7 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); assert( p->nOpAlloc==0 ); + assert( pParse->szOpAlloc==0 ); sqlite3VdbeAddOp2(p, OP_Init, 0, 1); return p; } @@ -89911,6 +90398,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlag p->prepFlags = prepFlags; if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ p->expmask = 0; + p->smimask = 0; } assert( p->zSql==0 ); p->zSql = sqlite3DbStrNDup(p->db, z, n); @@ -89984,7 +90472,8 @@ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ pA->zNormSql = pB->zNormSql; pB->zNormSql = zTmp; #endif - pB->expmask = pA->expmask; + pB->expmask |= pA->expmask; + pB->smimask |= pA->smimask; pB->prepFlags = pA->prepFlags; memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter)); pB->aCounter[SQLITE_STMTSTATUS_REPREPARE]++; @@ -90030,7 +90519,8 @@ static int growOpArray(Vdbe *v, int nOp){ assert( nNew>=(v->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ - v->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); + p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); + v->nOpAlloc = p->szOpAlloc/sizeof(Op); v->aOp = pNew; } return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); @@ -90202,6 +90692,20 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( return i; } +/* Generate an opcode that loads a 64-bit integer into register iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeAddInt64(Vdbe *p, int iDest, i64 iVal){ + return sqlite3VdbeAddOp3(p, OP_Int64, LOWER32(iVal), iDest, UPPER32(iVal)); +} + +/* Generate an opcode that loads a 64-floating point value into register iDest. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddDouble(Vdbe *p, int iDest, double rVal){ + i64 iVal; + memcpy(&iVal,&rVal,8); + return sqlite3VdbeAddOp3(p, OP_Real, LOWER32(iVal), iDest, UPPER32(iVal)); +} + /* Generate code for an unconditional jump to instruction iDest */ SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ @@ -90305,24 +90809,6 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( return addr; } -/* -** Add an opcode that includes the p4 value with a P4_INT64 or -** P4_REAL type. -*/ -SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - const u8 *zP4, /* The P4 operand */ - int p4type /* P4 operand type */ -){ - char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8); - if( p4copy ) memcpy(p4copy, zP4, 8); - return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); -} - #ifndef SQLITE_OMIT_EXPLAIN /* ** Return the address of the current EXPLAIN QUERY PLAN baseline. @@ -90636,6 +91122,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || opcode==OP_Function || opcode==OP_PureFunc || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) + || (opcode==OP_MustBeInt && pOp->p2==0) ){ hasAbort = 1; break; @@ -91229,8 +91716,6 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ freeP4FuncCtx(db, (sqlite3_context*)p4); break; } - case P4_REAL: - case P4_INT64: case P4_DYNAMIC: case P4_INTARRAY: { if( p4 ) sqlite3DbNNFreeNN(db, p4); @@ -91584,6 +92069,9 @@ static int translateP(char c, const Op *pOp){ ** "PX@PY" -> "r[X..X+Y-1]" or "r[x]" if y is 0 or 1 ** "PX@PY+1" -> "r[X..X+Y]" or "r[x]" if y is 0 ** "PY..PY" -> "r[X..Y]" or "r[x]" if y<=x +** "PINT13" -> int(P1|P3<<32) +** "PDBL13" -> real(P1|P3<<32) +** "PHEX23" -> hex(P2|P3<<32) */ SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( sqlite3 *db, /* Optional - Oom error reporting only */ @@ -91619,6 +92107,20 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( seenCom = 1; break; } + }else if( strncmp(&zSynopsis[ii],"INT13",5)==0 ){ + sqlite3_str_appendf(&x,"%lld",INT32_TO_64(pOp->p1,pOp->p3)); + ii += 4; +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + }else if( strncmp(&zSynopsis[ii],"HEX23",5)==0 ){ + sqlite3_str_appendf(&x,"0x%llx",INT32_TO_64(pOp->p2,pOp->p3)); + ii += 4; +#endif + }else if( strncmp(&zSynopsis[ii],"DBL13",5)==0 ){ + i64 iVal = INT32_TO_64(pOp->p1,pOp->p3); + double r; + memcpy(&r, &iVal, 8); + sqlite3_str_appendf(&x,"%.17g",r); + ii += 4; }else{ int v1 = translateP(c, pOp); int v2; @@ -91796,18 +92298,10 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } - case P4_INT64: { - sqlite3_str_appendf(&x, "%lld", *pOp->p4.pI64); - break; - } case P4_INT32: { sqlite3_str_appendf(&x, "%d", pOp->p4.i); break; } - case P4_REAL: { - sqlite3_str_appendf(&x, "%.16g", *pOp->p4.pReal); - break; - } case P4_MEM: { Mem *pMem = pOp->p4.pMem; if( pMem->flags & MEM_Str ){ @@ -92533,7 +93027,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( n = ROUND8P(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); - x.nFree = ROUNDDOWN8((p->nOpAlloc-p->nOp)*sizeof(Op)); /* Bytes unused mem */ + x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes unused memory */ assert( x.nFree>=0 ); assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); @@ -93071,6 +93565,22 @@ static void checkActiveVdbeCnt(sqlite3 *db){ #define checkActiveVdbeCnt(x) #endif +/* +** Return a bitmask of all schemas referenced by active VDBEs, other +** than pExcept. +*/ +SQLITE_PRIVATE void sqlite3VdbeSchemaInUse(sqlite3 *db, Vdbe *pExcept, yDbMask *pM){ + Vdbe *p; + yDbMask m; + DbMaskZero(m); + for(p=db->pVdbe; p; p=p->pVNext){ + if( p==pExcept ) continue; + if( p->eVdbeState!=VDBE_RUN_STATE ) continue; + DbMaskUnion(m,p->btreeMask); + } + memcpy(pM, &m, sizeof(m)); +} + /* ** If the Vdbe passed as the first argument opened a statement-transaction, ** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or @@ -93380,6 +93890,13 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3ConnectionUnlocked(db); } + /* Do any pending schema resets that were deferred because multiple + ** VDBEs where active at the same time. + */ + if( db->bPendingSchemaReset && db->nVdbeActive==0 ){ + sqlite3ResetOneSchema(db, -1, 0); + } + assert( db->nVdbeActive>0 || db->autoCommit==0 || db->nStatement==0 ); return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); } @@ -94974,7 +95491,7 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ testcase( flags & MEM_Null ); testcase( flags & MEM_Blob ); if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 - && p->pKeyInfo->aColl[0]==0 + && sqlite3IsBinary(p->pKeyInfo->aColl[0]) ){ assert( flags & MEM_Str ); p->u.z = p->aMem[0].z; @@ -95147,6 +95664,23 @@ SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ } } +/* +** Expire (urgently, with iCode 0) every prepared statement that is using +** a particular schema (iDb). +*/ +SQLITE_PRIVATE void sqlite3ExpireStmtsUsingSchema( + sqlite3 *db, /* The database connection statements belong to */ + int iDb, /* Index of the schema they are using */ + Vdbe *pExcept /* Except, don't expire this one */ +){ + Vdbe *p; + for(p = db->pVdbe; p; p=p->pVNext){ + if( p==pExcept ) continue; + if( !DbMaskTest(p->btreeMask, iDb) ) continue; + p->expired = 1; + } +} + /* ** Return the database associated with the Vdbe. */ @@ -95190,17 +95724,30 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff /* ** Configure SQL variable iVar so that binding a new value to it signals ** to sqlite3_reoptimize() that re-preparing the statement may result -** in a better query plan. +** in a better query plan. If parameter bSmallint is true, then the +** statement is only re-prepared if the new value is integer value 0 or 1. +** +** The v->expmask bit is always set. expmask means that a reprepare is +** possible. The v->smimask bit is only set if we want to restrict +** reprepare when the value changes from (0,1) to something else, or from +** something else to (0,1). */ -SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ +SQLITE_PRIVATE void sqlite3VdbeReprepareOnBind(Vdbe *v, int iVar, int bSmallint){ + u32 m; assert( iVar>0 ); assert( (v->db->flags & SQLITE_EnableQPSG)==0 || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); - if( iVar>=32 ){ - v->expmask |= 0x80000000; + + m = (iVar>=32) ? 0x80000000 : ((u32)1 << (iVar-1)); + if( bSmallint ){ + if( (v->expmask & m)==0 ) v->smimask |= m; }else{ - v->expmask |= ((u32)1 << (iVar-1)); + v->smimask &= ~m; } + v->expmask |= m; + + /* smimask is always a subset of expmask */ + assert( (v->smimask & v->expmask) == v->smimask ); } /* @@ -95770,6 +96317,7 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ int i; int rc = SQLITE_OK; Vdbe *p = (Vdbe*)pStmt; + u32 maskSmallInt = 0; /* Bitmask of bindings with value 0 or 1 */ #if SQLITE_THREADSAFE sqlite3_mutex *mutex; #endif @@ -95781,13 +96329,18 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ #if SQLITE_THREADSAFE mutex = p->db->mutex; #endif + sqlite3_mutex_enter(mutex); for(i=0; inVar; i++){ - sqlite3VdbeMemRelease(&p->aVar[i]); - p->aVar[i].flags = MEM_Null; + Mem *pVar = &p->aVar[i]; + if( (pVar->flags & MEM_Int)!=0 && pVar->u.i>=0 && pVar->u.i<=1 ){ + maskSmallInt |= i>=31 ? 0x80000000 : ((u32)1)<flags = MEM_Null; } assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || p->expmask==0 ); - if( p->expmask ){ + if( (p->expmask&~p->smimask)!=0 || (p->smimask&maskSmallInt)!=0 ){ p->expired = 1; } sqlite3_mutex_leave(mutex); @@ -96299,6 +96852,7 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ } #ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); + pOut->enc = pCtx->enc; return SQLITE_OK; #else return sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); @@ -97350,10 +97904,8 @@ static int vdbeUnbind(Vdbe *p, unsigned int i){ sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } + pVar = &p->aVar[i]; - sqlite3VdbeMemRelease(pVar); - pVar->flags = MEM_Null; - p->db->errCode = SQLITE_OK; /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. @@ -97365,9 +97917,22 @@ static int vdbeUnbind(Vdbe *p, unsigned int i){ ** following any change to the bindings of that parameter. */ assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || p->expmask==0 ); + assert( (p->expmask & p->smimask)==p->smimask ); if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<expired = 1; + /* If the p->smimask bit is also set, then only expire the statement if + ** the value being clobbered was integer value 0 or 1. */ + if( (p->smimask==0) + || (p->smimask & (i>=31 ? 0x80000000 : (u32)1<flags & MEM_Int)!=0 && pVar->u.i>=0 && pVar->u.i<=1) + ){ + p->expired = 1; + } } + + sqlite3VdbeMemRelease(pVar); + pVar->flags = MEM_Null; + p->db->errCode = SQLITE_OK; + return SQLITE_OK; } @@ -97380,12 +97945,13 @@ static int bindText( const void *zData, /* Pointer to the data to be bound */ i64 nData, /* Number of bytes of data to be bound */ void (*xDel)(void*), /* Destructor for the data */ - u8 encoding /* Encoding for the data */ + u8 encoding /* Encoding for the data + optional ENC_KEEPBOM */ ){ Vdbe *p = (Vdbe *)pStmt; Mem *pVar; int rc; + assert( (encoding&ENC_KEEPBOM)==0 || (encoding&ENC_MASK)>SQLITE_UTF8 ); rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ @@ -97464,15 +98030,46 @@ SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ return sqlite3_bind_int64(p, i, (i64)iValue); } SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ - int rc; - Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, (u32)(i-1)); - if( rc==SQLITE_OK ){ - assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ - sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); + Mem *pVar; + Vdbe *p = (Vdbe*)pStmt; + if( vdbeSafetyNotNull(p) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(p->db->mutex); + if( p->eVdbeState!=VDBE_READY_STATE ){ + sqlite3Error(p->db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(p->db->mutex); + sqlite3_log(SQLITE_MISUSE, + "bind on a busy prepared statement: [%s]", p->zSql); + return SQLITE_MISUSE_BKPT; } - return rc; + if( i<=0 || (--i)>=p->nVar ){ + sqlite3Error(p->db, SQLITE_RANGE); + sqlite3_mutex_leave(p->db->mutex); + return SQLITE_RANGE; + } + pVar = &p->aVar[i]; + if( p->expmask!=0 ){ + u32 expireMask = i>=32 ? 0x80000000 : (u32)1 << i; + assert( (p->expmask & p->smimask)==p->smimask ); + if( (p->expmask & expireMask)!=0 ){ + if( (p->smimask & expireMask)!=0 ){ + /* If the smimask bit is set, only expire the prepared statement + ** if the value is changing to or from (0,1) and something else */ + int sm1 = (pVar->flags & MEM_Int)!=0 && pVar->u.i>=0 && pVar->u.i<=1; + int sm2 = iValue>=0 && iValue<=1; + if( sm1!=sm2 ){ + p->expired = 1; + } + }else if( (pVar->flags & MEM_Int)==0 || iValue!=pVar->u.i ){ + /* Always expire if the smimask bit is clear */ + p->expired = 1; + } + } + } + sqlite3VdbeMemSetInt64(pVar, iValue); + sqlite3_mutex_leave(p->db->mutex); + return SQLITE_OK; } SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; @@ -97561,8 +98158,9 @@ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_valu break; } case SQLITE_TEXT: { + assert( pValue->enc!=0 ); rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE_TRANSIENT, - pValue->enc); + pValue->enc<=SQLITE_UTF8 ? pValue->enc : pValue->enc|ENC_KEEPBOM); break; } default: { @@ -97583,6 +98181,7 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ #else rc = sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); #endif + p->aVar[i-1].enc = ENC(p->db); sqlite3_mutex_leave(p->db->mutex); } return rc; @@ -97736,9 +98335,14 @@ SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){ v->explain = eMode; rc = SQLITE_OK; }else{ + int savedMode = v->explain; v->explain = eMode; rc = sqlite3Reprepare(v); - v->haveEqpOps = eMode==2; + if( rc ){ + v->explain = savedMode; + }else{ + v->haveEqpOps = eMode==2; + } } if( v->explain ){ v->nResColumn = 12 - 4*v->explain; @@ -97930,10 +98534,11 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa assert( p->pCsr->eCurType==CURTYPE_BTREE ); nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); - aRec = sqlite3DbMallocRaw(db, nRec); + aRec = sqlite3DbMallocRaw(db, (u64)nRec+8); if( !aRec ) goto preupdate_old_out; rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); if( rc==SQLITE_OK ){ + memset(aRec+nRec, 0, 8); p->pUnpacked = vdbeUnpackRecord(p->pKeyinfo, nRec, aRec); if( !p->pUnpacked ) rc = SQLITE_NOMEM; } @@ -98096,6 +98701,11 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa sqlite3VdbeMemSetInt64(pMem, p->iKey2); }else if( iStore>=pUnpack->nField ){ pMem = (sqlite3_value *)columnNullValue(); + }else if( (p->pTab->aCol) + && (p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL) + && (pMem->flags & (MEM_Int|MEM_IntReal)) + ){ + sqlite3VdbeMemRealify(pMem); } }else{ /* For an UPDATE, memory cell (p->iNewReg+1+iStore) contains the required @@ -98961,7 +99571,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ }else{ pRec->u.r = rValue; pRec->flags |= MEM_Real; - if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); + if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec, 1); } /* TEXT->NUMERIC is many->one. Hence, it is important to invalidate the ** string representation after computing a numeric equivalent, because the @@ -99005,7 +99615,7 @@ static void applyAffinity( if( (pRec->flags & (MEM_Real|MEM_IntReal))==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); }else if( affinity<=SQLITE_AFF_REAL ){ - sqlite3VdbeIntegerAffinity(pRec); + sqlite3VdbeIntegerAffinity(pRec, 0); } } }else if( affinity==SQLITE_AFF_TEXT ){ @@ -99395,12 +100005,13 @@ static SQLITE_NOINLINE int vdbeColumnFromOverflow( assert( t>=12 ); sqlite3RCStrRef(pBuf); if( t&1 ){ - rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding|ENC_KEEPBOM, sqlite3RCStrUnref); pDest->flags |= MEM_Term; }else{ rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, sqlite3RCStrUnref); + pDest->enc = encoding; } }else{ rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); @@ -100029,31 +100640,35 @@ case OP_Integer: { /* out2 */ break; } -/* Opcode: Int64 * P2 * P4 * -** Synopsis: r[P2]=P4 +/* Opcode: Int64 P1 P2 P3 * * +** Synopsis: r[P2]=PINT13 ** -** P4 is a pointer to a 64-bit integer value. -** Write that value into register P2. +** Combine P1 and P3 into a signed 64-bit integer. P1 is the least +** significant 32 bits and P3 is the most significant. Store the +** result in register P2. */ case OP_Int64: { /* out2 */ pOut = out2Prerelease(p, pOp); - assert( pOp->p4.pI64!=0 ); - pOut->u.i = *pOp->p4.pI64; + pOut->u.i = INT32_TO_64(pOp->p1,pOp->p3); break; } #ifndef SQLITE_OMIT_FLOATING_POINT -/* Opcode: Real * P2 * P4 * -** Synopsis: r[P2]=P4 +/* Opcode: Real P1 P2 P3 * * +** Synopsis: r[P2]=PDBL13 ** -** P4 is a pointer to a 64-bit floating point value. -** Write that value into register P2. +** P1 and P3 are combined to 64 bits with P1 being the lower the P3 +** the upper. The result is interpreted as a 64-bit floating-point +** and stored in register P2. */ case OP_Real: { /* same as TK_FLOAT, out2 */ + u64 ii; pOut = out2Prerelease(p, pOp); pOut->flags = MEM_Real; - assert( !sqlite3IsNaN(*pOp->p4.pReal) ); - pOut->u.r = *pOp->p4.pReal; + ii = (u64)INT32_TO_64(pOp->p1,pOp->p3); + swapMixedEndianFloat(ii); + memcpy(&pOut->u.r, &ii, 8); + assert( !sqlite3IsNaN(pOut->u.r) ); break; } #endif @@ -100760,7 +101375,18 @@ case OP_AddImm: { /* in1 */ case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ - applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); + if( (pIn1->flags & MEM_Real)!=0 ){ + sqlite3_int64 ix; + if( pIn1->u.r>=-9223372036854775808.0 + && pIn1->u.r<+9223372036854775808.0 + && (double)(ix = sqlite3RealToI64(pIn1->u.r))==pIn1->u.r + ){ + pIn1->u.i = ix; + MemSetTypeFlag(pIn1, MEM_Int); + } + }else{ + applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); + } if( (pIn1->flags & MEM_Int)==0 ){ VdbeBranchTaken(1, 2); if( pOp->p2==0 ){ @@ -102025,6 +102651,7 @@ case OP_TypeCheck: { int nCol; assert( pOp->p4type==P4_TABLE ); + if( p->expired==1 ) goto abort_due_to_expired; pTab = pOp->p4.pTab; assert( pTab->tabFlags & TF_Strict ); assert( pOp->p3>=0 && pOp->p3nCol+2 ); @@ -102130,22 +102757,9 @@ case OP_Affinity: { assert( zAffinity[0]==SQLITE_AFF_NONE || memIsValid(pIn1) ); applyAffinity(pIn1, zAffinity[0], encoding); if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ - /* When applying REAL affinity, if the result is still an MEM_Int - ** that will fit in 6 bytes, then change the type to MEM_IntReal - ** so that we keep the high-resolution integer value but know that - ** the type really wants to be REAL. */ - testcase( pIn1->u.i==140737488355328LL ); - testcase( pIn1->u.i==140737488355327LL ); - testcase( pIn1->u.i==-140737488355328LL ); - testcase( pIn1->u.i==-140737488355329LL ); - if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL ){ - pIn1->flags |= MEM_IntReal; - pIn1->flags &= ~MEM_Int; - }else{ - pIn1->u.r = (double)pIn1->u.i; - pIn1->flags |= MEM_Real; - pIn1->flags &= ~(MEM_Int|MEM_Str); - } + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~(MEM_Int|MEM_Str); } REGISTER_TRACE((int)(pIn1-aMem), pIn1); zAffinity++; @@ -102365,13 +102979,13 @@ case OP_MakeRecord: { len = (u32)pRec->n; serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); if( pRec->flags & MEM_Zero ){ - serial_type += (u32)pRec->u.nZero*2; if( nData ){ if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; len += pRec->u.nZero; }else{ nZero += pRec->u.nZero; } + serial_type += 2*(u32)pRec->u.nZero; } nData += len; nHdr += sqlite3VarintLen(serial_type); @@ -102957,7 +103571,7 @@ case OP_Transaction: { ** a v-table method. */ if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){ - sqlite3ResetOneSchema(db, pOp->p1); + sqlite3ResetOneSchema(db, pOp->p1, p); } p->expired = 1; rc = SQLITE_SCHEMA; @@ -103399,6 +104013,9 @@ case OP_OpenEphemeral: { /* ncycle */ p->apCsr[pOp->p1] = 0; /* Not required; helps with static analysis */ }else{ assert( sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) ); + if( pOp->opcode==OP_OpenAutoindex ){ + sqlite3BtreeCursorHintFlags(pCx->uc.pCursor, BTREE_SEEK_EQ); + } } } } @@ -103499,21 +104116,21 @@ case OP_Close: { /* ncycle */ } #ifdef SQLITE_ENABLE_COLUMN_USED_MASK -/* Opcode: ColumnsUsed P1 * * P4 * +/* Opcode: ColumnsUsed P1 P2 P3 * * +** Synopsis: Cursor P1 uses columns PHEX23 ** ** This opcode (which only exists if SQLite was compiled with ** SQLITE_ENABLE_COLUMN_USED_MASK) identifies which columns of the -** table or index for cursor P1 are used. P4 is a 64-bit integer -** (P4_INT64) in which the first 63 bits are one for each of the -** first 63 columns of the table or index that are actually used -** by the cursor. The high-order bit is set if any column after -** the 64th is used. +** table or index for cursor P1 are used. P2 and P3 combine to give +** a 64-bit unsigned integer mask. P2 stores the lower 32 bits and +** P3 stores the upper 32 bits. If the high-order bit of P3 is set +** that means that and 64-th or some later column is used. */ case OP_ColumnsUsed: { VdbeCursor *pC; pC = p->apCsr[pOp->p1]; assert( pC->eCurType==CURTYPE_BTREE ); - pC->maskUsed = *(u64*)pOp->p4.pI64; + pC->maskUsed = INT32_TO_64(pOp->p2,pOp->p3); break; } #endif @@ -103533,12 +104150,23 @@ case OP_ColumnsUsed: { ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this ** opcode will either land on a record that exactly matches the key, or ** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, -** this opcode must be followed by an IdxLE opcode with the same arguments. +** this opcode must be followed by an IdxLE opcode with the same arguments, +** except that the P2 argument (the jump destination) can be different. ** The IdxGT opcode will be skipped if this opcode succeeds, but the ** IdxGT opcode will be used on subsequent loop iterations. The ** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this ** is an equality search. ** +** The P2 of the IdxLE might be less than the P2 of this opcode. The P2 +** of the IdxLE will be the location to jump to if there are one or more +** matching rows and is the location to continue with the next row of the +** first containing loop. The P2 of this opcode will be the location to +** jump to if zero rows of cursor P1 match the key, and that will be the +** continuation address of the first outer loop that contributes to the +** key. The P2 of this opcode might be greater than the P2 of the +** subsequent IdxLE if the first outer loop that influences the key is +** outer to the immediately next outer loop. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. @@ -103600,12 +104228,23 @@ case OP_ColumnsUsed: { ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this ** opcode will either land on a record that exactly matches the key, or ** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, -** this opcode must be followed by an IdxLE opcode with the same arguments. +** this opcode must be followed by an IdxGE opcode with the same arguments +** except the P2 of IdxGE might be less than the P2 of this opcode. ** The IdxGE opcode will be skipped if this opcode succeeds, but the ** IdxGE opcode will be used on subsequent loop iterations. The ** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this ** is an equality search. ** +** The P2 of the IdxGE might be less than the P2 of this opcode. The P2 +** of the IdxGE will be the location to jump to if there are one or more +** matching rows and is the location to continue with the next row of the +** first containing loop. The P2 of this opcode will be the location to +** jump to if no rows of cursor P1 match the key, and that will be the +** continuation address of the first outer loop that contributes to the +** key. The P2 of this opcode might be greater than the P2 of the +** subsequent IdxLE if the first outer loop that influences the key is +** outer to the immediately next outer loop. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ case OP_SeekLT: /* jump0, in3, group, ncycle */ @@ -103714,7 +104353,9 @@ case OP_SeekGT: { /* jump0, in3, group, ncycle */ assert( pOp->opcode==OP_SeekGE || pOp[1].opcode==OP_IdxLT ); assert( pOp->opcode==OP_SeekLE || pOp[1].opcode==OP_IdxGT ); assert( pOp[1].p1==pOp[0].p1 ); - assert( pOp[1].p2==pOp[0].p2 ); + assert( pOp[1].p2<=pOp[0].p2 ); + testcase( pOp->opcode==OP_SeekGE && pOp[1].p2opcode==OP_SeekLE && pOp[1].p2p1>=0 && pOp->p1nCursor ); - pCur = p->apCsr[pOp->p1]; - VdbeBranchTaken(pCur==0 || pCur->nullRow, 2); - if( pCur==0 || pCur->nullRow ){ - goto jump_to_p2_and_check_for_interrupt; - } - break; -} - /* Opcode: Found P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -104305,7 +104928,9 @@ case OP_SeekRowid: { /* jump0, in3, ncycle */ }else if( (x.flags & MEM_Real)==0 || x.u.r < -9223372036854775808.0 - || x.u.r > 9223372036854775807.0 + || x.u.r > 9223372036854774784.0 + /* ^^^^^^^^^^^^^^^^^^^^^-- same value as every other double + ** between 9223372036854774263.0 and 923372036854775295.0 */ || (double)(iKey = sqlite3RealToI64(x.u.r))!=x.u.r ){ goto jump_to_p2; @@ -104572,6 +105197,7 @@ case OP_Insert: { if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ assert( pC->iDb>=0 ); + if( p->expired==1 ) goto abort_due_to_expired; zDb = db->aDb[pC->iDb].zDbSName; pTab = pOp->p4.pTab; assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) ); @@ -104714,6 +105340,7 @@ case OP_Delete: { #ifdef SQLITE_DEBUG if( pOp->p4type==P4_TABLE + && !p->expired && HasRowid(pOp->p4.pTab) && pOp->p5==0 && sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) @@ -104732,6 +105359,7 @@ case OP_Delete: { ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set ** VdbeCursor.movetoTarget to the current rowid. */ if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ + if( p->expired==1 ) goto abort_due_to_expired; assert( pC->iDb>=0 ); assert( pOp->p4.pTab!=0 ); zDb = db->aDb[pC->iDb].zDbSName; @@ -104989,14 +105617,17 @@ case OP_Rowid: { /* out2, ncycle */ break; } -/* Opcode: NullRow P1 * * * * +/* Opcode: NullRow P1 P2 * * * ** -** Move the cursor P1 to a null row. Any OP_Column operations +** Move the cursor P1 to a null row if P2 is 0. Any OP_Column operations ** that occur while the cursor is on the null row will always ** write a NULL. ** ** If cursor P1 is not previously opened, open it now to a special ** pseudo-cursor that always returns NULL for every column. +** +** If P2 is not 0, then the nullRow flag on the pseudo-cursor is +** turned off. */ case OP_NullRow: { VdbeCursor *pC; @@ -105013,7 +105644,7 @@ case OP_NullRow: { pC->noReuse = 1; pC->uc.pCursor = sqlite3BtreeFakeValidCursor(); } - pC->nullRow = 1; + pC->nullRow = (pOp->p2==0); pC->cacheStatus = CACHE_STALE; if( pC->eCurType==CURTYPE_BTREE ){ assert( pC->uc.pCursor!=0 ); @@ -105458,6 +106089,7 @@ case OP_IdxDelete: { rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); if( rc ) goto abort_due_to_error; if( res!=0 ){ + if( p->expired==1 ) goto abort_due_to_expired; rc = sqlite3VdbeFindIndexKey(pCrsr, pOp->p4.pIdx, &r, &res, 0); if( rc!=SQLITE_OK ) goto abort_due_to_error; if( res!=0 ){ @@ -105957,6 +106589,7 @@ case OP_ParseSchema: { #ifndef SQLITE_OMIT_ALTERTABLE if( pOp->p4.z==0 ){ assert( pOp->p5!=0 ); + sqlite3ExpireStmtsUsingSchema(db, iDb, p); sqlite3SchemaClear(db->aDb[iDb].pSchema); db->mDbFlags &= ~DBFLAG_SchemaKnownOk; rc = sqlite3InitOne(db, iDb, &p->zErrMsg, pOp->p5); @@ -105997,6 +106630,7 @@ case OP_ParseSchema: { } if( rc ){ sqlite3ResetAllSchemasOfConnection(db); + sqlite3ExpirePreparedStatements(db, 0); if( rc==SQLITE_NOMEM ){ goto no_mem; } @@ -106155,6 +106789,7 @@ case OP_IFindKey: { /* jump, in3 */ assert( pOp[-1].p3==pOp->p3 ); pC = p->apCsr[pOp->p1]; assert( pOp->p4type==P4_INDEX ); + if( p->expired==1 ) goto abort_due_to_expired; assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); @@ -108172,7 +108807,7 @@ default: { /* This is really OP_Noop, OP_Explain */ } rc = SQLITE_ERROR; if( resetSchemaOnFault>0 ){ - sqlite3ResetOneSchema(db, resetSchemaOnFault-1); + sqlite3ResetOneSchema(db, resetSchemaOnFault-1, p); } /* This is the only way out of this procedure. We have to @@ -108233,6 +108868,10 @@ default: { /* This is really OP_Noop, OP_Explain */ assert( AtomicLoad(&db->u1.isInterrupted) ); rc = SQLITE_INTERRUPT; goto abort_due_to_error; + +abort_due_to_expired: + rc = SQLITE_SCHEMA; + goto abort_due_to_error; } @@ -113432,6 +114071,9 @@ static int lookupName( if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ pTab = pUpsert->pUpsertSrc->a[0].pSTab; pExpr->iTable = EXCLUDED_TABLE_NUMBER; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); + } } } #endif /* SQLITE_OMIT_UPSERT */ @@ -114454,7 +115096,7 @@ static int resolveOrderByTermToExprList( int rc; /* Return code from subprocedures */ u8 savedSuppErr; /* Saved value of db->suppressErr */ - assert( sqlite3ExprIsInteger(pE, &i, 0)==0 ); + assert( sqlite3ExprIsInteger(pE, &i, 0, 1)==0 ); pEList = pSelect->pEList; /* Resolve all names in the ORDER BY term expression @@ -114553,7 +115195,7 @@ static int resolveCompoundOrderBy( if( pItem->fg.done ) continue; pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( NEVER(pE==0) ) continue; - if( sqlite3ExprIsInteger(pE, &iCol, 0) ){ + if( sqlite3ExprIsInteger(pE, &iCol, 0, 1) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr, pE); return 1; @@ -114736,7 +115378,7 @@ static int resolveOrderGroupBy( continue; } } - if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){ + if( sqlite3ExprIsInteger(pE2, &iCol, 0, 1) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ @@ -115426,23 +116068,29 @@ SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr){ ** and the pExpr parameter is returned unchanged. */ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( - const Parse *pParse, /* Parsing context */ + Parse *pParse, /* Parsing context */ Expr *pExpr, /* Add the "COLLATE" clause to this expression */ const Token *pCollName, /* Name of collating sequence */ int dequote /* True to dequote pCollName */ ){ - if( pCollName->n>0 ){ + if( pCollName->n>0 && pParse->nErr==0 ){ Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, dequote); + assert( pExpr!=0 ); if( pNew ){ pNew->pLeft = pExpr; pNew->flags |= EP_Collate|EP_Skip; +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = pExpr->nHeight+1; + sqlite3ExprCheckHeight(pParse, pNew->nHeight); + testcase( pParse->nErr>0 ); +#endif pExpr = pNew; } } return pExpr; } SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString( - const Parse *pParse, /* Parsing context */ + Parse *pParse, /* Parsing context */ Expr *pExpr, /* Add the "COLLATE" clause to this expression */ const char *zC /* The collating sequence name */ ){ @@ -115622,11 +116270,10 @@ static char comparisonAffinity(const Expr *pExpr){ pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || pExpr->op==TK_NE || pExpr->op==TK_IS || pExpr->op==TK_ISNOT ); assert( pExpr->pLeft ); + assert( !ExprUseXSelect(pExpr) ); aff = sqlite3ExprAffinity(pExpr->pLeft); if( pExpr->pRight ){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); - }else if( ExprUseXSelect(pExpr) ){ - aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); }else if( aff==0 ){ aff = SQLITE_AFF_BLOB; } @@ -116419,9 +117066,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse|EP_HasFunc))==EP_IsFalse && !IN_RENAME_OBJECT ){ + Expr *p; sqlite3ExprDeferredDelete(pParse, pLeft); sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3ExprInt32(db, 0); + p = sqlite3ExprInt32(db, 0); + if( p ) p->op2 = 1; + return p; }else{ return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } @@ -116622,6 +117272,8 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n if( x>pParse->nVar ){ pParse->nVar = (int)x; doAdd = 1; + }else if( xaVnbmc[x>>6] & MASKBIT64(x&63))==0; }else if( sqlite3VListNumToName(pParse->pVList, x)==0 ){ doAdd = 1; } @@ -116638,6 +117290,9 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n } if( doAdd ){ pParse->pVList = sqlite3VListAdd(db, pParse->pVList, z, n, x); + if( pParse->pVList!=0 && xaVnbmc[x>>6] |= MASKBIT64(x&63); + } } } pExpr->iColumn = x; @@ -116837,8 +117492,8 @@ static int dupedExprNodeSize(const Expr *p, int flags){ ** tree formed by Expr.pLeft and Expr.pRight, but not any other ** substructure such as Expr.x.pList, Expr.x.pSelect, and Expr.y.pWin. */ -static int dupedExprSize(const Expr *p){ - int nByte; +static u64 dupedExprSize(const Expr *p){ + u64 nByte; assert( p!=0 ); nByte = dupedExprNodeSize(p, EXPRDUP_REDUCE); if( p->pLeft ) nByte += dupedExprSize(p->pLeft); @@ -116895,7 +117550,7 @@ static Expr *exprDup( assert( sEdupBuf.zAlloc!=0 ); assert( dupFlags==EXPRDUP_REDUCE ); }else{ - int nAlloc; + u64 nAlloc; if( dupFlags ){ nAlloc = dupedExprSize(p); }else if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ @@ -117769,7 +118424,7 @@ static SQLITE_NOINLINE int exprNodeIsConstantFunction( if( pDef==0 || pDef->xFinalize!=0 || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 - || ExprHasProperty(pExpr, EP_WinFunc) + || NEVER(ExprHasProperty(pExpr, EP_WinFunc)) ){ pWalker->eCode = 0; return WRC_Abort; @@ -117790,6 +118445,7 @@ static SQLITE_NOINLINE int exprNodeIsConstantFunction( ** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 ** sqlite3ExprIsTableConstant() pWalker->eCode==3 ** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** sqlite3ExprListIsConstant() pWalker->eCode==1 or 6 ** ** In all cases, the callbacks set Walker.eCode=0 and abort if the expression ** is found to not be a constant. @@ -117816,19 +118472,25 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } switch( pExpr->op ){ - /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->eCode==4 or 5 or the function has the - ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( (pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc)) - && !ExprHasProperty(pExpr, EP_WinFunc) - ){ + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + pWalker->eCode = 0; /* Window functions are never constant */ + return WRC_Abort; + }else if( pWalker->eCode>=4 && pWalker->eCode<=5 ){ + /* Functions used within a CREATE TABLE statement are constant as + ** long as all of their arguments are constant */ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( ExprHasProperty(pExpr,EP_ConstFunc) ){ + /* Functions marked with EP_ConstFunc are constant as long as + ** all their arugments are constant and they are not window functions */ + return WRC_Continue; }else if( pWalker->pParse ){ + /* Functions are constant if they have SQLITE_FUNC_CONSTANT or + ** SQLITE_FUNC_SLOCHNG and if all arguments are constant */ return exprNodeIsConstantFunction(pWalker, pExpr); }else{ - pWalker->eCode = 0; + pWalker->eCode = 0; /* Function call is not constant */ return WRC_Abort; } case TK_ID: @@ -117862,6 +118524,13 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; + case TK_IS: + case TK_ISNOT: + if( pWalker->eCode==6 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Continue; case TK_VARIABLE: if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE @@ -117912,6 +118581,22 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ return exprIsConst(pParse, p, 1); } +/* +** Return true if all expressions in pList are constant. If parameter bNoIs +** is true, do not consider expressions that contain "IS" operators to be +** constant. IS operators are not always considered constant as expressions +** like "x IS TRUE" need to be transformed to TK_TRUTH expression nodes, +** which happens at the same time as column name resolution. +*/ +SQLITE_PRIVATE int sqlite3ExprListIsConstant(Parse *pParse, ExprList *pList, int bNoIs){ + const int iInit = bNoIs ? 6 : 1; + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==exprIsConst(pParse, pList->a[ii].pExpr, iInit) ) return 0; + } + return 1; +} + /* ** Walk an expression tree. Return non-zero if ** @@ -117954,7 +118639,7 @@ static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ ** Consider uncorrelated subqueries to be constants if the bAllowSubq ** parameter is true. */ -static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ Walker w; w.eCode = 3; w.pParse = 0; @@ -118134,7 +118819,6 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ return exprIsConst(0, p, 4+isInit); } -#ifdef SQLITE_ENABLE_CURSOR_HINTS /* ** Walk an expression tree. Return 1 if the expression contains a ** subquery of some kind. Return 0 if there are no subqueries. @@ -118150,68 +118834,96 @@ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ sqlite3WalkExpr(&w, p); return w.eCode==0; } -#endif /* -** If the expression p codes a constant integer that is small enough -** to fit in a 32-bit integer, return 1 and put the value of the integer -** in *pValue. If the expression is not an integer or if it is too big -** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** If the expression p codes a constant integer between 0 and 0x7fffffff, +** then return 1 and put the value of the integer in *pValue. If the +** expression is not an integer or if it is an integer that is out side +** the range of 0...0x7fffffff, then return 0 and leave *pValue unchanged. ** ** If the pParse pointer is provided, then allow the expression p to be -** a parameter (TK_VARIABLE) that is bound to an integer. -** But if pParse is NULL, then p must be a pure integer literal. -*/ -SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){ - int rc = 0; - if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ - - /* If an expression is an integer literal that fits in a signed 32-bit - ** integer, then the EP_IntValue flag will have already been set */ - assert( p->op!=TK_INTEGER || (p->flags & EP_IntValue)!=0 - || sqlite3GetInt32(p->u.zToken, &rc)==0 ); - - if( p->flags & EP_IntValue ){ - *pValue = p->u.iValue; - return 1; - } - switch( p->op ){ - case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0); - break; - } - case TK_UMINUS: { - int v = 0; - if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){ - assert( ((unsigned int)v)!=0x80000000 ); - *pValue = -v; - rc = 1; +** a parameter (TK_VARIABLE) that is bound to an integer between 0 and +** 0x7fffffff. Variables that hold anything other than integers, or that +** hold integers outside the range of 0..0x7fffffff are not seen. +** But if pParse is NULL, then p must be a pure integer literal between +** 0 and 0x7fffffff. +** +** If pParse is not NULL and expression p is a variable, then the variable +** is marked so as to cause the statement to be reprepared each time a new +** value is bound to it. Except, if parameter bRSI is true, then the statement +** will only be reprepared if the rebind changes the value to or from a +** "small integer" (either 0 or 1). +** +** If pParse is NULL but bRSI is non-zero, then to not accept the input +** as an integer if the TK_INTEGER opcode has its op2 field set to non-zero. +** A non-zero Expr.op2 indicates that the expression is a compile-time +** reduction of a constant expression. For example, "123 NOTNULL" would +** be rendered by the parser as a TK_INTEGER with value 1, but with op2 set. +** +** Summary: +** +** pParse bRsi What happens +** -------- ---- ------------------------------------------------ +** NULL 0 Reject if TK_VARIABLE. Accept if op2!=0 +** +** NULL 1 Reject if TK_VARIABLE or op2!=0 +** +** not-NULL 0 Accept TK_VARIABLE and/or op2!=0 but for TK_VARIABLE, +** cause the statement to reprepare on *any* +** binding change +** +** not-NULL 1 As previous, except reprepare only happens if a binding +** change move the variable value to or from an integer 0 +** or 1. +*/ +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse, int bRSI){ + int iSign = 1; /* Either +1 or -1. */ + while( 1/*exit-by-break*/ ){ + if( NEVER(p==0) ) return 0; + if( ExprUseUValue(p) ){ + assert( p->op==TK_INTEGER ); + *pValue = p->u.iValue*iSign; + if( bRSI && pParse==0 && p->op2 ){ + return 0; } - break; + return 1; } - case TK_VARIABLE: { - sqlite3_value *pVal; - if( pParse==0 ) break; + if( p->op==TK_UPLUS ){ + p = p->pLeft; + pParse = 0; + continue; + } + if( p->op==TK_UMINUS ){ + iSign = -iSign; + p = p->pLeft; + pParse = 0; + continue; + } + if( p->op==TK_VARIABLE && pParse!=0 ){ + sqlite3_value *pVal; /* The variable */ + int rc = 0; /* 1 if successful, 0 if failed */ + assert( iSign==1 ); if( NEVER(pParse->pVdbe==0) ) break; if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break; - sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn); pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn, SQLITE_AFF_BLOB); if( pVal ){ - if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){ - sqlite3_int64 vv = sqlite3_value_int64(pVal); - if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */ - *pValue = (int)vv; - rc = 1; - } + i64 vv; + if( sqlite3_value_type(pVal)==SQLITE_INTEGER + && (vv = sqlite3_value_int64(pVal))>=0 + && vv<=0x7fffffff + ){ + *pValue = (int)vv; + rc = 1; } sqlite3ValueFree(pVal); } - break; + sqlite3VdbeReprepareOnBind(pParse->pVdbe, p->iColumn, bRSI); + return rc; } - default: break; + break; } - return rc; + return 0; } /* @@ -118378,13 +119090,24 @@ static Select *isCandidateForInOpt(const Expr *pX){ ** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull ** to be set to NULL if iCur contains one or more NULL values. */ -static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ +static void sqlite3SetHasNullFlag( + Vdbe *v, /* Write new code into this statement under construction */ + int iCur, /* Cursor for the index */ + int regHasNull, /* Register in which to store hasNull flag */ + int eSortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ +){ int addr1; + int op; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + if( eSortOrder==SQLITE_SO_ASC ){ + op = OP_Rewind; + }else{ + op = OP_Last; + } + addr1 = sqlite3VdbeAddOp1(v, op, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - VdbeComment((v, "first_entry_in(%d)", iCur)); + VdbeComment((v, op==OP_Last?"last_entry_in(%d)":"first_entry_in(%d)", iCur)); sqlite3VdbeJumpHere(v, addr1); } #endif @@ -118645,13 +119368,14 @@ SQLITE_PRIVATE int sqlite3FindInIndex( if( prRhsHasNull ){ #ifdef SQLITE_ENABLE_COLUMN_USED_MASK - i64 mask = (1<nMem; if( nExpr==1 ){ - sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull, + pIdx->aSortOrder[0]); } } sqlite3VdbeJumpHere(v, iAddr); @@ -118700,7 +119424,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( } sqlite3CodeRhsOfIN(pParse, pX, iTab, bloomOk); if( rMayHaveNull ){ - sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); + sqlite3SetHasNullFlag(v, iTab, rMayHaveNull, SQLITE_SO_ASC); } pParse->nQueryLoop = savedNQueryLoop; } @@ -119235,6 +119959,12 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ } sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); + if( dest.eDest==SRT_Mem + && sqlite3ExprCanReturnSubtype(pParse, pSel->pEList->a[0].pExpr) + ){ + sqlite3VdbeAddOp1(v, OP_ClrSubtype, rReg); + } + /* Subroutine return */ assert( ExprUseYSub(pExpr) ); assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn @@ -119454,7 +120184,6 @@ static void sqlite3ExprCodeIN( testcase( aiMap[i]!=i ); sqlite3VdbeAddOp3(v, OP_Copy, rLhsOrig+i, rLhs+aiMap[i], 0); } - sqlite3ReleaseTempReg(pParse, rLhsOrig); } } @@ -119533,7 +120262,13 @@ static void sqlite3ExprCodeIN( ** of the RHS. */ if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); - addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, destIfFalse); + if( eType==IN_INDEX_INDEX_DESC ){ + addrTop = sqlite3VdbeAddOp2(v, OP_Last, iTab, destIfFalse); + }else{ + testcase( eType==IN_INDEX_EPH ); + testcase( eType==IN_INDEX_ROWID ); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, destIfFalse); + } VdbeCoverage(v); if( nVector>1 ){ destNotNull = sqlite3VdbeMakeLabel(pParse); @@ -119601,7 +120336,7 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ sqlite3AtoF(z, &value); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; - sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); + sqlite3VdbeAddDouble(v, iMem, value); } } #endif @@ -119642,7 +120377,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ #endif }else{ if( negFlag ){ value = c==3 ? SMALLEST_INT64 : -value; } - sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); + sqlite3VdbeAddInt64(v, iMem, value); } } } @@ -119723,12 +120458,12 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); VdbeComment((v, "%s.rowid", pTab->zName)); - }else{ - int op; + }else{ int x; if( IsVirtual(pTab) ){ - op = OP_VColumn; x = iCol; + sqlite3VdbeAddOp3(v, OP_VColumn, iTabCur, x, regOut); + return; #ifndef SQLITE_OMIT_GENERATED_COLUMNS }else if( (pCol = &pTab->aCol[iCol])->colFlags & COLFLAG_VIRTUAL ){ Parse *pParse = sqlite3VdbeParser(v); @@ -119748,13 +120483,11 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( }else if( !HasRowid(pTab) ){ testcase( iCol!=sqlite3TableColumnToStorage(pTab, iCol) ); x = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), iCol); - op = OP_Column; }else{ x = sqlite3TableColumnToStorage(pTab,iCol); testcase( x!=iCol ); - op = OP_Column; } - sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); + sqlite3VdbeAddOp3(v, OP_Column, iTabCur, x, regOut); sqlite3ColumnDefault(v, pTab, iCol, regOut); } } @@ -119986,7 +120719,7 @@ static int exprCodeInlineFunction( ** ** The only expressions that can return a subtype are: ** -** 1. A function +** 1. A function, including an aggregate function ** 2. The no-op "+" operator ** 3. A CASE...END expression ** 4. A CAST() expression @@ -120011,14 +120744,19 @@ static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ ){ return WRC_Continue; } - if( pExpr->op!=TK_FUNCTION ){ + if( pExpr->op!=TK_FUNCTION && pExpr->op!=TK_AGG_FUNCTION ){ return WRC_Prune; } assert( ExprUseXList(pExpr) ); db = pWalker->pParse->db; n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); - if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return WRC_Continue; +#else + assert( pDef!=0 ); +#endif + if( (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ pWalker->eCode = 1; return WRC_Abort; } @@ -120033,7 +120771,7 @@ static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ ** are acceptable as they only disable an optimization. False negatives, ** on the other hand, can lead to incorrect answers. */ -static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ +SQLITE_PRIVATE int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ Walker w; memset(&w, 0, sizeof(w)); w.pParse = pParse; @@ -120088,10 +120826,16 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( v = pParse->pVdbe; assert( v!=0 ); - if( p->bMaybeNullRow ){ - /* If the index is on a NULL row due to an outer join, then we - ** cannot extract the value from the index. The value must be - ** computed using the original expression. */ + if( p->bMaybeNullRow || sqlite3ExprIsConstant(pParse, pExpr) ){ + /* The value of pExpr should be computed from the original expression, + ** not from the index, if either: + ** + ** 1. The index might be on a NULL row due to an outer join. + ** + ** 2. The expression is a constant. This prevents an attempt to + ** use an index value outside of the loop that positions the + ** index. (Bug 2026-10-11T19:26:40Z). + */ int addr = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); VdbeCoverage(v); @@ -120137,6 +120881,11 @@ static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ ret = sqlite3ExprCodeTarget(pParse, p->pExpr, iTarget); sqlite3VdbeAddOp4(pParse->pVdbe, OP_Affinity, ret, 1, 0, (const char*)&p->aff, 1); + if( sqlite3ExprCanReturnSubtype(pParse, p->pExpr) ){ + /* If the expression value may have a sub-type, clear it. Values + ** read from columns do not have subtypes. */ + sqlite3VdbeAddOp1(pParse->pVdbe, OP_ClrSubtype, ret); + } if( addr ){ sqlite3VdbeJumpHere(v, addr); sqlite3VdbeChangeP3(v, addr, ret); @@ -121811,7 +122560,7 @@ static SQLITE_NOINLINE int exprCompareVariable( sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR); if( pR ){ iVar = pVar->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + sqlite3VdbeReprepareOnBind(pParse->pVdbe, iVar, 0); pL = sqlite3VdbeGetBoundValue(pParse->pReprepare, iVar, SQLITE_AFF_BLOB); if( pL ){ if( sqlite3_value_type(pL)==SQLITE_TEXT ){ @@ -122067,7 +122816,7 @@ static int sqlite3ExprIsNotTrue(Expr *pExpr){ if( pExpr->op==TK_NULL ) return 1; if( pExpr->op==TK_TRUEFALSE && sqlite3ExprTruthValue(pExpr)==0 ) return 1; v = 1; - if( sqlite3ExprIsInteger(pExpr, &v, 0) && v==0 ) return 1; + if( sqlite3ExprIsInteger(pExpr, &v, 0, 0) && v==0 ) return 1; return 0; } @@ -122722,6 +123471,11 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } if( i>=pSrcList->nSrc ) break; if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ + if( ExprHasProperty(pExpr, EP_SubtArg) + && sqlite3ExprCanReturnSubtype(pParse, pExpr) + ){ + break; + } if( pParse->nErr ){ return WRC_Abort; } /* If we reach this point, it means that expression pExpr can be @@ -122813,7 +123567,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( pOBList->nExpr==1 && nArg==1 && sqlite3ExprCompare(0,pOBList->a[0].pExpr, - pExpr->x.pList->a[0].pExpr,0)==0 + pExpr->x.pList->a[0].pExpr,-1)==0 ){ pItem->bOBPayload = 0; pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); @@ -122901,6 +123655,13 @@ SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ */ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ if( iReg ){ +#ifdef SQLITE_DEBUG + /* Verify that iReg is not already on the unused register list */ + int kk; + for(kk=0; kknTempReg; kk++){ + assert( pParse->aTempReg[kk]!=iReg ); + } +#endif sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0, 0); if( pParse->nTempRegaTempReg) ){ pParse->aTempReg[pParse->nTempReg++] = iReg; @@ -123318,6 +124079,30 @@ static void sqlite3ErrorIfNotEmpty( ); } +/* +** zCol is a column name used in an ALTER TABLE DROP, ADD or RENAME COLUMN +** operation. zOp identifies the specific operation - "drop", "add", "rename +** to" or "rename from". pTab is the table being altered. +** +** If pTab has a rowid and zCol is a rowid alias, then SQLITE_ERROR is +** returned and an error message left in pParse. Or, if zCol is not an alias +** for "rowid" or pTab is not an intkey table, then SQLITE_OK is returned. +*/ +static int isRowidAlias( + Parse *pParse, + Table *pTab, + const char *zCol, + const char *zOp +){ + if( HasRowid(pTab) && sqlite3IsRowid(zCol) ){ + sqlite3ErrorMsg(pParse, "cannot %s rowid alias: %s", zOp, zCol); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + + + /* ** This function is called after an "ALTER TABLE ... ADD" statement ** has been parsed. Argument pColDef contains the text of the new @@ -123363,9 +124148,9 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ #endif - /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. - ** If there is a NOT NULL constraint, then the default value for the - ** column must not be NULL. + /* Check that the new column is not specified as PRIMARY KEY or UNIQUE, + ** or a rowid alias. If there is a NOT NULL constraint, then the default + ** value for the column must not be NULL. */ if( pCol->colFlags & COLFLAG_PRIMKEY ){ sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); @@ -123376,6 +124161,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ "Cannot add a UNIQUE column"); return; } + if( isRowidAlias(pParse, pTab, pCol->zCnName, "add") ) return; if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ /* If the default value for the new column was specified with a ** literal NULL, then set pDflt to 0. This simplifies checking @@ -123464,6 +124250,7 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ || (pCol->notNull && (pCol->colFlags & COLFLAG_GENERATED)!=0) || (pTab->tabFlags & TF_Strict)!=0 ){ + pParse->colNamesSet = 1; sqlite3NestedParse(pParse, "SELECT CASE WHEN quick_check GLOB 'CHECK*'" " THEN raise(ABORT,'CHECK constraint failed')" @@ -123668,6 +124455,8 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn( sqlite3MayAbort(pParse); zNew = sqlite3NameFromToken(db, pNew); if( !zNew ) goto exit_rename_column; + if( isRowidAlias(pParse, pTab, zOld, "rename from") ) goto exit_rename_column; + if( isRowidAlias(pParse, pTab, zNew, "rename to") ) goto exit_rename_column; assert( pNew->n>0 ); bQuote = sqlite3Isquote(pNew->z[0]); sqlite3NestedParse(pParse, @@ -123679,13 +124468,14 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn( zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, pTab->zName ); - - sqlite3NestedParse(pParse, - "UPDATE temp." LEGACY_SCHEMA_TABLE " SET " - "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " - "WHERE type IN ('trigger', 'view')", - zDb, pTab->zName, iCol, zNew, bQuote - ); + if( iSchema!=1 ){ + sqlite3NestedParse(pParse, + "UPDATE temp." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " + "WHERE type IN ('trigger', 'view')", + zDb, pTab->zName, iCol, zNew, bQuote + ); + } /* Drop and reload the database schema. */ renameReloadSchema(pParse, iSchema, INITFLAG_AlterRename); @@ -125294,6 +126084,7 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pName); goto exit_drop_column; } + if( isRowidAlias(pParse, pTab, zCol, "drop") ) goto exit_drop_column; /* Do not allow the user to drop a PRIMARY KEY column or a column ** constrained by a UNIQUE constraint. */ @@ -125374,12 +126165,12 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ - char aff = pTab->aCol[i].affinity; - if( aff==SQLITE_AFF_REAL ){ - pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; - } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); - pTab->aCol[i].affinity = aff; + assert( pParse->nErr + || sqlite3VdbeGetOp(v,sqlite3VdbeCurrentAddr(v)-1)->opcode + != OP_RealAffinity + || pTab->aCol[i].affinity==SQLITE_AFF_REAL ); + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); } nField++; } @@ -125430,15 +126221,19 @@ static int getWhitespace(const u8 *z){ ** keyword. ** ** Argument eTok may be the token value of the first token of the constraint -** (e.g. TK_CHECK or TK_REFERENCES) or zero. If it is either TK_REFERENCES -** or TK_FOREIGN, special parsing is enabled to find the end of the foreign-key -** constraint definition. +** (e.g. TK_CHECK or TK_REFERENCES or TK_NOT) or zero. If it is either +** TK_REFERENCES or TK_FOREIGN, special parsing is enabled to find the +** end of the foreign-key constraint definition. If eTok is TK_NOT then +** special processing is enabled to find the end of a NOT NULL constraint +** including its optional ON CONFLICT clause. ** ** Return the number of bytes until the end of the constraint. */ static int getConstraint(const u8 *z, int eTok){ int iOff = 0; int t = 0; + int t2 = 0; + int prev = 0; #ifndef SQLITE_OMIT_FOREIGN_KEY if( eTok==TK_FOREIGN ){ @@ -125454,6 +126249,14 @@ static int getConstraint(const u8 *z, int eTok){ /* REFERENCES is followed by a table name. Gobble this up here in ** case the table name is a fallback token like TK_GENERATED. */ iOff += getConstraintToken(&z[iOff], &t); + + /* If the table names is followed by a parenthesized list of columns, + ** also skip that, lest the close ")" end the constraint in the + ** while loop below. */ + iOff += getWhitespace(&z[iOff]); + if( z[iOff]=='(' ){ + iOff += getConstraintToken(&z[iOff], &t); + } } #endif @@ -125464,17 +126267,34 @@ static int getConstraint(const u8 *z, int eTok){ ** COLLATE, REFERENCES, FOREIGN, GENERATED, AS, RP, or COMMA ** ** Also exit the loop if ILLEGAL turns up. + ** + ** Exceptions: + ** + ** * For NOT NULL constraints, also terminate at DEFERRABLE + ** + ** * Do not terminate at DEFAULT if the previous token was SET */ while( 1 ){ int n = getConstraintToken(&z[iOff], &t); - if( t==TK_CONSTRAINT || t==TK_PRIMARY || t==TK_NOT || t==TK_UNIQUE - || t==TK_CHECK || t==TK_DEFAULT || t==TK_COLLATE || t==TK_REFERENCES + if( t==TK_CONSTRAINT || t==TK_PRIMARY || t==TK_UNIQUE + || t==TK_CHECK || t==TK_COLLATE || t==TK_REFERENCES + || (t==TK_DEFAULT && prev!=TK_SET) || t==TK_FOREIGN || t==TK_RP || t==TK_COMMA || t==TK_ILLEGAL || t==TK_AS || t==TK_GENERATED + || (eTok==TK_NOT && t==TK_DEFERRABLE) ){ break; } + if( t==TK_NOT + && (eTok!=TK_REFERENCES + || (getConstraintToken(&z[iOff+n],&t2), t2!=TK_DEFERRABLE)) + ){ + /* NOT ends the constraint unless eTok is TK_REFERENCES and the next + ** token is DEFERRABLE */ + break; + } iOff += n; + prev = t; } return iOff; @@ -125641,8 +126461,12 @@ static void dropConstraintFunc( } }else if( t==TK_NOT && iNotNull==ii ){ - iEnd = iOff + getConstraint(&zSql[iOff], 0); - break; + int t2; + (void)getConstraintToken(zSql+iOff, &t2); + if( t2==TK_NULL ){ + iEnd = iOff + getConstraint(zSql+iOff, TK_NOT); + break; + } }else if( t==TK_RP || t==TK_ILLEGAL ){ iEnd = -1; break; @@ -126066,7 +126890,7 @@ SQLITE_PRIVATE void sqlite3AlterAddConstraint( /* Search for a constraint violation. Throw an exception if one is found. */ sqlite3NestedParse(pParse, "SELECT sqlite_fail('constraint failed', %d) " - "FROM %Q.%Q WHERE (%.*s) IS NOT TRUE", + "FROM %Q.%Q WHERE (%.*s) IS FALSE", SQLITE_CONSTRAINT, zDb, pTab->zName, nExpr, zExpr ); @@ -128248,6 +129072,23 @@ SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName){ ); } +/* +** If any TEMP triggers reference schema pSchema, move those triggers to +** reference the TEMP schema itself. +*/ +static void detachTempTriggers(sqlite3 *db, Schema *pSchema){ + HashElem *pEntry; + assert( db->aDb[1].pSchema ); + pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); + while( pEntry ){ + Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); + if( pTrig->pTabSchema==pSchema ){ + pTrig->pTabSchema = pTrig->pSchema; + } + pEntry = sqliteHashNext(pEntry); + } +} + /* ** An SQL user-function registered to do the work of an ATTACH statement. The ** three arguments to the function come directly from an attach statement: @@ -128317,9 +129158,16 @@ static void attachFunc( /* Both the Btree and the new Schema were allocated successfully. ** Close the old db and update the aDb[] slot with the new memdb ** values. */ + detachTempTriggers(db, pNew->pSchema); sqlite3BtreeClose(pNew->pBt); pNew->pBt = pNewBt; pNew->pSchema = pNewSchema; + if( db->init.iDb==0 ){ + /* Clear all eponymous virtual table instances, as they are holding + ** pointers to the schema object just freed by sqlite3BtreeClose() */ + sqlite3VtabEponymousTableClearAll(db); + } + sqlite3ExpirePreparedStatements(db, 1); }else{ sqlite3BtreeClose(pNewBt); rc = SQLITE_NOMEM; @@ -128491,7 +129339,6 @@ static void detachFunc( sqlite3 *db = sqlite3_context_db_handle(context); int i; Db *pDb = 0; - HashElem *pEntry; char zErr[128]; UNUSED_PARAMETER(NotUsed); @@ -128518,18 +129365,7 @@ static void detachFunc( goto detach_error; } - /* If any TEMP triggers reference the schema being detached, move those - ** triggers to reference the TEMP schema itself. */ - assert( db->aDb[1].pSchema ); - pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); - while( pEntry ){ - Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); - if( pTrig->pTabSchema==pDb->pSchema ){ - pTrig->pTabSchema = pTrig->pSchema; - } - pEntry = sqliteHashNext(pEntry); - } - + detachTempTriggers(db, pDb->pSchema); sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; @@ -129237,6 +130073,16 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ } #endif +/* +** Do the union all two yDbMask objects into the first. +*/ +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE void sqlite3DbMaskUnion(yDbMask m, yDbMask x){ + int i; + for(i=0; inSchemaLock is not zero. -** Deferred resets may be run by calling with iDb<0. +** TEMP schema. The reset is deferred if db->nSchemaLock is not zero +** or if an active VDBE other than pSafe is using iDb +** +** Deferred schema resets may be retried by calling with iDb<0. */ -SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb, Vdbe *pSafe){ int i; assert( iDbnDb ); @@ -129741,12 +130589,21 @@ SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ DbSetProperty(db, iDb, DB_ResetWanted); DbSetProperty(db, 1, DB_ResetWanted); db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + db->bPendingSchemaReset = 1; } if( db->nSchemaLock==0 ){ + yDbMask m; + sqlite3VdbeSchemaInUse(db, pSafe, &m); + db->bPendingSchemaReset = 0; for(i=0; inDb; i++){ - if( DbHasProperty(db, i, DB_ResetWanted) ){ + if( !DbHasProperty(db, i, DB_ResetWanted) ){ + /* no-op */ + }else if( DbMaskTest(m,i) ){ + db->bPendingSchemaReset = 1; + }else{ sqlite3SchemaClear(db->aDb[i].pSchema); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; } } } @@ -129769,6 +130626,7 @@ SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ } } } + sqlite3ExpirePreparedStatements(db, 0); db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk); sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); @@ -129810,10 +130668,41 @@ SQLITE_PRIVATE void sqlite3ColumnSetExpr( } } +#ifndef SQLITE_OMIT_AUTHORIZATION +/* +** If the expression contains any TK_FUNCTION, send it to the authorizer +** and if the authorization fails then raise an error. +*/ +static int authFunctions(Walker *pWalker, Expr *p){ + if( p->op==TK_FUNCTION ){ + Parse *pParse = pWalker->pParse; + int rc = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, p->u.zToken, 0); + if( rc ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %#T", p); + return WRC_Abort; + } + } + return WRC_Continue; +} +static void sqlite3FuncAuth(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = authFunctions; + w.pParse = pParse; + sqlite3WalkExpr(&w,pExpr); +} +#endif /* SQLITE_OMIT_AUTHORIZATION */ + /* ** Return the expression associated with a column. The expression might be ** the DEFAULT clause or the AS clause of a generated column. ** Return NULL if the column has no associated expression. +** +** There are two variants of this routine. sqlite3ColumnExpr() just +** returns the expression with no side effects. sqlite3ColumnExprAuth() +** takes the extra step of invoking the authorizer (if one is defined) +** and raising an error if the returned expression uses any unauthorized +** SQL function. */ SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ if( pCol->iDflt==0 ) return 0; @@ -129822,6 +130711,15 @@ SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ if( NEVER(pTab->u.tab.pDfltList->nExpriDflt) ) return 0; return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; } +SQLITE_PRIVATE Expr *sqlite3ColumnExprAuth(Table *pTab, Column *pCol, Parse *pParse){ + Expr *pExpr = sqlite3ColumnExpr(pTab,pCol); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pParse->db->xAuth!=0 ){ + sqlite3FuncAuth(pParse, pExpr); + } +#endif + return pExpr; +} /* ** Suppress false-positive warning message generated with -O3 in GCC @@ -130888,6 +131786,13 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( pDfltExpr = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); sqlite3DbFree(db, x.u.zToken); sqlite3ColumnSetExpr(pParse, p, pCol, pDfltExpr); +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Reject unauthorized functions from DEFAULT clauses */ + if( db->init.busy==0 && db->xAuth!=0 ){ + sqlite3FuncAuth(pParse, pExpr); + } +#endif /* SQLITE_OMIT_AUTHORIZER */ + } } if( IN_RENAME_OBJECT ){ @@ -131754,7 +132659,6 @@ static void markExprListImmutable(ExprList *pList){ #define markExprListImmutable(X) /* no-op */ #endif /* SQLITE_DEBUG */ - /* ** This routine is called to report the final ")" that terminates ** a CREATE TABLE statement. @@ -133793,8 +134697,8 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists db->aDb[iDb].zDbSName, pIndex->zName ); sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); - sqlite3ChangeCookie(pParse, iDb); destroyRootPage(pParse, pIndex->tnum, iDb); + sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); } @@ -136507,6 +137411,7 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] && pPrior->aiColumn[j]!=XN_EXPR @@ -136515,7 +137420,9 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( continue; } sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); - if( pIdx->aiColumn[j]>=0 ){ + if( (iTabCol = pIdx->aiColumn[j])>=0 + && (pIdx->pTable->aCol[iTabCol].colFlags & COLFLAG_VIRTUAL)==0 + ){ /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. @@ -136880,6 +137787,14 @@ static void printfFunc( } } +/* Return true if any of the eight bytes in word is NUL or non-ASCII. */ +static SQLITE_INLINE int containsNulOrNonAscii(u64 word){ + const u64 byteOnes = UINT64_C(0x0101010101010101); + const u64 byteHighBits = UINT64_C(0x8080808080808080); + /* NUL causes a borrow into a high bit; non-ASCII already has one set. */ + return ((word | (word - byteOnes)) & byteHighBits)!=0; +} + /* ** Implementation of the substr() function. ** @@ -136925,7 +137840,7 @@ static void substrFunc( p2 = sqlite3_value_int64(argv[2]); if( p2==0 && sqlite3_value_type(argv[2])==SQLITE_NULL ) return; }else{ - p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; + p2 = LARGEST_INT64; } if( p1==0 ){ #ifdef SQLITE_SUBSTR_COMPATIBILITY @@ -136963,6 +137878,20 @@ static void substrFunc( } assert( p1>=0 && p2>=0 ); if( p0type!=SQLITE_BLOB ){ + if( p1>=8 ){ + /* Skip 8 characters at a time while all 8 bytes are in 0x01..0x7f */ + const unsigned char *zEnd = z + sqlite3_value_bytes(argv[0]); + while( p1>=8 && zEnd-z>=8 ){ + u64 word; + memcpy(&word, z, sizeof(word)); + /* Handle NUL and UTF-8 with the scalar loop below. */ + if( containsNulOrNonAscii(word) ){ + break; + } + z += 8; + p1 -= 8; + } + } for( ; p1>0; p1--){ /* vvvvvv---- See tag-20260418-01 */ if( (u8)(z[0]-1)<(0x80-1) ){ @@ -137342,10 +138271,20 @@ static int patternCompare( if( c<0x80 ){ char zStop[3]; int bMatch; + /* If the next pattern character is an ordinary ASCII character, + ** then a candidate position can only match if the next input + ** character is nx or ny. Check that inline, to avoid most of the + ** recursive calls. nx==0 disables this check. */ + u8 nx = zPattern[0], ny = nx; + if( nx>=0x80 || nx==matchAll || nx==matchOne || nx==matchOther ){ + nx = 0; + } if( noCase ){ zStop[0] = sqlite3Toupper(c); zStop[1] = sqlite3Tolower(c); zStop[2] = 0; + nx = sqlite3Toupper(nx); + ny = sqlite3Tolower(ny); }else{ zStop[0] = c; zStop[1] = 0; @@ -137354,6 +138293,7 @@ static int patternCompare( zString += strcspn((const char*)zString, zStop); if( zString[0]==0 ) break; zString++; + if( nx && zString[0]!=nx && zString[0]!=ny ) continue; bMatch = patternCompare(zPattern,zString,pInfo,matchOther); if( bMatch!=SQLITE_NOMATCH ) return bMatch; } @@ -137631,14 +138571,20 @@ static void compileoptiongetFunc( int argc, sqlite3_value **argv ){ - int n; + i64 n; + const char *zRes; assert( argc==1 ); UNUSED_PARAMETER(argc); /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. */ - n = sqlite3_value_int(argv[0]); - sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); + n = sqlite3_value_int64(argv[0]); + /* Use a negative lower bound to assist testing of the + ** sqlite3_compileoption_get() C-language interface and its handling + ** of negative inputs. + ** vvvvv */ + zRes = (n>=(-10) && n<10000) ? sqlite3_compileoption_get((int)n) : 0; + sqlite3_result_text(context, zRes, -1, SQLITE_STATIC); } #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ @@ -138366,6 +139312,21 @@ static void soundexFunc( } #endif /* SQLITE_SOUNDEX */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_BUILTIN_OOM_FUNCTION) +/* +** This SQL function just invokes sqlite3OomFault() and then returns +** NULL. Used for testing only. +*/ +static void oomFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3OomFault(db); +} +#endif /* SQLITE_DEBUG || SQLITE_BUILTIN_OOM_FUNCTION */ + #ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** A function that loads a shared-library extension then returns NULL. @@ -138844,7 +139805,7 @@ static void groupConcatInverse( pGCC->str.nChar -= nVS; memmove(pGCC->str.zText, &pGCC->str.zText[nVS], pGCC->str.nChar); } - if( pGCC->str.nChar==0 ){ + if( pGCC->nAccum==0 ){ pGCC->str.mxAlloc = 0; sqlite3_free(pGCC->pnSepLengths); pGCC->pnSepLengths = 0; @@ -139072,11 +140033,11 @@ static void logFunc( return; } if( argc==2 ){ - switch( sqlite3_value_numeric_type(argv[0]) ){ + switch( sqlite3_value_numeric_type(argv[1]) ){ case SQLITE_INTEGER: case SQLITE_FLOAT: b = log(x); - if( b<=0.0 ) return; + if( b==0.0 ) return; x = sqlite3_value_double(argv[1]); if( x<=0.0 ) return; break; @@ -139736,6 +140697,48 @@ static void filestatFunc( } #endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_WALSTAT) +/* +** Implementation of sqlite_walstat(SCHEMA). +** +** Return JSON text that describes the current state of a WAL file. +** This function is for debugging and analysis only. It is not part +** of production builds. This function is not part of the SQLite API +** and is likely to change or be removed in future versions of SQLite. +*/ +static void walstatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDbName; + sqlite3_str *pStr; + Btree *pBtree; + + zDbName = (const char*)sqlite3_value_text(argv[0]); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + pStr = sqlite3_str_new(db); + if( sqlite3_str_errcode(pStr) ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_str_append(pStr, "{", 1); + sqlite3PagerWalStat(pPager, pStr); + sqlite3_str_append(pStr, "}", 1); + sqlite3_result_str(context, pStr, SQLITE_FINISH); + } + sqlite3BtreeLeave(pBtree); + }else{ + sqlite3_result_text(context, "{}", 2, SQLITE_STATIC); + } +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_WALSTAT */ + #ifdef SQLITE_DEBUG /* ** Implementation of fpdecode(x,y,z) function. @@ -139764,6 +140767,7 @@ static void fpdecodeFunc( y = sqlite3_value_int(argv[1]); z = sqlite3_value_int(argv[2]); if( z<=0 ) z = 1; + if( y<-100000000 ) y = -100000000; /* Same limit as printf() */ sqlite3FpDecode(&s, x, y, z); if( s.isSpecial==2 ){ sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); @@ -139896,6 +140900,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) FUNCTION(sqlite_filestat, 1, 0, 0, filestatFunc ), + FUNCTION(sqlite_walstat, 1, 0, 0, walstatFunc ), #endif FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), @@ -139924,6 +140929,9 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(fpdecode, 3, 0, 0, fpdecodeFunc ), FUNCTION(parseuri, -1, 0, 0, parseuriFunc ), #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_BUILTIN_OOM_FUNCTION) + FUNCTION(oom, -1, 0, 0, oomFunc ), +#endif #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), FUNCTION(round, 2, 0, 0, roundFunc ), @@ -140582,6 +141590,144 @@ static Expr *exprTableColumn( return pExpr; } +/* +** Column iCol1 of table p1 is to be compared to column iCol2 of table p2. +** Return non-zero if the comparison will use an affinity compatible +** with the affinity of column iCol1 of table p1. Or zero if it will not. +** +** This is used in cases where (p1.iCol1 = p2.iCol2) must be evaluated +** using the affinity of p1.iCol1 as part of foreign key processing. +*/ +static int fkUsesLhsAffinity(Table *p1, int iCol1, Table *p2, int iCol2){ + u8 aff1 = iCol1>=0 ? p1->aCol[iCol1].affinity : SQLITE_AFF_NUMERIC; + u8 aff2 = p2->aCol[iCol2].affinity; + + /* If either side has a numeric affinity, then the comparison uses numeric + ** affinity. So the affinity matches if aff1 is numeric, but not if aff1 + ** is not numeric but aff2 is. */ + if( sqlite3IsNumericAffinity(aff1) ) return 1; + if( sqlite3IsNumericAffinity(aff2) ) return 0; + + assert( aff1==SQLITE_AFF_TEXT || aff1==SQLITE_AFF_BLOB ); + assert( aff2==SQLITE_AFF_TEXT || aff2==SQLITE_AFF_BLOB ); + + /* At this point, either the affinities are the same or one is TEXT and + ** the other BLOB. In the second case the comparison will be done with + ** BLOB affinity, which does not match if aff1 is TEXT. */ + return (aff1==aff2 || aff1==SQLITE_AFF_BLOB); +} + +#ifndef SQLITE_OMIT_CAST +/* +** Return non-zero if a lookup on column iCol of table pTab, using collation +** sequence zColl, can be done using either the rowid or an index. That is, +** if iCol is the INTEGER PRIMARY KEY of pTab, or if pTab has an index +** whose left-most column is iCol and whose collation sequence for that +** column is zColl. Return zero otherwise. +*/ +static int fkChildKeyIsIndexed(Table *pTab, int iCol, const char *zColl){ + Index *pIdx; + if( iCol==pTab->iPKey ) return 1; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aiColumn[0]==iCol + && pIdx->pPartIdxWhere==0 + && sqlite3StrICmp(pIdx->azColl[0], zColl)==0 + ){ + return 1; + } + } + return 0; +} +#endif + +/* +** Return an expression equivalent to (pLeft = pRight), where pLeft is a +** parent key value and pRight is an unresolved reference to column +** iChildCol of child table pChild. The comparison uses the affinity and +** collation sequence of column iCol of parent table pTab. pLeft must +** already carry that affinity and collation sequence. Ownership of +** both pLeft and pRight passes to this routine. +** +** If the simple (pLeft = pRight) does not naturally use the affinity of +** the parent column, the comparison is coded as (pLeft = +pRight). But +** the unary + prevents any index on the child column from being used. +** So in that case, if the child column is indexed, an extra term is added: +** +** (pLeft = +pRight) AND +** (pRight COLLATE IN (+pLeft, +CAST(pLeft AS NUMERIC))) +** +** Each child value that matches pLeft under the affinity of the parent +** column is either equal to pLeft itself or numerically equal to +** CAST(pLeft AS NUMERIC). So the IN term matches a superset of the +** child rows that the exact (pLeft = +pRight) term does, and since it +** compares using the affinity of the child column, it may be used with +** an index on the child column. The unary + operators on the RHS of the +** IN ensure that no affinity other than that of the child column is +** applied. +*/ +static Expr *fkParentChildEq( + Parse *pParse, /* Parsing context */ + Expr *pLeft, /* Parent key value */ + Expr *pRight, /* Unresolved reference to child key column */ + Table *pTab, /* Parent table */ + int iCol, /* Parent key column, or <0 for rowid */ + Table *pChild, /* Child table */ + int iChildCol, /* Child key column */ + int bComposite /* This is part of a composite FK */ +){ + sqlite3 *db = pParse->db; + Expr *pEq; +#ifndef SQLITE_OMIT_CAST + Expr *pIn = 0; +#endif + + if( fkUsesLhsAffinity(pTab, iCol, pChild, iChildCol) ){ + return sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); + } + assert( iCol>=0 ); + +#ifndef SQLITE_OMIT_CAST + { + const char *zColl = sqlite3ColumnColl(&pTab->aCol[iCol]); + if( zColl==0 ) zColl = sqlite3StrBINARY; + + /* The IN term is only useful if it can drive a lookup on the child + ** table. Omit it otherwise, so that a full scan of the child table + ** does not have to evaluate it for every row. */ + if( bComposite || fkChildKeyIsIndexed(pChild, iChildCol, zColl) ){ + static const Token tNumeric = { "NUMERIC", 7 }; + Expr *pCast; + ExprList *pList; + + pCast = sqlite3ExprAlloc(db, TK_CAST, &tNumeric, 0); + sqlite3ExprAttachSubtrees(db, pCast, sqlite3ExprDup(db, pLeft, 0), 0); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(db, pLeft, 0), 0) + ); + pList = sqlite3ExprListAppend(pParse, pList, + sqlite3PExpr(pParse, TK_UPLUS, pCast, 0) + ); + pIn = sqlite3PExpr(pParse, TK_IN, sqlite3ExprAddCollateString(pParse, + sqlite3ExprDup(db, pRight, 0), zColl + ), 0); + if( pIn ){ + pIn->x.pList = pList; + sqlite3ExprSetHeightAndFlags(pParse, pIn); + }else{ + sqlite3ExprListDelete(db, pList); + } + } + } +#endif + + pRight = sqlite3PExpr(pParse, TK_UPLUS, pRight, 0); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); +#ifndef SQLITE_OMIT_CAST + pEq = sqlite3ExprAnd(pParse, pEq, pIn); +#endif + return pEq; +} + /* ** This function is called to generate code executed when a row is deleted ** from the parent table of foreign key constraint pFKey and, if pFKey is @@ -140650,16 +141796,18 @@ static void fkScanChildren( Expr *pLeft; /* Value from parent table row */ Expr *pRight; /* Column ref to child table */ Expr *pEq; /* Expression (pLeft = pRight) */ - i16 iCol; /* Index of column in child table */ + i16 iCol; /* Index of column in parent table */ + i16 iCol2; /* Index of column in child table */ const char *zCol; /* Name of column in child table */ iCol = pIdx ? pIdx->aiColumn[i] : -1; pLeft = exprTableRegister(pParse, pTab, regData, iCol); - iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; - assert( iCol>=0 ); - zCol = pFKey->pFrom->aCol[iCol].zCnName; + iCol2 = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iCol2>=0 ); + zCol = pFKey->pFrom->aCol[iCol2].zCnName; pRight = sqlite3Expr(db, TK_ID, zCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); + pEq = fkParentChildEq(pParse, pLeft, pRight, pTab, iCol, + pFKey->pFrom, iCol2, pFKey->nCol>0); pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); } @@ -140919,13 +142067,12 @@ static int fkParentIsModified( ** to trigger pFKey. */ static int isSetNullAction(Parse *pParse, FKey *pFKey){ - Parse *pTop = sqlite3ParseToplevel(pParse); - if( pTop->pTriggerPrg ){ - Trigger *p = pTop->pTriggerPrg->pTrigger; + Trigger *p = pParse->pTrigCoded; + if( p ){ if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) ){ - assert( (pTop->db->flags & SQLITE_FkNoAction)==0 ); + assert( (pParse->db->flags & SQLITE_FkNoAction)==0 ); return 1; } } @@ -141319,26 +142466,35 @@ static Trigger *fkActionTrigger( Token tFromCol; /* Name of column in child table */ Token tToCol; /* Name of column in parent table */ int iFromCol; /* Idx of column in child table */ - Expr *pEq; /* tFromCol = OLD.tToCol */ + int iToCol; /* Idx of column in parent table */ + Expr *pEq; /* OLD.tToCol = tFromCol */ + Expr *pLeft; /* OLD.tToCol */ + Expr *pRight; /* tFromCol */ iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); - sqlite3TokenInit(&tToCol, - pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zCnName); + iToCol = pIdx ? pIdx->aiColumn[i] : pTab->iPKey; + sqlite3TokenInit(&tToCol, pTab->aCol[iToCol].zCnName); sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zCnName); - /* Create the expression "OLD.zToCol = zFromCol". It is important - ** that the "OLD.zToCol" term is on the LHS of the = operator, so - ** that the affinity and collation sequence associated with the - ** parent table are used for the comparison. */ - pEq = sqlite3PExpr(pParse, TK_EQ, - sqlite3PExpr(pParse, TK_DOT, - sqlite3ExprAlloc(db, TK_ID, &tOld, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), - sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) + pLeft = sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) ); + pRight = sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0); + + /* OLD.* columns do not normally have an affinity. So set the affinity + ** of OLD.tToCol (the parent key) explicitly here. */ + if( pLeft ) pLeft->affExpr = pTab->aCol[iToCol].affinity; + + /* Create the expression "OLD.tToCol = tFromCol". It is important + ** that the "OLD.tToCol" term is on the LHS of the = operator, so + ** that the collation sequence associated with the parent table is + ** used for the comparison. */ + pEq = fkParentChildEq(pParse, pLeft, pRight, pTab, iToCol, + pFKey->pFrom, iFromCol, pFKey->nCol>1); pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); /* For ON UPDATE, construct the next term of the WHEN clause. @@ -141372,7 +142528,7 @@ static Trigger *fkActionTrigger( testcase( pCol->colFlags & COLFLAG_STORED ); pDflt = 0; }else{ - pDflt = sqlite3ColumnExpr(pFKey->pFrom, pCol); + pDflt = sqlite3ColumnExprAuth(pFKey->pFrom, pCol, pParse); } if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); @@ -142159,25 +143315,13 @@ SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ } } -/* -** Return true if all expressions in the expression-list passed as the -** only argument are constant. -*/ -static int exprListIsConstant(Parse *pParse, ExprList *pRow){ - int ii; - for(ii=0; iinExpr; ii++){ - if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; - } - return 1; -} - /* ** Return true if all expressions in the expression-list passed as the ** only argument are both constant and have no affinity. */ static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ int ii; - if( exprListIsConstant(pParse,pRow)==0 ) return 0; + if( sqlite3ExprListIsConstant(pParse, pRow, 0)==0 ) return 0; for(ii=0; iinExpr; ii++){ Expr *pExpr = pRow->a[ii].pExpr; assert( pExpr->op!=TK_RAISE ); @@ -142243,7 +143387,7 @@ SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList if( pParse->bHasWith /* condition (a) above */ || pParse->db->init.busy /* condition (b) above */ - || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || sqlite3ExprListIsConstant(pParse, pRow, 1)==0 /* condition (c) above */ || (pLeft->pSrc->nSrc==0 && exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ || IN_SPECIAL_PARSE @@ -142956,7 +144100,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* Hidden columns that are not explicitly named in the INSERT ** get their default value */ sqlite3ExprCodeFactorable(pParse, - sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + sqlite3ColumnExprAuth(pTab, &pTab->aCol[i], pParse), iRegStore); continue; } @@ -142968,7 +144112,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* A column not named in the insert column list gets its ** default value */ sqlite3ExprCodeFactorable(pParse, - sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + sqlite3ColumnExprAuth(pTab, &pTab->aCol[i], pParse), iRegStore); continue; } @@ -142976,7 +144120,7 @@ SQLITE_PRIVATE void sqlite3Insert( }else if( nColumn==0 ){ /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ sqlite3ExprCodeFactorable(pParse, - sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + sqlite3ColumnExprAuth(pTab, &pTab->aCol[i], pParse), iRegStore); continue; }else{ @@ -143027,6 +144171,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); + sqlite3MayAbort(pParse); } /* Copy the new data already generated. */ @@ -143095,6 +144240,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); + sqlite3MayAbort(pParse); } }else if( IsVirtual(pTab) || withoutRowid ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); @@ -143574,7 +144720,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( assert( (pCol->colFlags & COLFLAG_GENERATED)==0 ); nSeenReplace++; sqlite3ExprCodeCopy(pParse, - sqlite3ColumnExpr(pTab, pCol), iReg); + sqlite3ColumnExprAuth(pTab, pCol, pParse), iReg); sqlite3VdbeJumpHere(v, addr1); break; } @@ -144230,7 +145376,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** Hence, make a complete copy of the opcode, rather than using ** a pointer to the opcode. */ x = *sqlite3VdbeGetOp(v, addrConflictCk); - if( x.opcode!=OP_IdxRowid ){ + if( x.opcode!=OP_IdxRowid || isUpdate ){ int p2; /* New P2 value for copied conflict check opcode */ const char *zP4; if( sqlite3OpcodeProperty[x.opcode]&OPFLG_JUMP ){ @@ -145316,7 +146462,7 @@ struct sqlite3_api_routines { const char * (*libversion)(void); int (*libversion_number)(void); void *(*malloc)(int); - char * (*mprintf)(const char*,...); + char *(SQLITE_CDECL*mprintf)(const char*,...); int (*open)(const char*,sqlite3**); int (*open16)(const void*,sqlite3**); int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); @@ -145337,11 +146483,11 @@ struct sqlite3_api_routines { void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_value)(sqlite3_context*,sqlite3_value*); - void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + void *(*rollback_hook)(sqlite3*,void(*)(void*),void*); int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); - char * (*xsnprintf)(int,char*,const char*,...); + char *(SQLITE_CDECL*xsnprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, char const**,char const**,int*,int*,int*); @@ -145405,7 +146551,7 @@ struct sqlite3_api_routines { int (*xthreadsafe)(void); void (*result_zeroblob)(sqlite3_context*,int); void (*result_error_code)(sqlite3_context*,int); - int (*test_control)(int, ...); + int (SQLITE_CDECL*test_control)(int, ...); void (*randomness)(int,void*); sqlite3 *(*context_db_handle)(sqlite3_context*); int (*extended_result_codes)(sqlite3*,int); @@ -145425,11 +146571,11 @@ struct sqlite3_api_routines { void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*), void(*xDestroy)(void*)); - int (*db_config)(sqlite3*,int,...); + int (SQLITE_CDECL*db_config)(sqlite3*,int,...); sqlite3_mutex *(*db_mutex)(sqlite3*); int (*db_status)(sqlite3*,int,int*,int*,int); int (*extended_errcode)(sqlite3*); - void (*log)(int,const char*,...); + void (SQLITE_CDECL*log)(int,const char*,...); sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); const char *(*sourceid)(void); int (*stmt_status)(sqlite3_stmt*,int,int); @@ -145439,7 +146585,7 @@ struct sqlite3_api_routines { int (*wal_checkpoint)(sqlite3*,const char*); void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); - int (*vtab_config)(sqlite3*,int op,...); + int (SQLITE_CDECL*vtab_config)(sqlite3*,int op,...); int (*vtab_on_conflict)(sqlite3*); /* Version 3.7.16 and later */ int (*close_v2)(sqlite3*); @@ -145508,7 +146654,7 @@ struct sqlite3_api_routines { int (*keyword_check)(const char*,int); sqlite3_str *(*str_new)(sqlite3*); char *(*str_finish)(sqlite3_str*); - void (*str_appendf)(sqlite3_str*, const char *zFormat, ...); + void (SQLITE_CDECL*str_appendf)(sqlite3_str*, const char *zFormat, ...); void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list); void (*str_append)(sqlite3_str*, const char *zIn, int N); void (*str_appendall)(sqlite3_str*, const char *zIn); @@ -146797,25 +147943,32 @@ SQLITE_API int sqlite3_auto_extension( SQLITE_API int sqlite3_cancel_auto_extension( void (*xInit)(void) ){ + int n = 0; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ){ + return 0; + }else +#endif + { #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); #endif - int i; - int n = 0; - wsdAutoextInit; + int i; + wsdAutoextInit; #ifdef SQLITE_ENABLE_API_ARMOR - if( xInit==0 ) return 0; + if( xInit==0 ) return 0; #endif - sqlite3_mutex_enter(mutex); - for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ - if( wsdAutoext.aExt[i]==xInit ){ - wsdAutoext.nExt--; - wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; - n++; - break; + sqlite3_mutex_enter(mutex); + for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ + if( wsdAutoext.aExt[i]==xInit ){ + wsdAutoext.nExt--; + wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; + n++; + break; + } } + sqlite3_mutex_leave(mutex); } - sqlite3_mutex_leave(mutex); return n; } @@ -147153,7 +148306,7 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ SQLITE_CountRows }, #endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && defined(_WIN32) {/* zName: */ "data_store_directory", /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, /* ePragFlg: */ PragFlg_NoColumns1, @@ -147585,7 +148738,7 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; -/* Number of pragmas: 68 on by default, 79 total. */ +/* Number of pragmas: 67 on by default, 79 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ @@ -147638,7 +148791,7 @@ static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ /* on no off false yes true extra full */ int i, n; if( sqlite3Isdigit(*z) ){ - return (u8)sqlite3Atoi(z); + return sqlite3Atoi(z)!=0; } n = sqlite3Strlen30(z); for(i=0; isafety_level = iLevel; pDb->bSyncSet = 1; @@ -149405,14 +150564,9 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. - ** reg[8+i] counts the number of entries in the i-th index */ sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ - } - assert( pParse->nMem>=8+j ); - assert( sqlite3NoTempsInRange(pParse,1,7+j) ); + assert( sqlite3NoTempsInRange(pParse,1,7) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); @@ -149494,8 +150648,8 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ if( pCol->iDflt ){ sqlite3_value *pDfltValue = 0; - sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db), - pCol->affinity, &pDfltValue); + sqlite3ValueFromExpr(db, sqlite3ColumnExprAuth(pTab,pCol,pParse), + ENC(db), pCol->affinity, &pDfltValue); if( pDfltValue ){ p4 = sqlite3_value_type(pDfltValue); sqlite3ValueFree(pDfltValue); @@ -149625,7 +150779,6 @@ SQLITE_PRIVATE void sqlite3Pragma( r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, pPrior, r1); pPrior = pIdx; - sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ /* Verify that an index entry exists for the current table row */ sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); @@ -150209,9 +151362,10 @@ SQLITE_PRIVATE void sqlite3Pragma( ** disables the timeout. */ /*case PragTyp_BUSY_TIMEOUT*/ default: { + int n = 0; assert( pPragma->ePragTyp==PragTyp_BUSY_TIMEOUT ); - if( zRight ){ - sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + if( zRight && sqlite3GetInt32(zRight,&n) ){ + sqlite3_busy_timeout(db, n); } returnSingleInt(v, db->busyTimeout); break; @@ -150797,6 +151951,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char int rc; u8 saved_iDb = db->init.iDb; sqlite3_stmt *pStmt; + const char *zEnd; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ assert( db->init.busy ); @@ -150811,7 +151966,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char db->init.orphanTrigger = 0; db->init.azInit = (const char**)argv; pStmt = 0; - TESTONLY(rcp = ) sqlite3Prepare(db, argv[4], -1, 0, 0, &pStmt, 0); + TESTONLY(rcp = ) sqlite3Prepare(db, argv[4], -1, 0, 0, &pStmt, &zEnd); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); db->init.iDb = saved_iDb; @@ -150827,6 +151982,8 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char corruptSchema(pData, argv, sqlite3_errmsg(db)); } } + }else if( zEnd[0] ){ + corruptSchema(pData, argv, 0); } db->init.azInit = sqlite3StdType; /* Any array of string ptrs will do */ sqlite3_finalize(pStmt); @@ -151105,7 +152262,7 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ sqlite3OomFault(db); } - sqlite3ResetOneSchema(db, iDb); + sqlite3ResetOneSchema(db, iDb, 0); } db->init.busy = 0; return rc; @@ -151207,7 +152364,7 @@ static void schemaIsValid(Parse *pParse){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ if( DbHasProperty(db, iDb, DB_SchemaLoaded) ) pParse->rc = SQLITE_SCHEMA; - sqlite3ResetOneSchema(db, iDb); + sqlite3ResetOneSchema(db, iDb, 0); } /* Close the transaction, if one was opened. */ @@ -151549,7 +152706,7 @@ static int sqlite3LockAndPrepare( if( rc==SQLITE_OK || db->mallocFailed ) break; cnt++; }while( (rc==SQLITE_ERROR_RETRY && ALWAYS(cnt<=SQLITE_MAX_PREPARE_RETRY)) - || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt)==1) ); + || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1,0), cnt)==1) ); sqlite3BtreeLeaveAll(db); assert( rc!=SQLITE_ERROR_RETRY ); rc = sqlite3ApiExit(db, rc); @@ -152300,6 +153457,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ int i, j; /* Loop counters */ SrcItem *pLeft; /* Left table being joined */ SrcItem *pRight; /* Right table being joined */ + int bSeenTFunc = 0; /* True if have seen table-valued function */ pSrc = p->pSrc; pLeft = &pSrc->a[0]; @@ -152442,7 +153600,18 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ p->selFlags |= SF_OnToWhere; } - if( pRight->fg.isTabFunc && joinType==EP_OuterON && pRight->u1.pFuncArg ){ + if( bSeenTFunc==0 ){ + bSeenTFunc = (pLeft->fg.isTabFunc && pLeft->u1.pFuncArg); + } + + /* We also need to call sqlite3SelectCheckOnClauses() to verify that + ** table-valued function arguments do not illegally refer to any tables to + ** their right. This test is required if either (a) there is a RIGHT JOIN + ** in the SrcList, or (b) there is a LEFT JOIN to the right of a + ** table-valued function. */ + if( (pRight->fg.isTabFunc && joinType==EP_OuterON && pRight->u1.pFuncArg) + || (bSeenTFunc && joinType==EP_OuterON) + ){ p->selFlags |= SF_OnToWhere; } } @@ -153551,6 +154720,9 @@ static void generateSortTail( Table *pTab = pSort->aDefer[i].pTab; int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp2(v, OP_Rewind, pSort->aDefer[i].iCsr, + sqlite3VdbeCurrentAddr(v)+1); + VdbeCoverage(v); nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey); } #endif @@ -154070,7 +155242,7 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); testcase( aCol==0 ); - if( NEVER(nCol>32767) ) nCol = 32767; + if( nCol>32767 ) nCol = 32767; }else{ nCol = 0; aCol = 0; @@ -154257,6 +155429,20 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( pTab->szTabRow = 1; /* Any non-zero value works */ } +/* +** Check the current subquery nesting depth. Return true and set an +** error if it has gone too deep. +*/ +static int checkSubqueryNestingDepth(Parse *pParse){ +#if SQLITE_MAX_EXPR_DEPTH>0 + if( pParse->nNestSel >= pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH] ){ + sqlite3ErrorMsg(pParse, "VIEWs and/or subqueries nested too deep"); + return 1; + } +#endif + return 0; +} + /* ** Given a SELECT statement, generate a Table structure that describes ** the result set of that SELECT. @@ -154267,12 +155453,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, c u64 savedFlags; pParse->nNestSel++; -#if SQLITE_MAX_EXPR_DEPTH>0 - if( pParse->nNestSel >= db->aLimit[SQLITE_LIMIT_EXPR_DEPTH] ){ - sqlite3ErrorMsg(pParse, "VIEWs and/or subqueries nested too deep"); - return 0; - } -#endif + if( checkSubqueryNestingDepth(pParse) ) return 0; savedFlags = db->flags; db->flags &= ~(u64)SQLITE_FullColNames; db->flags |= SQLITE_ShortColNames; @@ -154307,9 +155488,7 @@ SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ if( pParse->pVdbe ){ return pParse->pVdbe; } - if( pParse->pToplevel==0 - && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) - ){ + if( pParse->pToplevel==0 ){ pParse->okConstFactor = 1; } return sqlite3VdbeCreate(pParse); @@ -154360,7 +155539,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse, 0) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -154373,6 +155552,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ sqlite3ExprCode(pParse, pLimit->pLeft, iLimit); sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); + sqlite3MayAbort(pParse); sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); } if( pLimit->pRight ){ @@ -154380,6 +155560,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ pParse->nMem++; /* Allocate an extra register for limit+offset */ sqlite3ExprCode(pParse, pLimit->pRight, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); + sqlite3MayAbort(pParse); VdbeComment((v, "OFFSET counter")); sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); VdbeComment((v, "LIMIT+OFFSET")); @@ -154727,11 +155908,33 @@ static int hasAnchor(Select *p){ } /* -** This routine is called to process a compound query form from +** Return TRUE if p is a UNION with a LIMIT of exactly 1 and no OFFSET +** clause. This is a precondition for a couple of related optimizations. +** +** False negatives are harmless (apart from resulting in a slower query). +** False positives can result in incorrect answers, however. To provoke +** false negatives for testing purposes, disable the SQLITE_UnionLimit +** optimization. +*/ +static int unionWithLimitOne(sqlite3 *db, Select *p){ + int v; + if( p->op!=TK_UNION ) return 0; + if( p->pLimit==0 ) return 0; + if( p->pLimit->pRight ) return 0; /* No OFFSET */ + v = 0; + assert( p->pLimit->pLeft!=0 ); + if( sqlite3ExprIsInteger(p->pLimit->pLeft, &v, 0, 0)==0 ) return 0; + if( v!=1 ) return 0; + if( OptimizationDisabled(db, SQLITE_UnionLimit) ) return 0; + return 1; +} + +/* +** This routine is called to process a compound query formed from ** two or more separate queries using UNION, UNION ALL, EXCEPT, or ** INTERSECT ** -** "p" points to the right-most of the two queries. the query on the +** "p" points to the right-most of the two queries. The query on the ** left is p->pPrior. The left query could also be a compound query ** in which case this routine will be called recursively. ** @@ -154815,7 +156018,7 @@ static int multiSelect( /* If the compound has an ORDER BY clause, then always use the merge ** algorithm. */ return multiSelectByMerge(pParse, p, pDest); - }else if( p->op!=TK_ALL ){ + }else if( p->op!=TK_ALL && !unionWithLimitOne(db,p) ){ /* If the compound is EXCEPT, INTERSECT, or UNION (anything other than ** UNION ALL) then also always use the merge algorithm. However, the ** multiSelectByMerge() routine requires that the compound have an @@ -154827,7 +156030,8 @@ static int multiSelect( p->pOrderBy->a[0].u.x.iOrderByCol = 1; return multiSelectByMerge(pParse, p, pDest); }else{ - /* For a UNION ALL compound without ORDER BY, simply run the left + /* For a UNION ALL compound without ORDER BY, or for a UNION with a + ** LIMIT of exactly 1 and no OFFSET and no ORDER BY, simply run the left ** query, then run the right query */ int addr = 0; int nLimit = 0; /* Initialize to suppress harmless compiler warning */ @@ -154868,7 +156072,7 @@ static int multiSelect( p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse, 0) && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ p->nSelectRow = sqlite3LogEst((u64)nLimit); @@ -155283,9 +156487,12 @@ static int multiSelectByMerge( /* For operators other than UNION ALL we have to make sure that ** the ORDER BY clause covers every term of the result set. Add - ** terms to the ORDER BY clause as necessary. + ** terms to the ORDER BY clause as necessary. Do the same for UNION ALL + ** if any operator to the left is not UNION ALL, so that the ORDER BY + ** carries the collation of every column into that sub-compound. */ - if( op!=TK_ALL ){ + for(pPrior=p; pPrior && pPrior->op==TK_ALL; pPrior=pPrior->pPrior){} + if( pPrior && pPrior->pPrior ){ for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ struct ExprList_item *pItem; for(j=0, pItem=pOrderBy->a; jaColl[i] = multiSelectCollSeq(pParse, p, i); - pKeyDup->aSortFlags[i] = 0; + /* Use the ORDER BY collations, not multiSelectCollSeq(p), because + ** p might be just one part of a larger compound, in which case + ** the ORDER BY carries the COLLATE of the larger compound. */ + for(i=0; ALWAYS(pKeyMerge) && iaColl[pOrderBy->a[i].u.x.iOrderByCol-1] = pKeyMerge->aColl[i]; } } } @@ -155378,7 +156587,7 @@ static int multiSelectByMerge( /* Compute the limit registers */ computeLimitRegisters(pParse, p, labelEnd); - if( p->iLimit && op==TK_ALL ){ + if( p->iLimit && (op==TK_ALL || unionWithLimitOne(db,p)) ){ regLimitA = ++pParse->nMem; regLimitB = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, @@ -155859,7 +157068,9 @@ static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ if( op==TK_COLUMN || op==TK_IF_NULL_ROW ){ renumberCursorDoMapping(pWalker, &pExpr->iTable); } - if( ExprHasProperty(pExpr, EP_OuterON) ){ + if( ExprHasProperty(pExpr, (EP_OuterON|EP_InnerON)) ){ + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + testcase( ExprHasProperty(pExpr, EP_InnerON) ); renumberCursorDoMapping(pWalker, &pExpr->w.iJoin); } return WRC_Continue; @@ -156406,7 +157617,7 @@ static int flattenSubquery( pSub = pSub1; for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; - u8 jointype = pSubitem->fg.jointype; + u8 jointype = pParent->pSrc->a[iFrom].fg.jointype; assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ @@ -156431,7 +157642,6 @@ static int flattenSubquery( pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); if( pSrc==0 ) break; pParent->pSrc = pSrc; - pSubitem = &pSrc->a[iFrom]; } /* Transfer the FROM clause terms from the subquery into the @@ -156449,7 +157659,7 @@ static int flattenSubquery( pItem->fg.jointype |= (jointype & JT_LTORJ); memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSubitem->fg.jointype |= jointype; + pSrc->a[iFrom].fg.jointype |= jointype; /* Begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -156564,12 +157774,21 @@ struct WhereConst { }; /* -** Add a new entry to the pConst object. Except, do not add duplicate -** pColumn entries. Also, do not add if doing so would not be appropriate. +** Add a new entry to the pConst object, if appropriate. +** +** * Do not add if pValue is not constant. (This is enforced by +** the caller) +** +** * Do not add duplicate pColumn entries +** +** * Do not add if pValue has an affinity ** -** The caller guarantees the pColumn is a column and pValue is a constant. -** This routine has to do some additional checks before completing the -** insert. +** * Do not add if the comparison uses a collating sequence other +** than binary. +** +** * Do not add if pValue is a function that might return a subtype +** +** The caller guarantees the pColumn is a column. */ static void constInsert( WhereConst *pConst, /* The WhereConst into which we are inserting */ @@ -156580,12 +157799,12 @@ static void constInsert( int i; assert( pColumn->op==TK_COLUMN ); assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); - if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){ return; } + if( sqlite3ExprCanReturnSubtype(pConst->pParse, pValue) ) return; /* 2018-10-25 ticket [cf5ed20f] ** Make sure the same pColumn is not inserted more than once */ @@ -156731,7 +157950,7 @@ static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ ** ** If the WHERE clause contains terms of the form COLUMN=CONSTANT or ** CONSTANT=COLUMN that are top-level AND-connected terms that are not -** part of a ON clause from a LEFT JOIN, then throughout the query +** part of a ON clause from an OUTER JOIN, then throughout the query ** replace all other occurrences of COLUMN with CONSTANT. ** ** For example, the query: @@ -157391,8 +158610,8 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); p->op = TK_SELECT; p->pWhere = 0; - pNew->pGroupBy = 0; - pNew->pHaving = 0; + p->pGroupBy = 0; + p->pHaving = 0; pNew->pOrderBy = 0; p->pPrior = 0; p->pNext = 0; @@ -157400,7 +158619,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ #ifndef SQLITE_OMIT_WINDOWFUNC p->pWinDefn = 0; #endif - p->selFlags &= ~(u32)SF_Compound; + p->selFlags &= ~(u32)(SF_Compound|SF_Distinct); assert( (p->selFlags & SF_Converted)==0 ); p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); @@ -157488,15 +158707,16 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ return pWith; } +static int selectExpanderWalk(Walker*, Select*); + /* -** This function checks if argument pFrom refers to a CTE declared by +** This function checks to see if argument pFrom refers to a CTE declared by ** a WITH clause on the stack currently maintained by the parser (on the -** pParse->pWith linked list). And if currently processing a CTE -** CTE expression, through routine checks to see if the reference is -** a recursive reference to the CTE. +** pParse->pWith linked list). And if currently processing a CTE expression, +** it also checks to see if the reference is a recursive reference to the CTE. ** -** If pFrom matches a CTE according to either of these two above, pFrom->pSTab -** and other fields are populated accordingly. +** If pFrom matches a CTE, pFrom->pSTab and other fields are populated +** accordingly. ** ** Return 0 if no match is found. ** Return 1 if a match is found. @@ -157625,6 +158845,16 @@ static int resolveFromTermToCte( pRecTerm = pRecTerm->pPrior; } + if( pParse->nTab >= db->aLimit[SQLITE_LIMIT_CTES] + && db->aLimit[SQLITE_LIMIT_CTES]>0 + ){ + sqlite3ErrorMsg(pParse, "too many CTEs"); + return 2; + } +#if SQLITE_MAX_EXPR_DEPTH>0 + pParse->nNestSel++; + if( checkSubqueryNestingDepth(pParse) ) return 2; +#endif pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; @@ -157643,12 +158873,16 @@ static int resolveFromTermToCte( return 2; } }else{ - if( sqlite3WalkSelect(pWalker, pSel) ){ + if( selectExpanderWalk(pWalker, pSel) ){ pParse->pWith = pSavedWith; return 2; } } pParse->pWith = pWith; +#if SQLITE_MAX_EXPR_DEPTH>0 + pParse->nNestSel--; + assert( pParse->nNestSel>=0 ); +#endif for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; @@ -157901,7 +159135,10 @@ static int selectExpander(Walker *pWalker, Select *p){ pTab->nCol = -1; pWalker->eCode = 1; /* Turn on Select.selId renumbering */ if( pFrom->fg.isSubquery ){ - sqlite3WalkSelect(pWalker, pFrom->u4.pSubq->pSelect); + bft hasCompound = pParse->hasCompound; + pParse->hasCompound = 1; + selectExpanderWalk(pWalker, pFrom->u4.pSubq->pSelect); + pParse->hasCompound = hasCompound; } pWalker->eCode = eCodeOrig; pTab->nCol = nCol; @@ -157984,6 +159221,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( ExprUseWOfst(pE->pLeft) ); iErrOfst = pE->pRight->w.iOfst; }else{ + assert( pE->pLeft==0 ); assert( ExprUseWOfst(pE) ); iErrOfst = pE->w.iOfst; } @@ -158095,6 +159333,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pRight = sqlite3Expr(db, TK_ID, zName); if( (pTabList->nSrc>1 && ( (pFrom->fg.jointype & JT_LTORJ)==0 + || zTName!=0 || (selFlags & SF_NestedFrom)!=0 || !inAnyUsingClause(zName,pFrom,pTabList->nSrc-i-1) ) @@ -158105,6 +159344,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pLeft = sqlite3Expr(db, TK_ID, zTabName); pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); if( IN_RENAME_OBJECT && pE->pLeft ){ + assert( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ); sqlite3RenameTokenRemap(pParse, pLeft, pE->pLeft); } if( zSchemaName ){ @@ -158188,6 +159428,27 @@ SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker *NotUsed, Select *NotUsed2){ assert( 0 ); } #endif + +/* +** The first argument is a Walker object created by sqlite3SelectExpand(). +** This function uses the walker to expand the SELECT statement passed +** as the second argument. If necessary, it also converts any compound +** SELECT statements that have COLLATE clauses on ORDER BY expressions +** to sub-queries. Return the return value of sqlite3WalkSelect(). +*/ +static int selectExpanderWalk(Walker *pWalker, Select *pSelect){ + assert( pWalker->xExprCallback==sqlite3ExprWalkNoop ); + assert( pWalker->pParse!=0 ); + if( OK_IF_ALWAYS_TRUE(pWalker->pParse->hasCompound) ){ + pWalker->xSelectCallback = convertCompoundSelectToSubquery; + pWalker->xSelectCallback2 = 0; + sqlite3WalkSelect(pWalker, pSelect); + } + pWalker->xSelectCallback = selectExpander; + pWalker->xSelectCallback2 = sqlite3SelectPopWith; + return sqlite3WalkSelect(pWalker, pSelect); +} + /* ** This routine "expands" a SELECT statement and all of its subqueries. ** For additional information on what it means to "expand" a SELECT @@ -158205,15 +159466,8 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; - if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){ - w.xSelectCallback = convertCompoundSelectToSubquery; - w.xSelectCallback2 = 0; - sqlite3WalkSelect(&w, pSelect); - } - w.xSelectCallback = selectExpander; - w.xSelectCallback2 = sqlite3SelectPopWith; w.eCode = 0; - sqlite3WalkSelect(&w, pSelect); + selectExpanderWalk(&w, pSelect); } @@ -159025,6 +160279,10 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ }else{ pExpr = sqlite3PExpr(pParse, TK_PLUS, pTerm, pExpr); } + if( pParse->nErr ){ + sqlite3ExprDelete(db, pExpr); + pExpr = 0; + } pSub = pPrior; } p->pEList->a[0].pExpr = pExpr; @@ -159220,7 +160478,7 @@ static SQLITE_NOINLINE void existsToJoin( typedef struct CheckOnCtx CheckOnCtx; struct CheckOnCtx { SrcList *pSrc; /* SrcList for this context */ - int iJoin; /* Cursors must be left of this one, if not zero */ + int iJoin; /* Cursors must be left of this one, if not negative */ int bFuncArg; /* True for table-function arg */ CheckOnCtx *pParent; /* Parent context */ }; @@ -159253,11 +160511,11 @@ static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ ** set it to the cursor number of the RHS of the join to which this ** ON expression was attached and then iterate through the entire ** expression. */ - assert( pCtx->iJoin==0 || pCtx->iJoin==pExpr->w.iJoin ); - if( pCtx->iJoin==0 ){ + assert( pCtx->iJoin<0 || pCtx->iJoin==pExpr->w.iJoin ); + if( pCtx->iJoin<0 ){ pCtx->iJoin = pExpr->w.iJoin; sqlite3WalkExprNN(pWalker, pExpr); - pCtx->iJoin = 0; + pCtx->iJoin = -1; return WRC_Prune; } } @@ -159275,7 +160533,7 @@ static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ for(ii=0; iia[ii].iCursor!=iTab; ii++){} if( iiiJoin ){ + if( pCtx->iJoin>=0 ){ for(ii--; ii>=0 && pSrc->a[ii].iCursor!=pCtx->iJoin; ii--){} if( ii>=0 ){ /* Table iJoin appears to the left of table iTab in the SrcList. @@ -159307,6 +160565,7 @@ static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ memset(&sCtx, 0, sizeof(sCtx)); sCtx.pSrc = pSelect->pSrc; sCtx.pParent = pCtx; + sCtx.iJoin = -1; pWalker->u.pCheckOnCtx = &sCtx; sqlite3WalkSelect(pWalker, pSelect); pWalker->u.pCheckOnCtx = pCtx; @@ -159332,20 +160591,41 @@ SQLITE_PRIVATE void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ w.u.pCheckOnCtx = &sCtx; memset(&sCtx, 0, sizeof(sCtx)); sCtx.pSrc = pSelect->pSrc; + sCtx.iJoin = -1; sqlite3WalkExpr(&w, pSelect->pWhere); pSelect->selFlags &= ~SF_OnToWhere; /* Check for any table-function args that are attached to virtual tables - ** on the RHS of an outer join. They are subject to the same constraints - ** as ON clauses. */ + ** on the RHS of an outer join. They are subject to similar constraints + ** as ON clauses. Specifically: + ** + ** * If the table-valued function is to the left of any RIGHT JOIN, or if + ** it is the RHS of a RIGHT JOIN, then the table-valued function arguments + ** may not refer to any tables to the right of this one. + ** + ** * If the table-valued function is to the left of a LEFT JOIN, then it + ** may not refer to any table that occurs to the right of the LEFT JOIN. + */ sCtx.bFuncArg = 1; for(ii=0; iipSrc->nSrc; ii++){ SrcItem *pItem = &pSelect->pSrc->a[ii]; - if( pItem->fg.isTabFunc - && (pItem->fg.jointype & JT_OUTER) - ){ - sCtx.iJoin = pItem->iCursor; - sqlite3WalkExprList(&w, pItem->u1.pFuncArg); + if( pItem->fg.isTabFunc ){ + sCtx.iJoin = -1; + if( (pItem->fg.jointype & (JT_LTORJ|JT_RIGHT)) ){ + sCtx.iJoin = pItem->iCursor; + }else{ + int jj; + for(jj=ii+1; jjpSrc->nSrc; jj++){ + SrcItem *pItem2 = &pSelect->pSrc->a[jj]; + if( pItem2->fg.jointype & JT_OUTER ){ + sCtx.iJoin = pItem2[-1].iCursor; + break; + } + } + } + if( sCtx.iJoin>=0 ){ + sqlite3WalkExprList(&w, pItem->u1.pFuncArg); + } } } } @@ -159912,11 +161192,21 @@ SQLITE_PRIVATE int sqlite3Select( ** then make the pItem->iCursor be a copy of the ephemeral table that ** holds the result of the materialization. tag-select-0484 */ CteUse *pCteUse = pItem->u2.pCteUse; + /* The materialization is computed once per run of the statement or + ** trigger program and does not change after that. So the subroutine + ** call and the OP_OpenDup are needed on the first pass only. The + ** OP_NullRow runs on every pass and leaves the duplicate cursor not + ** positioned on a row, as OP_OpenDup does. */ + int addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); if( pItem->iCursor!=pCteUse->iCur ){ sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur); VdbeComment((v, "%!S", pItem)); } + sqlite3VdbeJumpHere(v, addrOnce); + if( pItem->iCursor!=pCteUse->iCur ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pItem->iCursor); + } pSub->nSelectRow = pCteUse->nRowEst; }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in @@ -161346,6 +162636,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( DbFixer sFix; /* Fixer object */ int iDb; /* Database containing the trigger */ Token nameToken; /* Trigger name for error reporting */ + i64 n; /* Number of steps */ pParse->pNewTrigger = 0; if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; @@ -161353,10 +162644,14 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); assert( iDb>=00 && iDbnDb ); pTrig->step_list = pStepList; - while( pStepList ){ + for(n=0; pStepList; n++){ pStepList->pTrig = pTrig; pStepList = pStepList->pNext; } + if( n>pParse->db->aLimit[SQLITE_LIMIT_TRIGGER_STEPS] ){ + sqlite3ErrorMsg(pParse, "trigger \"%w\" contains too many steps", zName); + goto triggerfinish_cleanup; + } sqlite3TokenInit(&nameToken, pTrig->zName); sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) @@ -162298,6 +163593,7 @@ static TriggerPrg *codeRowTrigger( sNC.pParse = &sSubParse; sSubParse.pTriggerTab = pTab; sSubParse.pToplevel = pTop; + sSubParse.pTrigCoded = pTrigger; sSubParse.zAuthContext = pTrigger->zName; sSubParse.eTriggerOp = pTrigger->op; sSubParse.nQueryLoop = pParse->nQueryLoop; @@ -162651,26 +163947,37 @@ static void updateVirtualTable( ** integer. In that case, add an OP_RealAffinity opcode to make sure ** it has been converted into REAL. */ -SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ +static SQLITE_NOINLINE void columnDefaultUncommonCase( + Vdbe *v, /* Byte code under construction */ + Table *pTab, /* The table */ + Column *pCol /* Which column of the table */ +){ + sqlite3_value *pValue = 0; + u8 enc = ENC(sqlite3VdbeDb(v)); + Parse *pParse = sqlite3VdbeParser(v); + assert( !IsView(pTab) ); + sqlite3ValueFromExpr(sqlite3VdbeDb(v), + sqlite3ColumnExprAuth(pTab,pCol,pParse), enc, + pCol->affinity, &pValue); + if( pValue ){ + sqlite3VdbeAppendP4(v, pValue, P4_MEM); + } +} +SQLITE_PRIVATE void sqlite3ColumnDefault( + Vdbe *v, /* Byte code under construction */ + Table *pTab, /* The table */ + int i, /* Which column of the table */ + int iReg /* Register in which results are stored */ +){ Column *pCol; assert( pTab!=0 ); assert( pTab->nCol>i ); pCol = &pTab->aCol[i]; if( pCol->iDflt ){ - sqlite3_value *pValue = 0; - u8 enc = ENC(sqlite3VdbeDb(v)); - assert( !IsView(pTab) ); - VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); - assert( inCol ); - sqlite3ValueFromExpr(sqlite3VdbeDb(v), - sqlite3ColumnExpr(pTab,pCol), enc, - pCol->affinity, &pValue); - if( pValue ){ - sqlite3VdbeAppendP4(v, pValue, P4_MEM); - } + columnDefaultUncommonCase(v,pTab,pCol); } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + if( pCol->affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif @@ -163112,10 +164419,14 @@ SQLITE_PRIVATE void sqlite3Update( rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, j<0 ? "ROWID" : pTab->aCol[j].zCnName, db->aDb[iDb].zDbSName); - if( rc==SQLITE_DENY ){ - goto update_cleanup; - }else if( rc==SQLITE_IGNORE ){ - aXRef[j] = -1; + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DENY ){ + goto update_cleanup; + }else if( j<0 || j==pTab->iPKey ){ + chngRowid = 0; + }else{ + aXRef[j] = -1; + } } } #endif @@ -163496,6 +164807,7 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeAddOp3(v, OP_Column, iEph, iRowidExpr, regNewRowid); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v); + sqlite3MayAbort(pParse); } /* Compute the old pre-UPDATE content of the row being changed, if that @@ -165535,7 +166847,17 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ rc = growVTrans(db); if( rc==SQLITE_OK ){ - addToVTrans(db, sqlite3GetVTable(db, pTab)); + VTable *pVTab = sqlite3GetVTable(db, pTab); + const sqlite3_module *pModule = pMod->pModule; + int iSvpt = db->nStatement + db->nSavepoint; + addToVTrans(db, pVTab); + /* As in sqlite3VtabBegin(): open the current savepoint on the new + ** table, so that a ROLLBACK TO that undoes the CREATE also invokes + ** its xRollbackTo method. */ + if( iSvpt && pModule->iVersion>=2 && pModule->xSavepoint ){ + pVTab->iSavepoint = iSvpt; + rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1); + } } } @@ -166046,6 +167368,17 @@ SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ } } +/* +** Erase all eponymous virtual table instances associated with database +** handle db. +*/ +SQLITE_PRIVATE void sqlite3VtabEponymousTableClearAll(sqlite3 *db){ + HashElem *pElem; + for(pElem=sqliteHashFirst(&db->aModule); pElem; pElem=sqliteHashNext(pElem)){ + sqlite3VtabEponymousTableClear(db, (Module*)sqliteHashData(pElem)); + } +} + /* ** Return the ON CONFLICT resolution mode in effect for the virtual ** table update operation currently in progress. @@ -166223,6 +167556,7 @@ struct WhereLevel { int addrNxt; /* Jump here to start the next IN combination */ int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ + int addrDeleteCont; /* Delete auto-index entry then same as addrCont */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ int regBignull; /* big-null flag reg. True if a NULL-scan is needed */ @@ -167378,6 +168712,7 @@ static Expr *removeUnindexableInClauseTerms( ExprList *pRhs = 0; /* New RHS after modifications */ ExprList *pLhs = 0; /* New LHS after mods */ int i; /* Loop counter */ + int nRhs = 0; /* Number of RHS terms added so far */ assert( ExprUseXSelect(pNew) ); pOrigRhs = pSelect->pEList; @@ -167391,6 +168726,9 @@ static Expr *removeUnindexableInClauseTerms( int iField; assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); iField = pLoop->aLTerm[i]->u.x.iField - 1; + if( iField!=nRhs ){ + ExprClearProperty(pNew, EP_Subrtn); + } if( NEVER(pOrigRhs->a[iField].pExpr==0) ){ continue; /* Duplicate PK column */ } @@ -167402,8 +168740,12 @@ static Expr *removeUnindexableInClauseTerms( pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr); pOrigLhs->a[iField].pExpr = 0; } + nRhs++; } } + if( nRhsnExpr ){ + ExprClearProperty(pNew, EP_Subrtn); /* See bug 2026-10-16T22:07:15Z */ + } sqlite3ExprListDelete(db, pOrigRhs); if( pOrigLhs ){ sqlite3ExprListDelete(db, pOrigLhs); @@ -167501,8 +168843,20 @@ static SQLITE_NOINLINE void codeINTerm( }else{ sqlite3 *db = pParse->db; Expr *pXMod = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( nEq>1 ){ + /* If this IN(SELECT ...) expression drives more than one column of + ** the index, disable the seek-scan optimization. The reasons for this + ** are that (a) it is only possible to make this happen by populating + ** the sqlite_stat1 table with inconsistent information, and (b) it + ** would require sqlite3FindInIndex() to find an index that is not + ** only unique for the columns in question, but also delivers them + ** in sorted order (requires checking asc/desc, and rejecting cases + ** where the indexed columns are not in the right order). */ + pLoop->wsFlags &= ~WHERE_IN_SEEKSCAN; + } if( !db->mallocFailed ){ - aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nEq); + int nCol = pX->x.pSelect->pEList->nExpr; + aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nCol); eType = sqlite3FindInIndex(pParse, pXMod, IN_INDEX_LOOP, 0, aiMap, &iTab); } sqlite3ExprDelete(db, pXMod); @@ -167749,17 +169103,7 @@ static int codeAllEqualityTerms( testcase( pTerm->wtFlags & TERM_VIRTUAL ); r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); if( r1!=regBase+j ){ - /* If this routine is being called as part of a RIGHT JOIN loop, then - ** register r1 may be used by the body of the loop that the RIGHT JOIN - ** will jump back into (e.g. if pTerm is a sub-query). This can cause - ** problems if (say) the affinity of r1 is modified by the caller of - ** this routine. So, always take a copy of the value in this case. */ - if( nReg==1 && pParse->withinRJSubrtn==0 ){ - sqlite3ReleaseTempReg(pParse, regBase); - regBase = r1; - }else{ - sqlite3VdbeAddOp2(v, OP_Copy, r1, regBase+j); - } + sqlite3VdbeAddOp2(v, OP_Copy, r1, regBase+j); } if( pTerm->eOperator & WO_IN ){ if( pTerm->pExpr->flags & EP_xIsSelect ){ @@ -168257,6 +169601,170 @@ static int whereLoopIsOneRow(WhereLoop *pLoop){ return 0; } +/* +** This is called while coding loop pLevel, which scans FROM clause item +** pTabItem. pE is an expression for which all the prerequisites are +** available. This function tests if pE can be coded as part of the current +** loop, or whether it needs to be deferred to ensure outer joins are +** processed correctly. This function returns non-zero if the expression +** can be coded as part of the loop, or 0 if it must be deferred. +** +** The expression must be deferred if: +** +** * There are any RIGHT joins in the FROM clause and the expression +** was not part of an ON clause, or was part of an ON clause to the +** right of the join. +** +** * The table is the RHS of a LEFT JOIN and the expression was not +** part of an ON clause on an OUTER join, or was part of an ON clause +** on an OUTER join to the right of the join. +*/ +static int whereExprIsReady( + WhereInfo *pWInfo, + WhereLevel *pLevel, + SrcItem *pTabItem, + Expr *pE +){ + u8 jtype = pTabItem->fg.jointype; + if( jtype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ + if( !ExprHasProperty(pE,EP_OuterON|EP_InnerON) ){ + /* Defer processing WHERE clause constraints until after outer + ** join processing. tag-20220513a */ + return 0; + }else if( (jtype & JT_LEFT) && !ExprHasProperty(pE,EP_OuterON) ){ + return 0; + }else{ + Bitmask m = sqlite3WhereGetMask(&pWInfo->sMaskSet, pE->w.iJoin); + if( m & pLevel->notReady ){ + /* An ON clause that is not ripe */ + return 0; + } + } + } + return 1; +} + +/* +** Find the best address for an inner loop to jump to if its join +** constraint matches no rows. +** +** This is an optimization. The default behavior is to continue with +** the inner-most of the outer loops. But if the join constraint is not +** influenced by the inner-most outer loop, we might as well skip that +** loop too, and continue with the second inner-most outer loop. And +** so forth through subsequent outer loops. +** +** Return 0 to use the default behavior (continue with the inner-most +** outer loop. Or return an optimized address to jump to instead if +** we can do better. +** +** Conditions for returning non-zero: +** +** 1. The SQLITE_JoinMiss optimization must be enabled. +** +** 2. The iLevel loop must have at least two outer loops, otherwise +** this optimization is pointless. +** +** 3. We are not attempting to do one-pass +** +** 4. The join constraints may not contain subqueries +** +** If the inner-most outer loop that influences iLevel is also the +** only loop that influences iLevel, and if that loop uses an +** automatic index, then also arrange to delete the currently used +** entry from that automatic index, since we know that it can never +** be successful. +** +** This optimization inspired by the paper: "TreeTracker Join: Simple, +** Optimal, Fast" by Hu, Wang, and Miranker, 2025. +*/ +static int findNoMatchContinuation( + WhereInfo *pWInfo, /* WHERE clause */ + int iLevel /* Which loop we are talking about */ +){ + WhereLoop *pLoop; /* Description of this loop */ + WhereLevel *pParent; /* Controlling parent loop */ + SrcList *pTabList; /* Complete FROM clause */ + Bitmask mKey; /* Mask of dependent tables */ + int iParent; /* Inner-most loop affecting the key to iLevel */ + int i; /* Loop counter */ + + if( iLevel<2 ){ + /* If there are less than two outer loops, this optimization is no help */ + return 0; + } + + /* The OP_SeekGE and OP_SeekLE opcodes only work right for this + ** optimization on a pure equality constraint, so skip the optimization + ** for anything else. + */ + pLoop = pWInfo->a[iLevel].pWLoop; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=WHERE_COLUMN_EQ ) return 0; + if( pLoop->wsFlags & WHERE_SKIPSCAN ) return 0; + if( pLoop->wsFlags & WHERE_BIGNULL_SORT ) return 0; + assert( (pLoop->wsFlags & (WHERE_VIRTUALTABLE|WHERE_MULTI_OR))==0 ); + assert( pLoop->u.btree.nEq>0 ); + + /* Find a mask of all outer tables that influence the key. At the same + ** time, check to see if any keys values are computed from subqueries and + ** fail out (return 0) if there are any. */ + mKey = 0; + for(i=0; inLTerm; i++){ + if( sqlite3ExprContainsSubquery(pLoop->aLTerm[i]->pExpr) ){ + return 0; + } + mKey |= pLoop->aLTerm[i]->prereqRight; + } + + /* Look back through outer loops. Find the inner-most outer loop + ** that influences the key. Make iParent be the index of that inner-most + ** outer loop. + */ + pTabList = pWInfo->pTabList; + iParent = iLevel-1; + while( iParent>0 && (pWInfo->a[iParent].pWLoop->maskSelf & mKey)==0 ){ + iParent--; + } + assert( iParent>=0 && iParent<=iLevel ); + + /* Everything needs to be an INNER join. Fail if there are any + ** OUTER joins in between iParent and iLevel. */ + for(i=iParent; i<=iLevel; i++){ + if( pTabList->a[pWInfo->a[i].iFrom].fg.jointype & JT_OUTER ) return 0; + } + + /* If we reach this point, that means the conditions of the optimization + ** have been met. Even so, disallow the optimization if it is disabled. + */ + if( OptimizationDisabled(pWInfo->pParse->db, SQLITE_JoinMiss) ){ + return 0; + } + + /* If the inner-most outer loop that influences the key for iLevel + ** is controlled by an automatic index and if the automatic index is + ** the only outer loop that influences the key, then arrange to delete + ** the automatic index entry if there is no match on the key. + */ + pParent = &pWInfo->a[iParent]; + if( (pParent->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 + && (mKey & (mKey-1))==0 + ){ + for(i=0; inLTerm; i++){ + Expr *pX = pLoop->aLTerm[i]->pExpr; + if( !sqlite3ExprIsTableConstant(pX->pRight,pParent->iTabCur,0) ) break; + } + if( i==pLoop->nLTerm ){ + if( pParent->addrDeleteCont==0 ){ + pParent->addrDeleteCont = sqlite3VdbeMakeLabel(pWInfo->pParse); + } + return pParent->addrDeleteCont; + } + } + + testcase( iParentaddrCont; +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -168275,11 +169783,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int bRev; /* True if we need to scan in reverse order */ WhereLoop *pLoop; /* The WhereLoop object being coded */ WhereClause *pWC; /* Decomposition of the entire WHERE clause */ - WhereTerm *pTerm; /* A WHERE clause term */ - sqlite3 *db; /* Database connection */ - SrcItem *pTabItem; /* FROM clause term being coded */ - int addrBrk; /* Jump here to break out of the loop */ - int addrCont; /* Jump here to continue with next cycle */ + WhereTerm *pTerm; /* A WHERE clause term */ + sqlite3 *db; /* Database connection */ + SrcItem *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrCont; /* Jump here to continue with next cycle */ + int addrMiss; /* addrCont enhanced for no-matching rows */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ Index *pIdx = 0; /* Index used by loop (if any) */ @@ -168324,6 +169833,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( */ addrBrk = pLevel->addrNxt = pLevel->addrBrk; addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); + addrMiss = findNoMatchContinuation(pWInfo, iLevel); /* If this is the right table of a LEFT OUTER JOIN, allocate and ** initialize a memory cell that records if this table matches any @@ -168336,6 +169846,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN match flag")); + if( pTabItem->fg.viaCoroutine ){ + sqlite3VdbeAddOp2(v, OP_NullRow, pTabItem->iCursor, 1); + } } /* Special case of a FROM clause subquery implemented as a co-routine */ @@ -168496,15 +170009,16 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; + if( addrMiss==0 ) addrMiss = addrNxt; if( pLevel->regFilter ){ - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrMiss); VdbeCoverage(v); - sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, + sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrMiss, iRowidReg, 1); VdbeCoverage(v); filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady); } - sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrMiss, iRowidReg); VdbeCoverage(v); pLevel->op = OP_Noop; }else if( (pLoop->wsFlags & WHERE_IPK)!=0 @@ -168707,9 +170221,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pRangeStart!=0 ); /* LIKE opt constraints */ assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ pLevel->iLikeRepCntr = (u32)++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)pLevel->iLikeRepCntr); - VdbeComment((v, "LIKE loop counter")); - pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); /* iLikeRepCntr actually stores 2x the counter register number. The ** bottom bit indicates whether the search order is ASC or DESC. */ testcase( bRev ); @@ -168760,7 +170271,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( SWAP(u8, nBtm, nTop); } - if( iLevel>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 ){ + if( (iLevel>0 || (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0) + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 + ){ /* In case OP_SeekScan is used, ensure that the index cursor does not ** point to a valid row for the first iteration of this loop. */ sqlite3VdbeAddOp1(v, OP_NullRow, iIdxCur); @@ -168773,10 +170286,25 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd); regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( pLevel->iLikeRepCntr ){ + /* The LIKE optimization loop must be the innermost of any IN(...) + ** or skip-scan loops. Otherwise rows may not be visited in index + ** order. This means it needs an addrNxt label for the same reason + ** IN(...) scans do. */ + if( pLevel->addrNxt==pLevel->addrBrk ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + } + sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)(pLevel->iLikeRepCntr>>1)); + VdbeComment((v, "LIKE loop counter")); + pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); + } +#endif if( zStartAff && nTop ){ zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); } addrNxt = (regBignull ? pLevel->addrBignull : pLevel->addrNxt); + if( addrMiss==0 ) addrMiss = addrNxt; testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); @@ -168831,7 +170359,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } if( pLevel->regFilter ){ assert( sqlite3WhereLoopBloomable(pLoop) ); - sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, + sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrMiss, regBase, nEq); VdbeCoverage(v); filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady); @@ -168858,7 +170386,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } VdbeCoverage(v); } - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrMiss, regBase, nConstraint); VdbeCoverage(v); VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); @@ -169187,6 +170715,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; if( ExprHasProperty(pExpr, EP_Subquery) ) continue; /* tag-20220303a */ + if( whereExprIsReady(pWInfo, pLevel, pTabItem, pExpr)==0 ) continue; pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); } @@ -169427,22 +170956,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } pE = pTerm->pExpr; assert( pE!=0 ); - if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ - if( !ExprHasProperty(pE,EP_OuterON|EP_InnerON) ){ - /* Defer processing WHERE clause constraints until after outer - ** join processing. tag-20220513a */ - continue; - }else if( (pTabItem->fg.jointype & JT_LEFT)==JT_LEFT - && !ExprHasProperty(pE,EP_OuterON) ){ - continue; - }else{ - Bitmask m = sqlite3WhereGetMask(&pWInfo->sMaskSet, pE->w.iJoin); - if( m & pLevel->notReady ){ - /* An ON clause that is not ripe */ - continue; - } - } - } + if( whereExprIsReady(pWInfo, pLevel, pTabItem, pE)==0 ) continue; if( iLoop==1 && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ iNext = 2; continue; @@ -169595,9 +171109,11 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** loop will run that invokes this subroutine for unmatched rows ** of pTab, with all tables to left begin set to NULL. */ - WhereRightJoin *pRJ = pLevel->pRJ; - sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pRJ->regReturn); - pRJ->addrSubrtn = sqlite3VdbeCurrentAddr(v); + { + WhereRightJoin *pRJ = pLevel->pRJ; + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pRJ->regReturn); + pRJ->addrSubrtn = sqlite3VdbeCurrentAddr(v); + } assert( pParse->withinRJSubrtn < 255 ); pParse->withinRJSubrtn++; @@ -169715,17 +171231,25 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( pFrom->nAlloc = 1; memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); pFrom->a[0].fg.jointype = 0; - assert( pParse->withinRJSubrtn < 100 ); + if( pParse->withinRJSubrtn >= 100 ){ + /* This limit --------------^^^ + ** is based on an historical assert(). It is not compile-time or + ** run-time configurable. It could perhaps be raised as high as 254, + ** but only an attack robot would ever do even 100 RIGHT JOINS within + ** a single query, so we'll just leave it as it is. */ + sqlite3ErrorMsg(pParse, "too many RIGHT JOINs"); + return; + } pParse->withinRJSubrtn++; pSubWInfo = sqlite3WhereBegin(pParse, pFrom, pSubWhere, 0, 0, 0, WHERE_RIGHT_JOIN, 0); if( pSubWInfo ){ - int iCur = pLevel->iTabCur; - int r = ++pParse->nMem; - int nPk; - int jmp = 0; + int iCur = pLevel->iTabCur; /* Table on RHS of RIGHT JOIN &*/ + int r = ++pParse->nMem; /* Register range to hold primary key */ + int nPk; /* Number of values in the primary key */ + int r2; /* Register holding record for primary key */ int addrCont = sqlite3WhereContinueLabel(pSubWInfo); - Table *pTab = pTabItem->pSTab; + Table *pTab = pTabItem->pSTab; /* Table on RHS of RIGHT JOIN */ if( HasRowid(pTab) ){ sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r); nPk = 1; @@ -169739,13 +171263,31 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); } } + + /* Generate code that checks to see if the current row of the RHS table + ** has appeared in any prior output row. */ if( pRJ->regBloom ){ - jmp = sqlite3VdbeAddOp4Int(v, OP_Filter, pRJ->regBloom, 0, r, nPk); + sqlite3VdbeAddOp4Int(v, OP_Filter, pRJ->regBloom, + sqlite3VdbeCurrentAddr(v)+2, r, nPk); VdbeCoverage(v); } sqlite3VdbeAddOp4Int(v, OP_Found, pRJ->iMatch, addrCont, r, nPk); VdbeCoverage(v); - if( jmp ) sqlite3VdbeJumpHere(v, jmp); + r2 = sqlite3GetTempReg(pParse); + + /* Generate code that inserts the PK of the RHS table into the + ** pRH->iMatch index to indicate that the current row has appeared + ** in the output set. */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, r2); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pRJ->iMatch, r2, r, nPk); + sqlite3ReleaseTempReg(pParse, r2); + if( pRJ->regBloom ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pRJ->regBloom, 0, r, nPk); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Invoke the subroutine that actually puts the current RHS table row + ** into the output set, with NULLs for the LHS. */ sqlite3VdbeAddOp2(v, OP_Gosub, pRJ->regReturn, pRJ->addrSubrtn); sqlite3WhereEnd(pSubWInfo); } @@ -169971,7 +171513,7 @@ static int isLikeOrGlob( if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = sqlite3_value_text(pVal); } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); + sqlite3VdbeReprepareOnBind(pParse->pVdbe, iCol, 0); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ assert( !ExprHasProperty(pRight, EP_IntValue) ); @@ -170076,7 +171618,7 @@ static int isLikeOrGlob( ** reprepare the statement when that parameter is rebound */ if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); + sqlite3VdbeReprepareOnBind(v, pRight->iColumn, 0); assert( !ExprHasProperty(pRight, EP_IntValue) ); if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current @@ -170711,7 +172253,7 @@ static void exprAnalyzeOrTerm( }else{ pDup = sqlite3ExprDup(db, pLeft, 0); if( sqlite3ExprCollSeq(pParse, pDup)!=pCollSeq - && ALWAYS(pCollSeq!=0) + && pCollSeq!=0 ){ assert( pCollSeq->zName!=0 ); pDup = sqlite3ExprAddCollateString(pParse, pDup, pCollSeq->zName); @@ -171427,7 +172969,7 @@ static void whereAddLimitExpr( Expr *pNew; int iVal = 0; - if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ + if( sqlite3ExprIsInteger(pExpr, &iVal, pParse, 0) && iVal>=0 ){ Expr *pVal = sqlite3ExprInt32(db, iVal); if( pVal==0 ) return; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); @@ -171982,14 +173524,16 @@ static int whereOrInsert( } if( pSet->na[pSet->n++]; - p->nOut = nOut; }else{ + /* The set is full. Replace the most expensive entry, but only if the + ** new entry is less expensive than it. */ p = pSet->a; for(i=1; in; i++){ - if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + if( p->rRuna[i].rRun ) p = pSet->a + i; } if( p->rRun<=rRun ) return 0; } + p->nOut = nOut; whereOrInsert_done: p->prereq = prereq; p->rRun = rRun; @@ -172085,12 +173629,16 @@ static SQLITE_NOINLINE const char *indexInAffinityOk( assert( pTerm->eOperator & WO_IN ); - if( sqlite3ExprIsVector(pX->pLeft) ){ - int iField = pTerm->u.x.iField - 1; + if( ExprUseXSelect(pX) ){ + int iField = 0; inexpr.flags = 0; inexpr.op = TK_EQ; - inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; - assert( ExprUseXSelect(pX) ); + if( sqlite3ExprIsVector(pX->pLeft) ){ + iField = pTerm->u.x.iField - 1; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + }else{ + inexpr.pLeft = pX->pLeft; + } inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; pX = &inexpr; } @@ -173010,9 +174558,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** for pLevel. ** ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER -** flag set, initialize a Bloomfilter for them as well. Except don't do -** this recursive initialization if the SQLITE_BloomPulldown optimization has -** been turned off. +** flag set, initialize a Bloomfilter for them as well. ** ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared ** from the loop, but the regFilter value is set to a register that implements @@ -173121,7 +174667,6 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrTop); pLoop->wsFlags &= ~WHERE_BLOOMFILTER; - if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break; while( ++iLevel < pWInfo->nLevel ){ const SrcItem *pTabItem; pLevel = &pWInfo->a[iLevel]; @@ -174801,6 +176346,13 @@ static void whereLoopOutputAdjust( int i, j; LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ + /* Skip all this if the FROM clause of the query is a single table and + ** there is no ORDER BY. In this case it doesn't matter how accurate + ** the WhereLoop.nOut values are. */ + if( pWC->pWInfo->pTabList->nSrc<=1 && pWC->pWInfo->pOrderBy==0 ){ + return; + } + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ assert( pTerm!=0 ); @@ -174848,7 +176400,7 @@ static void whereLoopOutputAdjust( Parse *pParse = pWC->pWInfo->pParse; int k = 0; testcase( pOpExpr->op==TK_IS ); - if( sqlite3ExprIsInteger(pRight, &k, pParse) && k>=(-1) && k<=1 ){ + if( sqlite3ExprIsInteger(pRight, &k, pParse, 1) && k>=(-1) && k<=1 ){ k = 10; }else{ k = 20; @@ -175689,35 +177241,51 @@ static void wherePartIdxExpr( if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){ Expr *pLeft = pPart->pLeft; Expr *pRight = pPart->pRight; - u8 aff; + Table *pLeftTab; /* Table of the left operand */ + Column *pLeftCol; /* Column of the left operand */ + u8 affRight, affLeft; if( pLeft->op!=TK_COLUMN ) return; if( !sqlite3ExprIsConstant(0, pRight) ) return; if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; if( pLeft->iColumn<0 ) return; - aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; - if( aff>=SQLITE_AFF_TEXT ){ - if( pItem ){ - sqlite3 *db = pParse->db; - IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p)); - if( p ){ - int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0; - p->pExpr = sqlite3ExprDup(db, pRight, 0); - p->iDataCur = pItem->iCursor; - p->iIdxCur = iIdxCur; - p->iIdxCol = pLeft->iColumn; - p->bMaybeNullRow = bNullRow; - p->pIENext = pParse->pIdxPartExpr; - p->aff = aff; - pParse->pIdxPartExpr = p; - if( p->pIENext==0 ){ - void *pArg = (void*)&pParse->pIdxPartExpr; - sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); - } + pLeftTab = pIdx->pTable; + pLeftCol = &pLeftTab->aCol[pLeft->iColumn]; + affLeft = pLeftCol->affinity; + assert( SQLITE_AFF_BLOBdb; + IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p)); + if( p ){ + int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0; + pRight = sqlite3ExprDup(db, pRight, 0); + if( (pLeftTab->tabFlags & TF_Strict)!=0 ){ + Token t; + t.z = sqlite3ColumnType(pLeftCol, 0); + assert( t.z!=0 ); /* Because affLeft>=SQLITE_AFF_TEXT */ + t.n = (int)strlen(t.z); + p->pExpr = sqlite3ExprAlloc(db, TK_CAST, &t, 1); + sqlite3ExprAttachSubtrees(db, p->pExpr, pRight, 0); + }else{ + p->pExpr = pRight; + } + p->iDataCur = pItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = pLeft->iColumn; + p->bMaybeNullRow = bNullRow; + p->pIENext = pParse->pIdxPartExpr; + p->aff = affLeft; + pParse->pIdxPartExpr = p; + if( p->pIENext==0 ){ + void *pArg = (void*)&pParse->pIdxPartExpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); } - }else if( pLeft->iColumn<(BMS-1) ){ - *pMask &= ~((Bitmask)1 << pLeft->iColumn); } + }else if( pLeft->iColumn<(BMS-1) ){ + *pMask &= ~((Bitmask)1 << pLeft->iColumn); } } } @@ -175772,7 +177340,7 @@ static int whereLoopAddBtree( SrcItem *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ - int iSortIdx = 1; /* Index number */ + int iSortIdx; /* Index number */ int b; /* A boolean value */ LogEst rSize; /* number of rows in the table */ WhereClause *pWC; /* The parsed WHERE clause */ @@ -175791,7 +177359,11 @@ static int whereLoopAddBtree( /* An INDEXED BY clause specifies a particular index to use */ pProbe = pSrc->u2.pIBIndex; }else if( !HasRowid(pTab) ){ - pProbe = pTab->pIndex; + if( pSrc->fg.notIndexed ){ + pProbe = sqlite3PrimaryKeyIndex(pTab); + }else{ + pProbe = pTab->pIndex; + } }else{ /* There is no INDEXED BY clause. Create a fake Index object in local ** variable sPk to represent the rowid primary key index. Make this @@ -175810,11 +177382,7 @@ static int whereLoopAddBtree( aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pSTab->pIndex; - if( pSrc->fg.notIndexed==0 ){ - /* The real indices of the table are only considered if the - ** NOT INDEXED qualifier is omitted from the FROM clause */ - sPk.pNext = pFirst; - } + sPk.pNext = pFirst; pProbe = &sPk; } rSize = pTab->nRowLogEst; @@ -175876,8 +177444,10 @@ static int whereLoopAddBtree( /* Loop over all indices. If there was an INDEXED BY clause, then only ** consider index pProbe. */ - for(; rc==SQLITE_OK && pProbe; - pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ + for(iSortIdx=1; + rc==SQLITE_OK && pProbe; + pProbe=(pSrc->fg.isIndexedBy||pSrc->fg.notIndexed)?0:pProbe->pNext, + iSortIdx++ ){ if( pProbe->pPartIdxWhere!=0 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC, @@ -175950,7 +177520,9 @@ static int whereLoopAddBtree( pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; - if( pProbe->pPartIdxWhere ){ + if( pProbe->pPartIdxWhere + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + ){ wherePartIdxExpr( pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0 ); @@ -176595,6 +178167,8 @@ static int whereLoopAddOr( for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + && (!ExprHasProperty(pTerm->pExpr, EP_OuterON) + || pTerm->pExpr->w.iJoin==iCur) ){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; @@ -176834,13 +178408,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** direction because there is no way to run a co-routine backwards. */ static SQLITE_NOINLINE int wherePathMatchSubqueryOB( - WhereInfo *pWInfo, /* The WHERE clause */ WhereLoop *pLoop, /* The nested loop term that is a subquery */ int iLoop, /* Which level of the nested loop. 0==outermost */ int iCur, /* Cursor used by the this loop */ ExprList *pOrderBy, /* The ORDER BY clause on the whole query */ Bitmask *pRevMask, /* When loops need to go in reverse order */ - Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */ + Bitmask *pOBSat, /* Which terms of pOrderBy are satisfied so far */ + u16 wctrlFlags /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ ){ int iOB; /* Index into pOrderBy->a[] */ int jSub; /* Index into pSubOB->a[] */ @@ -176858,7 +178432,7 @@ static SQLITE_NOINLINE int wherePathMatchSubqueryOB( if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; if( pOBExpr->iTable!=iCur ) break; if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break; - if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + if( (wctrlFlags & WHERE_GROUPBY)==0 ){ u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */ u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */ if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){ @@ -177018,12 +178592,20 @@ static i8 wherePathSatisfiesOrderBy( if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ Parse *pParse = pWInfo->pParse; CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr); - CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); + const Expr *pCExpr = pTerm->pExpr; + CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pCExpr); + char affRight; assert( pColl1 ); if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){ continue; } - testcase( pTerm->pExpr->op==TK_IS ); + affRight = sqlite3ExprAffinity(pCExpr->pRight); + /* Left operand must be a column, hence always has affinity */ + assert( sqlite3ExprAffinity(pCExpr->pLeft)!=0 ); + if( affRight!=0 && affRight!=sqlite3ExprAffinity(pCExpr->pLeft) ){ + continue; /* An affinity conversion would be required */ + } + testcase( pCExpr->op==TK_IS ); } obSat |= MASKBIT(i); } @@ -177032,8 +178614,8 @@ static i8 wherePathSatisfiesOrderBy( if( pLoop->wsFlags & WHERE_IPK ){ if( pLoop->u.btree.pOrderBy && OptimizationEnabled(db, SQLITE_OrderBySubq) - && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur, - pOrderBy,pRevMask, &obSat) + && wherePathMatchSubqueryOB(pLoop,iLoop,iCur, + pOrderBy,pRevMask, &obSat, wctrlFlags) ){ nColumn = 0; isOrderDistinct = 0; @@ -177041,7 +178623,7 @@ static i8 wherePathSatisfiesOrderBy( nColumn = 1; } pIndex = 0; - nKeyCol = 0; + nKeyCol = 1; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ @@ -177182,7 +178764,14 @@ static i8 wherePathSatisfiesOrderBy( } } if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ - if( j==pLoop->u.btree.nEq ){ + if( j==pLoop->u.btree.nEq + && (jpTable)||!IsPrimaryKeyIndex(pIndex)) + ){ + /* If this is the PK of a WITHOUT ROWID table, do not use any + ** field past nKeyCol for a NULLS LAST sort. If ALTER TABLE has + ** been used to add columns to the table, this field may not + ** be present in all records, which causes problems for the seeks + ** used by NULLS LAST sorts. */ pLoop->wsFlags |= WHERE_BIGNULL_SORT; }else{ isMatch = 0; @@ -177196,7 +178785,8 @@ static i8 wherePathSatisfiesOrderBy( obSat |= MASKBIT(i); }else{ /* No match found */ - if( j==0 || j0 ); + if( jiTab>iFromIdx ) break; if( pWLoop->rRun>mxRun ) mxRun = pWLoop->rRun; } - if( ALWAYS(mxRuniCursor, 0, 0, - (const u8*)&pTabItem->colUsed, P4_INT64); + sqlite3VdbeAddOp3(v, OP_ColumnsUsed, pTabItem->iCursor, + LOWER32(pTabItem->colUsed), UPPER32(pTabItem->colUsed)); #endif if( ii>=2 && (pTabItem[0].fg.jointype & (JT_LTORJ|JT_LEFT))==0 @@ -179139,8 +180729,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; colUsed |= ((u64)1)<<(ii<63 ? ii : 63); } - sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, - (u8*)&colUsed, P4_INT64); + sqlite3VdbeAddOp3(v, OP_ColumnsUsed, iIndexCur, + LOWER32(colUsed),UPPER32(colUsed)); } #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -179357,6 +180947,17 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** successful row, it should break out of itself. */ sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); VdbeComment((v, "EXISTS break %d", i)); + }else if( pLevel->addrDeleteCont ){ + /* An inner loop matched no rows on this auto-index driven loop and + ** there are outer loops. So delete the auto-index entry to avoid + ** future unnecessary work. */ + int iBypass = sqlite3VdbeAddOp0(v, OP_Goto); + assert( pLoop->wsFlags & WHERE_AUTO_INDEX ); + sqlite3VdbeResolveLabel(v, pLevel->addrDeleteCont); + sqlite3VdbeAddOp1(v, OP_Delete, pLevel->iIdxCur); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + VdbeComment((v, "delete useless auto-index entry")); + sqlite3VdbeJumpHere(v, iBypass); } sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ @@ -179366,22 +180967,43 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); - if( pLevel->regBignull ){ - sqlite3VdbeResolveLabel(v, pLevel->addrBignull); - sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); - VdbeCoverage(v); - } #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT if( addrSeek ){ sqlite3VdbeJumpHere(v, addrSeek); addrSeek = 0; } #endif + if( pLevel->regBignull ){ + sqlite3VdbeResolveLabel(v, pLevel->addrBignull); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); + VdbeCoverage(v); + } + } + if( pLevel->addrNxt!=pLevel->addrBrk ){ + /* This occurs in three cases - (a) for a LIKE/GLOB scan that must visit + ** ranges of text and blob values separately, and (b) when there is + ** an IN(...) constraint driving the index, and (c) when a prior error + ** such as an OOM has left us in an unusual state. This is asserted() + ** in the next block, inside the #ifndef. */ + sqlite3VdbeResolveLabel(v, pLevel->addrNxt); } +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + assert( (pLevel->addrNxt==pLevel->addrBrk) + || (pLevel->addrLikeRep) /* (a) */ + || ((pLoop->wsFlags & WHERE_IN_ABLE) && pLevel->u.in.nIn>0) /* (b) */ + || pWInfo->pParse->nErr>0 /* (c) */ + ); + if( pLevel->addrLikeRep ){ + /* The LIKE optimization loop is nested inside any IN(...) and + ** skip-scan loops, so it is closed here, before either of those. */ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), + pLevel->addrLikeRep); + VdbeCoverage(v); + } +#endif if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; - sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull || pParse->db->mallocFailed ); @@ -179392,16 +181014,14 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0; if( pLevel->iLeftJoin ){ - /* For LEFT JOIN queries, cursor pIn->iCur may not have been - ** opened yet. This occurs for WHERE clauses such as - ** "a = ? AND b IN (...)", where the index is on (a, b). If - ** the RHS of the (a=?) is NULL, then the "b IN (...)" may - ** never have been coded, but the body of the loop run to - ** return the null-row. So, if the cursor is not open yet, - ** jump over the OP_Next or OP_Prev instruction about to - ** be coded. */ - sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, - sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut); + /* For LEFT JOIN queries, cursor pIn->iCur might have been + ** disabled by OP_NullRow. This occurs for WHERE clauses such + ** as "a = ? AND b IN (...)", where the index is on (a, b). + ** When the OP_IfNullRow jump is taken, the pIn->iBase register + ** is NULLed out as a side effect. But as that register is not + ** used when the jump is taken, the side-effect is harmless. */ + sqlite3VdbeAddOp3(v, OP_IfNullRow, pIn->iCur, + sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut, pIn->iBase); VdbeCoverage(v); } if( bEarlyOut ){ @@ -179435,13 +181055,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } -#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS - if( pLevel->addrLikeRep ){ - sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), - pLevel->addrLikeRep); - VdbeCoverage(v); - } -#endif if( pLevel->iLeftJoin ){ int ws = pLoop->wsFlags; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); @@ -179470,6 +181083,25 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } + if( ws & WHERE_IN_ABLE ){ + int m; + for(m=0; mu.in.nIn; m++){ + struct InLoop *pIn = &pLevel->u.in.aInLoop[m]; + if( pIn->eEndLoopOp!=OP_Noop ){ + sqlite3VdbeAddOp1(v, OP_NullRow, pIn->iCur); + } + } + } +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( pLevel->addrLikeRep ){ + /* The LIKE loop counter is initialized after the == and IN terms + ** are coded, so a NULL == term or an empty IN list can reach this + ** point without ever setting it. Set it to 0 so that the + ** OP_DecrJumpZero at the bottom of the loop falls through after + ** the NULL row instead of reading an undefined register. */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, (int)(pLevel->iLikeRepCntr>>1)); + } +#endif if( pLevel->op==OP_Return ){ sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); }else{ @@ -180003,6 +181635,7 @@ static void percent_rankInvFunc( UNUSED_PARAMETER(nArg); assert( nArg==0 ); UNUSED_PARAMETER(apArg); p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + assert( p!=0 ); p->nStep++; } static void percent_rankValueFunc(sqlite3_context *pCtx){ @@ -180048,6 +181681,7 @@ static void cume_distInvFunc( UNUSED_PARAMETER(nArg); assert( nArg==0 ); UNUSED_PARAMETER(apArg); p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + assert( p!=0 ); p->nStep++; } static void cume_distValueFunc(sqlite3_context *pCtx){ @@ -180104,6 +181738,7 @@ static void ntileInvFunc( assert( nArg==1 ); UNUSED_PARAMETER(nArg); UNUSED_PARAMETER(apArg); p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + assert( p!=0 ); p->iRow++; } static void ntileValueFunc(sqlite3_context *pCtx){ @@ -180409,11 +182044,16 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ /* If this function is being called from within a scalar sub-select ** that used by the SELECT statement being processed, only process - ** TK_COLUMN expressions that refer to it (the outer SELECT). Do - ** not process aggregates or window functions at all, as they belong - ** to the scalar sub-select. */ + ** TK_COLUMN and TK_AGGREGATE expressions that refer to it (the outer + ** SELECT). Do not process window functions at all, as they belong to the + ** scalar sub-select. */ if( p->pSubSelect ){ - if( pExpr->op!=TK_COLUMN ){ + if( pExpr->op==TK_AGG_FUNCTION ){ + /* If op2==walkerDepth, then this expression aggregates over the rows + ** of the SELECT being processed, even though it is within a scalar + ** sub-select. In that case proceed, otherwise skip this node. */ + if( pExpr->op2!=pWalker->walkerDepth ) return WRC_Continue; + }else if( pExpr->op!=TK_COLUMN ){ return WRC_Continue; }else{ int nSrc = p->pSrc->nSrc; @@ -180461,7 +182101,9 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ p->pSub = sqlite3ExprListAppend(pParse, p->pSub, pDup); } if( p->pSub ){ - int f = pExpr->flags & EP_Collate; + int f = pExpr->flags & (EP_Collate|EP_OuterON|EP_InnerON); + int iJoin = pExpr->w.iJoin; + assert( ExprHasProperty(pExpr, EP_Static)==0 ); ExprSetProperty(pExpr, EP_Static); sqlite3ExprDelete(pParse->db, pExpr); @@ -180473,6 +182115,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ pExpr->iTable = p->pWin->iEphCsr; pExpr->y.pTab = p->pTab; pExpr->flags = f; + if( (f & (EP_OuterON|EP_InnerON)) ) pExpr->w.iJoin = iJoin; } if( pParse->db->mallocFailed ) return WRC_Abort; break; @@ -180491,7 +182134,9 @@ static int selectWindowRewriteSelectCb(Walker *pWalker, Select *pSelect){ return WRC_Continue; }else{ p->pSubSelect = pSelect; + pWalker->walkerDepth++; sqlite3WalkSelect(pWalker, pSelect); + pWalker->walkerDepth--; p->pSubSelect = pSave; } return WRC_Prune; @@ -180565,7 +182210,7 @@ static ExprList *exprListAppendList( int iDummy; Expr *pSub; pSub = sqlite3ExprSkipCollateAndLikely(pDup); - if( sqlite3ExprIsInteger(pSub, &iDummy, 0) ){ + if( sqlite3ExprIsInteger(pSub, &iDummy, 0, 0) ){ pSub->op = TK_NULL; pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); pSub->u.zToken = 0; @@ -180595,8 +182240,15 @@ static int sqlite3WindowExtraAggFuncDepth(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } +/* +** Walker callback used to detect aggregate functions in the ORDER BY +** clause of an otherwise non-aggregate SELECT. +*/ static int disallowAggregatesInOrderByCb(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION && pExpr->pAggInfo==0 ){ + if( pExpr->op==TK_AGG_FUNCTION + && pExpr->pAggInfo==0 + && pExpr->op2==pWalker->walkerDepth + ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pWalker->pParse, "misuse of aggregate: %s()", pExpr->u.zToken); @@ -180643,7 +182295,9 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ sqlite3WalkSelect(&w, p); if( (p->selFlags & SF_Aggregate)==0 ){ w.xExprCallback = disallowAggregatesInOrderByCb; - w.xSelectCallback = 0; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + w.walkerDepth = 0; sqlite3WalkExprList(&w, p->pOrderBy); } @@ -183019,7 +184673,9 @@ static void updateDeleteLimitError( case TK_FLOAT: case TK_BLOB: sqlite3ExprDeferredDelete(pParse, pLeft); - return sqlite3ExprInt32(pParse->db, op==TK_NOTNULL); + p = sqlite3ExprInt32(pParse->db, op==TK_NOTNULL); + if( p ) p->op2 = 1; + return p; default: break; } @@ -187088,6 +188744,7 @@ static YYACTIONTYPE yy_reduce( SrcItem *pOld = yymsp[-3].minor.yy563->a; assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; + pOld->zName = 0; assert( pOld->fg.fixedSchema==0 ); if( pOld->fg.isSubquery ){ pNew->fg.isSubquery = 1; @@ -187107,8 +188764,13 @@ static YYACTIONTYPE yy_reduce( pOld->u1.pFuncArg = 0; pOld->fg.isTabFunc = 0; pNew->fg.isTabFunc = 1; + }else if( pOld->fg.isIndexedBy ){ + pNew->u1.zIndexedBy = pOld->u1.zIndexedBy; + pOld->u1.zIndexedBy = 0; + pOld->fg.isIndexedBy = 0; + pNew->fg.isIndexedBy = 1; } - pOld->zName = 0; + pNew->fg.notIndexed = pOld->fg.notIndexed; } sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy563); }else{ @@ -187635,7 +189297,11 @@ static YYACTIONTYPE yy_reduce( } }else{ Expr *pRHS = yymsp[-1].minor.yy402->a[0].pExpr; - if( yymsp[-1].minor.yy402->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy590->op!=TK_VECTOR ){ + if( yymsp[-1].minor.yy402->nExpr==1 + && sqlite3ExprIsConstant(pParse,pRHS) + && ExprHasProperty(pRHS, EP_Collate)==0 + && yymsp[-4].minor.yy590->op!=TK_VECTOR + ){ yymsp[-1].minor.yy402->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); @@ -190008,10 +191674,10 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; /* ** Return zero if the given SQL string is complete - if all comments, ** string and blob literals, and quoted identifiers have been closed and -** if the entire string ends with ";" and possible with ";END;" if the +** if the entire string ends with ";" and possibly with ";END;" if the ** string is a CREATE TRIGGER statement. A non-zero return indicates -** that the string is incomplete. Bits of the return value indicate -** what is missing and is needed to close out the statement. +** that the string is empty or incomplete. Bits of the return value +** indicate what is missing and is needed to close out the statement. ** ** Special handling is require for CREATE TRIGGER statements. ** Whenever the CREATE TRIGGER keywords are seen, the statement @@ -190023,11 +191689,12 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** R = 0xwwwwwwww00xxyyzz ** ** In other words, zz is the least significant byte, yy is the next -** most significant byte, xx is the third byte, wwwwwwww is a 32-bit -** value from the middle. +** most significant byte, xx is the third byte, wwwwwwww is the upper +** four bytes of the return value. ** ** zz == SQLITE_OK Input is complete ** zz == SQLITE_ERROR Input is incomplete +** zz == SQLITE_EMPTY Input contains only whitespace ** zz == SQLITE_MISUSE Input is a NULL pointer ** zz != 0 New values for zz may be added in the future ** @@ -190053,7 +191720,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** ** This implementation uses a state machine with 8 states: ** -** (0) INVALID We have not yet seen a non-whitespace character. +** (0) NOTHING We have not yet seen a non-whitespace character. ** ** (1) START At the beginning or end of an SQL statement. This routine ** returns 1 if it ends in the START state and 0 if it ends @@ -190110,7 +191777,7 @@ SQLITE_API sqlite3_int64 sqlite3_incomplete(const char *zSql){ static const u8 trans[8][8] = { /* Token: */ /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 0 NOTHING: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, @@ -190126,14 +191793,14 @@ SQLITE_API sqlite3_int64 sqlite3_incomplete(const char *zSql){ static const u8 trans[3][3] = { /* Token: */ /* State: ** SEMI WS OTHER */ - /* 0 INVALID: */ { 1, 0, 2, }, + /* 0 NOTHING: */ { 1, 0, 2, }, /* 1 START: */ { 1, 1, 2, }, /* 2 NORMAL: */ { 1, 2, 2, }, }; #endif /* SQLITE_OMIT_TRIGGER */ /* Mapping state number to yy value for the return */ static const u8 statemap[8] = { - /* 0 INVALID */ 1, + /* 0 NOTHING */ 0, /* 1 START */ 0, /* 2 NORMAL */ 1, /* 3 EXPLAIN */ 1, @@ -190185,7 +191852,7 @@ SQLITE_API sqlite3_int64 sqlite3_incomplete(const char *zSql){ } while( *zSql && *zSql!='\n' ){ zSql++; } if( *zSql==0 ){ - if( state!=1 ) pending = '-'; + pending = '-'; goto incomplete_finish; } token = tkWS; @@ -190288,11 +191955,12 @@ SQLITE_API sqlite3_int64 sqlite3_incomplete(const char *zSql){ zSql++; } incomplete_finish: + if( state==0 && pending==0 ) return SQLITE_EMPTY; if( state==1 ) nParen = 0; return (i64)((((u64)nParen)<<32) | ((u64)pending<<16) | ((u64)statemap[state]<<8) | - (state!=1)); + (state>1 || pending!=0)); } SQLITE_API int sqlite3_complete(const char *zSql){ return sqlite3_incomplete(zSql)==0; @@ -190787,7 +192455,7 @@ SQLITE_API int sqlite3_initialize(void){ ** then we have to invoke the (application-supplied) WSD initialization ** routine before doing anything else. */ #ifdef SQLITE_OMIT_WSD - rc = sqlite3_wsd_init(4096, 24); + int rc = sqlite3_wsd_init(4096, 24); if( rc!=SQLITE_OK ){ return rc; } @@ -191483,7 +193151,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ ** This is the default collating function named "BINARY" which is always ** available. */ -static int binCollFunc( +SQLITE_PRIVATE int sqlite3BinaryCompare( void *NotUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 @@ -191515,16 +193183,19 @@ static int rtrimCollFunc( const u8 *pK2 = (const u8*)pKey2; while( nKey1 && pK1[nKey1-1]==' ' ) nKey1--; while( nKey2 && pK2[nKey2-1]==' ' ) nKey2--; - return binCollFunc(pUser, nKey1, pKey1, nKey2, pKey2); + return sqlite3BinaryCompare(pUser, nKey1, pKey1, nKey2, pKey2); } /* ** Return true if CollSeq is the default built-in BINARY. */ +#if !SQLITE_USES_INLINE SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq *p){ - assert( p==0 || p->xCmp!=binCollFunc || strcmp(p->zName,"BINARY")==0 ); - return p==0 || p->xCmp==binCollFunc; + assert( p==0 || p->xCmp!=sqlite3BinaryCompare + || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==sqlite3BinaryCompare; } +#endif /* ** Another built-in collating sequence: NOCASE. @@ -192476,7 +194147,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( if( db->nVdbeActive ){ sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); - assert( !db->mallocFailed ); + assert( !db->mallocFailed || db->nVdbeExec>0 ); return SQLITE_BUSY; }else{ sqlite3ExpirePreparedStatements(db, 0); @@ -193418,6 +195089,8 @@ static const int aHardLimit[] = { SQLITE_MAX_WORKER_THREADS, SQLITE_MAX_PARSER_DEPTH, SQLITE_MAX_SCHEMA, + SQLITE_MAX_TRIGGER_STEPS, + SQLITE_MAX_CTES, }; /* @@ -193459,6 +195132,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 # error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 #endif +#if SQLITE_MAX_TRIGGER_STEPS<0 || SQLITE_MAX_TRIGGER_STEPS>65535 +# error SQLITE_MAX_TRIGGER_STEPS must be between 0 and 65535 +#endif /* @@ -193501,7 +195177,9 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); assert( aHardLimit[SQLITE_LIMIT_SCHEMA]==SQLITE_MAX_SCHEMA ); - assert( SQLITE_LIMIT_SCHEMA==(SQLITE_N_LIMIT-1) ); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_STEPS]==SQLITE_MAX_TRIGGER_STEPS ); + assert( aHardLimit[SQLITE_LIMIT_CTES]==SQLITE_MAX_CTES ); + assert( SQLITE_LIMIT_CTES==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -193993,9 +195671,12 @@ static int openDatabase( ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating ** functions: */ - createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, + sqlite3BinaryCompare, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, + sqlite3BinaryCompare, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, + sqlite3BinaryCompare, 0); createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); createCollation(db, "RTRIM", SQLITE_UTF8, 0, rtrimCollFunc, 0); if( db->mallocFailed ){ @@ -194613,9 +196294,14 @@ SQLITE_API int sqlite3_sleep(int ms){ if( pVfs==0 ) return 0; /* This function works in milliseconds, but the underlying OsSleep() - ** API uses microseconds. Hence the 1000's. + ** API uses microseconds. Hence the range limitation and the "*1000". */ - rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000); + if( ms<0 ){ + ms = 0; + }else if( ms>2147483 ){ + ms = 2147483; /* Maximum sleep time of 35 minutes and 47.484 seconds */ + } + rc = sqlite3OsSleep(pVfs, ms*1000)/1000; return rc; } @@ -195212,6 +196898,28 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_ATOF, sqlite3 *db, int iDb) + ** + ** Test access to the sqlite3ExpireStmtsUsingSchema() routine. All + ** prepared statements associated with database connection db and that + ** use schema iDb (0==main, 1==temp, etc.) are marked as expired. + ** Or if iDb is exactly -1, then expire all prepared statements. + ** + ** If iDb is out of range, this test-control is a harmless no-op. + */ + case SQLITE_TESTCTRL_EXPIRE_STMTS: { + sqlite3 *db = va_arg(ap,sqlite3*); + int iDb = va_arg(ap,int); + assert( db!=0 ); + if( iDb>=0 && iDbnDb ){ + sqlite3ExpireStmtsUsingSchema(db, iDb, 0); + }else if( iDb==(-1) ){ + sqlite3ExpirePreparedStatements(db, 0); + } + rc = SQLITE_OK; + break; + } + #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) ** @@ -196944,8 +198652,8 @@ struct Fts3Table { int nPrefix; /* Prefix length (0 for main terms index) */ Fts3Hash hPending; /* Pending terms table for this index */ } *aIndex; - int nMaxPendingData; /* Max pending data before flush to disk */ - int nPendingData; /* Current bytes of pending data */ + sqlite_int64 nMaxPendingData; /* Max pending data before flush to disk */ + sqlite_int64 nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ int iPrevLangid; /* Langid of recently inserted document */ int bPrevDelete; /* True if last operation was a delete */ @@ -198170,7 +199878,10 @@ static int fts3ContentColumns( ** nul-terminator byte. */ nCol = sqlite3_column_count(pStmt); for(i=0; ibase.pVtab; switch( nVal ){ - case 6: nToken = sqlite3_value_int(apVal[5]); + case 6: nToken = sqlite3_value_int64(apVal[5]); /* no break */ deliberate_fall_through - case 5: iCol = sqlite3_value_int(apVal[4]); + case 5: iCol = sqlite3_value_int64(apVal[4]); /* no break */ deliberate_fall_through case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); /* no break */ deliberate_fall_through @@ -200796,6 +202508,9 @@ static void fts3SnippetFunc( }else if( nToken==0 || iCol>=pTab->nColumn ){ sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + if( iCol<0 ) iCol = -1; + if( nToken<-64 ) nToken = -64; + if( nToken>64 ) nToken = 64; sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); } } @@ -203407,6 +205122,14 @@ static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ #define FTS4AUX_GE_CONSTRAINT 2 #define FTS4AUX_LE_CONSTRAINT 4 +/* +** Return true if constraint iCons of pInfo uses the "binary" collation +** sequence. Or false it it uses anything else. +*/ +static int fts3auxIsBinary(sqlite3_index_info *pInfo, int iCons){ + return 0==sqlite3_stricmp("binary", sqlite3_vtab_collation(pInfo, iCons)); +} + /* ** xBestIndex - Analyze a WHERE and ORDER BY clause. */ @@ -203434,7 +205157,7 @@ static int fts3auxBestIndexMethod( /* Search for equality and range constraints on the "term" column. ** And equality constraints on the hidden "languageid" column. */ for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable ){ + if( pInfo->aConstraint[i].usable && fts3auxIsBinary(pInfo, i) ){ int op = pInfo->aConstraint[i].op; int iCol = pInfo->aConstraint[i].iColumn; @@ -203605,7 +205328,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ /* State 3. The integer just read is a column number. */ default: assert( eState==3 ); iCol = (int)v; - if( iCol<1 || iCol>(pFts3->nColumn+1) ){ + if( iCol<1 || iCol>32767 ){ rc = SQLITE_CORRUPT_VTAB; break; } @@ -203680,7 +205403,7 @@ static int fts3auxFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - if( iEq>=0 || iGe>=0 ){ + if( (iEq>=0 || iGe>=0) && sqlite3_value_type(apVal[0])==SQLITE_TEXT ){ const unsigned char *zStr = sqlite3_value_text(apVal[0]); assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); if( zStr ){ @@ -203690,7 +205413,7 @@ static int fts3auxFilterMethod( } } - if( iLe>=0 ){ + if( iLe>=0 && sqlite3_value_type(apVal[0])==SQLITE_TEXT ){ pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); if( pCsr->zStop==0 ) return SQLITE_NOMEM; pCsr->nStop = (int)strlen(pCsr->zStop); @@ -206151,10 +207874,11 @@ static int porterNext( if( c->iOffset>iStartOffset ){ int n = c->iOffset-iStartOffset; if( n>c->nAllocated ){ + i64 nNew = n + 20; char *pNew; - c->nAllocated = n+20; - pNew = sqlite3_realloc64(c->zToken, c->nAllocated); + pNew = sqlite3_realloc64(c->zToken, nNew); if( !pNew ) return SQLITE_NOMEM; + c->nAllocated = (int)nNew; c->zToken = pNew; } porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); @@ -208194,8 +209918,8 @@ static int fts3PendingTermsAddOne( pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); if( pList ){ assert( (i64)pList->nData+(i64)nToken+(i64)sizeof(Fts3HashElem) - <= (i64)p->nPendingData ); - p->nPendingData -= (int)(pList->nData + nToken + sizeof(Fts3HashElem)); + <= p->nPendingData ); + p->nPendingData -= pList->nData + nToken + sizeof(Fts3HashElem); } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ @@ -208208,9 +209932,9 @@ static int fts3PendingTermsAddOne( } } if( rc==SQLITE_OK ){ - assert( (i64)p->nPendingData + pList->nData + nToken + assert( p->nPendingData + pList->nData + nToken + sizeof(Fts3HashElem) <= 0x3fffffff ); - p->nPendingData += (int)(pList->nData + nToken + sizeof(Fts3HashElem)); + p->nPendingData += pList->nData + nToken + sizeof(Fts3HashElem); } return rc; } @@ -212682,38 +214406,42 @@ static u64 fts3ChecksumIndex( rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); } - if( rc==SQLITE_OK ){ - while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ - char *pCsr = csr.aDoclist; - char *pEnd = &pCsr[csr.nDoclist]; + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p,&csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; - i64 iDocid = 0; - i64 iCol = 0; - u64 iPos = 0; + i64 iDocid = 0; + int iCol = 0; + u64 iPos = 0; - pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); - while( pCsrbDescIdx ){ + iDocid = (i64)((u64)iDocid - iVal); }else{ - pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal); - if( p->bDescIdx ){ - iDocid = (i64)((u64)iDocid - iVal); - }else{ - iDocid = (i64)((u64)iDocid + iVal); - } + iDocid = (i64)((u64)iDocid + iVal); } + } + }else{ + iPos += (iVal - 2); + if( iPos>0x7FFFFFFF ){ + rc = SQLITE_CORRUPT_VTAB; + break; }else{ - iPos += (iVal - 2); cksum = cksum ^ fts3ChecksumEntry( csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, - (int)iCol, (int)iPos + iCol, (int)iPos ); } } @@ -212740,6 +214468,7 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ u64 cksum1 = 0; /* Checksum based on FTS index contents */ u64 cksum2 = 0; /* Checksum based on %_content contents */ sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ + int bContentless = (p->zContentTbl && p->zContentTbl[0]=='\0'); /* This block calculates the checksum according to the FTS index. */ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); @@ -212759,7 +214488,7 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ } /* This block calculates the checksum according to the %_content table */ - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && !bContentless ){ sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; sqlite3_stmt *pStmt = 0; char *zSql; @@ -212817,7 +214546,7 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ rc = SQLITE_OK; *pbOk = 0; }else{ - *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + *pbOk = (rc==SQLITE_OK && (bContentless || cksum1==cksum2)); } return rc; } @@ -213806,6 +215535,7 @@ static int fts3BestSnippet( sqlite3_int64 nByte; /* Number of bytes of space to allocate */ int iBestScore = -1; /* Best snippet score found so far */ int i; /* Loop counter */ + Fts3Expr *pExpr = pCsr->pExpr; memset(&sIter, 0, sizeof(sIter)); @@ -213834,9 +215564,12 @@ static int fts3BestSnippet( sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - rc = sqlite3Fts3ExprIterate( - pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter - ); + rc = sqlite3Fts3ExprIterate(pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ + /* Iterate through the expression twice, in case pointers garnered during + ** the first iteration are invalidated by a call to fts5EvalRestart(). */ + rc = sqlite3Fts3ExprIterate(pExpr, fts3SnippetFindPositions, (void*)&sIter); + } if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ @@ -214745,9 +216478,8 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( return; } - /* Limit the snippet length to 64 tokens. */ - if( nToken<-64 ) nToken = -64; - if( nToken>+64 ) nToken = +64; + /* The snippet length should already have been limited to 64 tokens. */ + assert( nToken>=-64 && nToken<=64 ); for(nSnippet=1; 1; nSnippet++){ @@ -215084,13 +216816,11 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( /* #include "fts3_tokenizer.h" */ /* -** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied -** from the sqlite3 source file utf.c. If this file is compiled as part -** of the amalgamation, they are not required. +** The following two macros - FTS3_READ_UTF8 and FTS3_WRITE_UTF8 - have +** been copied from the sqlite3 source file utf.c. */ -#ifndef SQLITE_AMALGAMATION -static const unsigned char sqlite3Utf8Trans1[] = { +static const unsigned char fts3Utf8Trans1[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, @@ -215101,10 +216831,10 @@ static const unsigned char sqlite3Utf8Trans1[] = { 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, }; -#define READ_UTF8(zIn, zTerm, c) \ +#define FTS3_READ_UTF8(zIn, zTerm, c) \ c = *(zIn++); \ if( c>=0xc0 ){ \ - c = sqlite3Utf8Trans1[c-0xc0]; \ + c = fts3Utf8Trans1[c-0xc0]; \ while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ c = (c<<6) + (0x3f & *(zIn++)); \ } \ @@ -215113,28 +216843,26 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } -#define WRITE_UTF8(zOut, c) { \ - if( c<0x00080 ){ \ - *zOut++ = (u8)(c&0xFF); \ - } \ - else if( c<0x00800 ){ \ - *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ - else if( c<0x10000 ){ \ - *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - }else{ \ - *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ - *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ +#define FTS3_WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ } -#endif /* ifndef SQLITE_AMALGAMATION */ - typedef struct unicode_tokenizer unicode_tokenizer; typedef struct unicode_cursor unicode_cursor; @@ -215200,7 +216928,7 @@ static int unicodeAddExceptions( assert( bAlnum==0 || bAlnum==1 ); while( zeRemoveDiacritic); if( iOut ){ - WRITE_UTF8(zOut, iOut); + FTS3_WRITE_UTF8(zOut, iOut); } /* If the cursor is not at EOF, read the next character */ if( z>=zTerm ) break; - READ_UTF8(z, zTerm, iCode); + FTS3_READ_UTF8(z, zTerm, iCode); }while( unicodeIsAlnum(p, (int)iCode) || sqlite3FtsUnicodeIsdiacritic((int)iCode) ); @@ -216242,6 +217970,7 @@ struct JsonParse { */ #define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ #define JSON_KEEPERROR 0x02 /* Return non-NULL even if there is an error */ +#define JSON_NOCACHE 0x04 /* Do not use the JSON cache */ /************************************************************************** ** Forward references @@ -217959,12 +219688,125 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ } } -/* The byte at index i is a node type-code. This routine +/* +** This a helper routine for jsonbPayloadSize() and +** jsonbPayloadSizeSemiInline(). This routine is called using tail +** recursion to handle the (relatively uncommon) cases where the +** the payload size is a 2, 4, or 8 byte integer. +*/ +static u32 jsonbPayloadSizeWide( + const JsonParse *pParse, /* JSON parsing context */ + u32 i, /* Index of the node type-code */ + u32 *pSz, /* Write payload size here */ + u8 x /* pParse->aBlob[i]>>4 */ +){ + u32 sz; + u32 n; + assert( inBlob ); + assert( x==(pParse->aBlob[i]>>4) ); + assert( x>=13 ); + if( x==13 ){ + if( i+2>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 3; + }else if( x==14 ){ + if( i+4>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = ((u32)pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 5; + }else{ + if( i+8>=pParse->nBlob + || pParse->aBlob[i+1]!=0 + || pParse->aBlob[i+2]!=0 + || pParse->aBlob[i+3]!=0 + || pParse->aBlob[i+4]!=0 + ){ + *pSz = 0; + return 0; + } + sz = ((u32)pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; + n = 9; + } + testcase( (i64)i+sz+n > pParse->nBlob ); + testcase( (i64)i+sz+n > pParse->nBlob-pParse->delta + && (i64)i+sz+n > pParse->nBlob ); + testcase( pParse->delta>0 ); + testcase( pParse->delta<0 ); + /* Quirks with -Os and gcov cause the NO_TEST line below to show up as + ** false-positive coverage miss. The testcases() macros above are + ** sufficient to prove that the branch is in fact covered */ + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta /*NO_TEST*/ + ){ + *pSz = 0; + return 0; + } + *pSz = sz; + return n; +} + +/* +** This is the main routine for determining the size of a node in JSONB. +** The jsonbPayloadSizeWide() above is a helper. The two routines +** jsonbPayloadSizeInline() and jsonbPayloadSizeSemiInline() below are +** optional optimizations. It is important to keep all these routines in +** sync. Agents reading this code: Help us humans to remember that! +** +** The byte at index i is a node type-code. This routine ** determines the payload size for that node and writes that ** payload size in to *pSz. It returns the offset from i to the ** beginning of the payload. Return 0 on error. */ -static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ +static u32 jsonbPayloadSize( + const JsonParse *pParse, /* JSON parsing context */ + u32 i, /* Index of the node type-code */ + u32 *pSz /* Write payload size here */ +){ + u8 x; + u32 sz; + u32 n; + if( i>=pParse->nBlob ){ + *pSz = 0; + return 0; + }else if( (x = pParse->aBlob[i]>>4)<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else{ + return jsonbPayloadSizeWide(pParse, i, pSz, x); + } + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta + ){ + *pSz = 0; + return 0; + } + *pSz = sz; + return n; +} + +#if SQLITE_USES_INLINE +/* A separate inline version of jsonbPayloadSize(), used in one particulary +** performance-critical place. +*/ +static SQLITE_INLINE u32 jsonbPayloadSizeInline( + const JsonParse *pParse, /* JSON parsing context */ + u32 i, /* Index of the node type-code */ + u32 *pSz /* Write payload size here */ +){ u8 x; u32 sz; u32 n; @@ -218019,7 +219861,61 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ *pSz = sz; return n; } +#else /* if !SQLITE_USES_INLINE */ + /* On compilers that do not support in-lining, just call the original + ** jsonbPayloadSize(). It's slower, but it gets the right answer. */ +# define jsonbPayloadSizeInline(a,b,c) jsonbPayloadSize(a,b,c) +#endif + +#if SQLITE_USES_INLINE +/* A separate inline version of jsonbPayloadSize() that implements the more +** common paths inline but then calls out to a subroutine for the uncommon +** paths. +** +** * It is guaranteed that the i parameter is a valid index for +** pParse->aBlob[]. There is no possibility of running of the end +** of the allocation. +** +** * Large sizes (greater than 255) are uncommon and are handled +** by a subroutine call to the jsonbPayloadSizeWide(). +*/ +static SQLITE_INLINE u32 jsonbPayloadSizeSemiInline( + const JsonParse *pParse, /* JSON parsing context */ + u32 i, /* Index of the node type-code */ + u32 *pSz /* Write payload size here */ +){ + u8 x; + u32 sz; + u32 n; + assert( inBlob ); + if( (x = pParse->aBlob[i]>>4)<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else{ + return jsonbPayloadSizeWide(pParse,i,pSz,x); + } + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta + ){ + *pSz = 0; + return 0; + } + *pSz = sz; + return n; +} +#else /* if !SQLITE_USES_INLINE */ + /* On compilers that do not support in-lining, just call the original + ** jsonbPayloadSize(). It's slower, but it gets the right answer. */ +# define jsonbPayloadSizeSemiInline(a,b,c) jsonbPayloadSize(a,b,c) +#endif /* ** Translate the binary JSONB representation of JSON beginning at @@ -218041,7 +219937,7 @@ static u32 jsonTranslateBlobToText( ){ u32 sz, n, j, iEnd; - n = jsonbPayloadSize(pParse, i, &sz); + n = jsonbPayloadSizeInline(pParse, i, &sz); if( n==0 ){ pOut->eErr |= JSTRING_MALFORMED; return pParse->nBlob+1; @@ -218321,7 +220217,10 @@ static u32 jsonTranslateBlobToPrettyText( while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToPrettyText(pPretty, j); - if( j>=iEnd ) break; + if( j>=iEnd ){ + if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + break; + } jsonAppendRawNZ(pOut, ",\n", 2); } jsonAppendChar(pOut, '\n'); @@ -218381,7 +220280,7 @@ static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ n = jsonbPayloadSize(pParse, iRoot, &sz); iEnd = iRoot+n+sz; for(i=iRoot+n; n>0 && iaBlob[j] & 0x0f; if( xJSONB_TEXTRAW ) return JSON_LOOKUP_ERROR; - n = jsonbPayloadSize(pParse, j, &sz); + n = jsonbPayloadSizeSemiInline(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; k = j+n; /* k is the index of the label text */ if( k+sz>=iEnd ) return JSON_LOOKUP_ERROR; @@ -218913,7 +220812,7 @@ static u32 jsonLookupStep( } j = k+sz; if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; - n = jsonbPayloadSize(pParse, j, &sz); + n = jsonbPayloadSizeSemiInline(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } @@ -219002,7 +220901,7 @@ static u32 jsonLookupStep( return rc; } kk--; - n = jsonbPayloadSize(pParse, j, &sz); + n = jsonbPayloadSizeSemiInline(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } @@ -219176,7 +221075,12 @@ static void jsonReturnFromBlob( if( c=='\\' ){ u32 v; u32 szEscape = jsonUnescapeOneChar(&z[iIn], sz-iIn, &v); - if( v<=0x7f ){ + if( v==0 && iIn+szEscape==sz + && (z[iIn+1]=='\n' || z[iIn+1]=='\r' || (u8)z[iIn+1]==0xe2) + ){ + /* A JSON5 line continuation at the end of the string + ** contributes no characters */ + }else if( v<=0x7f ){ zOut[iOut++] = (char)v; }else if( v<=0x7ff ){ assert( szEscape>=2 ); @@ -219515,7 +221419,9 @@ static JsonParse *jsonParseFuncArg( if( eType==SQLITE_NULL ){ return 0; } - pFromCache = jsonCacheSearch(ctx, pArg); + if( (flgs & JSON_NOCACHE)==0 ){ + pFromCache = jsonCacheSearch(ctx, pArg); + } if( pFromCache ){ pFromCache->nJPRef++; if( (flgs & JSON_EDITABLE)==0 ){ @@ -220590,7 +222496,7 @@ static void jsonValidFunc( if( (flags & 0x3)==0 ) break; memset(&px, 0, sizeof(px)); - p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); + p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR|JSON_NOCACHE); if( p ){ if( p->oom ){ sqlite3_result_error_nomem(ctx); @@ -221183,17 +223089,16 @@ static int jsonEachPathLength(JsonEachCursor *p){ u32 n = p->path.nUsed; char *z = p->path.zBuf; if( p->iRowid==0 && p->bRecursive && n>=2 ){ - while( n>1 ){ + while( ALWAYS(n>1) ){ n--; if( z[n]=='[' || z[n]=='.' ){ - u32 x, sz = 0; + u32 x; char cSaved = z[n]; z[n] = 0; assert( p->sParse.eEdit==0 ); x = jsonLookupStep(&p->sParse, 0, z+1, 0); z[n] = cSaved; - if( JSON_LOOKUP_ISERROR(x) ) continue; - if( x + jsonbPayloadSize(&p->sParse, x, &sz) == p->i ) break; + if( !JSON_LOOKUP_ISERROR(x) ) break; } } } @@ -221281,7 +223186,7 @@ static int jsonEachColumn( break; } default: { - sqlite3_result_text(ctx, p->path.zBuf, p->nRoot, SQLITE_STATIC); + sqlite3_result_text(ctx, p->path.zBuf, p->nRoot, SQLITE_TRANSIENT); break; } case JEACH_JSON: { @@ -222214,6 +224119,18 @@ static int writeInt64(u8 *p, i64 i){ return 8; } +/* +** Translate pVal into a 32-bit integer. If pVal is an integer +** that is larger than 32 bits, the cap it at the largest or +** smallest 32-bit integer. +*/ +static int rtreeValueInt32(sqlite3_value *pVal){ + i64 v64 = sqlite3_value_int64(pVal); + if( v64>2147483647 ) return 2147483647; + if( v64<-2147483648LL ) return (int)-2147483648LL; + return (int)v64; +} + /* ** Increment the reference count of node p. */ @@ -222494,7 +224411,17 @@ static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ sqlite3_bind_null(p, 2); if( pNode->iNode==0 && rc==SQLITE_OK ){ pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); - nodeHashInsert(pRtree, pNode); + if( pNode->iNode==0 ){ + /* If the SQL statement has succeeded but sqlite3_last_insert_rowid() + ** returns 0, then the shadow table schema has been corrupted somehow + ** (e.g. the %_node table has been replaced by a WITHOUT ROWID table). + ** It would be dangerous to continue in this case as the hash table + ** implementation assumes iNode==0 means that the node is not part + ** of the hash table. */ + rc = SQLITE_CORRUPT_VTAB; + }else{ + nodeHashInsert(pRtree, pNode); + } } } return rc; @@ -223372,6 +225299,10 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( NEVER(p==0) ) return SQLITE_OK; if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ + /* If this is an UPDATE that does not assign a value to the first + ** column, leave it as a "no-change" value. This allows rtreeUpdate() + ** to take the new rowid from aData[1] instead. */ + if( sqlite3_vtab_nochange(ctx) ) return SQLITE_OK; sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else if( i<=pRtree->nDim2 ){ nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); @@ -223470,6 +225401,20 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); +/* +** Argument pVal is the right-hand side of a "rowid = ?" constraint. If +** there is an integer that compares equal to pVal, set *piRowid to it and +** return 1. Otherwise - pVal is NULL, a blob, a non-numeric text value or +** a real value that is not an integer - return 0. +*/ +static int rtreeRowidEqValue(sqlite3_value *pVal, i64 *piRowid){ + int eType = sqlite3_value_numeric_type(pVal); + *piRowid = sqlite3_value_int64(pVal); + return eType==SQLITE_INTEGER + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(*piRowid, sqlite3_value_double(pVal))); +} + /* ** Rtree virtual table module xFilter method. */ @@ -223495,13 +225440,9 @@ static int rtreeFilter( /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ RtreeSearchPoint *p; /* Search point for the leaf */ - i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iRowid; i64 iNode = 0; - int eType = sqlite3_value_numeric_type(argv[0]); - if( eType==SQLITE_INTEGER - || (eType==SQLITE_FLOAT - && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) - ){ + if( rtreeRowidEqValue(argv[0], &iRowid) ){ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); }else{ rc = SQLITE_OK; @@ -224617,12 +226558,6 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ return rc; } -/* -** Rounding constants for float->double conversion. -*/ -#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ -#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ - #if !defined(SQLITE_RTREE_INT_ONLY) /* ** Convert an sqlite3_value into an RtreeValue (presumably a float) @@ -224632,7 +226567,10 @@ static RtreeValue rtreeValueDown(sqlite3_value *v){ double d = sqlite3_value_double(v); float f = (float)d; if( f>d ){ - f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); + unsigned int x; + memcpy(&x, &f, 4); + x = (x & 0x80000000)!=0 ? x+1 : x-1; + memcpy(&f, &x, 4); } return f; } @@ -224640,12 +226578,40 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){ double d = sqlite3_value_double(v); float f = (float)d; if( f1 ){ int ii; int nn = nData - 4; + sqlite3_value *pNewRowid; /* Value of the new rowid, or NULL */ if( nn > pRtree->nDim2 ) nn = pRtree->nDim2; /* Populate the cell.aCoord[] array. The first coordinate is aData[3]. @@ -224758,8 +226725,8 @@ static int rtreeUpdate( #endif { for(ii=0; iicell.aCoord[ii+1].i ){ rc = rtreeConstraintError(pRtree, ii+1); goto constraint; @@ -224767,10 +226734,24 @@ static int rtreeUpdate( } } + /* The new rowid is normally the value of the first column, aData[2]. + ** But SQLite does not treat that column as an alias for the rowid, + ** so a new rowid assigned using the name "rowid", "oid" or "_rowid_" + ** is passed in aData[1]. Use aData[1] if aData[2] is a "no-change" + ** value (an UPDATE that does not assign the first column - see + ** rtreeColumn()), or if aData[2] is NULL in an INSERT. */ + pNewRowid = aData[2]; + if( sqlite3_value_nochange(pNewRowid) + || (sqlite3_value_type(aData[0])==SQLITE_NULL + && sqlite3_value_type(pNewRowid)==SQLITE_NULL) + ){ + pNewRowid = aData[1]; + } + /* If a rowid value was supplied, check if it is already present in ** the table. If so, the constraint has failed. */ - if( sqlite3_value_type(aData[2])!=SQLITE_NULL ){ - cell.iRowid = sqlite3_value_int64(aData[2]); + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL ){ + cell.iRowid = sqlite3_value_int64(pNewRowid); if( sqlite3_value_type(aData[0])==SQLITE_NULL || sqlite3_value_int64(aData[0])!=cell.iRowid ){ @@ -225361,12 +227342,14 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ int nData; int errCode; sqlite3_str *pOut; + i64 nDim64; UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); - tree.nDim = (u8)sqlite3_value_int(apArg[0]); - if( tree.nDim<1 || tree.nDim>5 ) return; + nDim64 = sqlite3_value_int64(apArg[0]); + if( nDim64<1 || nDim64>5 ) return; + tree.nDim = (u8)nDim64; tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); @@ -226164,7 +228147,7 @@ static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ break; } if( geopolySkipSpace(&s)==']' - && s.nVertex>=4 + && s.nVertex>=4 && s.nVertex<16777216 && s.a[0]==s.a[s.nVertex*2-2] && s.a[1]==s.a[s.nVertex*2-1] && (s.z++, geopolySkipSpace(&s)==0) @@ -226188,6 +228171,8 @@ static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ s.nErr++; rc = SQLITE_ERROR; } + }else{ + rc = SQLITE_ERROR; } parse_json_err: if( pRc ) *pRc = rc; @@ -226238,9 +228223,11 @@ static GeoPoly *geopolyFuncParam( } p->hdr[0] ^= 1; } + if( pRc ) *pRc = SQLITE_OK; } + }else{ + if( pRc ) *pRc = SQLITE_ERROR; } - if( pRc ) *pRc = SQLITE_OK; return p; }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ const unsigned char *zJson = sqlite3_value_text(pVal); @@ -226298,7 +228285,7 @@ static void geopolyJsonFunc( sqlite3_str_appendf(x, "[%!g,%!g],", GeoX(p,i), GeoY(p,i)); } sqlite3_str_appendf(x, "[%!g,%!g]]", GeoX(p,0), GeoY(p,0)); - sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_result_str(context, x, SQLITE_FINISH); sqlite3_free(p); } } @@ -226335,7 +228322,7 @@ static void geopolySvgFunc( } } sqlite3_str_appendf(x, ">"); - sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_result_str(context, x, SQLITE_FINISH); sqlite3_free(p); } } @@ -226490,6 +228477,8 @@ static double geopolySine(double r){ ** ** Construct a simple, convex, regular polygon centered at X, Y ** with circumradius R and with N sides. +** +** Maximum N is 1000. Maximum R is 1.0e+300. */ static void geopolyRegularFunc( sqlite3_context *context, @@ -226499,12 +228488,12 @@ static void geopolyRegularFunc( double x = sqlite3_value_double(argv[0]); double y = sqlite3_value_double(argv[1]); double r = sqlite3_value_double(argv[2]); - int n = sqlite3_value_int(argv[3]); + i64 n = sqlite3_value_int64(argv[3]); int i; GeoPoly *p; (void)argc; - if( n<3 || r<=0.0 ) return; + if( n<3 || r<=0.0 || r>=1e300 ) return; if( n>1000 ) n = 1000; p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) ); if( p==0 ){ @@ -227293,9 +229282,13 @@ static int geopolyFilter( /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ RtreeSearchPoint *p; /* Search point for the leaf */ - i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iRowid; i64 iNode = 0; - rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rtreeRowidEqValue(argv[0], &iRowid) ){ + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + }else{ + pLeaf = 0; + } if( rc==SQLITE_OK && pLeaf!=0 ){ p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); assert( p!=0 ); /* Always returns pCsr->sPoint */ @@ -227525,6 +229518,8 @@ static int geopolyUpdate( i64 newRowid; /* The new rowid */ int newRowidValid; /* True if newRowid is valid */ int coordChange = 0; /* Change in coordinates */ + sqlite3_value *pShape = 0; /* New _shape value */ + sqlite3_value *pOldShape = 0; /* Copy of the unchanged _shape */ if( pRtree->nNodeRef ){ /* Unable to write to the btree while another cursor is reading from it, @@ -227541,13 +229536,38 @@ static int geopolyUpdate( newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0; cell.iRowid = newRowid; + if( nData>1 ){ + pShape = aData[2]; + if( oldRowidValid && oldRowid!=newRowid && sqlite3_value_nochange(pShape) ){ + /* The rowid changes but _shape does not. aData[2] holds no value in + ** this case, so read the current _shape from the %_rowid table. It + ** is needed for the bounding box of the new r-tree entry and for the + ** %_rowid record of the new rowid. */ + sqlite3_stmt *pRead = 0; + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pRead, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pRead, 1, oldRowid); + if( sqlite3_step(pRead)==SQLITE_ROW ){ + pOldShape = sqlite3_value_dup(sqlite3_column_value(pRead, 2)); + if( pOldShape==0 ) rc = SQLITE_NOMEM; + pShape = pOldShape; + } + rc2 = sqlite3_finalize(pRead); + if( rc==SQLITE_OK ) rc = rc2; + } + if( rc ) goto geopoly_update_end; + } + } + if( nData>1 /* not a DELETE */ && (!oldRowidValid /* INSERT */ - || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */ + || !sqlite3_value_nochange(pShape) /* UPDATE _shape */ || oldRowid!=newRowid) /* Rowid change */ ){ - assert( aData[2]!=0 ); - geopolyBBox(0, aData[2], cell.aCoord, &rc); + assert( pShape!=0 ); + geopolyBBox(0, pShape, cell.aCoord, &rc); if( rc ){ if( rc==SQLITE_ERROR ){ pVtab->zErrMsg = @@ -227613,17 +229633,17 @@ static int geopolyUpdate( int nChange = 0; sqlite3_bind_int64(pUp, 1, cell.iRowid); assert( pRtree->nAux>=1 ); - if( sqlite3_value_nochange(aData[2]) ){ + if( sqlite3_value_nochange(pShape) ){ sqlite3_bind_null(pUp, 2); }else{ GeoPoly *p = 0; - if( sqlite3_value_type(aData[2])==SQLITE_TEXT - && (p = geopolyFuncParam(0, aData[2], &rc))!=0 + if( sqlite3_value_type(pShape)==SQLITE_TEXT + && (p = geopolyFuncParam(0, pShape, &rc))!=0 && rc==SQLITE_OK ){ sqlite3_bind_blob(pUp, 2, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); }else{ - sqlite3_bind_value(pUp, 2, aData[2]); + sqlite3_bind_value(pUp, 2, pShape); } sqlite3_free(p); nChange = 1; @@ -227639,6 +229659,7 @@ static int geopolyUpdate( } geopoly_update_end: + sqlite3_value_free(pOldShape); rtreeRelease(pRtree); return rc; } @@ -227769,6 +229790,12 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "rtreecheck", -1, utf8, 0,rtreecheck, 0,0); } +#if defined(SQLITE_DEBUG) && !defined(SQLITE_RTREE_INT_ONLY) + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "rtree_round", 2, utf8, 0, + rtreeRoundFunc, 0, 0); + } +#endif /* SQLITE_DEBUG && !SQLITE_RTREE_INT_ONLY */ if( rc==SQLITE_OK ){ #ifdef SQLITE_RTREE_INT_ONLY void *c = (void *)RTREE_COORD_INT32; @@ -230853,15 +232880,27 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ /* Check that all non-HIDDEN columns in the destination table are also ** present in the input table. Populate the abTblPk[], azTblType[] and - ** aiTblOrder[] arrays at the same time. */ + ** aiTblOrder[] arrays at the same time. + ** + ** RBU does not support tables with GENERATED columns. If the target + ** table has any, return an error. */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, - sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + sqlite3_mprintf("PRAGMA table_xinfo(%Q)", pIter->zTbl) ); } while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + int eHidden = sqlite3_column_int(pStmt, 6); + if( zName==0 ) break; /* An OOM. Bail out */ + if( eHidden==1 ) continue; /* Hidden column. Ignore */ + if( eHidden!=0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "generated columns not supported: %s.%s", pIter->zTbl, zName + ); + break; + } for(i=iOrder; inTblCol; i++){ if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; } @@ -231714,7 +233753,7 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ char c = zSql[i]; /* If necessary, grow the pIter->aIdxCol[] array */ - if( iIdxCol==nIdxAlloc ){ + if( iIdxCol>=(nIdxAlloc-1) ){ RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64( pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan) ); @@ -232314,7 +234353,7 @@ static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ p->nRbu = 0; p->pRbuFd = 0; rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); - if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( rc!=SQLITE_NOTFOUND || p->pRbuFd==0 ) p->rc = rc; if( p->eStage>=RBU_STAGE_MOVE ){ bOpen = 1; }else{ @@ -235516,12 +237555,14 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ while( p->iCellnCell ){ StatCell *pCell = &p->aCell[p->iCell]; while( pCell->iOvflnOvfl ){ - int nUsable, iOvfl; + int nUsable; + int iOvfl = pCell->iOvfl; sqlite3BtreeEnter(pBt); nUsable = sqlite3BtreeGetPageSize(pBt) - sqlite3BtreeGetReserveNoMutex(pBt); sqlite3BtreeLeave(pBt); pCsr->nPage++; + pCsr->iPageno = pCell->aOvfl[iOvfl]; statSizeAndOffset(pCsr); if( pCell->iOvflnOvfl-1 ){ pCsr->nPayload += nUsable - 4; @@ -235529,11 +237570,9 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ pCsr->nPayload += pCell->nLastOvfl; pCsr->nUnused += nUsable - 4 - pCell->nLastOvfl; } - iOvfl = pCell->iOvfl; pCell->iOvfl++; if( !pCsr->isAgg ){ pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); - pCsr->iPageno = pCell->aOvfl[iOvfl]; pCsr->zPagetype = "overflow"; pCsr->zPath = z = sqlite3_mprintf( "%s%.3x+%.6x", p->zPath, p->iCell, iOvfl @@ -236537,8 +238576,14 @@ static int carrayColumn( default: { const struct iovec *p = (struct iovec*)pCur->pPtr; assert( pCur->eType==CARRAY_BLOB ); - sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base, - (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); + p += pCur->iRowid-1; + if( p->iov_len>0x7fffffff ){ + sqlite3_result_error_toobig(ctx); + return SQLITE_TOOBIG; + }else{ + sqlite3_result_blob(ctx, p->iov_base, (int)p->iov_len, + SQLITE_TRANSIENT); + } return SQLITE_OK; } } @@ -236802,7 +238847,7 @@ SQLITE_API int sqlite3_carray_bind_v2( } } - pNew->aData = sqlite3_malloc64( sz ); + pNew->aData = sqlite3_malloc64( sz>0 ? sz : 1 ); if( pNew->aData==0 ){ rc = SQLITE_NOMEM; goto carray_bind_error; @@ -236833,7 +238878,7 @@ SQLITE_API int sqlite3_carray_bind_v2( z += n; memcpy(p[i].iov_base, ((struct iovec*)aData)[i].iov_base, n); } - }else{ + }else if( sz>0 ){ memcpy(pNew->aData, aData, sz); } pNew->xDel = sqlite3_free; @@ -237532,7 +239577,8 @@ static int sessionSerialLen(const u8 *a){ e = *a; if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; if( e==SQLITE_TEXT || e==SQLITE_BLOB ){ - return sessionVarintGet(&a[1], &n) + 1 + n; + const int v = sessionVarintGet(&a[1], &n); + return v + 1 + n; } return 1; } @@ -239752,6 +241798,33 @@ static int sessionPrepare( return rc; } +/* +** Prepare the statement specified by printf format zFmt and its trailing +** arguments. +*/ +static void sessionPrepareMprintf( + int *pRc, + sqlite3 *db, + sqlite3_stmt **pp, + char **pzErrmsg, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zSql; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sessionPrepare(db, pp, pzErrmsg, zSql); + sqlite3_free(zSql); + } + va_end(ap); + } +} + /* ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. @@ -240830,9 +242903,15 @@ static int sessionChangesetNextOne( ** ** Update for bedrock branch: Do not do this for changebatch_add() on ** this branch, as changesets generated by sqlite3sessions_fullchangeset() - ** also have this property. */ + ** also have this property. + ** + ** Also check that all PK fields are present in the old.* record. If + ** they are not, the changeset is corrupt. */ if( p->bChangebatch==0 && p->bPatchset==0 && p->op==SQLITE_UPDATE){ for(i=0; inCol; i++){ + if( p->abPK[i] && p->apValue[i]==0 ){ + return (p->rc = SQLITE_CORRUPT_BKPT); + } if( p->abPK[i]==0 && p->apValue[i+p->nCol]==0 ){ sqlite3ValueFree(p->apValue[i]); p->apValue[i] = 0; @@ -242084,19 +244163,22 @@ static int sessionApplyOneWithRetry( assert( pIter->op==SQLITE_INSERT ); rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); if( rc==SQLITE_OK ){ + int rc2; rc = sessionBindRow(pIter, sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete); sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); - } - if( rc==SQLITE_OK ){ - sqlite3_step(pApply->pDelete); - rc = sqlite3_reset(pApply->pDelete); - } - if( rc==SQLITE_OK ){ - rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pApply->pDelete); + rc = sqlite3_reset(pApply->pDelete); + } + if( rc==SQLITE_OK ){ + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3_exec(db, "ROLLBACK TO replace_op", 0, 0, 0); + } + rc2 = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + if( rc==SQLITE_OK ) rc = rc2; } } } @@ -242178,161 +244260,215 @@ static int sessionApplyRetryBuffer( } /* -** Check if table zTab in the "main" database of db is a WITHOUT ROWID -** table. +** Buffer aUnique[] is pApply->nCol entries in size. This function sets +** aUnique[i] to true if column i of table zTab in the "main" database +** of db is part of at least one UNIQUE constraint, or to false otherwise. ** -** If no error occurs, return SQLITE_OK and set output variable (*pbWR) to -** true if zTab is a WITHOUT ROWID table, or false otherwise. Or, if an -** error does occur, return an SQLite error code. The final value of (*pbWR) -** is undefined in this case. +** If the table has a unique index on an expression, or a partial unique +** index, all entries of aUnique[] are set to true. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int sessionTableIsWithoutRowid(sqlite3 *db, const char *zTab, int *pbWR){ - sqlite3_stmt *pList = 0; - char *zSql = 0; +static int sessionUniqueColumns( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *pApply, /* Apply context */ + u8 *aUnique /* OUT: Array of pApply->nCol flags */ +){ + sqlite3_stmt *pList = 0; /* PRAGMA index_list */ int rc = SQLITE_OK; - zSql = sqlite3_mprintf("PRAGMA table_list = %Q", zTab); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &pList, 0); - sqlite3_free(zSql); - } + /* Ordinary PRAGMA statements are used here instead of the equivalent + ** table-valued functions (pragma_index_list() etc.) so that this works + ** in builds with SQLITE_OMIT_VIRTUALTABLE defined. */ + memset(aUnique, 0, pApply->nCol); + sessionPrepareMprintf(&rc, db, &pList, &pApply->zErr, + "PRAGMA main.index_list(%Q)", zTab + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + /* Columns of PRAGMA index_list are (seq, name, unique, origin, partial) */ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bUnique = sqlite3_column_int(pList, 2); + int bPartial = sqlite3_column_int(pList, 4); + sqlite3_stmt *pInfo = 0; /* PRAGMA index_xinfo */ + if( bUnique==0 ) continue; + sessionPrepareMprintf(&rc, db, &pInfo, &pApply->zErr, + "PRAGMA main.index_xinfo(%Q)", zIdx + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pInfo) ){ + /* Columns of PRAGMA index_xinfo are (seqno, cid, name, desc, coll, key)*/ + int iCid = sqlite3_column_int(pInfo, 1); + const char *zCol = (const char*)sqlite3_column_text(pInfo, 2); + int bKey = sqlite3_column_int(pInfo, 5); + int ii; + if( bKey==0 ) continue; + for(ii=0; iinCol; ii++){ + if( bPartial + || iCid==-2 + || (zCol && 0==sqlite3_stricmp(zCol, pApply->azCol[ii])) + ){ + aUnique[ii] = 1; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_finalize(pInfo); + }else{ + sqlite3_finalize(pInfo); + } + } if( rc==SQLITE_OK ){ - sqlite3_step(pList); - *pbWR = sqlite3_column_int(pList, 4); rc = sqlite3_finalize(pList); + }else{ + sqlite3_finalize(pList); } return rc; } /* -** Iterator pUp points to an UPDATE change. This function deletes the -** affected row from the database and creates an INSERT statement that -** may be used to reinsert the row as it is after the UPDATE change -** has been applied. +** Iterator pUp points to an UPDATE change. This function updates the +** affected row to set each column modified by the UPDATE change that +** is also part of at least one UNIQUE constraint to a "random" value, +** and creates another UPDATE statement that may be used to later set +** all columns modified by pUp to the actual values required by pUp. ** -** If successful, SQLITE_OK is returned and output variable (*ppInsert) -** is left pointing to a prepared INSERT statement. It is the responsibility +** Each random value has the same type as the final value required by +** pUp for the same column: +** +** NULL -> NULL +** INTEGER -> random() +** REAL -> random() +** TEXT -> CAST(random() AS TEXT) +** BLOB -> unhex(hex(random())) +** +** If none of the columns modified by pUp are part of a UNIQUE constraint, +** then SQLITE_CONSTRAINT is returned. +** +** If successful, SQLITE_OK is returned and output variable (*ppUpdate) +** is left pointing to a prepared UPDATE statement. It is the responsibility ** of the caller to eventually free this statement using sqlite3_finalize(). -** Or, if an error occurs, an SQLite error code is returned and (*ppInsert) +** Or, if an error occurs, an SQLite error code is returned and (*ppUpdate) ** set to NULL. pApply->zErr may be set to an error message in this case. +** Except - it is guaranteed that pApply->zErr is not set if SQLITE_CONSTRAINT +** is returned. */ -static int sessionUpdateToDeleteInsert( +static int sessionUpdateToUpdate( sqlite3 *db, /* Database to write to */ const char *zTab, /* Table name */ SessionApplyCtx *pApply, /* Apply context */ sqlite3_changeset_iter *pUp, /* Iterator pointing to UPDATE change */ - sqlite3_stmt **ppInsert /* OUT: INSERT statement */ + sqlite3_stmt **ppUpdate /* OUT: UPDATE statement */ ){ - sqlite3_stmt *pRet = 0; /* The INSERT statement */ - sqlite3_stmt *pSelect = 0; /* SELECT to read current values of row */ + sqlite3_stmt *pRet = 0; /* UPDATE to set the final values */ + sqlite3_stmt *pRand = 0; /* UPDATE to set random values */ int rc = SQLITE_OK; - int bWR = 0; - - rc = sessionTableIsWithoutRowid(db, zTab, &bWR); - if( rc==SQLITE_OK ){ - char *zSelect = 0; - char *zInsert = 0; - SessionBuffer cols = {0, 0, 0}; - SessionBuffer insbind = {0, 0, 0}; - SessionBuffer pkcols = {0, 0, 0}; - SessionBuffer selbind = {0, 0, 0}; - - const char *zComma = ""; - const char *zComma2 = ""; - int ii; - for(ii=0; iinCol; ii++){ - sessionAppendStr(&cols, zComma, &rc); - sessionAppendIdent(&cols, pApply->azCol[ii], &rc); - sessionAppendStr(&insbind, zComma, &rc); - sessionAppendStr(&insbind, "?", &rc); - zComma = ", "; - - if( pApply->abPK[ii] ){ - sessionAppendStr(&pkcols, zComma2, &rc); - sessionAppendIdent(&pkcols, pApply->azCol[ii], &rc); - sessionAppendStr(&selbind, zComma2, &rc); - sessionAppendPrintf(&selbind, &rc, "?%d", ii+1); - zComma2 = ", "; - } - } - if( bWR==0 ){ - sessionAppendStr(&cols, zComma, &rc); - sessionAppendStr(&cols, SESSIONS_ROWID, &rc); - sessionAppendStr(&insbind, zComma, &rc); - sessionAppendStr(&insbind, "?", &rc); - } - - if( rc==SQLITE_OK ){ - zSelect = sqlite3_mprintf("SELECT %s FROM %Q WHERE (%s) IS (%s)", - cols.aBuf, zTab, pkcols.aBuf, selbind.aBuf - ); - if( zSelect==0 ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - zInsert = sqlite3_mprintf("INSERT INTO %Q(%s) VALUES(%s)", - zTab, cols.aBuf, insbind.aBuf - ); - if( zInsert==0 ) rc = SQLITE_NOMEM; - } + u8 *aUnique = 0; /* aUnique[i] true if col i is UNIQUE */ + SessionBuffer randset = {0, 0, 0}; + SessionBuffer upset = {0, 0, 0}; + SessionBuffer where = {0, 0, 0}; + int ii; - if( rc==SQLITE_OK ){ - rc = sessionPrepare(db, &pSelect, &pApply->zErr, zSelect); - } - if( rc==SQLITE_OK ){ - rc = sessionPrepare(db, &pRet, &pApply->zErr, zInsert); - } + aUnique = (u8*)sqlite3_malloc64(pApply->nCol); + if( aUnique==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sessionUniqueColumns(db, zTab, pApply, aUnique); + } - sqlite3_free(zSelect); - sqlite3_free(zInsert); - sqlite3_free(cols.aBuf); - sqlite3_free(insbind.aBuf); - sqlite3_free(pkcols.aBuf); - sqlite3_free(selbind.aBuf); + /* Assuming a table structure like this: + ** + ** CREATE TABLE x(a, b UNIQUE, c, d UNIQUE, e, PRIMARY KEY(a, c)); + ** + ** and an UPDATE change that sets column b to an integer, column d to + ** a text value and column e to any value, the two statements prepared + ** are: + ** + ** UPDATE main.x SET b = random(), d = CAST(random() AS TEXT) + ** WHERE a IS ?1 AND c IS ?3 + ** UPDATE main.x SET b = ?2, d = ?4, e = ?5 WHERE a IS ?1 AND c IS ?3 + */ + for(ii=0; rc==SQLITE_OK && iinCol; ii++){ + sqlite3_value *pNew = 0; + if( pApply->abPK[ii] ){ + if( where.nBuf>0 ) sessionAppendStr(&where, " AND ", &rc); + sessionAppendIdent(&where, pApply->azCol[ii], &rc); + sessionAppendPrintf(&where, &rc, " IS ?%d", ii+1); + }else if( (pNew = sessionChangesetNew(pUp, ii))!=0 ){ + if( aUnique[ii] ){ + const char *zRand = 0; + switch( sqlite3_value_type(pNew) ){ + case SQLITE_NULL: zRand = "NULL"; break; + case SQLITE_TEXT: zRand = "CAST(random() AS TEXT)"; break; + case SQLITE_BLOB: zRand = "unhex(hex(random()))"; break; + default: zRand = "random()"; break; + } + if( randset.nBuf>0 ) sessionAppendStr(&randset, ", ", &rc); + sessionAppendIdent(&randset, pApply->azCol[ii], &rc); + sessionAppendStr(&randset, " = ", &rc); + sessionAppendStr(&randset, zRand, &rc); + } + if( upset.nBuf>0 ) sessionAppendStr(&upset, ", ", &rc); + sessionAppendIdent(&upset, pApply->azCol[ii], &rc); + sessionAppendPrintf(&upset, &rc, " = ?%d", ii+1); + } + } + + if( rc==SQLITE_OK && randset.nBuf==0 ){ + /* None of the columns modified by this change are part of a UNIQUE + ** constraint. So setting them to random values cannot help resolve + ** a constraint conflict. Report this as a constraint failure. */ + rc = SQLITE_CONSTRAINT; } + sessionPrepareMprintf(&rc, db, &pRand, &pApply->zErr, + "UPDATE main.\"%w\" SET %s WHERE %s", + zTab, (char*)randset.aBuf, (char*)where.aBuf + ); + sessionPrepareMprintf(&rc, db, &pRet, &pApply->zErr, + "UPDATE main.\"%w\" SET %s WHERE %s", + zTab, (char*)upset.aBuf, (char*)where.aBuf + ); + + /* Bind the PK values to both statements, and the new.* values to pRet. */ if( rc==SQLITE_OK ){ rc = sessionBindRow( - pUp, sqlite3changeset_old, pApply->nCol, pApply->abPK, pSelect + pUp, sqlite3changeset_old, pApply->nCol, pApply->abPK, pRand ); } - - if( rc==SQLITE_OK && sqlite3_step(pSelect)==SQLITE_ROW ){ - int iCol; - for(iCol=0; iColnCol; iCol++){ - sqlite3_value *pVal = pUp->apValue[iCol+pApply->nCol]; - if( pVal==0 ){ - pVal = sqlite3_column_value(pSelect, iCol); - } - rc = sqlite3_bind_value(pRet, iCol+1, pVal); - } - if( bWR==0 ){ - sqlite3_bind_int64(pRet, iCol+1, sqlite3_column_int64(pSelect, iCol)); - } - } - sessionFinalizeStmt(pSelect, &rc); - - /* Delete the row from the database. */ if( rc==SQLITE_OK ){ rc = sessionBindRow( - pUp, sqlite3changeset_old, pApply->nCol, pApply->abPK, pApply->pDelete + pUp, sqlite3changeset_old, pApply->nCol, pApply->abPK, pRet ); - sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); } + for(ii=0; rc==SQLITE_OK && iinCol; ii++){ + sqlite3_value *pNew = sessionChangesetNew(pUp, ii); + if( pApply->abPK[ii]==0 && pNew ){ + rc = sessionBindValue(pRet, ii+1, pNew); + } + } + + /* Evaluate the statement to set each modified column of the row to a random + ** value. If this fails with a constraint error, SQLITE_CONSTRAINT is + ** returned to the caller, but pApply->zErr is not set. */ if( rc==SQLITE_OK ){ - sqlite3_step(pApply->pDelete); - rc = sqlite3_reset(pApply->pDelete); + sqlite3_step(pRand); + rc = sqlite3_reset(pRand); } + sqlite3_finalize(pRand); if( rc!=SQLITE_OK ){ sqlite3_finalize(pRet); pRet = 0; } + *ppUpdate = pRet; + + sqlite3_free(aUnique); + sqlite3_free(randset.aBuf); + sqlite3_free(upset.aBuf); + sqlite3_free(where.aBuf); - *ppInsert = pRet; return rc; } @@ -242349,14 +244485,17 @@ static int sessionUpdateToDeleteInsert( ** 2) For each UPDATE change in the buffer, try the following in a ** savepoint transaction: ** -** a) DELETE the affected row, +** a) UPDATE the affected row, setting each column modified by the +** UPDATE change to a random value, ** b) Attempt step (1) with remaining changes, -** c) Attempt to INSERT a row equivalent to the one that would be -** created by applying this UPDATE change. +** c) Attempt to UPDATE the affected row again, this time setting +** each modified column to the value required by the UPDATE change. ** -** If the INSERT in (c) succeeds, the savepoint is committed and all +** If both (a) and (c) succeed, the savepoint is committed and all ** successfully applied changes are removed from the buffer. Step (2) -** is then repeated. +** is then repeated. Otherwise, if either (a) or (c) fails with +** SQLITE_CONSTRAINT, the savepoint is rolled back and the next UPDATE +** in the buffer is tried. ** ** 3) Once step (2) has been attempted for each UPDATE in the change, ** a final attempt is made to apply each remaining change. This time, @@ -242395,8 +244534,9 @@ static int sessionRetryConstraints( while( rc==SQLITE_OK && pApply->constraints.nBuf && !pApply->bNoUpdateLoop ){ SessionBuffer cons = {0, 0, 0}; sqlite3_changeset_iter *pUp = 0; - sqlite3_stmt *pInsert = 0; + sqlite3_stmt *pUpdate = 0; int iSkip = 0; + int bConstraint = 0; /* True if an SQLITE_CONSTRAINT occurs */ rc = sessionRetryIterInit( &pApply->constraints, bPatchset, zTab, pApply, &pUp @@ -242411,7 +244551,11 @@ static int sessionRetryConstraints( if( iThis==iUpdate ){ rc = sqlite3_exec(db, "SAVEPOINT update_op", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = sessionUpdateToDeleteInsert(db, zTab, pApply, pUp, &pInsert); + rc = sessionUpdateToUpdate(db, zTab, pApply, pUp, &pUpdate); + if( (rc&0xff)==SQLITE_CONSTRAINT ){ + bConstraint = 1; + rc = SQLITE_OK; + } } } sqlite3changeset_finalize(pUp); @@ -242420,6 +244564,9 @@ static int sessionRetryConstraints( if( rc==SQLITE_OK ){ cons = pApply->constraints; + if( bConstraint ){ + memset(&pApply->constraints, 0, sizeof(SessionBuffer)); + } while( rc==SQLITE_OK && pApply->constraints.nBuf>0 ){ SessionBuffer app = pApply->constraints; @@ -242439,9 +244586,16 @@ static int sessionRetryConstraints( iUpdate++; if( rc==SQLITE_OK ){ - sqlite3_step(pInsert); - rc = sqlite3_finalize(pInsert); - if( (rc&0xff)==SQLITE_CONSTRAINT ){ + if( bConstraint==0 ){ + sqlite3_step(pUpdate); + rc = sqlite3_finalize(pUpdate); + pUpdate = 0; + if( (rc&0xff)==SQLITE_CONSTRAINT ){ + bConstraint = 1; + rc = SQLITE_OK; + } + } + if( bConstraint ){ rc = sqlite3_exec(db, "ROLLBACK TO update_op", 0, 0, 0); sqlite3_free(pApply->constraints.aBuf); pApply->constraints = cons; @@ -242452,9 +244606,8 @@ static int sessionRetryConstraints( if( rc==SQLITE_OK ){ rc = sqlite3_exec(db, "RELEASE update_op", 0, 0, 0); } - }else{ - sqlite3_finalize(pInsert); } + sqlite3_finalize(pUpdate); sqlite3_free(cons.aBuf); } @@ -242503,10 +244656,10 @@ static int sessionChangesetApply( int schemaMismatch = 0; int rc = SQLITE_OK; /* Return code */ const char *zTab = 0; /* Name of current table */ - int nTab = 0; /* Result of sqlite3Strlen30(zTab) */ SessionApplyCtx sApply; /* changeset_apply() context object */ int bPatchset; u64 savedFlag = db->flags & SQLITE_FkNoAction; + int bNew = 0; /* True for first entry of new table */ assert( xConflict!=0 ); @@ -242528,15 +244681,16 @@ static int sessionChangesetApply( if( rc==SQLITE_OK ){ rc = sqlite3_exec(db, "PRAGMA defer_foreign_keys = 1", 0, 0, 0); } - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter) ){ + while( rc==SQLITE_OK && SQLITE_ROW==sessionChangesetNext(pIter, 0, 0,&bNew) ){ int nCol; int op; const char *zNew; sqlite3changeset_op(pIter, &zNew, &nCol, &op, 0); - if( zTab==0 || sqlite3_strnicmp(zNew, zTab, nTab+1) ){ + if( bNew ){ u8 *abPK; + bNew = 0; rc = sessionRetryConstraints( db, pIter->bPatchset, zTab, &sApply, xConflict, pCtx @@ -242571,7 +244725,6 @@ static int sessionChangesetApply( rc = SQLITE_NOMEM; break; } - nTab = (int)strlen(zTab); sApply.azCol = (const char **)zTab; }else{ int nMinCol = 0; @@ -242624,7 +244777,6 @@ static int sessionChangesetApply( sApply.bStat1 = 0; } } - nTab = sqlite3Strlen30(zTab); } } @@ -243030,7 +245182,7 @@ static int sessionChangeMerge( } }else if( bRebase ){ if( pExist->op==SQLITE_DELETE && pExist->bIndirect ){ - *ppNew = pExist; + pNew = pExist; }else{ sqlite3_int64 nByte = nRec + pExist->nRecord + sizeof(SessionChange); pNew = (SessionChange*)sqlite3_malloc64(nByte); @@ -243303,7 +245455,7 @@ static int sessionChangesetFindTable( if( pIter ){ sqlite3changeset_pk(pIter, &abPK, &nCol); }else if( !pTab && !pGrp->db ){ - return SQLITE_OK; + return SQLITE_ERROR; } /* If one was not found above, create a new table now */ @@ -243329,7 +245481,7 @@ static int sessionChangesetFindTable( if( rc || pTab->nCol==0 ){ sqlite3_free(pTab->azCol); sqlite3_free(pTab); - return rc; + return rc ? rc : SQLITE_ERROR; } } @@ -243884,8 +246036,8 @@ static void sessionAppendPartialUpdate( a2 = aChange; for(i=0; inCol; i++){ int n1 = sessionSerialLen(a1); - int n2 = sessionSerialLen(a2); - if( pIter->abPK[i] || a2[0]!=0xFF ){ + int n2 = (a2>=a2Eof) ? 0 : sessionSerialLen(a2); + if( pIter->abPK[i] || n2<=0 || a2[0]!=0xFF ){ memcpy(pOut, a1, n1); pOut += n1; }else{ @@ -243944,6 +246096,9 @@ static int sessionRebase( if( pIter->bPatchset ){ rc = SQLITE_ERROR; } + if( pTab && pTab->nCol>pIter->nCol ){ + rc = SQLITE_SCHEMA; + } /* Append a table header to the output for this new table */ sessionAppendByte(&sOut, pIter->bPatchset ? 'P' : 'T', &rc); @@ -244040,6 +246195,7 @@ static int sessionRebase( } } sqlite3_free(sOut.aBuf); + if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -244168,44 +246324,40 @@ SQLITE_API int sqlite3changegroup_change_begin( rc = SQLITE_ERROR; }else{ rc = sessionChangesetFindTable(pGrp, zTab, 0, &pTab); + if( pTab==0 && pzErr ){ + *pzErr = sqlite3_mprintf("no such table: %s", zTab); + } } if( rc==SQLITE_OK ){ - if( pTab==0 ){ - if( pzErr ){ - *pzErr = sqlite3_mprintf("no such table: %s", zTab); - } - rc = SQLITE_ERROR; - }else{ - int nReq = pTab->nCol * (eOp==SQLITE_UPDATE ? 2 : 1); - pGrp->cd.pTab = pTab; - pGrp->cd.eOp = eOp; - pGrp->cd.bIndirect = bIndirect; - - if( pGrp->cd.nBufAlloccd.aBuf, nReq * sizeof(SessionBuffer) + int nReq = pTab->nCol * (eOp==SQLITE_UPDATE ? 2 : 1); + pGrp->cd.pTab = pTab; + pGrp->cd.eOp = eOp; + pGrp->cd.bIndirect = bIndirect; + + if( pGrp->cd.nBufAlloccd.aBuf, nReq * sizeof(SessionBuffer) + ); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&aBuf[pGrp->cd.nBufAlloc], 0, + sizeof(SessionBuffer) * (nReq - pGrp->cd.nBufAlloc) ); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(&aBuf[pGrp->cd.nBufAlloc], 0, - sizeof(SessionBuffer) * (nReq - pGrp->cd.nBufAlloc) - ); - pGrp->cd.aBuf = aBuf; - pGrp->cd.nBufAlloc = nReq; - } + pGrp->cd.aBuf = aBuf; + pGrp->cd.nBufAlloc = nReq; } + } #ifdef SQLITE_DEBUG - { - /* Assert that all column values are currently undefined */ - int ii; - for(ii=0; iicd.nBufAlloc; ii++){ - assert( pGrp->cd.aBuf[ii].nBuf==0 ); - } + { + /* Assert that all column values are currently undefined */ + int ii; + for(ii=0; iicd.nBufAlloc; ii++){ + assert( pGrp->cd.aBuf[ii].nBuf==0 ); } -#endif } +#endif } return rc; @@ -247876,7 +250028,22 @@ static int fts5CInstIterInit( return rc; } - +/* +** Load error code rc into the context handle pCtx. Except, if +** rc==SQLITE_ERROR and there is an error message in the associated db +** handle, return that error message as well. +*/ +static void fts5AuxResultError(sqlite3_context *pCtx, int rc){ + if( rc==SQLITE_ERROR ){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + if( sqlite3_errcode(db)==SQLITE_ERROR ){ + const char *zErr = sqlite3_errmsg(db); + sqlite3_result_error(pCtx, zErr, -1); + return; + } + } + sqlite3_result_error_code(pCtx, rc); +} /************************************************************************* ** Start of highlight() implementation. @@ -247999,6 +250166,16 @@ static int fts5HighlightCb( return rc; } +/* +** Return the integer value of pVal, clamped to the range [-1, 0x7FFFFFFF]. +** This is used for the column-number arguments of auxiliary functions, +** so that 64-bit values that are too large or too small to fit in a +** 32-bit signed integer are not truncated to what might be a valid +** column number. +*/ +static int fts5ValueClampedInt(sqlite3_value *pVal){ + return (int)MIN(MAX(sqlite3_value_int64(pVal), -1), 0x7FFFFFFF); +} /* ** Implementation of highlight() function. @@ -248020,7 +250197,7 @@ static void fts5HighlightFunction( return; } - iCol = sqlite3_value_int(apVal[0]); + iCol = fts5ValueClampedInt(apVal[0]); memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); @@ -248055,7 +250232,7 @@ static void fts5HighlightFunction( sqlite3_free(ctx.zOut); } if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); + fts5AuxResultError(pCtx, rc); } } /* @@ -248219,7 +250396,7 @@ static void fts5SnippetFunction( nCol = pApi->xColumnCount(pFts); memset(&ctx, 0, sizeof(HighlightContext)); - iCol = sqlite3_value_int(apVal[0]); + iCol = fts5ValueClampedInt(apVal[0]); ctx.zOpen = fts5ValueToText(apVal[1]); ctx.zClose = fts5ValueToText(apVal[2]); ctx.iRangeEnd = -1; @@ -248349,7 +250526,7 @@ static void fts5SnippetFunction( if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); }else{ - sqlite3_result_error_code(pCtx, rc); + fts5AuxResultError(pCtx, rc); } sqlite3_free(ctx.zOut); sqlite3_free(aSeen); @@ -248515,7 +250692,7 @@ static void fts5Bm25Function( } sqlite3_result_double(pCtx, -1.0 * score); }else{ - sqlite3_result_error_code(pCtx, rc); + fts5AuxResultError(pCtx, rc); } } @@ -248551,15 +250728,15 @@ static void fts5GetLocaleFunction( return; } - iCol = sqlite3_value_int(apVal[0]); + iCol = fts5ValueClampedInt(apVal[0]); if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ - sqlite3_result_error_code(pCtx, SQLITE_RANGE); + fts5AuxResultError(pCtx, SQLITE_RANGE); return; } rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); + fts5AuxResultError(pCtx, rc); return; } @@ -249110,6 +251287,9 @@ static const char *fts5ConfigSkipLiteral(const char *pIn){ if( *p=='\'' ){ p++; if( *p!='\'' ) break; + }else if( *p==0 ){ + p = 0; + break; } p++; if( *p==0 ) p = 0; @@ -250534,7 +252714,7 @@ static int sqlite3Fts5ExprPattern( zExpr[iOut++] = '"'; zExpr[iOut++] = ' '; } - if( zText[i]==aSpec[2] ){ + if( inPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ return 1; } + + /* Not a match. Clear the position lists of all phrases, as some may + ** have been populated before the loop above exited early, and others + ** may still contain data left over from a previous row. */ + for(i=0; inPhrase; i++){ + pNear->apPhrase[i]->poslist.n = 0; + } return 0; } } @@ -251189,6 +253376,24 @@ static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ } } +/* +** Zero the position lists of all phrases within the sub-tree headed by +** pNode that are currently positioned at rowid iRowid. Phrases positioned +** at other rowids are not modified - their position lists belong to rows +** that have not yet been tested by the parent node. +*/ +static void fts5ExprNodeZeroPoslistAt(Fts5ExprNode *pNode, i64 iRowid){ + if( pNode->iRowid!=iRowid ) return; + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + fts5ExprNodeZeroPoslist(pNode); + }else{ + int i; + for(i=0; inChild; i++){ + fts5ExprNodeZeroPoslistAt(pNode->apChild[i], iRowid); + } + } +} + /* @@ -251507,10 +253712,10 @@ static int fts5ExprNodeTest_AND( } }while( bMatch==0 ); + pAnd->iRowid = iLast; if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ - fts5ExprNodeZeroPoslist(pAnd); + fts5ExprNodeZeroPoslistAt(pAnd, iLast); } - pAnd->iRowid = iLast; return SQLITE_OK; } @@ -251656,6 +253861,7 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ default: assert( pNode->eType==FTS5_NOT ); pNode->bEof = pNode->apChild[0]->bEof; + if( pNode->bEof ) fts5ExprNodeZeroPoslist(pNode->apChild[1]); break; } } @@ -254296,7 +256502,12 @@ static void sqlite3Fts5HashScanEntry( #define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ #define FTS5_DATA_DLI_B 1 /* Doclist-index flag (1 bit) */ #define FTS5_DATA_HEIGHT_B 5 /* Max dlidx tree height of 32 */ -#define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ +#define FTS5_DATA_PAGE_B 31 /* 31-bit page numbers */ + +/* Maximum page number within a segment. Declare any structure read from +** the database with a page number greater than this corrupt. It is not +** actually possible to create a database this large. */ +#define FTS5_MAX_PGNO 0x7FFFFF00 #define fts5_dri(segid, dlidx, height, pgno) ( \ ((i64)(segid) << (FTS5_DATA_PAGE_B+FTS5_DATA_HEIGHT_B+FTS5_DATA_DLI_B)) + \ @@ -254563,6 +256774,9 @@ struct Fts5SegIter { i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ u8 bDel; /* True if the delete flag is set */ + + /* True if the current entry shadows an entry that has the delete flag set */ + u8 bDelDup; }; static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){ @@ -254907,8 +257121,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ sqlite3_free(pRet); pRet = 0; }else{ - pRet->p[nByte] = 0x00; - pRet->p[nByte+1] = 0x00; + memset(&pRet->p[nByte], 0, FTS5_DATA_PADDING); } } p->rc = rc; @@ -255209,7 +257422,7 @@ static int fts5StructureDecode( i += fts5GetVarint(&pData[i], &pSeg->nEntry); nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); } - if( pSeg->pgnoLastpgnoFirst ){ + if( pSeg->pgnoLastpgnoFirst || pSeg->pgnoLast>FTS5_MAX_PGNO ){ rc = FTS5_CORRUPT; break; } @@ -255597,6 +257810,17 @@ static void fts5StructurePromote( } } +/* +** Return the page number iPgno+nDelta. If this is larger than FTS5_MAX_PGNO, +** which can only happen if the doclist-index record is corrupt, return +** FTS5_MAX_PGNO+1 instead. No leaf page with that number may exist, so +** this causes the corruption to be detected when the caller attempts to +** use the page number. +*/ +static int fts5AddPgno(int iPgno, int nDelta){ + i64 iRet = (i64)iPgno + nDelta; + return (int)(iRet>FTS5_MAX_PGNO ? FTS5_MAX_PGNO+1 : iRet); +} /* ** Advance the iterator passed as the only argument. If the end of the @@ -255619,7 +257843,7 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ if( iOffnn ){ u64 iVal; - pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; + pLvl->iLeafPgno = fts5AddPgno(pLvl->iLeafPgno, iOff-pLvl->iOff+1); iOff += fts5GetVarint(&pData->p[iOff], &iVal); pLvl->iRowid += iVal; pLvl->iOff = iOff; @@ -255729,7 +257953,7 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ ii += sqlite3Fts5GetVarint(&a[ii], &delta); if( ii>=iOff ) break; - pLvl->iLeafPgno += nZero+1; + pLvl->iLeafPgno = fts5AddPgno(pLvl->iLeafPgno, nZero+1); pLvl->iRowid += delta; pLvl->iOff = ii; } @@ -255805,6 +258029,9 @@ static Fts5DlidxIter *fts5DlidxIterInit( bDone = 1; } pIter->nLvl = i+1; + if( pIter->nLvl > (1<rc = FTS5_CORRUPT; + } } } @@ -256759,7 +258986,7 @@ static void fts5SegIterSeekInit( if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; } - if( bDlidx ){ + if( bDlidx && iPg==pIter->iTermLeafPgno ){ fts5SegIterLoadDlidx(p, pIter); } if( flags & FTS5INDEX_QUERY_DESC ){ @@ -257108,6 +259335,10 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ + /* p2 is about to be advanced past this entry. If it, or an entry + ** it shadowed, has the delete flag set, record this on p1. */ + p1->bDelDup |= (p2->bDel | p2->bDelDup); + p2->bDelDup = 0; return i2; } res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; @@ -257159,6 +259390,13 @@ static void fts5SegIterGotoPage( fts5SegIterLoadNPos(p, pIter); } break; + }else if( !fts5LeafIsTermless(pIter->pLeaf) ){ + /* There are no rowids on this page, but there is another term. + ** Since there are no more rowids for the current term, set the + ** iterator to EOF. */ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + break; } } } @@ -257796,6 +260034,7 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ UNUSED_PARAM(pSeg); pIter->base.nData = 0; + pIter->base.bEof = 1; } /* @@ -258862,6 +261101,14 @@ static void fts5IndexMergeLevel( int nPos; /* position-list size field value */ int nTerm; const u8 *pTerm; + int bDel; /* True to set delete flag on output */ + + /* If this entry has the delete flag set, or shadows an entry with the + ** delete flag set, set the delete flag on the output. Unless the output + ** will be the oldest segment in the table - in that case the delete flag + ** may be discarded. */ + bDel = bOldest ? 0 : (pSegIter->bDel || pSegIter->bDelDup); + pSegIter->bDelDup = 0; pTerm = fts5MultiIterTerm(pIter, &nTerm); if( nTerm!=term.n || fts5Memcmp(pTerm, term.p, nTerm) ){ @@ -258873,7 +261120,7 @@ static void fts5IndexMergeLevel( } /* Check for key annihilation. */ - if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; + if( pSegIter->nPos==0 && (bOldest || bDel==0) ) continue; if( p->rc==SQLITE_OK && bTermWritten==0 ){ /* This is a new term. Append a term to the output segment. */ @@ -258886,7 +261133,7 @@ static void fts5IndexMergeLevel( fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); if( eDetail==FTS5_DETAIL_NONE ){ - if( pSegIter->bDel ){ + if( bDel ){ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); if( pSegIter->nPos>0 ){ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); @@ -258894,7 +261141,7 @@ static void fts5IndexMergeLevel( } }else{ /* Append the position-list data to the output */ - nPos = pSegIter->nPos*2 + pSegIter->bDel; + nPos = pSegIter->nPos*2 + bDel; fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); } @@ -259333,7 +261580,8 @@ static void fts5DoSecureDelete( assert_nc( iSOP<=pSeg->iLeafOffset ); if( bDetailNone ){ - while( iSOPiLeafOffset ){ + int iEnd = pSeg->iLeafOffset - (pSeg->bDel ? 1 + pSeg->nPos : 0); + while( iSOPbDel ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta); - aPg[iOff++] = 0x01; + aPg[iOff++] = bDetailNone ? 0x00 : 0x01; }else if( bLastInDoclist==0 ){ if( iNextOff!=iPgIdx ){ u64 iNextDelta = 0; @@ -259526,34 +261774,38 @@ static void fts5DoSecureDelete( if( p->rc==SQLITE_OK ){ const int nMove = nPg - iNextOff; /* Number of bytes to move */ int nShift = iNextOff - iOff; /* Distance to move them */ - + int nNewPg = 0; int iPrevKeyOut = 0; - int iKeyIn = 0; + i64 iKeyIn = 0; if( nMove>0 ){ memmove(&aPg[iOff], &aPg[iNextOff], nMove); } iPgIdx -= nShift; - nPg = iPgIdx; + nNewPg = iPgIdx; fts5PutU16(&aPg[2], iPgIdx); for(iIdx=0; iIdxnPg ){ + FTS5_CORRUPT_IDX(p); + break; + } if( iKeyIn!=iDelKeyOff ){ int iKeyOut = (iKeyIn - (iKeyIn>iOff ? nShift : 0)); - nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut); + nNewPg += sqlite3Fts5PutVarint(&aPg[nNewPg], iKeyOut - iPrevKeyOut); iPrevKeyOut = iKeyOut; } } - if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + if( p->rc==SQLITE_OK && iPgIdx==nNewPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); } - assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); - fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + assert_nc( nNewPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nNewPg); } sqlite3_free(aIdx); } @@ -259701,7 +261953,6 @@ static void fts5FlushOneHash(Fts5Index *p){ iOff++; if( iOff=pIter->aEof ){ pIter->aPoslist = 0; }else{ - i64 iDelta; + u64 iDelta; - p += fts5GetVarint(p, (u64*)&iDelta); + p += fts5GetVarint(p, &iDelta); pIter->iRowid += iDelta; /* Read position list size */ @@ -261247,11 +263498,9 @@ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ if( p->base.bEof==0 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidbase.bEof==0 - && p->base.iRowidrc==SQLITE_OK - ){ + if( bFrom ){ + fts5MultiIterNextFrom(pIndex, p, iFrom); + }else{ fts5MultiIterNext(pIndex, p, 0, 0); } } @@ -261448,6 +263697,15 @@ static int sqlite3Fts5IndexQuery( bTokendata = 0; } + /* fts5SetupTokendataIter() flushes the in-memory hash table to disk. + ** The flush frees the cached Fts5Structure object, which iterators + ** opened earlier by this function point into without holding a + ** reference (for example one opened for a prefix index query in the + ** same MATCH expression). So, on a tokendata=1 table, flush first. */ + if( pConfig->bTokendata ){ + fts5IndexFlush(p); + } + /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to ** greater than pConfig->nPrefix to indicate that the query will be @@ -262299,6 +264557,9 @@ static int fts5QueryCksum( } /* +** This function is purely an internal test. It does not contribute to +** FTS functionality, or even the integrity-check, in any way. +** ** Check if buffer z[], size n bytes, contains as series of valid utf-8 ** encoded codepoints. If so, return 0. Otherwise, if the buffer does not ** contain valid utf-8, return non-zero. @@ -262320,8 +264581,8 @@ static int fts5TestUtf8(const char *z, int n){ }else if( (z[i] & 0xF8)==0xF0 ){ if( i+3>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; - if( (z[i+2] & 0xC0)!=0x80 ) return 1; - i += 3; + if( (z[i+3] & 0xC0)!=0x80 ) return 1; + i += 4; }else{ return 1; } @@ -262598,7 +264859,9 @@ static void fts5IndexIntegrityCheckSegment( ){ /* Check any rowid-less pages that occur before the current leaf. */ - for(iPg=iPrevLeaf+1; iPgrc==SQLITE_OK && iPgaConstraintUsage[i].omit = 1; } }else if( p->usable ){ - if( iCol>=0 && iCol=0 && iColabUnindexed[iCol]==0 + ){ assert( p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB ); idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE ? 'L' : 'G'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); @@ -264274,7 +266539,6 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ idxStr[iIdxStr++] = '='; bSeenEq = 1; pInfo->aConstraintUsage[i].argvIndex = ++iCons; - pInfo->aConstraintUsage[i].omit = 1; } } } @@ -265925,15 +268189,17 @@ static int fts5CsrPoslist( rc = fts5SeekCursor(pCsr, 0); } for(i=0; inCol && rc==SQLITE_OK; i++){ - const char *z = 0; - int n = 0; - rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5ExprPopulatePoslists( - pConfig, pCsr->pExpr, aPopulator, i, z, n - ); + if( pConfig->abUnindexed[i]==0 ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aPopulator, i, z, n + ); + } + sqlite3Fts5ClearLocale(pConfig); } - sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -267215,7 +269481,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2026-08-10 20:51:10 2137918d6aa8cc5963af92ba7302c8944e403b5f9ef892e2c7ec031962f72dbd", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2026-11-09 15:34:47 52082d4b80537268de4a7b7c1606a6077d96a3e1638a34a6931d22d8afed084a", -1, SQLITE_TRANSIENT); } /* @@ -267575,10 +269841,12 @@ static int fts5StorageGetStmt( Fts5Storage *p, /* Storage handle */ int eStmt, /* FTS5_STMT_XXX constant */ sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ - char **pzErrMsg /* OUT: Error message (if any) */ + char **pzErrMsg /* OUT: Error message (if any) (not used) */ ){ int rc = SQLITE_OK; + (void)pzErrMsg; + /* If there is no %_docsize table, there should be no requests for ** statements to operate on it. */ assert( p->pConfig->bColumnsize || ( @@ -267694,13 +269962,13 @@ static int fts5StorageGetStmt( rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); p->pConfig->bLock--; sqlite3_free(zSql); - if( rc!=SQLITE_OK && pzErrMsg ){ - *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); - } if( rc==SQLITE_ERROR && eStmt>FTS5_STMT_LOOKUP2 && eStmtpConfig->pzErrmsg ){ + assert( *p->pConfig->pzErrmsg==0 ); + *p->pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); } } } @@ -267754,11 +270022,15 @@ static int sqlite3Fts5DropAll(Fts5Config *pConfig){ pConfig->zDb, pConfig->zName ); } - if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ - rc = fts5ExecPrintf(pConfig->db, 0, - "DROP TABLE IF EXISTS %Q.'%q_content';", - pConfig->zDb, pConfig->zName - ); + if( rc==SQLITE_OK ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->bContentlessUnindexed + ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DROP TABLE IF EXISTS %Q.'%q_content';", + pConfig->zDb, pConfig->zName + ); + } } return rc; } @@ -267787,7 +270059,9 @@ static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ if( pConfig->bColumnsize ){ fts5StorageRenameOne(pConfig, &rc, "docsize", zName); } - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->bContentlessUnindexed + ){ fts5StorageRenameOne(pConfig, &rc, "content", zName); } return rc; @@ -267867,7 +270141,7 @@ static int sqlite3Fts5StorageOpen( sqlite3_str_appendf(pDefn, ", c%d", i); } } - if( pConfig->bLocale ){ + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ for(i=0; inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ sqlite3_str_appendf(pDefn, ", l%d", i); @@ -268108,6 +270382,64 @@ static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ pStorage->pSavedRow = 0; } +/* +** Load the contents of the "averages" record from disk into the +** p->nTotalRow and p->aTotalSize[] variables. If successful, and if +** argument bCache is true, set the p->bTotalsValid flag to indicate +** that the contents of aTotalSize[] and nTotalRow are valid until +** further notice. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ + int rc = SQLITE_OK; + if( p->bTotalsValid==0 ){ + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); + p->bTotalsValid = bCache; + } + return rc; +} + +/* +** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** variables in the "averages" record on disk. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageSaveTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + int i; + Fts5Buffer buf; + int rc = SQLITE_OK; + memset(&buf, 0, sizeof(buf)); + + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); + for(i=0; iaTotalSize[i]); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); + } + sqlite3_free(buf.p); + + return rc; +} + +static int fts5StorageDecodeSizeArray( + int *aCol, int nCol, /* Array to populate */ + const u8 *aBlob, int nBlob /* Record to read varints from */ +){ + int i; + int iOff = 0; + for(i=0; i=nBlob ) return 1; + iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]); + } + return (iOff!=nBlob); +} + /* ** This function is called to process a DELETE on a contentless_delete=1 ** table. It adds the tombstone required to delete the entry with rowid @@ -268124,19 +270456,41 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED ); + rc = fts5StorageLoadTotals(p, 1); + if( rc!=SQLITE_OK ) return rc; + /* Look up the origin of the document in the %_docsize table. Store ** this in stack variable iOrigin. */ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); if( rc==SQLITE_OK ){ + int rc2; sqlite3_bind_int64(pLookup, 1, iDel); if( SQLITE_ROW==sqlite3_step(pLookup) ){ + int *aCol = (int*)sqlite3_malloc64(p->pConfig->nCol * sizeof(int)); + if( aCol==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *aBlob = sqlite3_column_blob(pLookup, 0); + int nBlob = sqlite3_column_bytes(pLookup, 0); + int ii; + if( fts5StorageDecodeSizeArray(aCol, p->pConfig->nCol, aBlob, nBlob) ){ + rc = SQLITE_CORRUPT_VTAB; + }else{ + for(ii=0; iipConfig->nCol; ii++){ + p->aTotalSize[ii] -= aCol[ii]; + } + } + sqlite3_free(aCol); + } iOrigin = sqlite3_column_int64(pLookup, 1); } - rc = sqlite3_reset(pLookup); + rc2 = sqlite3_reset(pLookup); + if( rc==SQLITE_OK ) rc = rc2; } if( rc==SQLITE_OK && iOrigin!=0 ){ rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + p->nTotalRow--; } return rc; @@ -268177,51 +270531,6 @@ static int fts5StorageInsertDocsize( return rc; } -/* -** Load the contents of the "averages" record from disk into the -** p->nTotalRow and p->aTotalSize[] variables. If successful, and if -** argument bCache is true, set the p->bTotalsValid flag to indicate -** that the contents of aTotalSize[] and nTotalRow are valid until -** further notice. -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ - int rc = SQLITE_OK; - if( p->bTotalsValid==0 ){ - rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); - p->bTotalsValid = bCache; - } - return rc; -} - -/* -** Store the current contents of the p->nTotalRow and p->aTotalSize[] -** variables in the "averages" record on disk. -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -static int fts5StorageSaveTotals(Fts5Storage *p){ - int nCol = p->pConfig->nCol; - int i; - Fts5Buffer buf; - int rc = SQLITE_OK; - memset(&buf, 0, sizeof(buf)); - - sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); - for(i=0; iaTotalSize[i]); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); - } - sqlite3_free(buf.p); - - return rc; -} - /* ** Remove a row from the FTS table. */ @@ -268883,19 +271192,6 @@ static void sqlite3Fts5StorageStmtRelease( } } -static int fts5StorageDecodeSizeArray( - int *aCol, int nCol, /* Array to populate */ - const u8 *aBlob, int nBlob /* Record to read varints from */ -){ - int i; - int iOff = 0; - for(i=0; i=nBlob ) return 1; - iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]); - } - return (iOff!=nBlob); -} - /* ** Argument aCol points to an array of integers containing one entry for ** each table column. This function reads the %_docsize record for the @@ -269197,13 +271493,10 @@ static int fts5AsciiTokenize( /* -** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied -** from the sqlite3 source file utf.c. If this file is compiled as part -** of the amalgamation, they are not required. +** The following two macros - FTS5_READ_UTF8 and FTS5_WRITE_UTF8 - have +** been copied from the sqlite3 source file utf.c. */ -#ifndef SQLITE_AMALGAMATION - -static const unsigned char sqlite3Utf8Trans1[] = { +static const unsigned char fts5Utf8Trans1[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, @@ -269214,45 +271507,43 @@ static const unsigned char sqlite3Utf8Trans1[] = { 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, }; -#define READ_UTF8(zIn, zTerm, c) \ +#define FTS5_READ_UTF8(zIn, zTerm, c) \ c = *(zIn++); \ if( c>=0xc0 ){ \ - c = sqlite3Utf8Trans1[c-0xc0]; \ + c = fts5Utf8Trans1[c-0xc0]; \ while( zIn>6)&0x1F); \ - *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ - } \ - else if( c<0x10000 ){ \ - *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \ - *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ - }else{ \ - *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \ - *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \ - *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ - } \ +#define FTS5_WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (unsigned char)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \ + } \ } -#endif /* ifndef SQLITE_AMALGAMATION */ - -#define FTS5_SKIP_UTF8(zIn) { \ - if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \ - while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \ - } \ +#define FTS5_SKIP_UTF8(zIn) { \ + if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \ + while( (((unsigned char)*zIn) & 0xc0)==0x80 ){zIn++;} \ + } \ } typedef struct Unicode61Tokenizer Unicode61Tokenizer; @@ -269291,7 +271582,7 @@ static int fts5UnicodeAddExceptions( while( zCsraTokenChar[iCode] = (unsigned char)bTokenChars; }else{ @@ -269495,7 +271786,7 @@ static int fts5UnicodeTokenize( /* A character outside of the ascii range. Skip past it if it is ** a separator character. Or break out of the loop if it is not. */ is = zCsr - (unsigned char*)pText; - READ_UTF8(zCsr, zTerm, iCode); + FTS5_READ_UTF8(zCsr, zTerm, iCode); if( fts5UnicodeIsAlnum(p, iCode) ){ goto non_ascii_tokenchar; } @@ -269531,11 +271822,11 @@ static int fts5UnicodeTokenize( if( *zCsr & 0x80 ){ /* An non-ascii-range character. Fold it into the output buffer if ** it is a token character, or break out of the loop if it is not. */ - READ_UTF8(zCsr, zTerm, iCode); + FTS5_READ_UTF8(zCsr, zTerm, iCode); if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ non_ascii_tokenchar: iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); - if( iCode ) WRITE_UTF8(zOut, iCode); + if( iCode ) FTS5_WRITE_UTF8(zOut, iCode); }else{ break; } @@ -270390,10 +272681,10 @@ static int fts5TriTokenize( do { aStart[ii] = zIn - (const unsigned char*)pText; if( zIn>=zEof ) return SQLITE_OK; - READ_UTF8(zIn, zEof, iCode); + FTS5_READ_UTF8(zIn, zEof, iCode); if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); }while( iCode==0 ); - WRITE_UTF8(zOut, iCode); + FTS5_WRITE_UTF8(zOut, iCode); } /* At the start of each iteration of this loop: @@ -270415,7 +272706,7 @@ static int fts5TriTokenize( iCode = 0; break; } - READ_UTF8(zIn, zEof, iCode); + FTS5_READ_UTF8(zIn, zEof, iCode); if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); }while( iCode==0 ); @@ -270429,7 +272720,7 @@ static int fts5TriTokenize( FTS5_SKIP_UTF8(z1); memmove(aBuf, z1, zOut - z1); zOut -= (z1 - aBuf); - WRITE_UTF8(zOut, iCode); + FTS5_WRITE_UTF8(zOut, iCode); /* Update the aStart[] array */ aStart[0] = aStart[1]; @@ -271727,9 +274018,10 @@ struct Fts5VocabCursor { #define FTS5_VOCAB_ROW 1 #define FTS5_VOCAB_INSTANCE 2 -#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" -#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" -#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset" +#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt, PRIMARY KEY(term, col)" +#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt, PRIMARY KEY(term)" +#define FTS5_VOCAB_INST_SCHEMA \ + "term, doc, col, offset, PRIMARY KEY(term, doc, col, offset)" /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. @@ -271821,9 +274113,9 @@ static int fts5VocabInitVtab( char **pzErr /* Write any error message here */ ){ const char *azSchema[] = { - "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", - "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")", - "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")" + "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ") WITHOUT ROWID", + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ") WITHOUT ROWID", + "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ") WITHOUT ROWID" }; Fts5VocabTable *pRet = 0; @@ -271895,6 +274187,14 @@ static int fts5VocabCreateMethod( return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); } +/* +** Return true if constraint iCons of pInfo uses the "binary" collation +** sequence. Or false it it uses anything else. +*/ +static int fts5VocabIsBinary(sqlite3_index_info *pInfo, int iCons){ + return 0==sqlite3_stricmp("binary", sqlite3_vtab_collation(pInfo, iCons)); +} + /* ** Implementation of the xBestIndex method. ** @@ -271925,7 +274225,7 @@ static int fts5VocabBestIndexMethod( for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; if( p->usable==0 ) continue; - if( p->iColumn==0 ){ /* term column */ + if( p->iColumn==0 && fts5VocabIsBinary(pInfo, i) ){ /* term column */ if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i; if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i; if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i; @@ -272264,6 +274564,26 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ return rc; } +/* +** If value pVal is an SQLITE_TEXT value, return a pointer to a buffer +** containing the UTF-8 representation of the value, and set output +** parameter (*pnText) to the size of the buffer contents in bytes. Or, +** if pVal is some other type of value, return NULL and set (*pnText) +** to 0 before returning. +*/ +static const char *fts5VocabValueText(sqlite3_value *pVal, int *pnText){ + const char *zRet = 0; + int nRet = 0; + + if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ + zRet = (const char*)sqlite3_value_text(pVal); + nRet = sqlite3_value_bytes(pVal); + } + + *pnText = nRet; + return zRet; +} + /* ** This is the xFilter implementation for the virtual table. */ @@ -272291,24 +274611,30 @@ static int fts5VocabFilterMethod( UNUSED_PARAM2(zUnused, nUnused); fts5VocabResetCursor(pCsr); - if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; - if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; - if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + if( idxNum & FTS5_VOCAB_TERM_EQ ){ + pEq = apVal[iVal++]; + if( sqlite3_value_type(pEq)!=SQLITE_TEXT ) pEq = 0; + } + if( idxNum & FTS5_VOCAB_TERM_GE ){ + pGe = apVal[iVal++]; + if( sqlite3_value_type(pGe)!=SQLITE_TEXT ) pGe = 0; + } + if( idxNum & FTS5_VOCAB_TERM_LE ){ + pLe = apVal[iVal++]; + if( sqlite3_value_type(pLe)!=SQLITE_TEXT ) pLe = 0; + } pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK); if( pEq ){ - zTerm = (const char *)sqlite3_value_text(pEq); - nTerm = sqlite3_value_bytes(pEq); + zTerm = fts5VocabValueText(pEq, &nTerm); f = FTS5INDEX_QUERY_NOTOKENDATA; }else{ if( pGe ){ - zTerm = (const char *)sqlite3_value_text(pGe); - nTerm = sqlite3_value_bytes(pGe); + zTerm = fts5VocabValueText(pGe, &nTerm); } if( pLe ){ - const char *zCopy = (const char *)sqlite3_value_text(pLe); + const char *zCopy = fts5VocabValueText(pLe, &pCsr->nLeTerm); if( zCopy==0 ) zCopy = ""; - pCsr->nLeTerm = sqlite3_value_bytes(pLe); pCsr->zLeTerm = sqlite3_malloc64((i64)pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ rc = SQLITE_NOMEM; diff --git a/deps/sqlite3/sqlite3.h b/deps/sqlite3/sqlite3.h index 907d6ac..d11de5c 100644 --- a/deps/sqlite3/sqlite3.h +++ b/deps/sqlite3/sqlite3.h @@ -146,12 +146,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.54.0" -#define SQLITE_VERSION_NUMBER 3054000 -#define SQLITE_SOURCE_ID "2026-08-10 20:51:10 2137918d6aa8cc5963af92ba7302c8944e403b5f9ef892e2c7ec031962f72dbd" +#define SQLITE_VERSION "3.55.0" +#define SQLITE_VERSION_NUMBER 3055000 +#define SQLITE_SOURCE_ID "2026-11-09 15:34:47 52082d4b80537268de4a7b7c1606a6077d96a3e1638a34a6931d22d8afed084a" #define SQLITE_SCM_BRANCH "bedrock" #define SQLITE_SCM_TAGS "" -#define SQLITE_SCM_DATETIME "2026-08-10T20:51:10.738Z" +#define SQLITE_SCM_DATETIME "2026-11-09T15:34:47.743Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -463,7 +463,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_FULL 13 /* Insertion failed because database is full */ #define SQLITE_CANTOPEN 14 /* Unable to open the database file */ #define SQLITE_PROTOCOL 15 /* Database lock protocol error */ -#define SQLITE_EMPTY 16 /* Internal use only */ +#define SQLITE_EMPTY 16 /* No content */ #define SQLITE_SCHEMA 17 /* The database schema changed */ #define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ #define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ @@ -2975,10 +2975,14 @@ SQLITE_API int sqlite3_is_interrupted(sqlite3*); ** SQLITE_NOMEM is returned. ** ** The [sqlite3_incomplete(X)] routine is similar to [sqlite3_complete(X)] -** except that sqlite3_incomplete(X) returns 0 if the input X is complete -** and non-zero if X is incomplete. The non-zero return from +** except that sqlite3_incomplete(X) returns SQLITE_OK if the input X is +** complete, SQLITE_EMPTY if input X consists entirely of whitespace and/or +** comments, SQLITE_MISUSE if X is a NULL pointer, and some other non-zero +** value if X is incomplete. The non-zero return from ** sqlite3_incomplete(X) contains additional information about what is -** needed to complete the input X. The sqlite3_incomplete(X) interface +** needed to complete the input X. See the header comment on the +** implemenation of the sqlite3_incomplete() routine for a full description +** of its non-zero return values. The sqlite3_incomplete(X) interface ** is only available for UTF-8 text. ** ** ^None of these routines do a full parse the SQL statements and thus @@ -3523,6 +3527,11 @@ SQLITE_API int sqlite3_set_authorizer( ** is the name of the inner-most trigger or view that is responsible for ** the access attempt or NULL if this access attempt is directly from ** top-level SQL code. +** +** The case of strings in the 3rd through the 6th argument to the +** authorization callback is arbitrary. Authorization callbacks +** implementations should use [sqlite3_stricmp()] or similar when +** doing comparisons against those values. */ /******************************************* 3rd ************ 4th ***********/ #define SQLITE_CREATE_INDEX 1 /* Index Name Table Name */ @@ -4071,7 +4080,7 @@ SQLITE_API int sqlite3_open_v2( ** ** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a ** 64-bit signed integer and returns that integer, or D if P does not -** exist or ff the value of P is something other than an integer. +** exist or if the value of P is something other than an integer. ** ** The sqlite3_uri_key(F,N) returns a pointer to the name (not ** the value) of the N-th query parameter for filename F, or a NULL @@ -4436,6 +4445,19 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [[SQLITE_LIMIT_SCHEMA]] ^(
SQLITE_LIMIT_SCHEMA
**
The maximum number of objects (tables, indexes, triggers, and views) ** defined by the database schema.
)^ +** +** [[SQLITE_LIMIT_TRIGGER_STEPS]] ^(
SQLITE_LIMIT_TRIGGER_STEPS
+**
The maximum number of SQL statements that can be contained within +** a single trigger.
)^ +** +** [[SQLITE_LIMIT_CTES]] ^(
SQLITE_LIMIT_CTES
+**
The maximum number of [common table expressions] (CTEs) allowed +** in a single SQL statements. This limit is used to limit possible +** exponential growth in the number of required cursors required for +** nested joins of CTEs. The limit is an absolute upper bound, and +** SQLite may refuse to compile a statement if the number of CTEs simply +** approaches this limit. Set this limit to 0 to disable the restriction. +**
)^ **
*/ #define SQLITE_LIMIT_LENGTH 0 @@ -4452,6 +4474,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_WORKER_THREADS 11 #define SQLITE_LIMIT_PARSER_DEPTH 12 #define SQLITE_LIMIT_SCHEMA 13 +#define SQLITE_LIMIT_TRIGGER_STEPS 14 +#define SQLITE_LIMIT_CTES 15 /* ** CAPI3REF: Prepare Flags @@ -5864,6 +5888,10 @@ SQLITE_API int sqlite3_create_window_function( ** These constants define integer codes that represent the various ** text encodings supported by SQLite. ** +** If any value other than the ones defined here are passed into the +** "encoding" argument of an API function, then the results are +** unspecified and subject to change. +** **
** [[SQLITE_UTF8]]
SQLITE_UTF8
Text is encoding as UTF-8
** @@ -6846,6 +6874,10 @@ SQLITE_API void sqlite3_activate_cerod( ** at all. ^In SQLite version 3.42.0 and later, a negative ** argument passed into sqlite3_sleep() is changed to zero before it is relayed ** down into the xSleep method of the VFS. +** +** The maximum sleep time is 2147483 milliseconds (about 35 minutes and +** 47 seconds). If the argument to sqlite3_sleep() is greater than 2147483, +** the sleep time is capped at 2147483 milliseconds. */ SQLITE_API int sqlite3_sleep(int); @@ -8761,7 +8793,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ #define SQLITE_TESTCTRL_ATOF 34 -#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_EXPIRE_STMTS 35 +#define SQLITE_TESTCTRL_LAST 35 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -11265,7 +11298,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_wal_info( ** ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory -** allocation error occurs. +** allocation or I/O error occurs. ** ** This interface is omitted if SQLite is compiled with the ** [SQLITE_OMIT_DESERIALIZE] option. diff --git a/deps/sqlite3/sqlite3ext.h b/deps/sqlite3/sqlite3ext.h index 47b886f..d377778 100644 --- a/deps/sqlite3/sqlite3ext.h +++ b/deps/sqlite3/sqlite3ext.h @@ -109,7 +109,7 @@ struct sqlite3_api_routines { const char * (*libversion)(void); int (*libversion_number)(void); void *(*malloc)(int); - char * (*mprintf)(const char*,...); + char *(SQLITE_CDECL*mprintf)(const char*,...); int (*open)(const char*,sqlite3**); int (*open16)(const void*,sqlite3**); int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); @@ -130,11 +130,11 @@ struct sqlite3_api_routines { void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_value)(sqlite3_context*,sqlite3_value*); - void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + void *(*rollback_hook)(sqlite3*,void(*)(void*),void*); int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); - char * (*xsnprintf)(int,char*,const char*,...); + char *(SQLITE_CDECL*xsnprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, char const**,char const**,int*,int*,int*); @@ -198,7 +198,7 @@ struct sqlite3_api_routines { int (*xthreadsafe)(void); void (*result_zeroblob)(sqlite3_context*,int); void (*result_error_code)(sqlite3_context*,int); - int (*test_control)(int, ...); + int (SQLITE_CDECL*test_control)(int, ...); void (*randomness)(int,void*); sqlite3 *(*context_db_handle)(sqlite3_context*); int (*extended_result_codes)(sqlite3*,int); @@ -218,11 +218,11 @@ struct sqlite3_api_routines { void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*), void(*xDestroy)(void*)); - int (*db_config)(sqlite3*,int,...); + int (SQLITE_CDECL*db_config)(sqlite3*,int,...); sqlite3_mutex *(*db_mutex)(sqlite3*); int (*db_status)(sqlite3*,int,int*,int*,int); int (*extended_errcode)(sqlite3*); - void (*log)(int,const char*,...); + void (SQLITE_CDECL*log)(int,const char*,...); sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); const char *(*sourceid)(void); int (*stmt_status)(sqlite3_stmt*,int,int); @@ -232,7 +232,7 @@ struct sqlite3_api_routines { int (*wal_checkpoint)(sqlite3*,const char*); void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); - int (*vtab_config)(sqlite3*,int op,...); + int (SQLITE_CDECL*vtab_config)(sqlite3*,int op,...); int (*vtab_on_conflict)(sqlite3*); /* Version 3.7.16 and later */ int (*close_v2)(sqlite3*); @@ -301,7 +301,7 @@ struct sqlite3_api_routines { int (*keyword_check)(const char*,int); sqlite3_str *(*str_new)(sqlite3*); char *(*str_finish)(sqlite3_str*); - void (*str_appendf)(sqlite3_str*, const char *zFormat, ...); + void (SQLITE_CDECL*str_appendf)(sqlite3_str*, const char *zFormat, ...); void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list); void (*str_append)(sqlite3_str*, const char *zIn, int N); void (*str_appendall)(sqlite3_str*, const char *zIn);