diff --git a/NEWS b/NEWS index c5841f2381c..cfe090a793e 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ PHP NEWS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| ?? ??? 2010, PHP 5.3.4 -- Upgraded bundled Sqlite3 to version 3.7.0.1. (Ilia) +- Upgraded bundled Sqlite3 to version 3.7.3. (Ilia) - Upgraded bundled PCRE to version 8.10. (Ilia) - Added follow_location (enabled by default) option for the http stream diff --git a/ext/sqlite3/libsqlite/sqlite3.c b/ext/sqlite3/libsqlite/sqlite3.c index 73d3a8c45c7..511c15d417a 100644 --- a/ext/sqlite3/libsqlite/sqlite3.c +++ b/ext/sqlite3/libsqlite/sqlite3.c @@ -4,7 +4,7 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.0.1. By combining all the individual C code files into this +** version 3.7.3. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a one translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -219,20 +219,21 @@ # define SQLITE_MAX_VARIABLE_NUMBER 999 #endif -/* Maximum page size. The upper bound on this value is 32768. This a limit -** imposed by the necessity of storing the value in a 2-byte unsigned integer -** and the fact that the page size must be a power of 2. +/* Maximum page size. The upper bound on this value is 65536. This a limit +** imposed by the use of 16-bit offsets within each page. ** -** If this limit is changed, then the compiled library is technically -** incompatible with an SQLite library compiled with a different limit. If -** a process operating on a database with a page-size of 65536 bytes -** crashes, then an instance of SQLite compiled with the default page-size -** limit will not be able to rollback the aborted transaction. This could -** lead to database corruption. +** Earlier versions of SQLite allowed the user to change this value at +** compile time. This is no longer permitted, on the grounds that it creates +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback +** the aborted transaction. This could lead to database corruption. */ -#ifndef SQLITE_MAX_PAGE_SIZE -# define SQLITE_MAX_PAGE_SIZE 32768 +#ifdef SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_PAGE_SIZE #endif +#define SQLITE_MAX_PAGE_SIZE 65536 /* @@ -357,15 +358,21 @@ #endif /* -** The SQLITE_THREADSAFE macro must be defined as either 0 or 1. +** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. +** 0 means mutexes are permanently disable and the library is never +** threadsafe. 1 means the library is serialized which is the highest +** level of threadsafety. 2 means the libary is multithreaded - multiple +** threads can use SQLite as long as no two threads try to use the same +** database connection at the same time. +** ** Older versions of SQLite used an optional THREADSAFE macro. -** We support that for legacy +** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) #if defined(THREADSAFE) # define SQLITE_THREADSAFE THREADSAFE #else -# define SQLITE_THREADSAFE 1 +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ #endif #endif @@ -637,7 +644,7 @@ extern "C" { ** ** Since version 3.6.18, SQLite source code has been stored in the ** Fossil configuration management -** system. ^The SQLITE_SOURCE_ID macro evalutes to +** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID ** string contains the date and time of the check-in (UTC) and an SHA1 @@ -647,9 +654,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.0.1" -#define SQLITE_VERSION_NUMBER 3007000 -#define SQLITE_SOURCE_ID "2010-08-04 12:31:11 042a1abb030a0711386add7eb6e10832cc8b0f57" +#define SQLITE_VERSION "3.7.3" +#define SQLITE_VERSION_NUMBER 3007003 +#define SQLITE_SOURCE_ID "2010-10-08 02:34:02 2677848087c9c090efb17c1893e77d6136a9111d" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -694,7 +701,7 @@ SQLITE_API int sqlite3_libversion_number(void); ** compile time. ^The SQLITE_ prefix may be omitted from the ** option name passed to sqlite3_compileoption_used(). ** -** ^The sqlite3_compileoption_get() function allows interating +** ^The sqlite3_compileoption_get() function allows iterating ** over the list of options that were defined at compile time by ** returning the N-th compile time option string. ^If N is out of range, ** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ @@ -702,7 +709,7 @@ SQLITE_API int sqlite3_libversion_number(void); ** sqlite3_compileoption_get(). ** ** ^Support for the diagnostic functions sqlite3_compileoption_used() -** and sqlite3_compileoption_get() may be omitted by specifing the +** and sqlite3_compileoption_get() may be omitted by specifying the ** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. ** ** See also: SQL functions [sqlite_compileoption_used()] and @@ -808,7 +815,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** ** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. ** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfullly destroyed and all associated resources are deallocated. +** successfully destroyed and all associated resources are deallocated. ** ** Applications must [sqlite3_finalize | finalize] all [prepared statements] ** and [sqlite3_blob_close | close] all [BLOB handles] associated with @@ -1235,12 +1242,21 @@ struct sqlite3_io_methods { ** is often close. The underlying VFS might choose to preallocate database ** file space based on this hint in order to help writes to the database ** file run faster. +** +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 /* ** CAPI3REF: Mutex Handle @@ -1288,15 +1304,19 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** -** SQLite will guarantee that the zFilename parameter to xOpen +** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained -** from xFullPathname(). SQLite further guarantees that +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 10 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. -** If the zFilename parameter is xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. Whenever the +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** @@ -1307,7 +1327,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** -** SQLite will also add one of the following flags to the xOpen() +** ^(SQLite will also add one of the following flags to the xOpen() ** call, depending on the object being opened: ** ** +**
  • [SQLITE_OPEN_WAL] +** )^ ** ** The file I/O implementation can use the object type flags to ** change the way it deals with files. For example, an application @@ -1337,10 +1358,11 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** ** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be -** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] -** will be set for TEMP databases, journals and for subjournals. +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. ** -** The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the @@ -1349,7 +1371,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** It is not used to indicate the file should be opened ** for exclusive access. ** -** At least szOsFile bytes of memory are allocated by SQLite +** ^At least szOsFile bytes of memory are allocated by SQLite ** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that @@ -1359,13 +1381,13 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** -** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] ** to test whether a file is at least readable. The file can be a ** directory. ** -** SQLite will always allocate at least mxPathname+1 bytes for the +** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer ** is also passed as a parameter to both methods. If the output buffer ** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is @@ -1379,10 +1401,10 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** of good-quality randomness into zOut. The return value is ** the actual number of bytes of randomness obtained. ** The xSleep() method causes the calling thread to sleep for at -** least the number of microseconds given. The xCurrentTime() +** least the number of microseconds given. ^The xCurrentTime() ** method returns a Julian Day Number for the current date and time as ** a floating point value. -** The xCurrentTimeInt64() method returns, as an integer, the Julian +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian ** Day Number multipled by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current @@ -1779,7 +1801,7 @@ struct sqlite3_mem_methods { ** )^ ** ^Memory allocation statistics are enabled by default unless SQLite is @@ -1793,15 +1815,14 @@ struct sqlite3_mem_methods { ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. The sz parameter should be a few bytes -** larger than the actual scratch space required due to internal overhead. +** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than one scratch buffer per thread. So -** N should be set to the expected maximum number of threads. ^SQLite will -** never require a scratch buffer that is more than 6 times the database -** page size. ^If SQLite needs needs additional scratch memory beyond -** what is provided by this configuration option, then +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed. ** **
    SQLITE_CONFIG_PAGECACHE
    @@ -1821,8 +1842,7 @@ struct sqlite3_mem_methods { ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** ^The implementation might use one or more of the N buffers to hold -** memory accounting information. The pointer in the first argument must +** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined. ** @@ -1951,8 +1971,14 @@ struct sqlite3_mem_methods { ** or equal to the product of the second and third arguments. The buffer ** must be aligned to an 8-byte boundary. ^If the second argument to ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller -** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE] +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^ ** ** */ @@ -2257,6 +2283,9 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries ** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. @@ -2277,7 +2306,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** It is not safe to pass a result table directly to [sqlite3_free()]. ** A result table should be deallocated using [sqlite3_free_table()]. ** -** As an example of the result table format, suppose a query result +** ^(As an example of the result table format, suppose a query result ** is as follows: ** **
    @@ -2301,7 +2330,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms);
     **        azResult[5] = "28";
     **        azResult[6] = "Cindy";
     **        azResult[7] = "21";
    -** 
    +** )^ ** ** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 @@ -2309,19 +2338,19 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** pointer given in its 3rd parameter. ** ** After the application has finished with the result from sqlite3_get_table(), -** it should pass the result table pointer to sqlite3_free_table() in order to +** it must pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** ^(The sqlite3_get_table() interface is implemented as a wrapper around +** The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not ** reflected in subsequent calls to [sqlite3_errcode()] or -** [sqlite3_errmsg()].)^ +** [sqlite3_errmsg()]. */ SQLITE_API int sqlite3_get_table( sqlite3 *db, /* An open database */ @@ -2473,7 +2502,9 @@ SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); ** is not freed. ** ** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() -** is always aligned to at least an 8 byte boundary. +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. ** ** In SQLite version 3.5.0 and 3.5.1, it was possible to define ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in @@ -2716,7 +2747,13 @@ SQLITE_API int sqlite3_set_authorizer( ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time -** of how long that statement took to run. +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. */ SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, @@ -2725,17 +2762,28 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, /* ** CAPI3REF: Query Progress Callbacks ** -** ^This routine configures a callback function - the -** progress callback - that is invoked periodically during long -** running calls to [sqlite3_exec()], [sqlite3_step()] and -** [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** ** ^If the progress callback returns non-zero, the operation is ** interrupted. This feature can be used to implement a ** "Cancel" button on a GUI progress dialog box. ** -** The progress handler must not do anything that will modify +** The progress handler callback must not do anything that will modify ** the database connection that invoked the progress handler. ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. @@ -2794,7 +2842,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** If the 3rd parameter to sqlite3_open_v2() is not one of the ** combinations shown above or one of the combinations shown above combined ** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags, +** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2919,17 +2967,22 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** [database connection] whose limit is to be set or queried. The ** second parameter is one of the [limit categories] that define a ** class of constructs to be size limited. The third parameter is the -** new limit for that construct. The function returns the old limit.)^ +** new limit for that construct.)^ ** ** ^If the new limit is a negative number, the limit is unchanged. -** ^(For the limit category of SQLITE_LIMIT_XYZ there is a +** ^(For each limit category SQLITE_LIMIT_NAME there is a ** [limits | hard upper bound] -** set by a compile-time C preprocessor macro named -** [limits | SQLITE_MAX_XYZ]. +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. ** (The "_LIMIT_" in the name is changed to "_MAX_".))^ ** ^Attempts to increase a limit above its hard upper bound are ** silently truncated to the hard upper bound. ** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** ** Run-time limits are intended for use in applications that manage ** both their own internal database and also databases that are controlled ** by untrusted external sources. An example application might be a @@ -2958,7 +3011,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** **
    ** ^(
    SQLITE_LIMIT_LENGTH
    -**
    The maximum size of any string or BLOB or table row.
    )^ +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_SQL_LENGTH
    **
    The maximum length of an SQL statement, in bytes.
    )^ @@ -2976,7 +3029,9 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement.
    )^ +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.)^ ** ** ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ @@ -2989,8 +3044,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [GLOB] operators.)^ ** ** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    -**
    The maximum number of variables in an SQL statement that can -** be bound.
    )^ +**
    The maximum index number of any [parameter] in an SQL statement.)^ ** ** ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ @@ -3062,12 +3116,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. ^If the schema has changed in -** a way that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text -** of the parsing error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. **
  • ** **
  • @@ -3080,11 +3129,16 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** **
  • -** ^If the value of a [parameter | host parameter] in the WHERE clause might -** change the query plan for a statement, then the statement may be -** automatically recompiled (as if there had been a schema change) on the first -** [sqlite3_step()] call following any change to the -** [sqlite3_bind_text | bindings] of the [parameter]. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** the **
  • ** */ @@ -3151,7 +3205,7 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** then there is no distinction between protected and unprotected ** sqlite3_value objects and they can be used interchangeably. However, ** for maximum code portability it is recommended that applications -** still make the distinction between between protected and unprotected +** still make the distinction between protected and unprotected ** sqlite3_value objects even when not strictly required. ** ** ^The sqlite3_value objects that are passed as parameters into the @@ -3197,7 +3251,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** ** In the templates above, NNN represents an integer literal, -** and VVV represents an alphanumeric identifer.)^ ^The values of these +** and VVV represents an alphanumeric identifier.)^ ^The values of these ** parameters (also called "host parameter names" or "SQL parameters") ** can be set using the sqlite3_bind_*() routines defined here. ** @@ -3346,6 +3400,8 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL ** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); @@ -3536,8 +3592,14 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set ** -** ^The sqlite3_data_count(P) the number of columns in the -** of the result set of [prepared statement] P. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** +** See also: [sqlite3_column_count()] */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); @@ -3617,18 +3679,26 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** ^The value returned does not include the zero terminator at the end -** of the string. ^For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero terminated. ^The return -** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary -** pointer, possibly even a NULL pointer. -** -** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 in native byte order instead of UTF-8. -** ^The zero terminator is not included in this count. +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. An unprotected sqlite3_value object @@ -3673,10 +3743,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** ^Note that when type conversions occur, pointers returned by prior +** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** ^(Type conversions and pointer invalidations might occur +** Type conversions and pointer invalidations might occur ** in the following cases: ** ** )^ +** ** ** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds +** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** ^(The safest and easiest to remember policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** ** )^ +** ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result @@ -3742,17 +3812,26 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. ^If execution of the statement failed then an -** [error code] or [extended error code] is returned. +** ^If the most recent evaluation of the statement encountered no errors or +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** ^This routine can be called at any point during the execution of the -** [prepared statement]. ^If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an [sqlite3_interrupt | interrupt]. -** ^Incomplete updates may be rolled back and transactions canceled, -** depending on the circumstances, and the -** [error code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); @@ -3788,23 +3867,25 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** ^These two functions (collectively known as "function creation routines") +** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only difference between the -** two is that the second parameter, the name of the (scalar) function or -** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 -** for sqlite3_create_function16(). +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database ** connection then application-defined SQL functions must be added ** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created or -** redefined. ^The length of the name is limited to 255 bytes, exclusive of -** the zero-terminator. Note that the name length limit is in bytes, not -** characters. ^Any attempt to create a function with a longer name -** will result in [SQLITE_ERROR] being returned. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** ** ^The third parameter (nArg) ** is the number of arguments that the SQL function or @@ -3814,10 +3895,10 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** parameter is less than -1 or greater than 127 then the behavior is ** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be +** its parameters. Every SQL function implementation must be able to work +** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. @@ -3829,13 +3910,21 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc -** callback only; NULL pointers should be passed as the xStep and xFinal +** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL for all three function callbacks. +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL poiners for all three function +** callbacks. +** +** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is invoked when the function is deleted, either by being +** overloaded or when the database connection closes. +** ^When the destructure callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the pointer which was +** the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -3851,11 +3940,6 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** between UTF8 and UTF16. ** ** ^Built-in functions may be overloaded by new application-defined functions. -** ^The first application-defined function with a given name overrides all -** built-in functions in the same [database connection] with the same name. -** ^Subsequent application-defined functions of the same name only override -** prior application-defined functions that are an exact match for the -** number of parameters and preferred encoding. ** ** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not @@ -3882,6 +3966,17 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -3976,7 +4071,7 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context ** -** Implementions of aggregate SQL functions use this +** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. ** ** ^The first time the sqlite3_aggregate_context(C,N) routine is called @@ -4228,46 +4323,70 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences ** -** These functions are used to add new collation sequences to the -** [database connection] specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** ^The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases -** the name is passed as the second function argument. +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. ** -** ^The third argument may be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The -** third argument might also be [SQLITE_UTF16] to indicate that the routine -** expects pointers to be UTF-16 strings in the native byte order, or the -** argument can be [SQLITE_UTF16_ALIGNED] if the -** the routine expects pointers to 16-bit word aligned strings -** of UTF-16 in the native byte order. +** ^(The third argument (eTextRep) must be one of the constants: +** )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. ** -** A pointer to the user supplied routine must be passed as the fifth -** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). -** ^Each time the application supplied function is invoked, it is passed -** as its first parameter a copy of the void* passed as the fourth argument -** to sqlite3_create_collation() or sqlite3_create_collation16(). +** ^The fourth argument, pArg, is a application data pointer that is passed +** through as the first argument to the collating function callback. ** -** ^The remaining arguments to the application-supplied routine are two strings, -** each represented by a (length, data) pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The application defined collation routine should -** return negative, zero or positive if the first string is less than, -** equal to, or greater than the second string. i.e. (STRING1 - STRING2). +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must alway return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** except that it takes an extra argument which is a destructor for -** the collation. ^The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). -** ^Collations are destroyed when they are overridden by later calls to the -** collation creation functions or when the [database connection] is closed -** using [sqlite3_close()]. +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ @@ -4275,14 +4394,14 @@ SQLITE_API int sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); SQLITE_API int sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); @@ -4290,7 +4409,7 @@ SQLITE_API int sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); @@ -4379,16 +4498,19 @@ SQLITE_API void sqlite3_activate_cerod( /* ** CAPI3REF: Suspend Execution For A Short Time ** -** ^The sqlite3_sleep() function causes the current thread to suspend execution +** The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** ^If the operating system does not support sleep requests with +** If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. ^The number of milliseconds of sleep actually +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. ** ** ^SQLite implements this interface by calling the xSleep() -** method of the default [sqlite3_vfs] object. +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. */ SQLITE_API int sqlite3_sleep(int); @@ -4610,40 +4732,73 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** pages to improve performance is an example of non-essential memory. ** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. */ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Impose A Limit On Heap Size ** -** ^The sqlite3_soft_heap_limit() interface places a "soft" limit -** on the amount of heap memory that may be allocated by SQLite. -** ^If an internal allocation is requested that would exceed the -** soft heap limit, [sqlite3_release_memory()] is invoked one or -** more times to free up some space before the allocation is performed. +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. ** -** ^The limit is called "soft" because if [sqlite3_release_memory()] -** cannot free sufficient memory to prevent the limit from being exceeded, -** the memory is allocated anyway and the current operation proceeds. +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. ** -** ^A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhausted. -** ^The default value for the soft heap limit is zero. +** ^If the argument N is zero then the soft heap limit is disabled. ** -** ^(SQLite makes a best effort to honor the soft heap limit. -** But if the soft heap limit cannot be honored, execution will -** continue without error or notification.)^ This is why the limit is -** called a "soft" limit. It is advisory only. +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: ** -** Prior to SQLite version 3.5.0, this routine only constrained the memory -** allocated by a single thread - the same thread in which this routine -** runs. Beginning with SQLite version 3.5.0, the soft heap limit is -** applied to all threads. The value specified for the soft heap limit -** is an upper bound on the total memory allocation for all threads. In -** version 3.5.0 there is no mechanism for limiting the heap usage for -** individual threads. +** )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. */ -SQLITE_API void sqlite3_soft_heap_limit(int); +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + /* ** CAPI3REF: Extract Metadata About A Column Of A Table @@ -4767,34 +4922,47 @@ SQLITE_API int sqlite3_load_extension( SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions +** CAPI3REF: Automatically Load Statically Linked Extensions ** -** ^This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new [database connections]. +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked SQLite extension +** that is to be automatically loaded into all new database connections. ** -** ^(This routine stores a pointer to the extension entry point -** in an array that is obtained from [sqlite3_malloc()]. That memory -** is deallocated by [sqlite3_reset_auto_extension()].)^ +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: ** -** ^This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** ^Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** ^Automatic extensions apply across all threads. +**
    +**    int xEntryPoint(
    +**      sqlite3 *db,
    +**      const char **pzErrMsg,
    +**      const struct sqlite3_api_routines *pThunk
    +**    );
    +** 
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()]. */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** ^(This function disables all previously registered automatic -** extensions. It undoes the effect of all prior -** [sqlite3_auto_extension()] calls.)^ -** -** ^This function disables automatic extensions in all threads. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ SQLITE_API void sqlite3_reset_auto_extension(void); @@ -4865,7 +5033,8 @@ struct sqlite3_module { ** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info ** -** The sqlite3_index_info structure and its substructures is used to +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to ** pass information into and receive the reply from the [xBestIndex] ** method of a [virtual table module]. The fields under **Inputs** are the ** inputs to xBestIndex and are read-only. xBestIndex inserts its @@ -4873,10 +5042,12 @@ struct sqlite3_module { ** ** ^(The aConstraint[] array records WHERE clause constraints of the form: ** -**
    column OP expr
    +**
    column OP expr
    ** ** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is -** stored in aConstraint[].op.)^ ^(The index of the column is stored in +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in ** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint ** is usable) and false if it cannot.)^ @@ -4936,6 +5107,15 @@ struct sqlite3_index_info { int orderByConsumed; /* True if output is already ordered */ double estimatedCost; /* Estimated cost of using this index */ }; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 #define SQLITE_INDEX_CONSTRAINT_LE 8 @@ -5421,7 +5601,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** ** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** ^The xMutexInit routine is calle by SQLite exactly once for each +** ^The xMutexInit routine is called by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** ** ^The xMutexEnd method defined by this structure is invoked as @@ -5454,7 +5634,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** it is passed a NULL pointer). ** ** The xMutexInit() method must be threadsafe. ^It must be harmless to -** invoke xMutexInit() mutiple times within the same process and without +** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** @@ -5618,13 +5798,14 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_LAST 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 +#define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information -** about the preformance of SQLite, and optionally to reset various +** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes ** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ @@ -5637,7 +5818,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** ^The sqlite3_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be @@ -5676,6 +5857,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.)^ ** +** ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations.
    )^ +** ** ^(
    SQLITE_STATUS_PAGECACHE_USED
    **
    This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using @@ -5684,7 +5868,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache -** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to @@ -5707,7 +5891,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory -** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values ** returned include overflows because the requested allocation was too ** larger (that is, because the requested allocation was larger than the @@ -5737,6 +5921,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 #define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status @@ -5746,7 +5931,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that -** determiness the parameter to interrogate. The set of +** determines the parameter to interrogate. The set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely ** to grow in future releases of SQLite. ** @@ -5755,6 +5940,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. ** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); @@ -5776,16 +5964,33 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r **
    This parameter returns the number of lookaside memory slots currently ** checked out.
    )^ ** -**
    SQLITE_DBSTATUS_CACHE_USED
    -**
    ^This parameter returns the approximate number of of bytes of heap -** memory used by all pager caches associated with the database connection. +** ^(
    SQLITE_DBSTATUS_CACHE_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** ^(
    SQLITE_DBSTATUS_STMT_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
    **
    */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 #define SQLITE_DBSTATUS_CACHE_USED 1 -#define SQLITE_DBSTATUS_MAX 1 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */ /* @@ -5864,32 +6069,42 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ The majority of the -** heap memory used by SQLite is used by the page cache to cache data read -** from, or ready to be written to, the database file. By implementing a -** custom page cache using this API, an application can control more -** precisely the amount of memory consumed by SQLite, the way in which +** instance of the sqlite3_pcache_methods structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which ** that memory is allocated and released, and the policies used to ** determine exactly which parts of a database file are cached and for ** how long. ** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** ** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** -** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ -** ^The xInit() method can set up up global structures and/or any mutexes +** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ ** -** ^The xShutdown() method is called from within [sqlite3_shutdown()], -** if the application invokes this API. It can be used to clean up +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up ** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. ** -** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. ^The +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. @@ -5897,47 +6112,50 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** ^The xCreate() method is used to construct a new cache instance. SQLite -** will typically create one cache instance for each open database file, +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. ^SQLite will use the +** increment (here called "R") of about 100 or 200. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. ** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. ^The cache implementation +** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** ^In other words, a cache created with bPurgeable set to false will +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** ^The xPagecount() method should return the number of pages currently -** stored in the cache. +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. ** -** ^The xFetch() method is used to fetch a page and return a pointer to it. -** ^A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. ^The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** The xFetch() method locates a page in the cache and returns a pointer to +** the page, or a NULL pointer. +** A "page", in this context, means a buffer of szPage bytes aligned at an +** 8-byte boundary. The page to be fetched is determined by the key. ^The +** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** ^If the requested page is already in the page cache, then the page cache +** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. ^(If the requested page is not already in the cache, then the -** behavior of the cache implementation is determined by the value of the -** createFlag parameter passed to xFetch, according to the following table: +** intact. If the requested page is not already in the cache, then the +** behavior of the cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: ** ** **
    createFlag Behaviour when page is not already in cache @@ -5946,36 +6164,35 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** Otherwise return NULL. **
    2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -**
    )^ +** ** -** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If -** a call to xFetch() with createFlag==1 returns NULL, then SQLite will +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of -** pinned pages to disk and synching the operating system disk cache. After -** attempting to unpin pages, the xFetch() method will be invoked again with -** a createFlag of 2. +** pinned pages to disk and synching the operating system disk cache. ** ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. ^(If the third parameter, discard, is non-zero, -** then the page should be evicted from the cache. In this case SQLite -** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is -** zero, then the page is considered to be unpinned. ^The cache implementation +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** ^(The cache is not required to perform any reference counting. A single +** The cache must not perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch().)^ +** to xFetch(). ** -** ^The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. ^If the cache -** previously contains an entry associated with newKey, it should be +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be ** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** ^When SQLite calls the xTruncate() method, the cache must discard all +** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). ^If any +** to the value of the iLimit parameter passed to xTruncate(). If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** @@ -6151,7 +6368,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source databae file. +** up and the total number of pages in the source database file. ** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces ** retrieve these two values, respectively. ** @@ -6247,7 +6464,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is cancelled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -6329,7 +6546,7 @@ SQLITE_API int sqlite3_unlock_notify( ** ** ^The [sqlite3_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a -** case-indendent fashion, using the same definition of case independence +** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); @@ -6453,6 +6670,62 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); #endif #endif +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + double *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -7023,6 +7296,7 @@ typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct ExprSpan ExprSpan; typedef struct FKey FKey; +typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; @@ -7129,12 +7403,11 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** NOTE: These values must match the corresponding PAGER_ values in ** pager.h. */ -#define BTREE_OMIT_JOURNAL 1 /* Do not use journal. No argument */ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ #define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ -#define BTREE_MEMORY 4 /* In-memory DB. No argument */ -#define BTREE_READONLY 8 /* Open the database in read-only mode */ -#define BTREE_READWRITE 16 /* Open for both reading and writing */ -#define BTREE_CREATE 32 /* Create the database if it does not exist */ +#define BTREE_MEMORY 4 /* This is an in-memory DB */ +#define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); @@ -7170,11 +7443,17 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); /* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR -** of the following flags: +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) */ #define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ -#define BTREE_ZERODATA 2 /* Table has keys only - no data */ -#define BTREE_LEAFDATA 4 /* Data stored in leaves only. Implies INTKEY */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); @@ -7263,6 +7542,10 @@ SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); #endif +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*); +#endif + /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the @@ -7392,8 +7675,8 @@ struct SubProgram { int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ - int nRef; /* Number of pointers to this structure */ void *token; /* id that may be used to recursive triggers */ + SubProgram *pNext; /* Next sub-program already visited */ }; /* @@ -7681,6 +7964,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); @@ -7698,7 +7982,6 @@ SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, int); SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); -SQLITE_PRIVATE void sqlite3VdbeProgramDelete(sqlite3 *, SubProgram *, int); SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetValue(Vdbe*, int, u8); SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE @@ -7709,6 +7992,10 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*, SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +#endif + #ifndef NDEBUG SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); @@ -7787,6 +8074,7 @@ typedef struct PgHdr DbPage; */ #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ +#define PAGER_MEMORY 0x0004 /* In-memory database */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). @@ -7827,7 +8115,7 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); -SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u16*, int); +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int); @@ -7854,9 +8142,10 @@ SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ -SQLITE_PRIVATE int sqlite3PagerPagecount(Pager*, int*); +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*); SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int); SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*); SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*); SQLITE_PRIVATE int sqlite3PagerRollback(Pager*); @@ -8420,8 +8709,8 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); #define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK #define sqlite3_mutex_leave(X) -#define sqlite3_mutex_held(X) 1 -#define sqlite3_mutex_notheld(X) 1 +#define sqlite3_mutex_held(X) ((void)(X),1) +#define sqlite3_mutex_notheld(X) ((void)(X),1) #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() @@ -8448,14 +8737,6 @@ struct Db { /* ** An instance of the following structure stores a database schema. -** -** If there are no virtual tables configured in this schema, the -** Schema.db variable is set to NULL. After the first virtual table -** has been added, it is set to point to the database connection -** used to create the connection. Once a virtual table has been -** added to the Schema structure and the Schema.db variable populated, -** only that database connection may use the Schema to prepare -** statements. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ @@ -8468,9 +8749,6 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3 *db; /* "Owner" connection. See comment above */ -#endif }; /* @@ -8658,6 +8936,7 @@ struct sqlite3 { int nStatement; /* Number of nested statement-transactions */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ + int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MASTER @@ -8753,6 +9032,27 @@ struct FuncDef { void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ char *zName; /* SQL name of the function. */ FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ +}; + +/* +** This structure encapsulates a user-function destructor callback (as +** configured using create_function_v2()) and a reference counter. When +** create_function_v2() is called to create a function with a destructor, +** a single object of this type is allocated. FuncDestructor.nRef is set to +** the number of FuncDef objects created (either 1 or 3, depending on whether +** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor +** member of each of the new FuncDef objects is set to point to the allocated +** FuncDestructor. +** +** Thereafter, when one of the FuncDef objects is deleted, the reference +** count on this object is decremented. When it reaches 0, the destructor +** is invoked and the FuncDestructor structure freed. +*/ +struct FuncDestructor { + int nRef; + void (*xDestroy)(void *); + void *pUserData; }; /* @@ -8793,15 +9093,15 @@ struct FuncDef { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ - pArg, 0, xFunc, 0, 0, #zName, 0} + pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ - {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0} + {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8, nc*SQLITE_FUNC_NEEDCOLL, \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0} + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at @@ -9015,13 +9315,13 @@ struct VTable { ** of a SELECT statement. */ struct Table { - sqlite3 *dbMem; /* DB connection used for lookaside allocations. */ char *zName; /* Name of the table or view */ int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ int tnum; /* Root BTree node for this table (see note above) */ + unsigned nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ u16 nRef; /* Number of pointers to this Table */ u8 tabFlags; /* Mask of TF_* values */ @@ -9152,9 +9452,9 @@ struct FKey { */ struct KeyInfo { sqlite3 *db; /* The database connection */ - u8 enc; /* Text encoding - one of the TEXT_Utf* values */ + u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ u16 nField; /* Number of entries in aColl[] */ - u8 *aSortOrder; /* If defined an aSortOrder[i] is true, sort DESC */ + u8 *aSortOrder; /* Sort order for each column. May be NULL */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; @@ -10124,7 +10424,7 @@ struct StrAccum { int nAlloc; /* Amount of space allocated in zText */ int mxAlloc; /* Maximum allowed string length */ u8 mallocFailed; /* Becomes true if any memory allocation fails */ - u8 useMalloc; /* True if zText is enlargeable using realloc */ + u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ u8 tooBig; /* Becomes true if string size exceeds limits */ }; @@ -10276,7 +10576,6 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); ** Internal function prototypes */ SQLITE_PRIVATE int sqlite3StrICmp(const char *, const char *); -SQLITE_PRIVATE int sqlite3IsNumber(const char*, int*, u8); SQLITE_PRIVATE int sqlite3Strlen30(const char*); #define sqlite3StrNICmp sqlite3_strnicmp @@ -10300,7 +10599,7 @@ SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); -SQLITE_PRIVATE int sqlite3MemoryAlarm(void (*)(void*, sqlite3_int64, int), void*, sqlite3_int64); +SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* ** On systems with ample stack space and that support alloca(), make @@ -10422,7 +10721,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); #endif SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); -SQLITE_PRIVATE void sqlite3DeleteTable(Table*); +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); @@ -10471,7 +10770,6 @@ SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*, int); SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); -SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse*,int,int); SQLITE_PRIVATE int sqlite3ExprCode(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); @@ -10591,17 +10889,14 @@ SQLITE_PRIVATE int sqlite3AuthReadCol(Parse*, const char *, const char *, int) #endif SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); -SQLITE_PRIVATE int sqlite3BtreeFactory(sqlite3 *db, const char *zFilename, - int omitJournal, int nCache, int flags, Btree **ppBtree); SQLITE_PRIVATE int sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*); +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); -SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *, int); SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); @@ -10647,7 +10942,7 @@ SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *, Table *); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); -SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*); +SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -10656,7 +10951,8 @@ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Parse *pParse, Expr *, Token *); +SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Expr*, CollSeq*); +SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); @@ -10706,7 +11002,7 @@ SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *, const char *); SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3*,int iDB); -SQLITE_PRIVATE void sqlite3DeleteIndexSamples(Index*); +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); @@ -10717,7 +11013,9 @@ SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *, Index *); SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), - void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*)); + void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + FuncDestructor *pDestructor +); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); @@ -10768,7 +11066,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) #else -SQLITE_PRIVATE void sqlite3VtabClear(Table*); +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **); SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); @@ -10821,9 +11119,9 @@ SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *); #define sqlite3FkRequired(a,b,c,d) 0 #endif #ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE void sqlite3FkDelete(Table*); +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *, Table*); #else - #define sqlite3FkDelete(a) + #define sqlite3FkDelete(a,b) #endif @@ -10916,17 +11214,18 @@ SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*,...); ** sqlite3MemdebugHasType() returns true if any of the bits in its second ** argument match the type set by the previous sqlite3MemdebugSetType(). ** sqlite3MemdebugHasType() is intended for use inside assert() statements. -** For example: ** -** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); +** sqlite3MemdebugNoType() returns true if none of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). ** ** Perhaps the most important point is the difference between MEMTYPE_HEAP -** and MEMTYPE_DB. If an allocation is MEMTYPE_DB, that means it might have -** been allocated by lookaside, except the allocation was too large or -** lookaside was already full. It is important to verify that allocations -** that might have been satisfied by lookaside are not passed back to -** non-lookaside free() routines. Asserts such as the example above are -** placed on the non-lookaside free() routines to verify this constraint. +** and MEMTYPE_LOOKASIDE. If an allocation is MEMTYPE_LOOKASIDE, that means +** it might have been allocated by lookaside, except the allocation was +** too large or lookaside was already full. It is important to verify +** that allocations that might have been satisfied by lookaside are not +** passed back to non-lookaside free() routines. Asserts such as the +** example above are placed on the non-lookaside free() routines to verify +** this constraint. ** ** All of this is no-op for a production build. It only comes into ** play when the SQLITE_MEMDEBUG compile-time option is used. @@ -10934,14 +11233,17 @@ SQLITE_PRIVATE void (*sqlite3IoTrace)(const char*,...); #ifdef SQLITE_MEMDEBUG SQLITE_PRIVATE void sqlite3MemdebugSetType(void*,u8); SQLITE_PRIVATE int sqlite3MemdebugHasType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); #else # define sqlite3MemdebugSetType(X,Y) /* no-op */ # define sqlite3MemdebugHasType(X,Y) 1 +# define sqlite3MemdebugNoType(X,Y) 1 #endif -#define MEMTYPE_HEAP 0x01 /* General heap allocations */ -#define MEMTYPE_DB 0x02 /* Associated with a database connection */ -#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ -#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_HEAP 0x01 /* General heap allocations */ +#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ +#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ #endif /* _SQLITEINT_H_ */ @@ -11572,14 +11874,454 @@ SQLITE_API const char *sqlite3_compileoption_get(int N){ ** This module implements the sqlite3_status() interface and related ** functionality. */ +/************** Include vdbeInt.h in the middle of status.c ******************/ +/************** Begin file vdbeInt.h *****************************************/ +/* +** 2003 September 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for information that is private to the +** VDBE. This information used to all be at the top of the single +** source code file "vdbe.c". When that file became too big (over +** 6000 lines long) it was split up into several smaller files and +** this header information was factored out. +*/ +#ifndef _VDBEINT_H_ +#define _VDBEINT_H_ + +/* +** SQL is translated into a sequence of instructions to be +** executed by a virtual machine. Each instruction is an instance +** of the following structure. +*/ +typedef struct VdbeOp Op; + +/* +** Boolean values +*/ +typedef unsigned char Bool; + +/* +** A cursor is a pointer into a single BTree within a database file. +** The cursor can seek to a BTree entry with a particular key, or +** loop over all entries of the Btree. You can also insert new BTree +** entries or retrieve the key or data from the entry that the cursor +** is currently pointing to. +** +** Every cursor that the virtual machine has open is represented by an +** instance of the following structure. +** +** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is +** really a single row that represents the NEW or OLD pseudo-table of +** a row trigger. The data for the row is stored in VdbeCursor.pData and +** the rowid is in VdbeCursor.iKey. +*/ +struct VdbeCursor { + BtCursor *pCursor; /* The cursor structure of the backend */ + int iDb; /* Index of cursor database in db->aDb[] (or -1) */ + i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ + Bool zeroed; /* True if zeroed out and ready for reuse */ + Bool rowidIsValid; /* True if lastRowid is valid */ + Bool atFirst; /* True if pointing to first entry */ + Bool useRandomRowid; /* Generate new record numbers semi-randomly */ + Bool nullRow; /* True if pointing to a row with no data */ + Bool deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + Bool isTable; /* True if a table requiring integer keys */ + Bool isIndex; /* True if an index containing keys only - no data */ + Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ + i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ + Btree *pBt; /* Separate file holding temporary table */ + int pseudoTableReg; /* Register holding pseudotable content. */ + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ + int nField; /* Number of fields in the header */ + i64 seqCount; /* Sequence counter */ + sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ + const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ + + /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or + ** OP_IsUnique opcode on this cursor. */ + int seekResult; + + /* Cached information about the header for the data record that the + ** cursor is currently pointing to. Only valid if cacheStatus matches + ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of + ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that + ** the cache is out of date. + ** + ** aRow might point to (ephemeral) data for the current row, or it might + ** be NULL. + */ + u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ + int payloadSize; /* Total number of bytes in the record */ + u32 *aType; /* Type values for all entries in the record */ + u32 *aOffset; /* Cached offsets to the start of each columns data */ + u8 *aRow; /* Data for the current row, if all on one page */ +}; +typedef struct VdbeCursor VdbeCursor; + +/* +** When a sub-program is executed (OP_Program), a structure of this type +** is allocated to store the current value of the program counter, as +** well as the current memory cell array and various other frame specific +** values stored in the Vdbe struct. When the sub-program is finished, +** these values are copied back to the Vdbe from the VdbeFrame structure, +** restoring the state of the VM to as it was before the sub-program +** began executing. +** +** Frames are stored in a linked list headed at Vdbe.pParent. Vdbe.pParent +** is the parent of the current frame, or zero if the current frame +** is the main Vdbe program. +*/ +typedef struct VdbeFrame VdbeFrame; +struct VdbeFrame { + Vdbe *v; /* VM this frame belongs to */ + int pc; /* Program Counter */ + Op *aOp; /* Program instructions */ + int nOp; /* Size of aOp array */ + Mem *aMem; /* Array of memory cells */ + int nMem; /* Number of entries in aMem */ + VdbeCursor **apCsr; /* Element of Vdbe cursors */ + u16 nCursor; /* Number of entries in apCsr */ + void *token; /* Copy of SubProgram.token */ + int nChildMem; /* Number of memory cells for child frame */ + int nChildCsr; /* Number of cursors for child frame */ + i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + int nChange; /* Statement changes (Vdbe.nChanges) */ + VdbeFrame *pParent; /* Parent of this frame */ +}; + +#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) + +/* +** A value for VdbeCursor.cacheValid that means the cache is always invalid. +*/ +#define CACHE_STALE 0 + +/* +** Internally, the vdbe manipulates nearly all SQL values as Mem +** structures. Each Mem struct may cache multiple representations (string, +** integer etc.) of the same value. A value (and therefore Mem structure) +** has the following properties: +** +** Each value has a manifest type. The manifest type of the value stored +** in a Mem struct is returned by the MemType(Mem*) macro. The type is +** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or +** SQLITE_BLOB. +*/ +struct Mem { + union { + i64 i; /* Integer value. */ + int nZero; /* Used when bit MEM_Zero is set in flags */ + FuncDef *pDef; /* Used only when flags==MEM_Agg */ + RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ + VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ + } u; + double r; /* Real value */ + sqlite3 *db; /* The associated database connection */ + char *z; /* String or BLOB value */ + int n; /* Number of characters in string value, excluding '\0' */ + u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ + u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ + u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ +#ifdef SQLITE_DEBUG + Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ + void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ +#endif + void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ + char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ +}; + +/* One or more of the following flags are set to indicate the validOK +** representations of the value stored in the Mem struct. +** +** If the MEM_Null flag is set, then the value is an SQL NULL value. +** No other flags may be set in this case. +** +** If the MEM_Str flag is set then Mem.z points at a string representation. +** Usually this is encoded in the same unicode encoding as the main +** database (see below for exceptions). If the MEM_Term flag is also +** set, then the string is nul terminated. The MEM_Int and MEM_Real +** flags may coexist with the MEM_Str flag. +** +** Multiple of these values can appear in Mem.flags. But only one +** at a time can appear in Mem.type. +*/ +#define MEM_Null 0x0001 /* Value is NULL */ +#define MEM_Str 0x0002 /* Value is a string */ +#define MEM_Int 0x0004 /* Value is an integer */ +#define MEM_Real 0x0008 /* Value is a real number */ +#define MEM_Blob 0x0010 /* Value is a BLOB */ +#define MEM_RowSet 0x0020 /* Value is a RowSet object */ +#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ +#define MEM_Invalid 0x0080 /* Value is undefined */ +#define MEM_TypeMask 0x00ff /* Mask of type bits */ + +/* Whenever Mem contains a valid string or blob representation, one of +** the following flags must be set to determine the memory management +** policy for Mem.z. The MEM_Term flag tells us whether or not the +** string is \000 or \u0000 terminated +*/ +#define MEM_Term 0x0200 /* String rep is nul terminated */ +#define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ +#define MEM_Static 0x0800 /* Mem.z points to a static string */ +#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ +#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ +#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ +#ifdef SQLITE_OMIT_INCRBLOB + #undef MEM_Zero + #define MEM_Zero 0x0000 +#endif + +/* +** Clear any existing type flags from a Mem and replace them with f +*/ +#define MemSetTypeFlag(p, f) \ + ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) + +/* +** Return true if a memory cell is not marked as invalid. This macro +** is for use inside assert() statements only. +*/ +#ifdef SQLITE_DEBUG +#define memIsValid(M) ((M)->flags & MEM_Invalid)==0 +#endif + + +/* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains +** additional information about auxiliary information bound to arguments +** of the function. This is used to implement the sqlite3_get_auxdata() +** and sqlite3_set_auxdata() APIs. The "auxdata" is some auxiliary data +** that can be associated with a constant argument to a function. This +** allows functions such as "regexp" to compile their constant regular +** expression argument once and reused the compiled code for multiple +** invocations. +*/ +struct VdbeFunc { + FuncDef *pFunc; /* The definition of the function */ + int nAux; /* Number of entries allocated for apAux[] */ + struct AuxData { + void *pAux; /* Aux data for the i-th argument */ + void (*xDelete)(void *); /* Destructor for the aux data */ + } apAux[1]; /* One slot for each function argument */ +}; + +/* +** The "context" argument for a installable function. A pointer to an +** instance of this structure is the first argument to the routines used +** implement the SQL functions. +** +** There is a typedef for this structure in sqlite.h. So all routines, +** even the public interface to SQLite, can use a pointer to this structure. +** But this file is the only place where the internal details of this +** structure are known. +** +** This structure is defined inside of vdbeInt.h because it uses substructures +** (Mem) which are only defined there. +*/ +struct sqlite3_context { + FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ + VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ + Mem s; /* The return value is stored here */ + Mem *pMem; /* Memory cell used to store aggregate context */ + int isError; /* Error code returned by the function. */ + CollSeq *pColl; /* Collating sequence */ +}; + +/* +** A Set structure is used for quick testing to see if a value +** is part of a small set. Sets are used to implement code like +** this: +** x.y IN ('hi','hoo','hum') +*/ +typedef struct Set Set; +struct Set { + Hash hash; /* A set is just a hash table */ + HashElem *prev; /* Previously accessed hash elemen */ +}; + +/* +** An instance of the virtual machine. This structure contains the complete +** state of the virtual machine. +** +** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile() +** is really a pointer to an instance of this structure. +** +** The Vdbe.inVtabMethod variable is set to non-zero for the duration of +** any virtual table method invocations made by the vdbe program. It is +** set to 2 for xDestroy method calls and 1 for all other methods. This +** variable is used for two purposes: to allow xDestroy methods to execute +** "DROP TABLE" statements and to prevent some nasty side effects of +** malloc failure when SQLite is invoked recursively by a virtual table +** method function. +*/ +struct Vdbe { + sqlite3 *db; /* The database connection that owns this statement */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + int nOp; /* Number of instructions in the program */ + int nOpAlloc; /* Number of slots allocated for aOp[] */ + Op *aOp; /* Space to hold the virtual machine's program */ + int nLabel; /* Number of labels used */ + int nLabelAlloc; /* Number of slots allocated in aLabel[] */ + int *aLabel; /* Space to hold the labels */ + Mem **apArg; /* Arguments to currently executing user function */ + Mem *aColName; /* Column names to return */ + Mem *pResultSet; /* Pointer to an array of results */ + u16 nResColumn; /* Number of columns in one row of the result set */ + u16 nCursor; /* Number of slots in apCsr[] */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + u8 errorAction; /* Recovery action to do in case of an error */ + u8 okVar; /* True if azVar[] has been initialized */ + ynVar nVar; /* Number of entries in aVar[] */ + Mem *aVar; /* Values for the OP_Variable opcode. */ + char **azVar; /* Name of variables */ + u32 magic; /* Magic number for sanity checking */ + int nMem; /* Number of memory locations currently allocated */ + Mem *aMem; /* The memory locations */ + u32 cacheCtr; /* VdbeCursor row cache generation counter */ + int pc; /* The program counter */ + int rc; /* Value to return */ + char *zErrMsg; /* Error message written here */ + u8 explain; /* True if EXPLAIN present on SQL command */ + u8 changeCntOn; /* True to update the change-counter */ + u8 expired; /* True if the VM needs to be recompiled */ + u8 runOnlyOnce; /* Automatically expire on reset */ + u8 minWriteFileFormat; /* Minimum file format for writable database files */ + u8 inVtabMethod; /* See comments above */ + u8 usesStmtJournal; /* True if uses a statement journal */ + u8 readOnly; /* True for read-only statements */ + u8 isPrepareV2; /* True if prepared with prepare_v2() */ + int nChange; /* Number of db changes made since last reset */ + int btreeMask; /* Bitmask of db->aDb[] entries referenced */ + i64 startTime; /* Time when query started - used for profiling */ + BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */ + int aCounter[3]; /* Counters used by sqlite3_stmt_status() */ + char *zSql; /* Text of the SQL statement that generated this */ + void *pFree; /* Free this when deleting the vdbe */ + i64 nFkConstraint; /* Number of imm. FK constraints this VM */ + i64 nStmtDefCons; /* Number of def. constraints when stmt started */ + int iStatement; /* Statement number (or 0 if has not opened stmt) */ +#ifdef SQLITE_DEBUG + FILE *trace; /* Write an execution trace here, if not NULL */ +#endif + VdbeFrame *pFrame; /* Parent frame */ + int nFrame; /* Number of frames in pFrame list */ + u32 expmask; /* Binding to these vars invalidates VM */ + SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ +}; + +/* +** The following are allowed values for Vdbe.magic +*/ +#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ +#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ +#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ +#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ + +/* +** Function prototypes +*/ +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); +void sqliteVdbePopStack(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); +#endif +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, int, Mem*, int); +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc*, int); + +int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); +SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); +#ifdef SQLITE_OMIT_FLOATING_POINT +# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 +#else +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); +#endif +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); +SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, int); +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); +SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe*,Mem*); +#endif + +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); +#else +# define sqlite3VdbeCheckFk(p,i) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3VdbeMutexArrayEnter(Vdbe *p); +#else +# define sqlite3VdbeMutexArrayEnter(p) +#endif + +SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); + +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); +#else + #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK +#endif + +#endif /* !defined(_VDBEINT_H_) */ + +/************** End of vdbeInt.h *********************************************/ +/************** Continuing where we left off in status.c *********************/ /* ** Variables in which to record status information. */ typedef struct sqlite3StatType sqlite3StatType; static SQLITE_WSD struct sqlite3StatType { - int nowValue[9]; /* Current value */ - int mxValue[9]; /* Maximum value */ + int nowValue[10]; /* Current value */ + int mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; @@ -11661,6 +12403,8 @@ SQLITE_API int sqlite3_db_status( int *pHighwater, /* Write high-water mark here */ int resetFlag /* Reset high-water mark if true */ ){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { *pCurrent = db->lookaside.nOut; @@ -11679,6 +12423,7 @@ SQLITE_API int sqlite3_db_status( case SQLITE_DBSTATUS_CACHE_USED: { int totalUsed = 0; int i; + sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ @@ -11686,15 +12431,80 @@ SQLITE_API int sqlite3_db_status( totalUsed += sqlite3PagerMemUsed(pPager); } } + sqlite3BtreeLeaveAll(db); *pCurrent = totalUsed; *pHighwater = 0; break; } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store the schema for all databases (main, temp, and any ATTACHed + ** databases. *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_SCHEMA_USED: { + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( ALWAYS(pSchema!=0) ){ + HashElem *p; + + nByte += sqlite3GlobalConfig.m.xRoundup(sizeof(HashElem)) * ( + pSchema->tblHash.count + + pSchema->trigHash.count + + pSchema->idxHash.count + + pSchema->fkeyHash.count + ); + nByte += sqlite3MallocSize(pSchema->tblHash.ht); + nByte += sqlite3MallocSize(pSchema->trigHash.ht); + nByte += sqlite3MallocSize(pSchema->idxHash.ht); + nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + + for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); + } + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTable(db, (Table *)sqliteHashData(p)); + } + } + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store all prepared statements. + ** *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_STMT_USED: { + struct Vdbe *pVdbe; /* Used to iterate through VMs */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + sqlite3VdbeDeleteObject(db, pVdbe); + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + + break; + } + default: { - return SQLITE_ERROR; + rc = SQLITE_ERROR; } } - return SQLITE_OK; + sqlite3_mutex_leave(db->mutex); + return rc; } /************** End of status.c **********************************************/ @@ -11831,12 +12641,6 @@ end_getDigits: return cnt; } -/* -** Read text from z[] and convert into a floating point number. Return -** the number of digits converted. -*/ -#define getValue sqlite3AtoF - /* ** Parse a timezone extension on the end of a date-time. ** The extension is of the form: @@ -12038,7 +12842,7 @@ static int parseDateOrTime( const char *zDate, DateTime *p ){ - int isRealNum; /* Return from sqlite3IsNumber(). Not used */ + double r; if( parseYyyyMmDd(zDate,p)==0 ){ return 0; }else if( parseHhMmSs(zDate, p)==0 ){ @@ -12046,9 +12850,7 @@ static int parseDateOrTime( }else if( sqlite3StrICmp(zDate,"now")==0){ setDateTimeToCurrent(context, p); return 0; - }else if( sqlite3IsNumber(zDate, &isRealNum, SQLITE_UTF8) ){ - double r; - getValue(zDate, &r); + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); p->validJD = 1; return 0; @@ -12269,8 +13071,9 @@ static int parseModifier(const char *zMod, DateTime *p){ ** weekday N where 0==Sunday, 1==Monday, and so forth. If the ** date is already on the appropriate weekday, this is a no-op. */ - if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + if( strncmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; @@ -12325,8 +13128,11 @@ static int parseModifier(const char *zMod, DateTime *p){ case '8': case '9': { double rRounder; - n = getValue(z, &r); - assert( n>=1 ); + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + rc = 1; + break; + } if( z[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds @@ -12980,6 +13786,12 @@ SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ } SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; + /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() + ** method to get the current date and time if that method is available + ** (if iVersion is 2 or greater and the function pointer is not NULL) and + ** will fall back to xCurrentTime() if xCurrentTimeInt64() is + ** unavailable. + */ if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ rc = pVfs->xCurrentTimeInt64(pVfs, pTimeOut); }else{ @@ -13363,7 +14175,7 @@ static int sqlite3MemSize(void *pPrior){ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); - nByte = ROUND8(nByte); + assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ p--; p = realloc(p, nByte+8 ); if( p ){ @@ -13769,6 +14581,7 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ struct MemBlockHdr *pOldHdr; void *pNew; assert( mem.disallow==0 ); + assert( (nByte & 7)==0 ); /* EV: R-46199-30249 */ pOldHdr = sqlite3MemsysGetHeader(pPrior); pNew = sqlite3MemMalloc(nByte); if( pNew ){ @@ -13803,7 +14616,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** Set the "type" of an allocation. */ SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ - if( p ){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); @@ -13822,23 +14635,38 @@ SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ */ SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ int rc = 1; - if( p ){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ - assert( (pHdr->eType & (pHdr->eType-1))==0 ); /* Only one type bit set */ if( (pHdr->eType&eType)==0 ){ - void **pBt; - pBt = (void**)pHdr; - pBt -= pHdr->nBacktraceSlots; - backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(stderr)); - fprintf(stderr, "\n"); rc = 0; } } return rc; } - + +/* +** Return TRUE if the mask of type in eType matches no bits of the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +*/ +SQLITE_PRIVATE int sqlite3MemdebugNoType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)!=0 ){ + rc = 0; + } + } + return rc; +} /* ** Set the number of backtrace levels kept for each allocation. @@ -15023,7 +15851,7 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ int nOld; void *p; assert( pPrior!=0 ); - assert( (nBytes&(nBytes-1))==0 ); + assert( (nBytes&(nBytes-1))==0 ); /* EV: R-46199-30249 */ assert( nBytes>=0 ); if( nBytes==0 ){ return 0; @@ -16081,6 +16909,7 @@ static void pthreadMutexEnter(sqlite3_mutex *p){ */ pthread_mutex_lock(&p->mutex); #if SQLITE_MUTEX_NREF + assert( p->nRef>0 || p->owner==0 ); p->owner = pthread_self(); p->nRef++; #endif @@ -16153,6 +16982,7 @@ static void pthreadMutexLeave(sqlite3_mutex *p){ assert( pthreadMutexHeld(p) ); #if SQLITE_MUTEX_NREF p->nRef--; + if( p->nRef==0 ) p->owner = 0; #endif assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); @@ -16417,7 +17247,7 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); - assert( p->nRef==0 ); + assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); @@ -16441,6 +17271,7 @@ static void winMutexEnter(sqlite3_mutex *p){ #endif EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG + assert( p->nRef>0 || p->owner==0 ); p->owner = tid; p->nRef++; if( p->trace ){ @@ -16494,6 +17325,7 @@ static void winMutexLeave(sqlite3_mutex *p){ assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; + if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif LeaveCriticalSection(&p->mutex); @@ -16543,6 +17375,66 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** Memory allocation functions used throughout sqlite. */ +/* +** Attempt to release up to n bytes of non-essential memory currently +** held by SQLite. An example of non-essential memory is memory used to +** cache database pages that are not currently in use. +*/ +SQLITE_API int sqlite3_release_memory(int n){ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + return sqlite3PcacheReleaseMemory(n); +#else + /* IMPLEMENTATION-OF: R-34391-24921 The sqlite3_release_memory() routine + ** is a no-op returning zero if SQLite is not compiled with + ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ + UNUSED_PARAMETER(n); + return 0; +#endif +} + +/* +** An instance of the following object records the location of +** each unused scratch buffer. +*/ +typedef struct ScratchFreeslot { + struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ +} ScratchFreeslot; + +/* +** State information local to the memory allocation subsystem. +*/ +static SQLITE_WSD struct Mem0Global { + sqlite3_mutex *mutex; /* Mutex to serialize access */ + + /* + ** The alarm callback and its arguments. The mem0.mutex lock will + ** be held while the callback is running. Recursive calls into + ** the memory subsystem are allowed, but no new callbacks will be + ** issued. + */ + sqlite3_int64 alarmThreshold; + void (*alarmCallback)(void*, sqlite3_int64,int); + void *alarmArg; + + /* + ** Pointers to the end of sqlite3GlobalConfig.pScratch memory + ** (so that a range test can be used to determine if an allocation + ** being freed came from pScratch) and a pointer to the list of + ** unused scratch allocations. + */ + void *pScratchEnd; + ScratchFreeslot *pScratchFree; + u32 nScratchFree; + + /* + ** True if heap is nearly "full" where "full" is defined by the + ** sqlite3_soft_heap_limit() setting. + */ + int nearlyFull; +} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; + +#define mem0 GLOBAL(struct Mem0Global, mem0) + /* ** This routine runs when the memory allocator sees that the ** total memory allocation is about to exceed the soft heap @@ -16557,79 +17449,67 @@ static void softHeapLimitEnforcer( sqlite3_release_memory(allocSize); } +/* +** Change the alarm callback +*/ +static int sqlite3MemoryAlarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + int nUsed; + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; + mem0.alarmThreshold = iThreshold; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3MemoryAlarm. +*/ +SQLITE_API int sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); +} +#endif + /* ** Set the soft heap-size limit for the library. Passing a zero or ** negative value indicates no limit. */ -SQLITE_API void sqlite3_soft_heap_limit(int n){ - sqlite3_uint64 iLimit; - int overage; - if( n<0 ){ - iLimit = 0; - }else{ - iLimit = n; - } +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; + sqlite3_int64 excess; #ifndef SQLITE_OMIT_AUTOINIT sqlite3_initialize(); #endif - if( iLimit>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, iLimit); + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.alarmThreshold; + sqlite3_mutex_leave(mem0.mutex); + if( n<0 ) return priorLimit; + if( n>0 ){ + sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); }else{ sqlite3MemoryAlarm(0, 0, 0); } - overage = (int)(sqlite3_memory_used() - (i64)n); - if( overage>0 ){ - sqlite3_release_memory(overage); - } + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); + return priorLimit; } - -/* -** Attempt to release up to n bytes of non-essential memory currently -** held by SQLite. An example of non-essential memory is memory used to -** cache database pages that are not currently in use. -*/ -SQLITE_API int sqlite3_release_memory(int n){ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - int nRet = 0; - nRet += sqlite3PcacheReleaseMemory(n-nRet); - return nRet; -#else - UNUSED_PARAMETER(n); - return SQLITE_OK; -#endif +SQLITE_API void sqlite3_soft_heap_limit(int n){ + if( n<0 ) n = 0; + sqlite3_soft_heap_limit64(n); } -/* -** State information local to the memory allocation subsystem. -*/ -static SQLITE_WSD struct Mem0Global { - /* Number of free pages for scratch and page-cache memory */ - u32 nScratchFree; - u32 nPageFree; - - sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; - - /* - ** Pointers to the end of sqlite3GlobalConfig.pScratch and - ** sqlite3GlobalConfig.pPage to a block of memory that records - ** which pages are available. - */ - u32 *aScratchFree; - u32 *aPageFree; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; - -#define mem0 GLOBAL(struct Mem0Global, mem0) - /* ** Initialize the memory allocation subsystem. */ @@ -16642,36 +17522,45 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); } if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 - && sqlite3GlobalConfig.nScratch>=0 ){ - int i; - sqlite3GlobalConfig.szScratch = ROUNDDOWN8(sqlite3GlobalConfig.szScratch-4); - mem0.aScratchFree = (u32*)&((char*)sqlite3GlobalConfig.pScratch) - [sqlite3GlobalConfig.szScratch*sqlite3GlobalConfig.nScratch]; - for(i=0; i0 ){ + int i, n, sz; + ScratchFreeslot *pSlot; + sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); + sqlite3GlobalConfig.szScratch = sz; + pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; + n = sqlite3GlobalConfig.nScratch; + mem0.pScratchFree = pSlot; + mem0.nScratchFree = n; + for(i=0; ipNext = (ScratchFreeslot*)(sz+(char*)pSlot); + pSlot = pSlot->pNext; + } + pSlot->pNext = 0; + mem0.pScratchEnd = (void*)&pSlot[1]; }else{ + mem0.pScratchEnd = 0; sqlite3GlobalConfig.pScratch = 0; sqlite3GlobalConfig.szScratch = 0; + sqlite3GlobalConfig.nScratch = 0; } - if( sqlite3GlobalConfig.pPage && sqlite3GlobalConfig.szPage>=512 - && sqlite3GlobalConfig.nPage>=1 ){ - int i; - int overhead; - int sz = ROUNDDOWN8(sqlite3GlobalConfig.szPage); - int n = sqlite3GlobalConfig.nPage; - overhead = (4*n + sz - 1)/sz; - sqlite3GlobalConfig.nPage -= overhead; - mem0.aPageFree = (u32*)&((char*)sqlite3GlobalConfig.pPage) - [sqlite3GlobalConfig.szPage*sqlite3GlobalConfig.nPage]; - for(i=0; i= mem0.alarmThreshold ){ + mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); + }else{ + mem0.nearlyFull = 0; } } p = sqlite3GlobalConfig.m.xMalloc(nFull); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( p==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } +#endif if( p ){ nFull = sqlite3MallocSize(p); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; return nFull; @@ -16790,7 +17655,9 @@ static int mallocWithAlarm(int n, void **pp){ */ SQLITE_PRIVATE void *sqlite3Malloc(int n){ void *p; - if( n<=0 || n>=0x7fffff00 ){ + if( n<=0 /* IMP: R-65312-04917 */ + || n>=0x7fffff00 + ){ /* A memory allocation of a number of bytes which is near the maximum ** signed integer value might cause an integer overflow inside of the ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving @@ -16804,6 +17671,7 @@ SQLITE_PRIVATE void *sqlite3Malloc(int n){ }else{ p = sqlite3GlobalConfig.m.xMalloc(n); } + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ return p; } @@ -16842,81 +17710,42 @@ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ void *p; assert( n>0 ); + sqlite3_mutex_enter(mem0.mutex); + if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ + p = mem0.pScratchFree; + mem0.pScratchFree = mem0.pScratchFree->pNext; + mem0.nScratchFree--; + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + n = mallocWithAlarm(n, &p); + if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3GlobalConfig.m.xMalloc(n); + } + sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); + } + assert( sqlite3_mutex_notheld(mem0.mutex) ); + + #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocation per thread - ** is outstanding at one time. (This is only checked in the + /* Verify that no more than two scratch allocations per thread + ** are outstanding at one time. (This is only checked in the ** single-threaded case since checking in the multi-threaded case ** would be much more complicated.) */ assert( scratchAllocOut<=1 ); -#endif - - if( sqlite3GlobalConfig.szScratch=(void*)mem0.aScratchFree ){ - assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - if( sqlite3GlobalConfig.bMemstat ){ - int iSize = sqlite3MallocSize(p); - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); - sqlite3GlobalConfig.m.xFree(p); - sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3GlobalConfig.m.xFree(p); - } - }else{ - int i; - i = (int)((u8*)p - (u8*)sqlite3GlobalConfig.pScratch); - i /= sqlite3GlobalConfig.szScratch; - assert( i>=0 && i=1 && scratchAllocOut<=2 ); - scratchAllocOut = 0; + scratchAllocOut--; #endif + if( p>=sqlite3GlobalConfig.pScratch && ppNext = mem0.pScratchFree; + mem0.pScratchFree = pSlot; + mem0.nScratchFree++; + assert( mem0.nScratchFree<=sqlite3GlobalConfig.nScratch ); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); + sqlite3_mutex_leave(mem0.mutex); + }else{ + /* Release memory back to the heap */ + assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + if( sqlite3GlobalConfig.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } } } } @@ -16936,7 +17792,7 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, void *p){ - return db && p && p>=db->lookaside.pStart && plookaside.pEnd; + return p && p>=db->lookaside.pStart && plookaside.pEnd; } #else #define isLookaside(A,B) 0 @@ -16948,15 +17804,17 @@ static int isLookaside(sqlite3 *db, void *p){ */ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ + if( db && isLookaside(db, p) ){ return db->lookaside.sz; }else{ - assert( sqlite3MemdebugHasType(p, - db ? (MEMTYPE_DB|MEMTYPE_HEAP) : MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); return sqlite3GlobalConfig.m.xSize(p); } } @@ -16965,11 +17823,13 @@ SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ SQLITE_API void sqlite3_free(void *p){ - if( p==0 ) return; + if( p==0 ) return; /* IMP: R-49053-54554 */ + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ @@ -16983,16 +17843,24 @@ SQLITE_API void sqlite3_free(void *p){ */ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; - db->lookaside.nOut--; - }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB|MEMTYPE_HEAP) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - sqlite3_free(p); + if( db ){ + if( db->pnBytesFreed ){ + *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + return; + } + if( isLookaside(db, p) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + db->lookaside.nOut--; + return; + } } + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); } /* @@ -17002,10 +17870,10 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ int nOld, nNew; void *pNew; if( pOld==0 ){ - return sqlite3Malloc(nBytes); + return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ } if( nBytes<=0 ){ - sqlite3_free(pOld); + sqlite3_free(pOld); /* IMP: R-31593-10574 */ return 0; } if( nBytes>=0x7fffff00 ){ @@ -17013,6 +17881,9 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ return 0; } nOld = sqlite3MallocSize(pOld); + /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second + ** argument to xRealloc is always a value returned by a prior call to + ** xRoundup. */ nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); if( nOld==nNew ){ pNew = pOld; @@ -17024,6 +17895,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ sqlite3MallocAlarm(nNew-nOld); } assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); if( pNew==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nBytes); @@ -17037,6 +17909,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ }else{ pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ return pNew; } @@ -17096,6 +17969,7 @@ SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, int n){ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){ void *p; assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( db==0 || db->pnBytesFreed==0 ); #ifndef SQLITE_OMIT_LOOKASIDE if( db ){ LookasideSlot *pBuf; @@ -17121,8 +17995,8 @@ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){ if( !p && db ){ db->mallocFailed = 1; } - sqlite3MemdebugSetType(p, - (db && db->lookaside.bEnabled) ? MEMTYPE_DB : MEMTYPE_HEAP); + sqlite3MemdebugSetType(p, MEMTYPE_DB | + ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); return p; } @@ -17148,14 +18022,16 @@ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ sqlite3DbFree(db, p); } }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc(p, n); if( !pNew ){ + sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); db->mallocFailed = 1; } - sqlite3MemdebugSetType(pNew, - db->lookaside.bEnabled ? MEMTYPE_DB : MEMTYPE_HEAP); + sqlite3MemdebugSetType(pNew, MEMTYPE_DB | + (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; @@ -18028,7 +18904,11 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ }else{ p->nAlloc = (int)szNew; } - zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + if( p->useMalloc==1 ){ + zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + }else{ + zNew = sqlite3_malloc(p->nAlloc); + } if( zNew ){ memcpy(zNew, p->zText, p->nChar); sqlite3StrAccumReset(p); @@ -18053,7 +18933,11 @@ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ p->zText[p->nChar] = 0; if( p->useMalloc && p->zText==p->zBase ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( p->useMalloc==1 ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + }else{ + p->zText = sqlite3_malloc(p->nChar+1); + } if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); }else{ @@ -18069,7 +18953,11 @@ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ */ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ if( p->zText!=p->zBase ){ - sqlite3DbFree(p->db, p->zText); + if( p->useMalloc==1 ){ + sqlite3DbFree(p->db, p->zText); + }else{ + sqlite3_free(p->zText); + } } p->zText = 0; } @@ -18151,6 +19039,7 @@ SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ if( sqlite3_initialize() ) return 0; #endif sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + acc.useMalloc = 2; sqlite3VXPrintf(&acc, 0, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; @@ -18444,429 +19333,6 @@ SQLITE_PRIVATE void sqlite3PrngResetState(void){ ** 0xfe 0xff big-endian utf-16 follows ** */ -/************** Include vdbeInt.h in the middle of utf.c *********************/ -/************** Begin file vdbeInt.h *****************************************/ -/* -** 2003 September 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the header file for information that is private to the -** VDBE. This information used to all be at the top of the single -** source code file "vdbe.c". When that file became too big (over -** 6000 lines long) it was split up into several smaller files and -** this header information was factored out. -*/ -#ifndef _VDBEINT_H_ -#define _VDBEINT_H_ - -/* -** SQL is translated into a sequence of instructions to be -** executed by a virtual machine. Each instruction is an instance -** of the following structure. -*/ -typedef struct VdbeOp Op; - -/* -** Boolean values -*/ -typedef unsigned char Bool; - -/* -** A cursor is a pointer into a single BTree within a database file. -** The cursor can seek to a BTree entry with a particular key, or -** loop over all entries of the Btree. You can also insert new BTree -** entries or retrieve the key or data from the entry that the cursor -** is currently pointing to. -** -** Every cursor that the virtual machine has open is represented by an -** instance of the following structure. -** -** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is -** really a single row that represents the NEW or OLD pseudo-table of -** a row trigger. The data for the row is stored in VdbeCursor.pData and -** the rowid is in VdbeCursor.iKey. -*/ -struct VdbeCursor { - BtCursor *pCursor; /* The cursor structure of the backend */ - int iDb; /* Index of cursor database in db->aDb[] (or -1) */ - i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ - Bool zeroed; /* True if zeroed out and ready for reuse */ - Bool rowidIsValid; /* True if lastRowid is valid */ - Bool atFirst; /* True if pointing to first entry */ - Bool useRandomRowid; /* Generate new record numbers semi-randomly */ - Bool nullRow; /* True if pointing to a row with no data */ - Bool deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ - Bool isTable; /* True if a table requiring integer keys */ - Bool isIndex; /* True if an index containing keys only - no data */ - i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - Btree *pBt; /* Separate file holding temporary table */ - int pseudoTableReg; /* Register holding pseudotable content. */ - KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ - int nField; /* Number of fields in the header */ - i64 seqCount; /* Sequence counter */ - sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ - const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ - - /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or - ** OP_IsUnique opcode on this cursor. */ - int seekResult; - - /* Cached information about the header for the data record that the - ** cursor is currently pointing to. Only valid if cacheStatus matches - ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of - ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that - ** the cache is out of date. - ** - ** aRow might point to (ephemeral) data for the current row, or it might - ** be NULL. - */ - u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ - int payloadSize; /* Total number of bytes in the record */ - u32 *aType; /* Type values for all entries in the record */ - u32 *aOffset; /* Cached offsets to the start of each columns data */ - u8 *aRow; /* Data for the current row, if all on one page */ -}; -typedef struct VdbeCursor VdbeCursor; - -/* -** When a sub-program is executed (OP_Program), a structure of this type -** is allocated to store the current value of the program counter, as -** well as the current memory cell array and various other frame specific -** values stored in the Vdbe struct. When the sub-program is finished, -** these values are copied back to the Vdbe from the VdbeFrame structure, -** restoring the state of the VM to as it was before the sub-program -** began executing. -** -** Frames are stored in a linked list headed at Vdbe.pParent. Vdbe.pParent -** is the parent of the current frame, or zero if the current frame -** is the main Vdbe program. -*/ -typedef struct VdbeFrame VdbeFrame; -struct VdbeFrame { - Vdbe *v; /* VM this frame belongs to */ - int pc; /* Program Counter */ - Op *aOp; /* Program instructions */ - int nOp; /* Size of aOp array */ - Mem *aMem; /* Array of memory cells */ - int nMem; /* Number of entries in aMem */ - VdbeCursor **apCsr; /* Element of Vdbe cursors */ - u16 nCursor; /* Number of entries in apCsr */ - void *token; /* Copy of SubProgram.token */ - int nChildMem; /* Number of memory cells for child frame */ - int nChildCsr; /* Number of cursors for child frame */ - i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ - int nChange; /* Statement changes (Vdbe.nChanges) */ - VdbeFrame *pParent; /* Parent of this frame */ -}; - -#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) - -/* -** A value for VdbeCursor.cacheValid that means the cache is always invalid. -*/ -#define CACHE_STALE 0 - -/* -** Internally, the vdbe manipulates nearly all SQL values as Mem -** structures. Each Mem struct may cache multiple representations (string, -** integer etc.) of the same value. A value (and therefore Mem structure) -** has the following properties: -** -** Each value has a manifest type. The manifest type of the value stored -** in a Mem struct is returned by the MemType(Mem*) macro. The type is -** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or -** SQLITE_BLOB. -*/ -struct Mem { - union { - i64 i; /* Integer value. */ - int nZero; /* Used when bit MEM_Zero is set in flags */ - FuncDef *pDef; /* Used only when flags==MEM_Agg */ - RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ - VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ - } u; - double r; /* Real value */ - sqlite3 *db; /* The associated database connection */ - char *z; /* String or BLOB value */ - int n; /* Number of characters in string value, excluding '\0' */ - u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ - u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ - u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ -}; - -/* One or more of the following flags are set to indicate the validOK -** representations of the value stored in the Mem struct. -** -** If the MEM_Null flag is set, then the value is an SQL NULL value. -** No other flags may be set in this case. -** -** If the MEM_Str flag is set then Mem.z points at a string representation. -** Usually this is encoded in the same unicode encoding as the main -** database (see below for exceptions). If the MEM_Term flag is also -** set, then the string is nul terminated. The MEM_Int and MEM_Real -** flags may coexist with the MEM_Str flag. -** -** Multiple of these values can appear in Mem.flags. But only one -** at a time can appear in Mem.type. -*/ -#define MEM_Null 0x0001 /* Value is NULL */ -#define MEM_Str 0x0002 /* Value is a string */ -#define MEM_Int 0x0004 /* Value is an integer */ -#define MEM_Real 0x0008 /* Value is a real number */ -#define MEM_Blob 0x0010 /* Value is a BLOB */ -#define MEM_RowSet 0x0020 /* Value is a RowSet object */ -#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ -#define MEM_TypeMask 0x00ff /* Mask of type bits */ - -/* Whenever Mem contains a valid string or blob representation, one of -** the following flags must be set to determine the memory management -** policy for Mem.z. The MEM_Term flag tells us whether or not the -** string is \000 or \u0000 terminated -*/ -#define MEM_Term 0x0200 /* String rep is nul terminated */ -#define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ -#define MEM_Static 0x0800 /* Mem.z points to a static string */ -#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ -#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ -#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ - -#ifdef SQLITE_OMIT_INCRBLOB - #undef MEM_Zero - #define MEM_Zero 0x0000 -#endif - - -/* -** Clear any existing type flags from a Mem and replace them with f -*/ -#define MemSetTypeFlag(p, f) \ - ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) - - -/* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains -** additional information about auxiliary information bound to arguments -** of the function. This is used to implement the sqlite3_get_auxdata() -** and sqlite3_set_auxdata() APIs. The "auxdata" is some auxiliary data -** that can be associated with a constant argument to a function. This -** allows functions such as "regexp" to compile their constant regular -** expression argument once and reused the compiled code for multiple -** invocations. -*/ -struct VdbeFunc { - FuncDef *pFunc; /* The definition of the function */ - int nAux; /* Number of entries allocated for apAux[] */ - struct AuxData { - void *pAux; /* Aux data for the i-th argument */ - void (*xDelete)(void *); /* Destructor for the aux data */ - } apAux[1]; /* One slot for each function argument */ -}; - -/* -** The "context" argument for a installable function. A pointer to an -** instance of this structure is the first argument to the routines used -** implement the SQL functions. -** -** There is a typedef for this structure in sqlite.h. So all routines, -** even the public interface to SQLite, can use a pointer to this structure. -** But this file is the only place where the internal details of this -** structure are known. -** -** This structure is defined inside of vdbeInt.h because it uses substructures -** (Mem) which are only defined there. -*/ -struct sqlite3_context { - FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ - VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ - Mem s; /* The return value is stored here */ - Mem *pMem; /* Memory cell used to store aggregate context */ - int isError; /* Error code returned by the function. */ - CollSeq *pColl; /* Collating sequence */ -}; - -/* -** A Set structure is used for quick testing to see if a value -** is part of a small set. Sets are used to implement code like -** this: -** x.y IN ('hi','hoo','hum') -*/ -typedef struct Set Set; -struct Set { - Hash hash; /* A set is just a hash table */ - HashElem *prev; /* Previously accessed hash elemen */ -}; - -/* -** An instance of the virtual machine. This structure contains the complete -** state of the virtual machine. -** -** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile() -** is really a pointer to an instance of this structure. -** -** The Vdbe.inVtabMethod variable is set to non-zero for the duration of -** any virtual table method invocations made by the vdbe program. It is -** set to 2 for xDestroy method calls and 1 for all other methods. This -** variable is used for two purposes: to allow xDestroy methods to execute -** "DROP TABLE" statements and to prevent some nasty side effects of -** malloc failure when SQLite is invoked recursively by a virtual table -** method function. -*/ -struct Vdbe { - sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ - int nOp; /* Number of instructions in the program */ - int nOpAlloc; /* Number of slots allocated for aOp[] */ - Op *aOp; /* Space to hold the virtual machine's program */ - int nLabel; /* Number of labels used */ - int nLabelAlloc; /* Number of slots allocated in aLabel[] */ - int *aLabel; /* Space to hold the labels */ - Mem **apArg; /* Arguments to currently executing user function */ - Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ - u16 nResColumn; /* Number of columns in one row of the result set */ - u16 nCursor; /* Number of slots in apCsr[] */ - VdbeCursor **apCsr; /* One element of this array for each open cursor */ - u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ - ynVar nVar; /* Number of entries in aVar[] */ - Mem *aVar; /* Values for the OP_Variable opcode. */ - char **azVar; /* Name of variables */ - u32 magic; /* Magic number for sanity checking */ - int nMem; /* Number of memory locations currently allocated */ - Mem *aMem; /* The memory locations */ - u32 cacheCtr; /* VdbeCursor row cache generation counter */ - int pc; /* The program counter */ - int rc; /* Value to return */ - char *zErrMsg; /* Error message written here */ - u8 explain; /* True if EXPLAIN present on SQL command */ - u8 changeCntOn; /* True to update the change-counter */ - u8 expired; /* True if the VM needs to be recompiled */ - u8 runOnlyOnce; /* Automatically expire on reset */ - u8 minWriteFileFormat; /* Minimum file format for writable database files */ - u8 inVtabMethod; /* See comments above */ - u8 usesStmtJournal; /* True if uses a statement journal */ - u8 readOnly; /* True for read-only statements */ - u8 isPrepareV2; /* True if prepared with prepare_v2() */ - int nChange; /* Number of db changes made since last reset */ - int btreeMask; /* Bitmask of db->aDb[] entries referenced */ - i64 startTime; /* Time when query started - used for profiling */ - BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */ - int aCounter[3]; /* Counters used by sqlite3_stmt_status() */ - char *zSql; /* Text of the SQL statement that generated this */ - void *pFree; /* Free this when deleting the vdbe */ - i64 nFkConstraint; /* Number of imm. FK constraints this VM */ - i64 nStmtDefCons; /* Number of def. constraints when stmt started */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ -#ifdef SQLITE_DEBUG - FILE *trace; /* Write an execution trace here, if not NULL */ -#endif - VdbeFrame *pFrame; /* Parent frame */ - int nFrame; /* Number of frames in pFrame list */ - u32 expmask; /* Binding to these vars invalidates VM */ -}; - -/* -** The following are allowed values for Vdbe.magic -*/ -#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ -#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ -#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ -#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ - -/* -** Function prototypes -*/ -SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); -void sqliteVdbePopStack(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); -#endif -SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, int, Mem*, int); -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc*, int); - -int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); -SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); -SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); -SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); -SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); -SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); -SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); -#ifdef SQLITE_OMIT_FLOATING_POINT -# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 -#else -SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); -#endif -SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, int); -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); -SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); -SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); -SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); -SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); -SQLITE_PRIVATE const char *sqlite3OpcodeName(int); -SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); -SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); -SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); -SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); -SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); - -#ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); -#else -# define sqlite3VdbeCheckFk(p,i) 0 -#endif - -#ifndef SQLITE_OMIT_SHARED_CACHE -SQLITE_PRIVATE void sqlite3VdbeMutexArrayEnter(Vdbe *p); -#else -# define sqlite3VdbeMutexArrayEnter(p) -#endif - -SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); -#endif -SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); - -#ifndef SQLITE_OMIT_INCRBLOB -SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); -#else - #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK -#endif - -#endif /* !defined(_VDBEINT_H_) */ - -/************** End of vdbeInt.h *********************************************/ -/************** Continuing where we left off in utf.c ************************/ #ifndef SQLITE_AMALGAMATION /* @@ -19607,6 +20073,12 @@ SQLITE_PRIVATE int sqlite3Dequote(char *z){ /* ** Some systems have stricmp(). Others have strcasecmp(). Because ** there is no consistency, we will define our own. +** +** IMPLEMENTATION-OF: R-20522-24639 The sqlite3_strnicmp() API allows +** applications and extensions to compare the contents of two buffers +** containing UTF-8 strings in a case-independent fashion, using the same +** definition of case independence that SQLite uses internally when +** comparing identifiers. */ SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; @@ -19624,121 +20096,111 @@ SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ } /* -** Return TRUE if z is a pure numeric string. Return FALSE and leave -** *realnum unchanged if the string contains any character which is not -** part of a number. +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. ** -** If the string is pure numeric, set *realnum to TRUE if the string -** contains the '.' character or an "E+000" style exponentiation suffix. -** Otherwise set *realnum to FALSE. Note that just becaue *realnum is -** false does not mean that the number can be successfully converted into -** an integer - it might be too big. +** The string z[] is length bytes in length (bytes, not characters) and +** uses the encoding enc. The string is not necessarily zero-terminated. ** -** An empty string is considered non-numeric. +** Return TRUE if the result is a valid real number (or integer) and FALSE +** if the string is empty or contains extraneous text. Valid numbers +** are in one of these formats: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Leading and trailing whitespace is ignored for the purpose of determining +** validity. +** +** If some prefix of the input string is a valid number, this routine +** returns FALSE but it still converts the prefix and writes the result +** into *pResult. */ -SQLITE_PRIVATE int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +#ifndef SQLITE_OMIT_FLOATING_POINT int incr = (enc==SQLITE_UTF8?1:2); - if( enc==SQLITE_UTF16BE ) z++; - if( *z=='-' || *z=='+' ) z += incr; - if( !sqlite3Isdigit(*z) ){ - return 0; - } - z += incr; - *realnum = 0; - while( sqlite3Isdigit(*z) ){ z += incr; } -#ifndef SQLITE_OMIT_FLOATING_POINT - if( *z=='.' ){ - z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z += incr; - if( *z=='+' || *z=='-' ) z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } -#endif - return *z==0; -} - -/* -** The string z[] is an ASCII representation of a real number. -** Convert this string to a double. -** -** This routine assumes that z[] really is a valid number. If it -** is not, the result is undefined. -** -** This routine is used instead of the library atof() function because -** the library atof() might want to use "," as the decimal point instead -** of "." depending on how locale is set. But that would cause problems -** for SQL. So this routine always uses "." regardless of locale. -*/ -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){ -#ifndef SQLITE_OMIT_FLOATING_POINT - const char *zBegin = z; + const char *zEnd = z + length; /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; + *pResult = 0.0; /* Default return value, in case of an error */ + + if( enc==SQLITE_UTF16BE ) z++; + /* skip leading spaces */ - while( sqlite3Isspace(*z) ) z++; + while( z=zEnd ) return 0; + /* get sign of significand */ if( *z=='-' ){ sign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } + /* skip leading zeroes */ - while( z[0]=='0' ) z++, nDigits++; + while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z++; + z+=incr; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ - while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ + while( z=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - z++; + z+=incr; + eValid = 0; + if( z>=zEnd ) goto do_atof_calc; /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } /* copy digits to exponent */ - while( sqlite3Isdigit(*z) ){ + while( z=zEnd && nDigits>0 && eValid; #else - return sqlite3Atoi64(z, pResult); + return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ } @@ -19808,20 +20270,26 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){ ** Compare the 19-character string zNum against the text representation ** value 2^63: 9223372036854775808. Return negative, zero, or positive ** if zNum is less than, equal to, or greater than the string. +** Note that zNum must contain exactly 19 characters. ** ** Unlike memcmp() this routine is guaranteed to return the difference ** in the values of the last digit if the only difference is in the ** last digit. So, for example, ** -** compare2pow63("9223372036854775800") +** compare2pow63("9223372036854775800", 1) ** ** will return -8. */ -static int compare2pow63(const char *zNum){ - int c; - c = memcmp(zNum,"922337203685477580",18)*10; +static int compare2pow63(const char *zNum, int incr){ + int c = 0; + int i; + /* 012345678901234567 */ + const char *pow63 = "922337203685477580"; + for(i=0; c==0 && i<18; i++){ + c = (zNum[i*incr]-pow63[i])*10; + } if( c==0 ){ - c = zNum[18] - '8'; + c = zNum[18*incr] - '8'; testcase( c==(-1) ); testcase( c==0 ); testcase( c==(+1) ); @@ -19831,94 +20299,60 @@ static int compare2pow63(const char *zNum){ /* -** Return TRUE if zNum is a 64-bit signed integer and write -** the value of the integer into *pNum. If zNum is not an integer -** or is an integer that is too large to be expressed with 64 bits, -** then return false. +** Convert zNum to a 64-bit signed integer and write +** the value of the integer into *pNum. +** If zNum is exactly 9223372036854665808, return 2. +** This is a special case as the context will determine +** if it is too big (used as a negative). +** If zNum is not an integer or is an integer that +** is too large to be expressed with 64 bits, +** then return 1. Otherwise return 0. ** -** When this routine was originally written it dealt with only -** 32-bit numbers. At that time, it was much faster than the -** atoi() library routine in RedHat 7.2. +** length is the number of bytes in the string (bytes, not characters). +** The string is not necessarily zero-terminated. The encoding is +** given by enc. */ -SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum){ +SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ + int incr = (enc==SQLITE_UTF8?1:2); i64 v = 0; - int neg; - int i, c; + int neg = 0; /* assume positive */ + int i; + int c = 0; const char *zStart; - while( sqlite3Isspace(*zNum) ) zNum++; + const char *zEnd = zNum + length; + if( enc==SQLITE_UTF16BE ) zNum++; + while( zNum=zEnd ) goto do_atoi_calc; if( *zNum=='-' ){ neg = 1; - zNum++; + zNum+=incr; }else if( *zNum=='+' ){ - neg = 0; - zNum++; - }else{ - neg = 0; + zNum+=incr; } +do_atoi_calc: zStart = zNum; - while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */ - for(i=0; (c=zNum[i])>='0' && c<='9'; i++){ + while( zNum='0' && c<='9'; i+=incr){ v = v*10 + c - '0'; } *pNum = neg ? -v : v; testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( c!=0 || (i==0 && zStart==zNum) || i>19 ){ + if( (c!=0 && &zNum[i]19*incr ){ /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranting that it is too large) */ - return 0; - }else if( i<19 ){ - /* Less than 19 digits, so we know that it fits in 64 bits */ + ** than 19 digits (thus guaranteeing that it is too large) */ return 1; + }else if( i<19*incr ){ + /* Less than 19 digits, so we know that it fits in 64 bits */ + return 0; }else{ /* 19-digit numbers must be no larger than 9223372036854775807 if positive ** or 9223372036854775808 if negative. Note that 9223372036854665808 - ** is 2^63. */ - return compare2pow63(zNum)='0' && zNum[0]<='9' ); /* zNum is an unsigned number */ - - if( negFlag ) neg = 1-neg; - while( *zNum=='0' ){ - zNum++; /* Skip leading zeros. Ticket #2454 */ - } - for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); } - testcase( i==18 ); - testcase( i==19 ); - testcase( i==20 ); - if( i<19 ){ - /* Guaranteed to fit if less than 19 digits */ - return 1; - }else if( i>19 ){ - /* Guaranteed to be too big if greater than 19 digits */ - return 0; - }else{ - /* Compare against 2^63. */ - return compare2pow63(zNum)0 ){ if( wrote<0 ){ /* lastErrno set by seekAndWrite */ @@ -25224,6 +25660,7 @@ static int unixWrite( return SQLITE_FULL; } } + return SQLITE_OK; } @@ -25425,12 +25862,23 @@ static int unixSync(sqlite3_file *id, int flags){ ** Truncate an open file to a specified size */ static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; int rc; - assert( id ); + assert( pFile ); SimulateIOError( return SQLITE_IOERR_TRUNCATE ); - rc = ftruncate(((unixFile*)id)->h, (off_t)nByte); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = ftruncate(pFile->h, (off_t)nByte); if( rc ){ - ((unixFile*)id)->lastErrno = errno; + pFile->lastErrno = errno; return SQLITE_IOERR_TRUNCATE; }else{ #ifndef NDEBUG @@ -25441,8 +25889,8 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ ** when restoring a database using the backup API from a zero-length ** source. */ - if( ((unixFile*)id)->inNormalWrite && nByte==0 ){ - ((unixFile*)id)->transCntrChng = 1; + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; } #endif @@ -25485,6 +25933,54 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ static int proxyFileControl(sqlite3_file*,int,void*); #endif +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. +** +** If the user has configured a chunk-size for this file, it could be +** that the file needs to be extended at this point. Otherwise, the +** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( fstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ + return SQLITE_IOERR_WRITE; + } +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. This + ** is the same technique used by glibc to implement posix_fallocate() + ** on systems that do not have a real fallocate() system call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ + int nWrite; /* Return value from seekAndWrite() */ + + if( ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_TRUNCATE; + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + do { + nWrite = seekAndWrite(pFile, iWrite, "", 1); + iWrite += nBlk; + } while( nWrite==1 && iWritelastErrno; return SQLITE_OK; } - case SQLITE_FCNTL_SIZE_HINT: { -#if 0 /* No performance advantage seen on Linux */ - sqlite3_int64 szFile = *(sqlite3_int64*)pArg; - unixFile *pFile = (unixFile*)id; - ftruncate(pFile->h, szFile); -#endif + case SQLITE_FCNTL_CHUNK_SIZE: { + ((unixFile*)id)->szChunk = *(int *)pArg; return SQLITE_OK; } + case SQLITE_FCNTL_SIZE_HINT: { + return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); + } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -25950,7 +26445,7 @@ static int unixShmMap( pShmNode->apRegion = apNew; while(pShmNode->nRegion<=iRegion){ void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, iRegion*szRegion + MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ rc = SQLITE_IOERR; @@ -26468,11 +26963,21 @@ static int fillInUnixFile( */ UNUSED_PARAMETER(isDelete); + /* Usually the path zFilename should not be a relative pathname. The + ** exception is when opening the proxy "conch" file in builds that + ** include the special Apple locking styles. + */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + assert( zFilename==0 || zFilename[0]=='/' + || pVfs->pAppData==(void*)&autolockIoFinder ); +#else + assert( zFilename==0 || zFilename[0]=='/' ); +#endif + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); pNew->h = h; pNew->dirfd = dirfd; pNew->fileFlags = 0; - assert( zFilename==0 || zFilename[0]=='/' ); /* Never a relative pathname */ pNew->zPath = zFilename; #if OS_VXWORKS @@ -27773,6 +28278,8 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); +#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ + && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 if( gethostuuid(pHostID, &timeout) ){ int err = errno; if( pError ){ @@ -27780,6 +28287,7 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ } return SQLITE_IOERR; } +#endif #ifdef SQLITE_TEST /* simulate multiple hosts by creating unique hostid file paths */ if( sqlite3_hostid_num != 0){ @@ -28370,7 +28878,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); sqlite3_free(pCtx->conchFile); } - sqlite3_free(pCtx->lockProxyPath); + sqlite3DbFree(0, pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); sqlite3_free(pCtx); } @@ -28561,9 +29069,9 @@ static int proxyClose(sqlite3_file *id) { if( rc ) return rc; sqlite3_free(conchFile); } - sqlite3_free(pCtx->lockProxyPath); + sqlite3DbFree(0, pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); - sqlite3_free(pCtx->dbPath); + sqlite3DbFree(0, pCtx->dbPath); /* restore the original locking context and pMethod then close it */ pFile->lockingContext = pCtx->oldLockingContext; pFile->pMethod = pCtx->pOldMethod; @@ -29009,6 +29517,7 @@ struct winFile { DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ @@ -29520,6 +30029,42 @@ static BOOL winceLockFileEx( ** by the sqlite3_io_methods object. ******************************************************************************/ +/* +** Some microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occured, it is also necessary to call + ** GetLastError(). + */ + dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ + pFile->lastErrno = GetLastError(); + return 1; + } + + return 0; +} + /* ** Close a file. ** @@ -29562,13 +30107,6 @@ static int winClose(sqlite3_file *id){ return rc ? SQLITE_OK : SQLITE_IOERR; } -/* -** Some microsoft compilers lack this definition. -*/ -#ifndef INVALID_SET_FILE_POINTER -# define INVALID_SET_FILE_POINTER ((DWORD)-1) -#endif - /* ** Read data from a file into a buffer. Return SQLITE_OK if all ** bytes were read successfully and SQLITE_IOERR if anything goes @@ -29580,32 +30118,27 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD got; + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &got, 0) ){ + if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ pFile->lastErrno = GetLastError(); return SQLITE_IOERR_READ; } - if( got==(DWORD)amt ){ - return SQLITE_OK; - }else{ + if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pBuf)[got], 0, amt-got); + memset(&((char*)pBuf)[nRead], 0, amt-nRead); return SQLITE_IOERR_SHORT_READ; } + + return SQLITE_OK; } /* @@ -29613,47 +30146,42 @@ static int winRead( ** or some other error code on failure. */ static int winWrite( - sqlite3_file *id, /* File to write into */ - const void *pBuf, /* The bytes to be written */ - int amt, /* Number of bytes to write */ - sqlite3_int64 offset /* Offset into the file to begin writing at */ + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD wrote = 0; + int rc; /* True if error has occured, else false */ + winFile *pFile = (winFile*)id; /* File handle */ - assert( id!=0 ); + assert( amt>0 ); + assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); + OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ - return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; + + rc = seekWinFile(pFile, offset); + if( rc==0 ){ + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + + while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = GetLastError(); + rc = 1; } } - assert( amt>0 ); - while( - amt>0 - && (rc = WriteFile(pFile->h, pBuf, amt, &wrote, 0))!=0 - && wrote>0 - ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; - } - if( !rc || amt>(int)wrote ){ - pFile->lastErrno = GetLastError(); + + if( rc ){ if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; } + return SQLITE_IOERR_WRITE; } return SQLITE_OK; } @@ -29662,26 +30190,32 @@ static int winWrite( ** Truncate an open file to a specified size */ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ - LONG upperBits = (LONG)((nByte>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(nByte & 0xffffffff); - DWORD dwRet; - winFile *pFile = (winFile*)id; - DWORD error; - int rc = SQLITE_OK; + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + + assert( pFile ); - assert( id!=0 ); OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( dwRet==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( seekWinFile(pFile, nByte) ){ rc = SQLITE_IOERR_TRUNCATE; - /* SetEndOfFile will fail if nByte is negative */ - }else if( !SetEndOfFile(pFile->h) ){ + }else if( 0==SetEndOfFile(pFile->h) ){ pFile->lastErrno = GetLastError(); rc = SQLITE_IOERR_TRUNCATE; } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc==SQLITE_OK ? "ok" : "failed")); + + OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; } @@ -30046,6 +30580,10 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = (int)((winFile*)id)->lastErrno; return SQLITE_OK; } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((winFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; + } case SQLITE_FCNTL_SIZE_HINT: { sqlite3_int64 sz = *(sqlite3_int64*)pArg; SimulateIOErrorBenign(1); @@ -30082,6 +30620,14 @@ static int winDeviceCharacteristics(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +SYSTEM_INFO winSysInfo; + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by @@ -30250,6 +30796,7 @@ static int winDelete(sqlite3_vfs *,const char*,int); static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode **pp; winShmNode *p; + BOOL bRc; assert( winShmMutexHeld() ); pp = &winShmNodeList; while( (p = *pp)!=0 ){ @@ -30257,8 +30804,14 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ int i; if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ - UnmapViewOfFile(p->aRegion[i].pMap); - CloseHandle(p->aRegion[i].hMap); + bRc = UnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); + bRc = CloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); } if( p->hFile.h != INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); @@ -30335,10 +30888,11 @@ static int winOpenSharedMemory(winFile *pDbFd){ rc = SQLITE_NOMEM; goto shm_open_err; } + rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ 0); if( SQLITE_OK!=rc ){ rc = SQLITE_CANTOPEN_BKPT; @@ -30646,10 +31200,18 @@ static int winShmMap( hMap = CreateFileMapping(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); + OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, - 0, 0, nByte + 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); + OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, + pMap ? "ok" : "failed")); } if( !pMap ){ pShmNode->lastErrno = GetLastError(); @@ -30666,8 +31228,10 @@ static int winShmMap( shmpage_out: if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; char *p = (char *)pShmNode->aRegion[iRegion].pMap; - *pp = (void *)&p[iRegion*szRegion]; + *pp = (void *)&p[iOffsetShift]; }else{ *pp = 0; } @@ -30894,9 +31458,60 @@ static int winOpen( int isTemp = 0; #endif winFile *pFile = (winFile*)id; - void *zConverted; /* Filename in OS encoding */ - const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ - char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); +#ifndef NDEBUG + int isReadonly = (flags & SQLITE_OPEN_READONLY); +#endif + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); assert( id!=0 ); UNUSED_PARAMETER(pVfs); @@ -30907,7 +31522,8 @@ static int winOpen( ** temporary file name to use */ if( !zUtf8Name ){ - int rc = getTempname(MAX_PATH+1, zTmpname); + assert(isDelete && !isOpenJournal); + rc = getTempname(MAX_PATH+1, zTmpname); if( rc!=SQLITE_OK ){ return rc; } @@ -30920,29 +31536,31 @@ static int winOpen( return SQLITE_NOMEM; } - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; }else{ dwDesiredAccess = GENERIC_READ; } + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is ** created. SQLite doesn't use it to indicate "exclusive access" ** as it is usually understood. */ - assert(!(flags & SQLITE_OPEN_EXCLUSIVE) || (flags & SQLITE_OPEN_CREATE)); - if( flags & SQLITE_OPEN_EXCLUSIVE ){ + if( isExclusive ){ /* Creates a new file, only if it does not already exist. */ /* If the file exists, it fails. */ dwCreationDisposition = CREATE_NEW; - }else if( flags & SQLITE_OPEN_CREATE ){ + }else if( isCreate ){ /* Open existing file, or create if it doesn't exist */ dwCreationDisposition = OPEN_ALWAYS; }else{ /* Opens a file, only if it exists. */ dwCreationDisposition = OPEN_EXISTING; } + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + + if( isDelete ){ #if SQLITE_OS_WINCE dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; isTemp = 1; @@ -30959,6 +31577,7 @@ static int winOpen( #if SQLITE_OS_WINCE dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; #endif + if( isNT() ){ h = CreateFileW((WCHAR*)zConverted, dwDesiredAccess, @@ -30984,26 +31603,30 @@ static int winOpen( ); #endif } + OSTRACE(("OPEN %d %s 0x%lx %s\n", h, zName, dwDesiredAccess, h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = GetLastError(); free(zConverted); - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~SQLITE_OPEN_READWRITE), pOutFlags); + ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); }else{ return SQLITE_CANTOPEN_BKPT; } } + if( pOutFlags ){ - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ *pOutFlags = SQLITE_OPEN_READWRITE; }else{ *pOutFlags = SQLITE_OPEN_READONLY; } } + memset(pFile, 0, sizeof(*pFile)); pFile->pMethod = &winIoMethod; pFile->h = h; @@ -31012,9 +31635,9 @@ static int winOpen( pFile->pShm = 0; pFile->zPath = zName; pFile->sectorSize = getSectorSize(pVfs, zUtf8Name); + #if SQLITE_OS_WINCE - if( (flags & (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB)) == - (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB) + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB && !winceCreateLock(zName, pFile) ){ CloseHandle(h); @@ -31028,8 +31651,9 @@ static int winOpen( { free(zConverted); } + OpenCounter(+1); - return SQLITE_OK; + return rc; } /* @@ -31548,6 +32172,13 @@ SQLITE_API int sqlite3_os_init(void){ winCurrentTimeInt64, /* xCurrentTimeInt64 */ }; +#ifndef SQLITE_OMIT_WAL + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&winSysInfo); + assert(winSysInfo.dwAllocationGranularity > 0); +#endif + sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; } @@ -31597,7 +32228,7 @@ SQLITE_API int sqlite3_os_end(void){ */ /* Size of the Bitvec structure in bytes. */ -#define BITVEC_SZ (sizeof(void*)*128) /* 512 on 32bit. 1024 on 64bit */ +#define BITVEC_SZ 512 /* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ @@ -32112,12 +32743,16 @@ static void pcacheUnpin(PgHdr *p){ */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ if( sqlite3GlobalConfig.pcache.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ sqlite3PCacheSetDefault(); } return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); } SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ if( sqlite3GlobalConfig.pcache.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); } } @@ -32578,8 +33213,13 @@ typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; -/* Pointers to structures of this type are cast and returned as -** opaque sqlite3_pcache* handles +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. +** +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. */ struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable @@ -32639,6 +33279,9 @@ static SQLITE_WSD struct PCacheGlobal { /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */ int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nFreeSlot; /* Number of unused pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ void *pStart, *pEnd; /* Bounds of pagecache malloc range */ PgFreeslot *pFree; /* Free page blocks */ int isInit; /* True if initialized */ @@ -32686,6 +33329,8 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ PgFreeslot *p; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; pcache1.pFree = 0; while( n-- ){ @@ -32707,11 +33352,13 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ static void *pcache1Alloc(int nByte){ void *p; assert( sqlite3_mutex_held(pcache1.mutex) ); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); if( nByte<=pcache1.szSlot && pcache1.pFree ){ assert( pcache1.isInit ); p = (PgHdr1 *)pcache1.pFree; pcache1.pFree = pcache1.pFree->pNext; - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + pcache1.nFreeSlot--; + assert( pcache1.nFreeSlot>=0 ); sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); }else{ @@ -32745,6 +33392,8 @@ static void pcache1Free(void *p){ pSlot = (PgFreeslot*)p; pSlot->pNext = pcache1.pFree; pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + assert( pcache1.nFreeSlot<=pcache1.nSlot ); }else{ int iSize; assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); @@ -32755,6 +33404,25 @@ static void pcache1Free(void *p){ } } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** Return the size of a pcache allocation +*/ +static int pcache1MemSize(void *p){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( p>=pcache1.pStart && pszPage<=pcache1.szSlot ){ + return pcache1.nFreeSlot=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage) || nPinned>=(pCache->nMax * 9 / 10) + || pcache1UnderMemoryPressure(pCache) )){ goto fetch_out; } @@ -33112,7 +33812,9 @@ static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ /* Step 4. Try to recycle a page buffer if appropriate. */ if( pCache->bPurgeable && pcache1.pLruTail && ( - (pCache->nPage+1>=pCache->nMax) || pcache1.nCurrentPage>=pcache1.nMaxPage + (pCache->nPage+1>=pCache->nMax) + || pcache1.nCurrentPage>=pcache1.nMaxPage + || pcache1UnderMemoryPressure(pCache) )){ pPage = pcache1.pLruTail; pcache1RemoveFromHash(pPage); @@ -33255,6 +33957,7 @@ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ */ static void pcache1Destroy(sqlite3_pcache *p){ PCache1 *pCache = (PCache1 *)p; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(); pcache1TruncateUnsafe(pCache, 0); pcache1.nMaxPage -= pCache->nMax; @@ -33302,8 +34005,8 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ if( pcache1.pStart==0 ){ PgHdr1 *p; pcache1EnterMutex(); - while( (nReq<0 || nFree READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ ** -** PAGER_SHARED The page cache is reading the database. -** Writing is not permitted. There can be -** multiple readers accessing the same database -** file at the same time. ** -** PAGER_RESERVED This process has reserved the database for writing -** but has not yet made any changes. Only one process -** at a time can reserve the database. The original -** database file has not been modified so other -** processes may still be reading the on-disk -** database file. +** List of state transitions and the C [function] that performs each: +** +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] ** -** PAGER_EXCLUSIVE The page cache is writing the database. -** Access is exclusive. No other processes or -** threads can be reading or writing while one -** process is writing. +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] ** -** PAGER_SYNCED The pager moves to this state from PAGER_EXCLUSIVE -** after all dirty pages have been written to the -** database file and the file has been synced to -** disk. All that remains to do is to remove or -** truncate the journal file and the transaction -** will be committed. +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** ** -** The page cache comes up in PAGER_UNLOCK. The first time a -** sqlite3PagerGet() occurs, the state transitions to PAGER_SHARED. -** After all pages have been released using sqlite_page_unref(), -** the state transitions back to PAGER_UNLOCK. The first time -** that sqlite3PagerWrite() is called, the state transitions to -** PAGER_RESERVED. (Note that sqlite3PagerWrite() can only be -** called on an outstanding page which means that the pager must -** be in PAGER_SHARED before it transitions to PAGER_RESERVED.) -** PAGER_RESERVED means that there is an open rollback journal. -** The transition to PAGER_EXCLUSIVE occurs before any changes -** are made to the database file, though writes to the rollback -** journal occurs with just PAGER_RESERVED. After an sqlite3PagerRollback() -** or sqlite3PagerCommitPhaseTwo(), the state can go back to PAGER_SHARED, -** or it can stay at PAGER_EXCLUSIVE if we are in exclusive access mode. +** OPEN: +** +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. +** +** READER: +** +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. +** +** WRITER_LOCKED: +** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. +** +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. +** +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occured, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). */ -#define PAGER_UNLOCK 0 -#define PAGER_SHARED 1 /* same as SHARED_LOCK */ -#define PAGER_RESERVED 2 /* same as RESERVED_LOCK */ -#define PAGER_EXCLUSIVE 4 /* same as EXCLUSIVE_LOCK */ -#define PAGER_SYNCED 5 +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 + +/* +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) /* ** A macro used for invoking the codec if there is one @@ -34101,33 +35030,28 @@ struct PagerSavepoint { }; /* -** A open page cache is an instance of the following structure. +** A open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: ** -** errCode +** eState ** -** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or -** or SQLITE_FULL. Once one of the first three errors occurs, it persists -** and is returned as the result of every major pager API call. The -** SQLITE_FULL return code is slightly different. It persists only until the -** next successful rollback is performed on the pager cache. Also, -** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() -** APIs, they may still be used successfully. +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. ** -** dbSizeValid, dbSize, dbOrigSize, dbFileSize +** eLock ** -** Managing the size of the database file in pages is a little complicated. -** The variable Pager.dbSize contains the number of pages that the database -** image currently contains. As the database image grows or shrinks this -** variable is updated. The variable Pager.dbFileSize contains the number -** of pages in the database file. This may be different from Pager.dbSize -** if some pages have been appended to the database image but not yet written -** out from the cache to the actual file on disk. Or if the image has been -** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable -** contains the number of pages in the database image when the current -** transaction was opened. The contents of all three of these variables is -** only guaranteed to be correct if the boolean Pager.dbSizeValid is true. +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. ** -** TODO: Under what conditions is dbSizeValid set? Cleared? +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. ** ** changeCountDone ** @@ -34146,67 +35070,123 @@ struct PagerSavepoint { ** need only update the change-counter once, for the first transaction ** committed. ** -** dbModified -** -** The dbModified flag is set whenever a database page is dirtied. -** It is cleared at the end of each transaction. -** -** It is used when committing or otherwise ending a transaction. If -** the dbModified flag is clear then less work has to be done. -** -** journalStarted -** -** This flag is set whenever the the main journal is opened and -** initialized -** -** The point of this flag is that it must be set after the -** first journal header in a journal file has been synced to disk. -** After this has happened, new pages appended to the database -** do not need the PGHDR_NEED_SYNC flag set, as they do not need -** to wait for a journal sync before they can be written out to -** the database file (see function pager_write()). -** ** setMaster ** -** This variable is used to ensure that the master journal file name -** (if any) is only written into the journal file once. +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a master-journal name to be written into the +** journal file before it is synced to disk. ** -** When committing a transaction, the master journal file name (if any) -** may be written into the journal file while the pager is still in -** PAGER_RESERVED state (see CommitPhaseOne() for the action). It -** then attempts to upgrade to an exclusive lock. If this attempt -** fails, then SQLITE_BUSY may be returned to the user and the user -** may attempt to commit the transaction again later (calling -** CommitPhaseOne() again). This flag is used to ensure that the -** master journal name is only written to the journal file the first -** time CommitPhaseOne() is called. +** Whether or not a journal file contains a master-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a master-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a master-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain master journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** master journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setMaster flag +** is cleared anyway (and the pager will move to ERROR state). ** ** doNotSpill, doNotSyncSpill ** -** When enabled, cache spills are prohibited. The doNotSpill variable -** inhibits all cache spill and doNotSyncSpill inhibits those spills that -** would require a journal sync. The doNotSyncSpill is set and cleared -** by sqlite3PagerWrite() in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. The -** doNotSpill value set to prevent pagerStress() from trying to use -** the journal during a rollback. +** These two boolean variables control the behaviour of cache-spills +** (calls made by the pcache module to the pagerStress() routine to +** write cached data to the file-system in order to free up memory). ** -** needSync -** -** TODO: It might be easier to set this variable in writeJournalHdr() -** and writeMasterJournal() only. Change its meaning to "unsynced data -** has been written to the journal". +** When doNotSpill is non-zero, writing to the database from pagerStress() +** is disabled altogether. This is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). +** +** If doNotSyncSpill is non-zero, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. ** ** subjInMemory ** ** This is a boolean variable. If true, then any required sub-journal ** is opened as an in-memory journal file. If false, then in-memory ** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ - u8 journalMode; /* On of the PAGER_JOURNALMODE_* values */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ u8 noReadlock; /* Do not bother to obtain readlocks */ u8 noSync; /* Do not sync the journal if true */ @@ -34216,30 +35196,26 @@ struct Pager { u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ - /* The following block contains those class members that are dynamically - ** modified during normal operations. The other variables in this structure - ** are either constant throughout the lifetime of the pager, or else - ** used to store configuration parameters that affect the way the pager - ** operates. - ** - ** The 'state' variable is described in more detail along with the - ** descriptions of the values it may take - PAGER_UNLOCK etc. Many of the - ** other variables in this block are described in the comment directly - ** above this class definition. + /************************************************************************** + ** The following block contains those class members that change during + ** routine opertion. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. */ - u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ - u8 dbModified; /* True if there are any changes to the Db */ - u8 needSync; /* True if an fsync() is needed on the journal */ - u8 journalStarted; /* True if header of journal is synced */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 doNotSyncSpill; /* Do not do a spill that requires jrnl sync */ - u8 dbSizeValid; /* Set when dbSize is correct */ u8 subjInMemory; /* True to use in-memory sub-journals */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ int errCode; /* One of several kinds of errors */ int nRec; /* Pages journalled since last j-header written */ u32 cksumInit; /* Quasi-random value added to every checksum */ @@ -34250,17 +35226,21 @@ struct Pager { sqlite3_file *sjfd; /* File descriptor for sub-journal */ i64 journalOff; /* Current write offset in the journal file */ i64 journalHdr; /* Byte offset to previous journal header */ - i64 journalSizeLimit; /* Size limit for persistent journal files */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - u32 sectorSize; /* Assumed sector size during rollback */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ u16 nExtra; /* Add this many bytes to each in-memory page */ i16 nReserve; /* Number of unused bytes at end of each page */ u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ int pageSize; /* Number of bytes in a page */ Pgno mxPgno; /* Maximum allowed size of the database */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ @@ -34278,7 +35258,6 @@ struct Pager { #endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ - sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ #ifndef SQLITE_OMIT_WAL Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ @@ -34357,22 +35336,221 @@ static const unsigned char aJournalMagic[] = { */ #define PAGER_MAX_PGNO 2147483647 +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods) + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +#ifndef SQLITE_OMIT_WAL +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); +} +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + #ifndef NDEBUG /* ** Usage: ** ** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. */ -static int assert_pager_state(Pager *pPager){ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; - /* A temp-file is always in PAGER_EXCLUSIVE or PAGER_SYNCED state. */ - assert( pPager->tempFile==0 || pPager->state>=PAGER_EXCLUSIVE ); + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); - /* The changeCountDone flag is always set for temp-files */ - assert( pPager->tempFile==0 || pPager->changeCountDone ); + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); + } + + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); + + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK || p->noReadlock ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setMaster==0 ); + break; + + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; + + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + break; + } return 1; } + +/* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** +** (gdb) printf "%s", print_pager_state(pPager) +*/ +static char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; +} #endif /* @@ -34425,6 +35603,7 @@ static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ */ #define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + /* ** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK ** on success or an error code is something goes wrong. @@ -34436,27 +35615,53 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ } /* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. ** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. */ -#define isOpen(pFd) ((pFd)->pMethods) +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( !pPager->exclusiveMode ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } + return rc; +} /* -** If file pFd is open, call sqlite3OsUnlock() on it. +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. */ -static int osUnlock(sqlite3_file *pFd, int eLock){ - if( !isOpen(pFd) ){ - return SQLITE_OK; +static int pagerLockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } } - return sqlite3OsUnlock(pFd, eLock); + return rc; } /* @@ -34532,13 +35737,14 @@ static void pager_set_pagehash(PgHdr *pPage){ #define CHECK_PAGE(x) checkPage(x) static void checkPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( !pPg->pageHash || pPager->errCode - || (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); } #else #define pager_datahash(X,Y) 0 #define pager_pagehash(X) 0 +#define pager_set_pagehash(X) #define CHECK_PAGE(x) #endif /* SQLITE_CHECK_PAGES */ @@ -34705,7 +35911,7 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ static int writeJournalHdr(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ - u32 nHeader = pPager->pageSize; /* Size of buffer pointed to by zHeader */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ u32 nWrite; /* Bytes of header sector written */ int ii; /* Loop counter */ @@ -34748,7 +35954,7 @@ static int writeJournalHdr(Pager *pPager){ ** that garbage data is never appended to the journal file. */ assert( isOpen(pPager->fd) || pPager->noSync ); - if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); @@ -34872,7 +36078,6 @@ static int readJournalHdr( if( pPager->journalOff==0 ){ u32 iPageSize; /* Page-size field of journal header */ u32 iSectorSize; /* Sector-size field of journal header */ - u16 iPageSize16; /* Copy of iPageSize in 16-bit variable */ /* Read the page-size and sector-size journal header fields. */ if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) @@ -34881,6 +36086,14 @@ static int readJournalHdr( return rc; } + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } + /* Check that the values read from the page-size and sector-size fields ** are within range. To be 'in range', both values need to be a power ** of two greater than or equal to 512 or 32, and not greater than their @@ -34902,10 +36115,8 @@ static int readJournalHdr( ** Use a testcase() macro to make sure that malloc failure within ** PagerSetPagesize() is tested. */ - iPageSize16 = (u16)iPageSize; - rc = sqlite3PagerSetPagesize(pPager, &iPageSize16, -1); + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); testcase( rc!=SQLITE_OK ); - assert( rc!=SQLITE_OK || iPageSize16==(u16)iPageSize ); /* Update the assumed sector-size to match the value used by ** the process that created this journal. If this journal was @@ -34947,7 +36158,10 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ i64 jrnlSize; /* Size of journal file on disk */ u32 cksum = 0; /* Checksum of string zMaster */ - if( !zMaster || pPager->setMaster + assert( pPager->setMaster==0 ); + assert( !pagerUseWal(pPager) ); + + if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_OFF ){ @@ -34983,7 +36197,6 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } pPager->journalOff += (nMaster+20); - pPager->needSync = !pPager->noSync; /* If the pager is in peristent-journal mode, then the physical ** journal-file may extend past the end of the master-journal name @@ -35019,17 +36232,11 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ } /* -** Unless the pager is in error-state, discard all in-memory pages. If -** the pager is in error-state, then this call is a no-op. -** -** TODO: Why can we not reset the pager while in error state? +** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ - if( SQLITE_OK==pPager->errCode ){ - sqlite3BackupRestart(pPager->pBackup); - sqlite3PcacheClear(pPager->pPCache); - pPager->dbSizeValid = 0; - } + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); } /* @@ -35072,34 +36279,39 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ } /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. -*/ -#ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); -} -#else -# define pagerUseWal(x) 0 -# define pagerRollbackWal(x) 0 -# define pagerWalFrames(v,w,x,y,z) 0 -# define pagerOpenWalIfPresent(z) SQLITE_OK -# define pagerBeginReadTransaction(z) SQLITE_OK -#endif - -/* -** Unlock the database file. This function is a no-op if the pager -** is in exclusive mode. +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. ** -** If the pager is currently in error state, discard the contents of -** the cache and reset the Pager structure internal state. If there is -** an open journal-file, then the next time a shared-lock is obtained -** on the pager file (by this or any other process), it will be -** treated as a hot-journal and rolled back. +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). */ static void pager_unlock(Pager *pPager){ - if( !pPager->exclusiveMode ){ - int rc = SQLITE_OK; /* Return code */ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; /* If the operating system support deletion of open files, then @@ -35119,58 +36331,56 @@ static void pager_unlock(Pager *pPager){ sqlite3OsClose(pPager->jfd); } - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - releaseAllSavepoints(pPager); - - /* If the file is unlocked, somebody else might change it. The - ** values stored in Pager.dbSize etc. might become invalid if - ** this happens. One can argue that this doesn't need to be cleared - ** until the change-counter check fails in PagerSharedLock(). - ** Clearing the page size cache here is being conservative. + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. */ - pPager->dbSizeValid = 0; - - if( pagerUseWal(pPager) ){ - sqlite3WalEndReadTransaction(pPager->pWal); - }else{ - rc = osUnlock(pPager->fd, NO_LOCK); + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; } - if( rc ){ - pPager->errCode = rc; - } - IOTRACE(("UNLOCK %p\n", pPager)) - /* If Pager.errCode is set, the contents of the pager cache cannot be - ** trusted. Now that the pager file is unlocked, the contents of the - ** cache can be discarded and the error code safely cleared. + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. */ - if( pPager->errCode ){ - if( rc==SQLITE_OK ){ - pPager->errCode = SQLITE_OK; - } - pager_reset(pPager); - } - + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); pPager->changeCountDone = 0; - pPager->state = PAGER_UNLOCK; - pPager->dbModified = 0; + pPager->eState = PAGER_OPEN; } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + if( pPager->errCode ){ + assert( !MEMDB ); + pager_reset(pPager); + pPager->changeCountDone = pPager->tempFile; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setMaster = 0; } /* -** This function should be called when an IOERR, CORRUPT or FULL error -** may have occurred. The first argument is a pointer to the pager -** structure, the second the error-code about to be returned by a pager -** API function. The value returned is a copy of the second argument -** to this function. +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. ** -** If the second argument is SQLITE_IOERR, SQLITE_CORRUPT, or SQLITE_FULL -** the error becomes persistent. Until the persistent error is cleared, -** subsequent API calls on this Pager will immediately return the same -** error code. +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. ** -** A persistent error indicates that the contents of the pager-cache +** The ERROR state indicates that the contents of the pager-cache ** cannot be trusted. This state can be cleared by completely discarding ** the contents of the pager-cache. If a transaction was active when ** the persistent error occurred, then the rollback journal may need @@ -35187,36 +36397,11 @@ static int pager_error(Pager *pPager, int rc){ ); if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ pPager->errCode = rc; + pPager->eState = PAGER_ERROR; } return rc; } -/* -** Execute a rollback if a transaction is active and unlock the -** database file. -** -** If the pager has already entered the error state, do not attempt -** the rollback at this time. Instead, pager_unlock() is called. The -** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and clear the error state. If this means that -** there is a hot-journal left in the file-system, the next connection -** to obtain a shared lock on the pager (which may be this one) will -** roll it back. -** -** If the pager has not already entered the error state, but an IO or -** malloc error occurs during a rollback, then this will itself cause -** the pager to enter the error state. Which will be cleared by the -** call to pager_unlock(), as described above. -*/ -static void pagerUnlockAndRollback(Pager *pPager){ - if( pPager->errCode==SQLITE_OK && pPager->state>=PAGER_RESERVED ){ - sqlite3BeginBenignMalloc(); - sqlite3PagerRollback(pPager); - sqlite3EndBenignMalloc(); - } - pager_unlock(pPager); -} - /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -35224,8 +36409,9 @@ static void pagerUnlockAndRollback(Pager *pPager){ ** the journal file or writing the very first journal-header of a ** database transaction. ** -** If the pager is in PAGER_SHARED or PAGER_UNLOCK state when this -** routine is called, it is a no-op (returns SQLITE_OK). +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. ** ** Otherwise, any active savepoints are released. ** @@ -35256,13 +36442,9 @@ static void pagerUnlockAndRollback(Pager *pPager){ ** DELETE and the pager is in exclusive mode, the method described under ** journalMode==PERSIST is used instead. ** -** After the journal is finalized, if running in non-exclusive mode, the -** pager moves to PAGER_SHARED state (and downgrades the lock on the -** database file accordingly). -** -** If the pager is running in exclusive mode and is in PAGER_SYNCED state, -** it moves to PAGER_EXCLUSIVE. No locks are downgraded when running in -** exclusive mode. +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. ** ** SQLITE_OK is returned if no error occurs. If an error occurs during ** any of the IO operations to finalize the journal file or unlock the @@ -35277,11 +36459,26 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ - if( pPager->stateeState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); @@ -35297,14 +36494,11 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ rc = sqlite3OsTruncate(pPager->jfd, 0); } pPager->journalOff = 0; - pPager->journalStarted = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ rc = zeroJournalHdr(pPager, hasMaster); - pager_error(pPager, rc); pPager->journalOff = 0; - pPager->journalStarted = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal @@ -35320,49 +36514,77 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } + } #ifdef SQLITE_CHECK_PAGES - sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); -#endif + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = pager_lookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnref(p); + } } +#endif + sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); - pPager->state = PAGER_SHARED; - - /* If the connection was in locking_mode=exclusive mode but is no longer, - ** drop the EXCLUSIVE lock held on the database file. - */ - if( !pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, 0) ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - } - }else if( !pPager->exclusiveMode ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - pPager->state = PAGER_SHARED; + } + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; - }else if( pPager->state==PAGER_SYNCED ){ - pPager->state = PAGER_EXCLUSIVE; } + pPager->eState = PAGER_READER; pPager->setMaster = 0; - pPager->needSync = 0; - pPager->dbModified = 0; - - /* TODO: Is this optimal? Why is the db size invalidated here - ** when the database file is not unlocked? */ - pPager->dbOrigSize = 0; - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } return (rc==SQLITE_OK?rc2:rc); } +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. +** +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0); + } + } + pager_unlock(pPager); +} + /* ** Parameter aData must point to a buffer of pPager->pageSize bytes ** of data. Compute and return a checksum based ont the contents of the @@ -35413,9 +36635,8 @@ static void pagerReportSize(Pager *pPager){ ** The page begins at offset *pOffset into the file. The *pOffset ** value is increased to the start of the next page in the journal. ** -** The isMainJrnl flag is true if this is the main rollback journal and -** false for the statement journal. The main rollback journal uses -** checksums - the statement journal does not. +** The main rollback journal uses checksums - the statement journal does +** not. ** ** If the page number of the page record read from the (sub-)journal file ** is greater than the current value of Pager.dbSize, then playback is @@ -35469,6 +36690,17 @@ static int pager_playback_one_page( assert( aData ); /* Temp storage must have already been allocated */ assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); + /* Read the page number and page data from the journal or sub-journal ** file. Return an error code to the caller if an IO error occurs. */ @@ -35505,7 +36737,6 @@ static int pager_playback_one_page( if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ return rc; } - assert( pPager->state==PAGER_RESERVED || pPager->state>=PAGER_EXCLUSIVE ); /* When playing back page 1, restore the nReserve setting */ @@ -35514,7 +36745,7 @@ static int pager_playback_one_page( pagerReportSize(pPager); } - /* If the pager is in RESERVED state, then there must be a copy of this + /* If the pager is in CACHEMOD state, then there must be a copy of this ** page in the pager cache. In this case just update the pager cache, ** not the database file. The page is left marked dirty in this case. ** @@ -35525,8 +36756,11 @@ static int pager_playback_one_page( ** either. So the condition described in the above paragraph is not ** assert()able. ** - ** If in EXCLUSIVE state, then we update the pager cache if it exists - ** and the main file. The page is then marked not dirty. + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. ** ** Ticket #1171: The statement journal might contain page content that is ** different from the page content at the start of the transaction. @@ -35552,6 +36786,7 @@ static int pager_playback_one_page( pPg = pager_lookup(pPager, pgno); } assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -35561,8 +36796,8 @@ static int pager_playback_one_page( }else{ isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); } - if( (pPager->state>=PAGER_EXCLUSIVE) - && isOpen(pPager->fd) + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) && isSynced ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; @@ -35638,9 +36873,8 @@ static int pager_playback_one_page( assert( !pagerUseWal(pPager) ); sqlite3PcacheMakeClean(pPg); } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); + /* If this was page 1, then restore the value of Pager.dbFileVers. ** Do this before any decoding. */ if( pgno==1 ){ @@ -35792,10 +37026,10 @@ delmaster_out: ** file in the file-system. This only happens when committing a transaction, ** or rolling back a transaction (including rolling back a hot-journal). ** -** If the main database file is not open, or an exclusive lock is not -** held, this function is a no-op. Otherwise, the size of the file is -** changed to nPage pages (nPage*pPager->pageSize bytes). If the file -** on disk is currently larger than nPage pages, then use the VFS +** If the main database file is not open, or the pager is not in either +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** ** Or, it might might be the case that the file on disk is smaller than @@ -35809,8 +37043,14 @@ delmaster_out: */ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; - if( pPager->state>=PAGER_EXCLUSIVE && isOpen(pPager->fd) ){ + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ i64 currentSize, newSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* TODO: Is it safe to use Pager.dbFileSize here? */ rc = sqlite3OsFileSize(pPager->fd, ¤tSize); newSize = pPager->pageSize*(i64)nPage; @@ -35934,7 +37174,7 @@ static int pager_playback(Pager *pPager, int isHot){ */ assert( isOpen(pPager->jfd) ); rc = sqlite3OsFileSize(pPager->jfd, &szJ); - if( rc!=SQLITE_OK || szJ==0 ){ + if( rc!=SQLITE_OK ){ goto end_playback; } @@ -35968,7 +37208,7 @@ static int pager_playback(Pager *pPager, int isHot){ while( 1 ){ /* Read the next journal header from the journal file. If there are ** not enough bytes left in the journal file for a complete header, or - ** it is corrupted, then a process must of failed while writing it. + ** it is corrupted, then a process must have failed while writing it. ** This indicates nothing more needs to be rolled back. */ rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); @@ -36082,10 +37322,9 @@ end_playback: rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); - } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ + if( rc==SQLITE_OK && !pPager->noSync + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } if( rc==SQLITE_OK ){ @@ -36127,7 +37366,7 @@ static int readDbPage(PgHdr *pPg){ int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ - assert( pPager->state>=PAGER_SHARED && !MEMDB ); + assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); if( NEVER(!isOpen(pPager->fd)) ){ @@ -36274,6 +37513,14 @@ static int pagerWalFrames( sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } } + +#ifdef SQLITE_CHECK_PAGES + { + PgHdr *p; + for(p=pList; p; p=p->pDirty) pager_set_pagehash(p); + } +#endif + return rc; } @@ -36290,6 +37537,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ int changed = 0; /* True if cache must be reset */ assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); /* sqlite3WalEndReadTransaction() was not called for the previous ** transaction in locking_mode=EXCLUSIVE. So call it now. If we @@ -36299,19 +37547,70 @@ static int pagerBeginReadTransaction(Pager *pPager){ sqlite3WalEndReadTransaction(pPager->pWal); rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); - if( rc==SQLITE_OK ){ - int dummy; - if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); - } - rc = sqlite3PagerPagecount(pPager, &dummy); + if( rc!=SQLITE_OK || changed ){ + pager_reset(pPager); } - pPager->state = PAGER_SHARED; return rc; } +#endif +/* +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. + */ + if( nPage==0 ){ + i64 n = 0; /* Size of db file in bytes */ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + } + nPage = (Pgno)(n / pPager->pageSize); + if( nPage==0 && n>0 ){ + nPage = 1; + } + } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + + *pnPage = nPage; + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager ** exists if the database is not empy, or verify that the *-wal file does @@ -36324,8 +37623,6 @@ static int pagerBeginReadTransaction(Pager *pPager){ ** ** Return SQLITE_OK or an error code. ** -** If the WAL file is opened, also open a snapshot (read transaction). -** ** The caller must hold a SHARED lock on the database file to call this ** function. Because an EXCLUSIVE lock on the db file is required to delete ** a WAL on a none-empty database, this ensures there is no race condition @@ -36334,11 +37631,14 @@ static int pagerBeginReadTransaction(Pager *pPager){ */ static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ - int nPage; /* Size of the database file */ - assert( pPager->state>=SHARED_LOCK ); - rc = sqlite3PagerPagecount(pPager, &nPage); + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); if( rc ) return rc; if( nPage==0 ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); @@ -36350,11 +37650,8 @@ static int pagerOpenWalIfPresent(Pager *pPager){ } if( rc==SQLITE_OK ){ if( isWal ){ - pager_reset(pPager); + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); rc = sqlite3PagerOpenWal(pPager, 0); - if( rc==SQLITE_OK ){ - rc = pagerBeginReadTransaction(pPager); - } }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } @@ -36406,7 +37703,8 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ int rc = SQLITE_OK; /* Return code */ Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ - assert( pPager->state>=PAGER_SHARED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); /* Allocate a bitvec to use to store the set of pages rolled back */ if( pSavepoint ){ @@ -36545,7 +37843,6 @@ SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFu pPager->noSync = (level==1 || pPager->tempFile) ?1:0; pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL); - if( pPager->noSync ) pPager->needSync = 0; } #endif @@ -36627,7 +37924,7 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( ** ** If the pager is in the error state when this function is called, it ** is a no-op. The value returned is the error state error code (i.e. -** one of SQLITE_IOERR, SQLITE_CORRUPT or SQLITE_FULL). +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). ** ** Otherwise, if all of the following are true: ** @@ -36651,28 +37948,48 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( ** function was called, or because the memory allocation attempt failed, ** then *pPageSize is set to the old, retained page size before returning. */ -SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ - int rc = pPager->errCode; +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; - if( rc==SQLITE_OK ){ - u16 pageSize = *pPageSize; - assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); - if( (pPager->memDb==0 || pPager->dbSize==0) - && sqlite3PcacheRefCount(pPager->pPCache)==0 - && pageSize && pageSize!=pPager->pageSize - ){ - char *pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ){ - rc = SQLITE_NOMEM; - }else{ - pager_reset(pPager); - pPager->pageSize = pageSize; - sqlite3PageFree(pPager->pTmpSpace); - pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); - } + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ + + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; + + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); } - *pPageSize = (u16)pPager->pageSize; + if( rc==SQLITE_OK ){ + pNew = (char *)sqlite3PageMalloc(pageSize); + if( !pNew ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + pager_reset(pPager); + pPager->dbSize = (Pgno)(nByte/pageSize); + pPager->pageSize = pageSize; + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + } + + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ if( nReserve<0 ) nReserve = pPager->nReserve; assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; @@ -36701,13 +38018,11 @@ SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager *pPager){ ** Regardless of mxPage, return the current maximum page count. */ SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ - int nPage; if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->state!=PAGER_UNLOCK ){ - sqlite3PagerPagecount(pPager, &nPage); - assert( (int)pPager->mxPgno>=nPage ); + if( pPager->eState!=PAGER_OPEN && pPager->mxPgnodbSize ){ + pPager->mxPgno = pPager->dbSize; } return pPager->mxPgno; } @@ -36772,66 +38087,16 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned cha } /* -** Return the total number of pages in the database file associated -** with pPager. Normally, this is calculated as (/). +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** ** However, if the file is between 1 and bytes in size, then ** this is considered a 1 page file. -** -** If the pager is in error state when this function is called, then the -** error state error code is returned and *pnPage left unchanged. Or, -** if the file system has to be queried for the size of the file and -** the query attempt returns an IO error, the IO error code is returned -** and *pnPage is left unchanged. -** -** Otherwise, if everything is successful, then SQLITE_OK is returned -** and *pnPage is set to the number of pages in the database. */ -SQLITE_PRIVATE int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ - Pgno nPage = 0; /* Value to return via *pnPage */ - - /* Determine the number of pages in the file. Store this in nPage. */ - if( pPager->dbSizeValid ){ - nPage = pPager->dbSize; - }else{ - int rc; /* Error returned by OsFileSize() */ - i64 n = 0; /* File size in bytes returned by OsFileSize() */ - - if( pagerUseWal(pPager) && pPager->state!=PAGER_UNLOCK ){ - sqlite3WalDbsize(pPager->pWal, &nPage); - } - - if( nPage==0 ){ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - if( SQLITE_OK!=(rc = sqlite3OsFileSize(pPager->fd, &n)) ){ - pager_error(pPager, rc); - return rc; - } - } - if( n>0 && npageSize ){ - nPage = 1; - }else{ - nPage = (Pgno)(n / pPager->pageSize); - } - } - if( pPager->state!=PAGER_UNLOCK ){ - pPager->dbSize = nPage; - pPager->dbFileSize = nPage; - pPager->dbSizeValid = 1; - } - } - - /* If the current number of pages in the file is greater than the - ** configured maximum pager number, increase the allowed limit so - ** that the file can be read. - */ - if( nPage>pPager->mxPgno ){ - pPager->mxPgno = (Pgno)nPage; - } - - /* Set the output variable and return SQLITE_OK */ - *pnPage = nPage; - return SQLITE_OK; +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; } @@ -36852,38 +38117,19 @@ SQLITE_PRIVATE int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ - /* The OS lock values must be the same as the Pager lock values */ - assert( PAGER_SHARED==SHARED_LOCK ); - assert( PAGER_RESERVED==RESERVED_LOCK ); - assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); - - /* If the file is currently unlocked then the size must be unknown. It - ** must not have been modified at this point. - */ - assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 ); - assert( pPager->state>=PAGER_SHARED || pPager->dbModified==0 ); - /* Check that this is either a no-op (because the requested lock is ** already held, or one of the transistions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ - assert( (pPager->state>=locktype) - || (pPager->state==PAGER_UNLOCK && locktype==PAGER_SHARED) - || (pPager->state==PAGER_RESERVED && locktype==PAGER_EXCLUSIVE) + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) ); - if( pPager->state>=locktype ){ - rc = SQLITE_OK; - }else{ - do { - rc = sqlite3OsLock(pPager->fd, locktype); - }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); - if( rc==SQLITE_OK ){ - pPager->state = (u8)locktype; - IOTRACE(("LOCK %p %d\n", pPager, locktype)) - } - } + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); return rc; } @@ -36928,9 +38174,8 @@ static void assertTruncateConstraint(Pager *pPager){ ** truncation will be done when the current transaction is committed. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ - assert( pPager->dbSizeValid ); assert( pPager->dbSize>=nPage ); - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; assertTruncateConstraint(pPager); } @@ -36980,7 +38225,7 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pPager->errCode = 0; + /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, @@ -36993,14 +38238,19 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ if( MEMDB ){ pager_unlock(pPager); }else{ - /* Set Pager.journalHdr to -1 for the benefit of the pager_playback() - ** call which may be made from within pagerUnlockAndRollback(). If it - ** is not -1, then the unsynced portion of an open journal file may - ** be played back into the database. If a power failure occurs while - ** this is happening, the database may become corrupt. + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. */ if( isOpen(pPager->jfd) ){ - pPager->errCode = pagerSyncHotJournal(pPager); + pager_error(pPager, pagerSyncHotJournal(pPager)); } pagerUnlockAndRollback(pPager); } @@ -37045,9 +38295,9 @@ SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ ** been written to the journal have actually reached the surface of the ** disk and can be restored in the event of a hot-journal rollback. ** -** If the Pager.needSync flag is not set, then this function is a -** no-op. Otherwise, the actions required depend on the journal-mode -** and the device characteristics of the the file-system, as follows: +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the the file-system, as follows: ** ** * If the journal file is an in-memory journal file, no action need ** be taken. @@ -37071,18 +38321,25 @@ SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ ** if( NOT SEQUENTIAL ) xSync(); ** } ** -** The Pager.needSync flag is never be set for temporary files, or any -** file operating in no-sync mode (Pager.noSync set to non-zero). -** ** If successful, this routine clears the PGHDR_NEED_SYNC flag of every ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ -static int syncJournal(Pager *pPager){ - if( pPager->needSync ){ +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ assert( !pPager->tempFile ); - if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ - int rc; /* Return code */ + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); assert( isOpen(pPager->jfd) ); @@ -37157,17 +38414,25 @@ static int syncJournal(Pager *pPager){ ); if( rc!=SQLITE_OK ) return rc; } - } - /* The journal file was just successfully synced. Set Pager.needSync - ** to zero and clear the PGHDR_NEED_SYNC flag on all pagess. - */ - pPager->needSync = 0; - pPager->journalStarted = 1; - pPager->journalHdr = pPager->journalOff; - sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } + }else{ + pPager->journalHdr = pPager->journalOff; + } } + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); return SQLITE_OK; } @@ -37204,27 +38469,12 @@ static int syncJournal(Pager *pPager){ ** be obtained, SQLITE_BUSY is returned. */ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ - /* At this point there may be either a RESERVED or EXCLUSIVE lock on the - ** database file. If there is already an EXCLUSIVE lock, the following - ** call is a no-op. - ** - ** Moving the lock from RESERVED to EXCLUSIVE actually involves going - ** through an intermediate state PENDING. A PENDING lock prevents new - ** readers from attaching to the database but is unsufficient for us to - ** write. The idea of a PENDING lock is to prevent new readers from - ** coming in while we wait for existing readers to clear. - ** - ** While the pager is in the RESERVED state, the original database file - ** is unchanged and we can rollback without having to playback the - ** journal into the original database file. Once we transition to - ** EXCLUSIVE, it means the database file has been changed and any rollback - ** will require a journal playback. - */ + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->state>=PAGER_RESERVED ); - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -37239,9 +38489,10 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>(pPager->dbOrigSize+1) ){ + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; } while( rc==SQLITE_OK && pList ){ @@ -37259,6 +38510,8 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ char *pData; /* Data to write */ + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + /* Encode the database */ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); @@ -37287,9 +38540,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } -#ifdef SQLITE_CHECK_PAGES - pList->pageHash = pager_pagehash(pList); -#endif + pager_set_pagehash(pList); pList = pList->pDirty; } @@ -37401,9 +38652,14 @@ static int pagerStress(void *p, PgHdr *pPg){ ** The doNotSpill flag inhibits all cache spilling regardless of whether ** or not a sync is required. This is set during a rollback. ** - ** Spilling is also inhibited when in an error state. + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementaton it + ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. */ - if( pPager->errCode ) return SQLITE_OK; + if( NEVER(pPager->errCode) ) return SQLITE_OK; if( pPager->doNotSpill ) return SQLITE_OK; if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ return SQLITE_OK; @@ -37421,16 +38677,10 @@ static int pagerStress(void *p, PgHdr *pPg){ }else{ /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC ){ - assert( !pPager->noSync ); - rc = syncJournal(pPager); - if( rc==SQLITE_OK && - !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && - !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) - ){ - pPager->nRec = 0; - rc = writeJournalHdr(pPager); - } + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); } /* If the page number of this page is larger than the current size of @@ -37468,6 +38718,7 @@ static int pagerStress(void *p, PgHdr *pPg){ /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); rc = pager_write_pagelist(pPager, pPg); } } @@ -37478,7 +38729,7 @@ static int pagerStress(void *p, PgHdr *pPg){ sqlite3PcacheMakeClean(pPg); } - return pager_error(pPager, rc); + return pager_error(pPager, rc); } @@ -37533,7 +38784,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ - u16 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). This @@ -37552,6 +38803,13 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + zFilename = 0; + } +#endif + /* Compute and store the full pathname in an allocated buffer pointed ** to by zPathname, length nPathname. Or, if this is a temporary file, ** leave both nPathname and zPathname set to 0. @@ -37562,17 +38820,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( if( zPathname==0 ){ return SQLITE_NOMEM; } -#ifndef SQLITE_OMIT_MEMORYDB - if( strcmp(zFilename,":memory:")==0 ){ - memDb = 1; - zPathname[0] = 0; - }else -#endif - { - zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ - rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); - } - + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); nPathname = sqlite3Strlen30(zPathname); if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by @@ -37627,19 +38876,15 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ if( zPathname ){ + assert( nPathname>0 ); pPager->zJournal = (char*)(pPtr += nPathname + 1); memcpy(pPager->zFilename, zPathname, nPathname); memcpy(pPager->zJournal, zPathname, nPathname); memcpy(&pPager->zJournal[nPathname], "-journal", 8); - if( pPager->zFilename[0]==0 ){ - pPager->zJournal[0] = 0; - } #ifndef SQLITE_OMIT_WAL - else{ - pPager->zWal = &pPager->zJournal[nPathname+8+1]; - memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal", 4); - } + pPager->zWal = &pPager->zJournal[nPathname+8+1]; + memcpy(pPager->zWal, zPathname, nPathname); + memcpy(&pPager->zWal[nPathname], "-wal", 4); #endif sqlite3_free(zPathname); } @@ -37648,9 +38893,10 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* Open the pager file. */ - if( zFilename && zFilename[0] && !memDb ){ + if( zFilename && zFilename[0] ){ int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); readOnly = (fout&SQLITE_OPEN_READONLY); /* If the file was successfully opened for read/write access, @@ -37668,7 +38914,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; }else{ - szPageDflt = (u16)pPager->sectorSize; + szPageDflt = (u32)pPager->sectorSize; } } #ifdef SQLITE_ENABLE_ATOMIC_WRITE @@ -37696,7 +38942,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** disk and uses an in-memory rollback journal. */ tempFile = 1; - pPager->state = PAGER_EXCLUSIVE; + pPager->eState = PAGER_READER; + pPager->eLock = EXCLUSIVE_LOCK; readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -37733,13 +38980,14 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* pPager->stmtOpen = 0; */ /* pPager->stmtInUse = 0; */ /* pPager->nRef = 0; */ - pPager->dbSizeValid = (u8)memDb; /* pPager->stmtSize = 0; */ /* pPager->stmtJSize = 0; */ /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ +#if 0 assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); +#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL @@ -37749,7 +38997,6 @@ SQLITE_PRIVATE int sqlite3PagerOpen( pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; - /* pPager->needSync = 0; */ assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; pPager->fullSync = pPager->noSync ?0:1; @@ -37814,10 +39061,10 @@ static int hasHotJournal(Pager *pPager, int *pExists){ int exists = 1; /* True if a journal file is present */ int jrnlOpen = !!isOpen(pPager->jfd); - assert( pPager!=0 ); assert( pPager->useJournal ); assert( isOpen(pPager->fd) ); - assert( pPager->state <= PAGER_SHARED ); + assert( pPager->eState==PAGER_OPEN ); + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN )); @@ -37827,7 +39074,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); } if( rc==SQLITE_OK && exists ){ - int locked; /* True if some process holds a RESERVED lock */ + int locked = 0; /* True if some process holds a RESERVED lock */ /* Race condition here: Another process might have been holding the ** the RESERVED lock and have a journal open at the sqlite3OsAccess() @@ -37839,7 +39086,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ */ rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); if( rc==SQLITE_OK && !locked ){ - int nPage; + Pgno nPage; /* Number of pages in database file */ /* Check the size of the database file. If it consists of 0 pages, ** then delete the journal file. See the header comment above for @@ -37847,13 +39094,13 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** a RESERVED lock to avoid race conditions and to avoid violating ** [H33020]. */ - rc = sqlite3PagerPagecount(pPager, &nPage); + rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ if( nPage==0 ){ sqlite3BeginBenignMalloc(); - if( sqlite3OsLock(pPager->fd, RESERVED_LOCK)==SQLITE_OK ){ + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); + pagerUnlockDb(pPager, SHARED_LOCK); } sqlite3EndBenignMalloc(); }else{ @@ -37906,7 +39153,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** ** The following operations are also performed by this function. ** -** 1) If the pager is currently in PAGER_UNLOCK state (no lock held +** 1) If the pager is currently in PAGER_OPEN state (no lock held ** on the database file), then an attempt is made to obtain a ** SHARED lock on the database file. Immediately after obtaining ** the SHARED lock, the file-system is checked for a hot-journal, @@ -37921,66 +39168,47 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** the contents of the page cache and rolling back any open journal ** file. ** -** If the operation described by (2) above is not attempted, and if the -** pager is in an error state other than SQLITE_FULL when this is called, -** the error state error code is returned. It is permitted to read the -** database when in SQLITE_FULL error state. -** -** Otherwise, if everything is successful, SQLITE_OK is returned. If an -** IO error occurs while locking the database, checking for a hot-journal -** file or rolling back a journal file, the IO error code is returned. +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. */ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int isErrorReset = 0; /* True if recovering from error state */ /* This routine is only called from b-tree and only when there are no - ** outstanding pages */ + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. + */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } - /* If this database is in an error-state, now is a chance to clear - ** the error. Discard the contents of the pager-cache and rollback - ** any hot journal in the file-system. - */ - if( pPager->errCode ){ - if( isOpen(pPager->jfd) || pPager->zJournal ){ - isErrorReset = 1; - } - pPager->errCode = SQLITE_OK; - pager_reset(pPager); - } + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ - if( pagerUseWal(pPager) ){ - rc = pagerBeginReadTransaction(pPager); - }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ - sqlite3_vfs * const pVfs = pPager->pVfs; - int isHotJournal = 0; assert( !MEMDB ); - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - if( pPager->noReadlock ){ - assert( pPager->readOnly ); - pPager->state = PAGER_SHARED; - }else{ + assert( pPager->noReadlock==0 || pPager->readOnly ); + + if( pPager->noReadlock==0 ){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); - return pager_error(pPager, rc); + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; } } - assert( pPager->state>=SHARED_LOCK ); /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. */ - if( !isErrorReset ){ - assert( pPager->state <= PAGER_SHARED ); - rc = hasHotJournal(pPager, &isHotJournal); - if( rc!=SQLITE_OK ){ - goto failed; - } + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); } - if( isErrorReset || isHotJournal ){ + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the @@ -37992,58 +39220,45 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** other process attempting to access the database file will get to ** this point in the code and fail to obtain its own EXCLUSIVE lock ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. */ - if( pPager->statefd, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } - pPager->state = PAGER_EXCLUSIVE; - } - - /* Open the journal for read/write access. This is because in - ** exclusive-access mode the file descriptor will be kept open and - ** possibly used for a transaction later on. On some systems, the - ** OsTruncate() call used in exclusive-access mode also requires - ** a read/write file handle. - */ - if( !isOpen(pPager->jfd) ){ - int res; - rc = sqlite3OsAccess(pVfs,pPager->zJournal,SQLITE_ACCESS_EXISTS,&res); - if( rc==SQLITE_OK ){ - if( res ){ - int fout = 0; - int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; - assert( !pPager->tempFile ); - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ - rc = SQLITE_CANTOPEN_BKPT; - sqlite3OsClose(pPager->jfd); - } - }else{ - /* If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it - ** may mean that the pager was in the error-state when this - ** function was called and the journal file does not exist. */ - rc = pager_end_transaction(pPager, 0); - } - } - } + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ goto failed; } - - /* Reset the journal status fields to indicates that we have no - ** rollback journal at this time. */ - pPager->journalStarted = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - /* Make sure the journal file has been synced to disk. */ + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. Purge the cache before @@ -38054,21 +39269,46 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** the journal before playing it back. */ if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ rc = pager_playback(pPager, 1); + pPager->eState = PAGER_OPEN; } - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); } - assert( (pPager->state==PAGER_SHARED) - || (pPager->exclusiveMode && pPager->state>PAGER_SHARED) + + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } + + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) ); } - if( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 ){ + if( !pPager->tempFile + && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) + ){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -38085,14 +39325,11 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** detected. The chance of an undetected change is so small that ** it can be neglected. */ - int nPage = 0; + Pgno nPage = 0; char dbFileVers[sizeof(pPager->dbFileVers)]; - sqlite3PagerPagecount(pPager, &nPage); - if( pPager->errCode ){ - rc = pPager->errCode; - goto failed; - } + rc = pagerPagecount(pPager, &nPage); + if( rc ) goto failed; if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); @@ -38108,18 +39345,32 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } } - assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); /* If there is a WAL file in the file-system, open this database in WAL ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL + assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif + } + + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); } failed: if( rc!=SQLITE_OK ){ - /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */ + assert( !MEMDB ); pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; } return rc; } @@ -38133,9 +39384,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) - && (!pPager->exclusiveMode || pPager->journalOff>0) - ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -38199,8 +39448,8 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( int rc; PgHdr *pPg; + assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( pPager->state>PAGER_UNLOCK ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -38208,7 +39457,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ - if( pPager->errCode!=SQLITE_OK && pPager->errCode!=SQLITE_FULL ){ + if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); @@ -38234,7 +39483,6 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( }else{ /* The pager cache has created a new page. Its content needs to ** be initialized. */ - int nMax; PAGER_INCR(pPager->nMiss); pPg = *ppPage; @@ -38247,12 +39495,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( goto pager_acquire_err; } - rc = sqlite3PagerPagecount(pPager, &nMax); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } - - if( MEMDB || nMax<(int)pgno || noContent || !isOpen(pPager->fd) ){ + if( MEMDB || pPager->dbSizefd) ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -38282,9 +39525,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( goto pager_acquire_err; } } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } return SQLITE_OK; @@ -38303,9 +39544,7 @@ pager_acquire_err: /* ** Acquire a page if it is already in the in-memory cache. Do ** not read the page from disk. Return a pointer to the page, -** or 0 if the page is not in cache. Also, return 0 if the -** pager is in PAGER_UNLOCK state when this function is called, -** or if the pager is in an error state other than SQLITE_FULL. +** or 0 if the page is not in cache. ** ** See also sqlite3PagerGet(). The difference between this routine ** and sqlite3PagerGet() is that _get() will go to the disk and read @@ -38318,7 +39557,7 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->state > PAGER_UNLOCK ); + assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); return pPg; } @@ -38363,12 +39602,10 @@ SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ */ static int pager_open_journal(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int nPage; /* Size of database file */ sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ - assert( pPager->state>=PAGER_RESERVED ); - assert( pPager->useJournal ); - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF ); + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); /* If already in the error state, this function is a no-op. But on @@ -38376,56 +39613,56 @@ static int pager_open_journal(Pager *pPager){ ** an error state. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - testcase( pPager->dbSizeValid==0 ); - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; - pPager->pInJournal = sqlite3BitvecCreate(nPage); - if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; - } - - /* Open the journal file if it is not already open. */ - if( !isOpen(pPager->jfd) ){ - if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ - sqlite3MemJournalOpen(pPager->jfd); - }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) - ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + ); + #else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); + #endif + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); } - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - } - - - /* Write the first journal header to the journal file and open - ** the sub-journal if necessary. - */ - if( rc==SQLITE_OK ){ - /* TODO: Check if all of these are really required. */ - pPager->dbOrigSize = pPager->dbSize; - pPager->journalStarted = 0; - pPager->needSync = 0; - pPager->nRec = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - rc = writeJournalHdr(pPager); } if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; } + return rc; } @@ -38438,14 +39675,6 @@ static int pager_open_journal(Pager *pPager){ ** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** -** If this is not a temporary or in-memory file and, the journal file is -** opened if it has not been already. For a temporary file, the opening -** of the journal file is deferred until there is an actual need to -** write to the journal. TODO: Why handle temporary files differently? -** -** If the journal file is opened (or if it is already open), then a -** journal-header is written to the start of it. -** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This ** has no effect if the sub-journal is already opened (as it may be when @@ -38456,20 +39685,20 @@ static int pager_open_journal(Pager *pPager){ */ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ int rc = SQLITE_OK; - assert( pPager->state!=PAGER_UNLOCK ); + + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - if( pPager->state==PAGER_SHARED ){ + if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); - assert( !MEMDB && !pPager->tempFile ); if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); - pPager->state = PAGER_SHARED; + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } @@ -38480,52 +39709,40 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. - ** - ** WAL mode sets Pager.state to PAGER_RESERVED when it has an open - ** transaction, but never to PAGER_EXCLUSIVE. This is because in - ** PAGER_EXCLUSIVE state the code to roll back savepoint transactions - ** may copy data from the sub-journal into the database file as well - ** as into the page cache. Which would be incorrect in WAL mode. */ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); - if( rc==SQLITE_OK ){ - pPager->dbOrigSize = pPager->dbSize; - pPager->state = PAGER_RESERVED; - pPager->journalOff = 0; - } - - assert( rc!=SQLITE_OK || pPager->state==PAGER_RESERVED ); - assert( rc==SQLITE_OK || pPager->state==PAGER_SHARED ); }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - if( rc==SQLITE_OK ){ - pPager->state = PAGER_RESERVED; - if( exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } } - /* No need to open the journal file at this time. It will be - ** opened before it is written to. If we defer opening the journal, - ** we might save the work of creating a file if the transaction - ** ends up being a no-op. - */ - - if( rc!=SQLITE_OK ){ - assert( !pPager->dbModified ); - /* Ignore any IO error that occurs within pager_end_transaction(). The - ** purpose of this call is to reset the internal state of the pager - ** sub-system. It doesn't matter if the journal-file is not properly - ** finalized at this point (since it is not a valid journal file anyway). + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. */ - pager_end_transaction(pPager, 0); + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); } PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); @@ -38544,80 +39761,86 @@ static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - /* This routine is not called unless a transaction has already been - ** started. + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. */ - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* If an error has been previously detected, report the same error - ** again. - */ + ** again. This should not happen, but the check provides robustness. */ if( NEVER(pPager->errCode) ) return pPager->errCode; /* Higher-level routines never call this function if database is not ** writable. But check anyway, just for robustness. */ if( NEVER(pPager->readOnly) ) return SQLITE_PERM; - assert( !pPager->setMaster ); - CHECK_PAGE(pPg); + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); + /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. */ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ assert( !pagerUseWal(pPager) ); - pPager->dbModified = 1; }else{ - - /* If we get this far, it means that the page needs to be - ** written to the transaction journal or the ckeckpoint journal - ** or both. - ** - ** Higher level routines should have already started a transaction, - ** which means they have acquired the necessary locks but the rollback - ** journal might not yet be open. - */ - assert( pPager->state>=RESERVED_LOCK ); - if( pPager->pInJournal==0 - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - && !pagerUseWal(pPager) - ){ - assert( pPager->useJournal ); - rc = pager_open_journal(pPager); - if( rc!=SQLITE_OK ) return rc; - } - pPager->dbModified = 1; /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to ** the transaction journal if it is not there already. */ - if( !pageInJournal(pPg) && isOpen(pPager->jfd) ){ - assert( !pagerUseWal(pPager) ); - if( pPg->pgno<=pPager->dbOrigSize ){ + if( !pageInJournal(pPg) && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ u32 cksum; char *pData2; + i64 iOff = pPager->journalOff; /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - assert( pPager->journalHdr <= pPager->journalOff ); + assert( pPager->journalHdr<=pPager->journalOff ); CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); cksum = pager_cksum(pPager, (u8*)pData2); - rc = write32bits(pPager->jfd, pPager->journalOff, pPg->pgno); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, - pPager->journalOff + 4); - pPager->journalOff += pPager->pageSize+4; - } - if( rc==SQLITE_OK ){ - rc = write32bits(pPager->jfd, pPager->journalOff, cksum); - pPager->journalOff += 4; - } + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, pPager->journalOff, pPager->pageSize)); PAGER_INCR(sqlite3_pager_writej_count); @@ -38625,25 +39848,7 @@ static int pager_write(PgHdr *pPg){ PAGERID(pPager), pPg->pgno, ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - /* Even if an IO or diskfull error occurred while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - if( !pPager->noSync ){ - pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; - } - - /* An error has occurred writing to the journal file. The - ** transaction will be rolled back by the layer above. - */ - if( rc!=SQLITE_OK ){ - return rc; - } - + pPager->journalOff += 8 + pPager->pageSize; pPager->nRec++; assert( pPager->pInJournal!=0 ); rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); @@ -38655,9 +39860,8 @@ static int pager_write(PgHdr *pPg){ return rc; } }else{ - if( !pPager->journalStarted && !pPager->noSync ){ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; } PAGERTRACE(("APPEND %d page %d needSync=%d\n", PAGERID(pPager), pPg->pgno, @@ -38677,7 +39881,6 @@ static int pager_write(PgHdr *pPg){ /* Update the database size and return. */ - assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } @@ -38705,6 +39908,10 @@ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ @@ -38726,19 +39933,17 @@ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ */ pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - rc = sqlite3PagerPagecount(pPager, (int *)&nPageCount); - if( rc==SQLITE_OK ){ - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); for(ii=0; iiflags&PGHDR_NEED_SYNC ){ needSync = 1; - assert(pPager->needSync); } sqlite3PagerUnref(pPage); } @@ -38770,7 +39974,7 @@ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage *pDbPage){ ** before any of them can be written out to the database file. */ if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB && pPager->noSync==0 ); + assert( !MEMDB ); for(ii=0; iineedSync); } assert( pPager->doNotSyncSpill==1 ); @@ -38820,9 +40023,7 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } } @@ -38845,6 +40046,11 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + /* Declare and initialize constant integer 'isDirect'. If the ** atomic-write optimization is enabled in this build, then isDirect ** is initialized to the value passed as the isDirectMode parameter @@ -38863,7 +40069,6 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - assert( pPager->state>=PAGER_RESERVED ); if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ u32 change_counter; /* Initial value of change-counter field */ @@ -38935,6 +40140,30 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){ return rc; } +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + return rc; +} + /* ** Sync the database file for the pager pPager. zMaster points to the name ** of a master journal file that should be written into the individual @@ -38968,22 +40197,29 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ){ int rc = SQLITE_OK; /* Return code */ - /* The dbOrigSize is never set if journal_mode=OFF */ - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF || pPager->dbOrigSize==0 ); + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); /* If a prior error occurred, report that error again. */ - if( pPager->errCode ) return pPager->errCode; + if( NEVER(pPager->errCode) ) return pPager->errCode; PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); - if( MEMDB && pPager->dbModified ){ + /* If no database changes have been made, return early. */ + if( pPager->eStatepBackup); - }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ + }else{ if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ @@ -39026,7 +40262,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ); if( !zMaster && isOpen(pPager->jfd) && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbFileSize + && pPager->dbSize>=pPager->dbOrigSize && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) ){ /* Update the db file change counter via the direct-write method. The @@ -39056,13 +40292,10 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** that it took at the start of the transaction. Otherwise, the ** calls to sqlite3PagerGet() return zeroed pages instead of ** reading data from the database file. - ** - ** When journal_mode==OFF the dbOrigSize is always zero, so this - ** block never runs if journal_mode=OFF. */ #ifndef SQLITE_OMIT_AUTOVACUUM if( pPager->dbSizedbOrigSize - && ALWAYS(pPager->journalMode!=PAGER_JOURNALMODE_OFF) + && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ Pgno i; /* Iterator variable */ const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ @@ -39089,14 +40322,20 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( rc = writeMasterJournal(pPager, zMaster); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Sync the journal file. If the atomic-update optimization is being - ** used, this call will not create the journal file or perform any - ** real IO. + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. */ - rc = syncJournal(pPager); + rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Write all dirty pages to the database file. */ rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); @@ -39109,7 +40348,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( */ if( pPager->dbSize!=pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); - assert( pPager->state>=PAGER_EXCLUSIVE ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } @@ -39120,11 +40359,12 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( } IOTRACE(("DBSYNC %p\n", pPager)) } - - pPager->state = PAGER_SYNCED; } commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } return rc; } @@ -39152,12 +40392,11 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - /* This function should not be called if the pager is not in at least - ** PAGER_RESERVED state. **FIXME**: Make it so that this test always - ** fails - make it so that we never reach this point if we do not hold - ** all necessary locks. - */ - if( NEVER(pPager->stateeState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); /* An optimization. If the database was not actually modified during ** this transaction, the pager is running in exclusive-mode and is @@ -39170,102 +40409,76 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** header. Since the pager is in exclusive mode, there is no need ** to drop any locks either. */ - if( pPager->dbModified==0 && pPager->exclusiveMode + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; return SQLITE_OK; } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - assert( pPager->state==PAGER_SYNCED || MEMDB || !pPager->dbModified ); rc = pager_end_transaction(pPager, pPager->setMaster); return pager_error(pPager, rc); } /* -** Rollback all changes. The database falls back to PAGER_SHARED mode. +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. ** -** This function performs two tasks: +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. +** +** Otherwise, in rollback mode, this function performs two functions: ** ** 1) It rolls back the journal file, restoring all database file and ** in-memory cache pages to the state they were in when the transaction ** was opened, and +** ** 2) It finalizes the journal file, so that it is not used for hot ** rollback at any point in the future. ** -** subject to the following qualifications: +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. ** -** * If the journal file is not yet open when this function is called, -** then only (2) is performed. In this case there is no journal file -** to roll back. -** -** * If in an error state other than SQLITE_FULL, then task (1) is -** performed. If successful, task (2). Regardless of the outcome -** of either, the error state error code is returned to the caller -** (i.e. either SQLITE_IOERR or SQLITE_CORRUPT). -** -** * If the pager is in PAGER_RESERVED state, then attempt (1). Whether -** or not (1) is successful, also attempt (2). If successful, return -** SQLITE_OK. Otherwise, enter the error state and return the first -** error code encountered. -** -** In this case there is no chance that the database was written to. -** So is safe to finalize the journal file even if the playback -** (operation 1) failed. However the pager must enter the error state -** as the contents of the in-memory cache are now suspect. -** -** * Finally, if in PAGER_EXCLUSIVE state, then attempt (1). Only -** attempt (2) if (1) is successful. Return SQLITE_OK if successful, -** otherwise enter the error state and return the error code from the -** failing operation. -** -** In this case the database file may have been written to. So if the -** playback operation did not succeed it would not be safe to finalize -** the journal file. It needs to be left in the file-system so that -** some other process can use it to restore the database state (by -** hot-journal rollback). +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. */ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + if( pagerUseWal(pPager) ){ int rc2; - rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; - rc = pager_error(pPager, rc); - }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){ - rc = pager_end_transaction(pPager, pPager->setMaster); - }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - if( pPager->state>=PAGER_EXCLUSIVE ){ - pager_playback(pPager, 0); - } - rc = pPager->errCode; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_end_transaction(pPager, 0); }else{ - if( pPager->state==PAGER_RESERVED ){ - int rc2; - rc = pager_playback(pPager, 0); - rc2 = pager_end_transaction(pPager, pPager->setMaster); - if( rc==SQLITE_OK ){ - rc = rc2; - } - }else{ - rc = pager_playback(pPager, 0); - } - - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } - - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error - ** persistent. - */ - rc = pager_error(pPager, rc); + rc = pager_playback(pPager, 0); } - return rc; + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); } /* @@ -39288,9 +40501,11 @@ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ ** used by the pager and its associated cache. */ SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ - int perPageSize = pPager->pageSize + pPager->nExtra + 20; + int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) + + 5*sizeof(void*); return perPageSize*sqlite3PcachePagecount(pPager->pPCache) - + sqlite3MallocSize(pPager); + + sqlite3MallocSize(pPager) + + pPager->pageSize; } /* @@ -39309,8 +40524,8 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ a[0] = sqlite3PcacheRefCount(pPager->pPCache); a[1] = sqlite3PcachePagecount(pPager->pPCache); a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); - a[3] = pPager->dbSizeValid ? (int) pPager->dbSize : -1; - a[4] = pPager->state; + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; a[5] = pPager->errCode; a[6] = pPager->nHit; a[7] = pPager->nMiss; @@ -39342,13 +40557,12 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + if( nSavepoint>nCurrent && pPager->useJournal ){ int ii; /* Iterator variable */ PagerSavepoint *aNew; /* New Pager.aSavepoint array */ - int nPage; /* Size of database file */ - - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM ** if the allocation fails. Otherwise, zero the new portion in case a @@ -39365,14 +40579,14 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ /* Populate the PagerSavepoint structures just allocated. */ for(ii=nCurrent; iidbSize; if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ aNew[ii].iOffset = pPager->journalOff; }else{ aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); } aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage); + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } @@ -39419,12 +40633,12 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = SQLITE_OK; + int rc = pPager->errCode; /* Return code */ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); - if( iSavepointnSavepoint ){ + if( rc==SQLITE_OK && iSavepointnSavepoint ){ int ii; /* Iterator variable */ int nNew; /* Number of remaining savepoints after this op. */ @@ -39460,8 +40674,8 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - } + return rc; } @@ -39559,6 +40773,10 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i Pgno origPgno; /* The original page number */ assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. @@ -39608,11 +40826,10 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i needSyncPgno = pPg->pgno; assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); - assert( pPager->needSync ); } /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PgHdr.needSync was set for + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ @@ -39624,7 +40841,6 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i if( MEMDB ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ - assert( pPager->dbSizeValid ); sqlite3PcacheMove(pPgOld, pPager->dbSize+1); }else{ sqlite3PcacheDrop(pPgOld); @@ -39634,14 +40850,23 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i origPgno = pPg->pgno; sqlite3PcacheMove(pPg, pgno); sqlite3PcacheMakeDirty(pPg); - pPager->dbModified = 1; + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( MEMDB ){ + assert( pPgOld ); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnref(pPgOld); + } if( needSyncPgno ){ /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. ** Currently, no such page exists in the page-cache and the ** "is journaled" bitvec flag has been set. This needs to be remedied by - ** loading the page into the pager-cache and setting the PgHdr.needSync + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC ** flag. ** ** If the attempt to load the page into the page-cache fails, (due @@ -39650,12 +40875,8 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** this transaction, it may be written to the database file before ** it is synced into the journal file. This way, it may end up in ** the journal file twice, but that is not a problem. - ** - ** The sqlite3PagerGet() call may cause the journal to sync. So make - ** sure the Pager.needSync flag is set too. */ PgHdr *pPgHdr; - assert( pPager->needSync ); rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ @@ -39664,23 +40885,11 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } return rc; } - pPager->needSync = 1; - assert( pPager->noSync==0 && !MEMDB ); pPgHdr->flags |= PGHDR_NEED_SYNC; sqlite3PcacheMakeDirty(pPgHdr); sqlite3PagerUnref(pPgHdr); } - /* - ** For an in-memory database, make sure the original page continues - ** to exist, in case the transaction needs to roll back. Use pPgOld - ** as the original page since it has already been allocated. - */ - if( MEMDB ){ - sqlite3PcacheMove(pPgOld, origPgno); - sqlite3PagerUnref(pPgOld); - } - return SQLITE_OK; } #endif @@ -39746,6 +40955,13 @@ SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ +#ifdef SQLITE_DEBUG + /* The print_pager_state() routine is intended to be used by the debugger + ** only. We invoke it once here to suppress a compiler warning. */ + print_pager_state(pPager); +#endif + + /* The eMode parameter is always valid */ assert( eMode==PAGER_JOURNALMODE_DELETE || eMode==PAGER_JOURNALMODE_TRUNCATE @@ -39771,20 +40987,13 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } if( eMode!=eOld ){ - /* When changing between rollback modes, close the journal file prior - ** to the change. But when changing from a rollback mode to WAL, keep - ** the journal open since there is a rollback-style transaction in play - ** used to convert the version numbers in the btree header. - */ - if( isOpen(pPager->jfd) && eMode!=PAGER_JOURNALMODE_WAL ){ - sqlite3OsClose(pPager->jfd); - } /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; /* When transistioning from TRUNCATE or PERSIST to any other journal - ** mode except WAL (and we are not in locking_mode=EXCLUSIVE) then + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); @@ -39805,24 +41014,30 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ ** database file. This ensures that the journal file is not deleted ** while it is in use by some other client. */ - int rc = SQLITE_OK; - int state = pPager->state; - if( statestate==PAGER_SHARED ){ - assert( rc==SQLITE_OK ); - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - } - if( rc==SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); } - if( rc==SQLITE_OK && state==PAGER_SHARED ){ - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); - }else if( state==PAGER_UNLOCK ){ - pager_unlock(pPager); - } - assert( state==pPager->state ); } } @@ -39843,7 +41058,8 @@ SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager *pPager){ ** is unmodified. */ SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ - if( pPager->dbModified ) return 0; + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; return 1; } @@ -39908,25 +41124,31 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ ** file (not a temp file or an in-memory database), and the WAL file ** is not already open, make an attempt to open it now. If successful, ** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pisOpen is +** not support the xShmXXX() methods, return an error code. *pbOpen is ** not modified in either case. ** ** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pisOpen to 1 and return SQLITE_OK +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK ** without doing anything. */ SQLITE_PRIVATE int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ - int *pisOpen /* OUT: Set to true if call is a no-op */ + int *pbOpen /* OUT: Set to true if call is a no-op */ ){ int rc = SQLITE_OK; /* Return code */ - assert( pPager->state>=PAGER_SHARED ); - assert( (pisOpen==0 && !pPager->tempFile && !pPager->pWal) || *pisOpen==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); + /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), unlock the database file and ** return an error code. @@ -39934,9 +41156,10 @@ SQLITE_PRIVATE int sqlite3PagerOpenWal( rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; } }else{ - *pisOpen = 1; + *pbOpen = 1; } return rc; @@ -39962,7 +41185,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ */ if( !pPager->pWal ){ int logexists = 0; - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_SHARED); + rc = pagerLockDb(pPager, SHARED_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3OsAccess( pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists @@ -39978,7 +41201,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, (pPager->noSync ? 0 : pPager->sync_flags), @@ -39988,7 +41211,7 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ }else{ /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock ** that we did get back to SHARED. */ - sqlite3OsUnlock(pPager->fd, SQLITE_LOCK_SHARED); + pagerUnlockDb(pPager, SQLITE_LOCK_SHARED); } } return rc; @@ -40308,6 +41531,10 @@ typedef struct WalCkptInfo WalCkptInfo; ** ** The actual header in the wal-index consists of two copies of this ** object. +** +** The szPage value can be any power of 2 between 512 and 32768, inclusive. +** Or it can be 1 to represent a 65536-byte page. The latter case was +** added in 3.7.1 when support for 64K pages was added. */ struct WalIndexHdr { u32 iVersion; /* Wal-index version */ @@ -40315,7 +41542,7 @@ struct WalIndexHdr { u32 iChange; /* Counter incremented each transaction */ u8 isInit; /* 1 when initialized */ u8 bigEndCksum; /* True if checksums in WAL are big-endian */ - u16 szPage; /* Database page size in bytes */ + u16 szPage; /* Database page size in bytes. 1==64K */ u32 mxFrame; /* Index of last valid frame in the WAL */ u32 nPage; /* Size of database in pages */ u32 aFrameCksum[2]; /* Checksum of last frame in log */ @@ -40426,7 +41653,7 @@ struct Wal { u32 iCallback; /* Value to pass to log callback (or 0) */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ - u16 szPage; /* Database page size */ + u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ @@ -41097,7 +42324,7 @@ static int walIndexRecover(Wal *pWal){ goto finished; } pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = (u16)szPage; + pWal->szPage = szPage; pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); @@ -41147,7 +42374,9 @@ static int walIndexRecover(Wal *pWal){ if( nTruncate ){ pWal->hdr.mxFrame = iFrame; pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)szPage; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } @@ -41172,6 +42401,17 @@ finished: pInfo->nBackfill = 0; pInfo->aReadMark[0] = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; + + /* If more than one frame was recovered from the log file, report an + ** event via sqlite3_log(). This is to help with identifying performance + ** problems caused by applications routinely shutting down without + ** checkpointing the log file. + */ + if( pWal->hdr.nPage ){ + sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", + pWal->hdr.nPage, pWal->zWalName + ); + } } recovery_error: @@ -41532,14 +42772,18 @@ static int walCheckpoint( u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ - int szPage = pWal->hdr.szPage; /* Database page-size */ + int szPage; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); if( pWal->hdr.mxFrame==0 ) return SQLITE_OK; /* Allocate the iterator */ @@ -41550,7 +42794,7 @@ static int walCheckpoint( assert( pIter ); /*** TODO: Move this test out to the caller. Make it an assert() here ***/ - if( pWal->hdr.szPage!=nBuf ){ + if( szPage!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; goto walcheckpoint_out; } @@ -41561,6 +42805,7 @@ static int walCheckpoint( ** cannot be backfilled from the WAL. */ mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; pInfo = walCkptInfo(pWal); for(i=1; iaReadMark[i]; @@ -41581,6 +42826,7 @@ static int walCheckpoint( if( pInfo->nBackfillnBackfill; /* Sync the WAL to disk */ @@ -41588,11 +42834,22 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } + /* If the database file may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue; + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); @@ -41699,7 +42956,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ /* The first page of the wal-index must be mapped at this point. */ assert( pWal->nWiData>0 && pWal->apWiData[0] ); - /* Read the header. This might happen currently with a write to the + /* Read the header. This might happen concurrently with a write to the ** same area of shared memory on a different CPU in a SMP, ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. @@ -41728,7 +42985,9 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ *pChanged = 1; memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); - pWal->szPage = pWal->hdr.szPage; + pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( pWal->szPage<=32768 ); + testcase( pWal->szPage>=65536 ); } /* The header was successfully read. Return zero. */ @@ -42047,6 +43306,7 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ ** read-lock. */ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); if( pWal->readLock>=0 ){ walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; @@ -42157,7 +43417,13 @@ SQLITE_PRIVATE int sqlite3WalRead( ** required page. Read and return data from the log file. */ if( iRead ){ - i64 iOffset = walFrameOffset(iRead, pWal->hdr.szPage) + WAL_FRAME_HDRSIZE; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; *pInWal = 1; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); @@ -42169,11 +43435,13 @@ SQLITE_PRIVATE int sqlite3WalRead( /* -** Set *pPgno to the size of the database file (or zero, if unknown). +** Return the size of the database in pages (or zero, if unknown). */ -SQLITE_PRIVATE void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ - assert( pWal->readLock>=0 || pWal->lockError ); - *pPgno = pWal->hdr.nPage; +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && ALWAYS(pWal->readLock>=0) ){ + return pWal->hdr.nPage; + } + return 0; } @@ -42249,7 +43517,7 @@ SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ */ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; - if( pWal->writeLock ){ + if( ALWAYS(pWal->writeLock) ){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; @@ -42337,7 +43605,7 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ ** ** SQLITE_OK is returned if no error is encountered (regardless of whether ** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned -** if some error +** if an error occurs. */ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; @@ -42370,6 +43638,8 @@ static int walRestartLog(Wal *pWal){ for(i=1; iaReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + }else if( rc!=SQLITE_BUSY ){ + return rc; } } walUnlockShared(pWal, WAL_READ_LOCK(0)); @@ -42438,7 +43708,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); - pWal->szPage = (u16)szPage; + pWal->szPage = szPage; pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; pWal->hdr.aFrameCksum[0] = aCksum[0]; pWal->hdr.aFrameCksum[1] = aCksum[1]; @@ -42449,7 +43719,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( return rc; } } - assert( pWal->szPage==szPage ); + assert( (int)pWal->szPage==szPage ); /* Write the log file. */ for(p=pList; p; p=p->pDirty){ @@ -42533,7 +43803,9 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc==SQLITE_OK ){ /* Update the private copy of the header. */ - pWal->hdr.szPage = (u16)szPage; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); pWal->hdr.mxFrame = iFrame; if( isCommit ){ pWal->hdr.iChange++; @@ -42745,7 +44017,7 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ ** ** The file is divided into pages. The first page is called page 1, ** the second is page 2, and so forth. A page number of zero indicates -** "no such page". The page size can be any power of 2 between 512 and 32768. +** "no such page". The page size can be any power of 2 between 512 and 65536. ** Each page can be either a btree page, a freelist page, an overflow ** page, or a pointer-map page. ** @@ -43107,18 +44379,19 @@ struct BtShared { u8 pageSizeFixed; /* True if the page size can no longer be changed */ u8 secureDelete; /* True if secure_delete is enabled */ u8 initiallyEmpty; /* Database is empty at start of transaction */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ #endif - u16 pageSize; /* Total number of bytes on a page */ - u16 usableSize; /* Number of usable bytes on each page */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ u8 inTransaction; /* Transaction state */ u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ int nTransaction; /* Number of open transactions (read + write) */ u32 nPage; /* Number of pages in the database */ void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ @@ -43717,7 +44990,16 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ # define TRACE(X) #endif - +/* +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. +*/ +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) #ifndef SQLITE_OMIT_SHARED_CACHE /* @@ -44406,11 +45688,16 @@ SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ ** Given a page number of a regular database page, return the page ** number for the pointer-map page that contains the entry for the ** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. */ static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ int nPagesPerMapPage; Pgno iPtrMap, ret; assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; nPagesPerMapPage = (pBt->usableSize/5)+1; iPtrMap = (pgno-2)/nPagesPerMapPage; ret = (iPtrMap*nPagesPerMapPage) + 2; @@ -44839,7 +46126,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); if( gap>top ) return SQLITE_CORRUPT_BKPT; testcase( gap+2==top ); testcase( gap+1==top ); @@ -44849,7 +46136,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* Always defragment highly fragmented pages */ rc = defragmentPage(pPage); if( rc ) return rc; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); }else if( gap+2<=top ){ /* Search the freelist looking for a free slot big enough to satisfy ** the request. The allocation is made from the first free slot in @@ -44891,7 +46178,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ if( gap+2+nByte>top ){ rc = defragmentPage(pPage); if( rc ) return rc; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); assert( gap+nByte<=top ); } @@ -45057,10 +46344,10 @@ static int btreeInitPage(MemPage *pPage){ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ BtShared *pBt; /* The main btree structure */ - u16 usableSize; /* Amount of usable space on each page */ + int usableSize; /* Amount of usable space on each page */ u16 cellOffset; /* Offset from start of page to first cell pointer */ - u16 nFree; /* Number of unused bytes on the page */ - u16 top; /* First byte of the cell content area */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ int iCellFirst; /* First allowable cell or freeblock offset */ int iCellLast; /* Last possible cell or freeblock offset */ @@ -45069,12 +46356,12 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; - assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); - pPage->maskPage = pBt->pageSize - 1; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ @@ -45173,13 +46460,13 @@ static void zeroPage(MemPage *pPage, int flags){ memset(&data[hdr+1], 0, 4); data[hdr+7] = 0; put2byte(&data[hdr+5], pBt->usableSize); - pPage->nFree = pBt->usableSize - first; + pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); pPage->hdrOffset = hdr; pPage->cellOffset = first; pPage->nOverflow = 0; - assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); - pPage->maskPage = pBt->pageSize - 1; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nCell = 0; pPage->isInit = 1; } @@ -45343,11 +46630,20 @@ static int btreeInvokeBusyHandler(void *pArg){ ** Open a database file. ** ** zFilename is the name of the database file. If zFilename is NULL -** a new database with a random name is created. This randomly named -** database file will be deleted when sqlite3BtreeClose() is called. +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. +** ** If zFilename is ":memory:" then an in-memory database is created ** that is automatically destroyed when it is closed. ** +** The "flags" parameter is a bitmask that might contain bits +** BTREE_OMIT_JOURNAL and/or BTREE_NO_READLOCK. The BTREE_NO_READLOCK +** bit is also set if the SQLITE_NoReadlock flags is set in db->flags. +** These flags are passed through into sqlite3PagerOpen() and must +** be the same values as PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK. +** ** If the database is already opened in the same database connection ** and we are in shared cache mode, then the open will fail with an ** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared @@ -45369,6 +46665,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( u8 nReserve; /* Byte of unused space on each page */ unsigned char zDbHeader[100]; /* Database header content */ + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; + /* Set the variable isMemdb to true for an in-memory database, or ** false for a file-based database. This symbol is only required if ** either of the shared-data or autovacuum features are compiled @@ -45378,13 +46677,30 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #ifdef SQLITE_OMIT_MEMORYDB const int isMemdb = 0; #else - const int isMemdb = zFilename && !strcmp(zFilename, ":memory:"); + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)); #endif #endif assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( db->flags & SQLITE_NoReadlock ){ + flags |= BTREE_NO_READLOCK; + } + if( isMemdb ){ + flags |= BTREE_MEMORY; + } + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } pVfs = db->pVfs; p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ @@ -45402,7 +46718,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( ** If this Btree is a candidate for shared cache, try to find an ** existing BtShared object that we can share with */ - if( isMemdb==0 && zFilename && zFilename[0] ){ + if( isMemdb==0 && isTempDb==0 ){ if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ int nFullPathname = pVfs->mxPathname+1; char *zFullPathname = sqlite3Malloc(nFullPathname); @@ -45477,6 +46793,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( if( rc!=SQLITE_OK ){ goto btree_open_out; } + pBt->openFlags = (u8)flags; pBt->db = db; sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); p->pBt = pBt; @@ -45487,7 +46804,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->secureDelete = 1; #endif - pBt->pageSize = get2byte(&zDbHeader[16]); + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ pBt->pageSize = 0; @@ -45581,6 +46898,14 @@ btree_open_out: sqlite3_free(pBt); sqlite3_free(p); *ppBtree = 0; + }else{ + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); + } } if( mutexOpen ){ assert( sqlite3_mutex_held(mutexOpen) ); @@ -45689,7 +47014,7 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } - sqlite3_free(pBt->pSchema); + sqlite3DbFree(0, pBt->pSchema); freeTempSpace(pBt); sqlite3_free(pBt); } @@ -45801,7 +47126,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); assert( !pBt->pPage1 && !pBt->pCursor ); - pBt->pageSize = (u16)pageSize; + pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); @@ -45936,15 +47261,13 @@ static int lockBtree(BtShared *pBt){ ** a valid database file. */ nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - if( (rc = sqlite3PagerPagecount(pBt->pPager, &nPageFile))!=SQLITE_OK ){; - goto page1_init_failed; - } + sqlite3PagerPagecount(pBt->pPager, &nPageFile); if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ nPage = nPageFile; } if( nPage>0 ){ - int pageSize; - int usableSize; + u32 pageSize; + u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; if( memcmp(page1, zMagicHeader, 16)!=0 ){ @@ -45995,15 +47318,16 @@ static int lockBtree(BtShared *pBt){ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } - pageSize = get2byte(&page1[16]); - if( ((pageSize-1)&pageSize)!=0 || pageSize<512 || - (SQLITE_MAX_PAGE_SIZE<32768 && pageSize>SQLITE_MAX_PAGE_SIZE) + pageSize = (page1[16]<<8) | (page1[17]<<16); + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 ){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); usableSize = pageSize - page1[20]; - if( pageSize!=pBt->pageSize ){ + if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size ** of BtShared.pageSize, we have discovered that the page-size is ** actually pageSize. Unlock the database, leave pBt->pPage1 at @@ -46011,8 +47335,8 @@ static int lockBtree(BtShared *pBt){ ** again with the correct page-size. */ releasePage(pPage1); - pBt->usableSize = (u16)usableSize; - pBt->pageSize = (u16)pageSize; + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); @@ -46025,8 +47349,8 @@ static int lockBtree(BtShared *pBt){ if( usableSize<480 ){ goto page1_init_failed; } - pBt->pageSize = (u16)pageSize; - pBt->usableSize = (u16)usableSize; + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); @@ -46042,14 +47366,14 @@ static int lockBtree(BtShared *pBt){ ** 9-byte nKey value ** 4-byte nData value ** 4-byte overflow page pointer - ** So a cell consists of a 2-byte poiner, a header which is as much as + ** So a cell consists of a 2-byte pointer, a header which is as much as ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow ** page pointer. */ - pBt->maxLocal = (pBt->usableSize-12)*64/255 - 23; - pBt->minLocal = (pBt->usableSize-12)*32/255 - 23; - pBt->maxLeaf = pBt->usableSize - 35; - pBt->minLeaf = (pBt->usableSize-12)*32/255 - 23; + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); pBt->pPage1 = pPage1; pBt->nPage = nPage; @@ -46102,7 +47426,8 @@ static int newDatabase(BtShared *pBt){ if( rc ) return rc; memcpy(data, zMagicHeader, sizeof(zMagicHeader)); assert( sizeof(zMagicHeader)==16 ); - put2byte(&data[16], pBt->pageSize); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); data[18] = 1; data[19] = 1; assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); @@ -47013,9 +48338,11 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; rc = newDatabase(pBt); pBt->nPage = get4byte(28 + pBt->pPage1->aData); - if( pBt->nPage==0 ){ - sqlite3PagerPagecount(pBt->pPager, (int*)&pBt->nPage); - } + + /* The database size was written into the offset 28 of the header + ** when the transaction started, so we know that the value at offset + ** 28 is nonzero. */ + assert( pBt->nPage>0 ); } sqlite3BtreeLeave(p); } @@ -48111,9 +49438,9 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( pCur->validNKey = 1; pCur->info.nKey = nCellKey; }else{ - /* The maximum supported page-size is 32768 bytes. This means that + /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree - ** page is at most 8198 bytes, which may be stored as a 2-byte + ** page is less than 16384 bytes and may be stored as a 2-byte ** varint. This information is used to attempt to avoid parsing ** the entire cell by checking for the cases where the record is ** stored entirely within the b-tree page by inspecting the first @@ -48476,6 +49803,10 @@ static int allocateBtreePage( if( !pPrevTrunk ){ memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); } }else{ @@ -48782,7 +50113,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ Pgno ovflPgno; int rc; int nOvfl; - u16 ovflPageSize; + u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); btreeParseCellPtr(pPage, pCell, &info); @@ -49007,7 +50338,7 @@ static int fillInCell( */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ int i; /* Loop counter */ - int pc; /* Offset to cell content of cell being deleted */ + u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ int rc; /* The return code */ @@ -49025,7 +50356,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ hdr = pPage->hdrOffset; testcase( pc==get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); - if( pc < get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ *pRC = SQLITE_CORRUPT_BKPT; return; } @@ -49082,7 +50413,7 @@ static void insertCell( if( *pRC ) return; assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); - assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); /* The cell should normally be sized correctly. However, when moving a @@ -49162,12 +50493,12 @@ static void assemblePage( assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); /* Check that the page has just been zeroed by zeroPage() */ assert( pPage->nCell==0 ); - assert( get2byte(&data[hdr+5])==nUsable ); + assert( get2byteNotZero(&data[hdr+5])==nUsable ); pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; @@ -49233,6 +50564,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); + /* This error condition is now caught prior to reaching this function */ if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of @@ -49562,7 +50894,7 @@ static int balance_nonroot( ** is allocated. */ if( pBt->secureDelete ){ int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); - if( (iOff+szNew[i])>pBt->usableSize ){ + if( (iOff+szNew[i])>(int)pBt->usableSize ){ rc = SQLITE_CORRUPT_BKPT; memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -49641,7 +50973,7 @@ static int balance_nonroot( szCell[nCell] = sz; pTemp = &aSpace1[iSpace1]; iSpace1 += sz; - assert( sz<=pBt->pageSize/4 ); + assert( sz<=pBt->maxLocal+23 ); assert( iSpace1<=pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); apCell[nCell] = pTemp+leafCorrection; @@ -49887,7 +51219,7 @@ static int balance_nonroot( } } iOvflSpace += sz; - assert( sz<=pBt->pageSize/4 ); + assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace<=pBt->pageSize ); insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); if( rc!=SQLITE_OK ) goto balance_cleanup; @@ -50524,11 +51856,12 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ ** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys ** BTREE_ZERODATA Used for SQL indices */ -static int btreeCreateTable(Btree *p, int *piTable, int flags){ +static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ BtShared *pBt = p->pBt; MemPage *pRoot; Pgno pgnoRoot; int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -50647,8 +51980,14 @@ static int btreeCreateTable(Btree *p, int *piTable, int flags){ } #endif assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - zeroPage(pRoot, flags | PTF_LEAF); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); *piTable = (int)pgnoRoot; return SQLITE_OK; } @@ -51131,7 +52470,7 @@ static void checkList( checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); } #endif - if( n>pCheck->pBt->usableSize/4-2 ){ + if( n>(int)pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, zContext, "freelist leaf count too big on page %d", iPage); N--; @@ -51342,7 +52681,7 @@ static int checkTreePage( if( hit==0 ){ pCheck->mallocFailed = 1; }else{ - u16 contentOffset = get2byte(&data[hdr+5]); + int contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ memset(hit+contentOffset, 0, usableSize-contentOffset); memset(hit, 1, contentOffset); @@ -51350,12 +52689,12 @@ static int checkTreePage( cellStart = hdr + 12 - 4*pPage->leaf; for(i=0; i=usableSize ){ + if( (int)(pc+size-1)>=usableSize ){ checkAppendMsg(pCheck, 0, "Corruption detected in cell %d on page %d",i,iPage); }else{ @@ -51448,6 +52787,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( sCheck.anRef[i] = 1; } sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist */ @@ -51546,6 +52886,29 @@ SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree *p){ return (p && (p->inTrans==TRANS_WRITE)); } +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +*/ +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + /* ** Return non-zero if a read (or write) transaction is active. */ @@ -51585,7 +52948,7 @@ SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); if( !pBt->pSchema && nBytes ){ - pBt->pSchema = sqlite3MallocZero(nBytes); + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); pBt->xFreeSchema = xFree; } sqlite3BtreeLeave(p); @@ -51884,7 +53247,10 @@ SQLITE_API sqlite3_backup *sqlite3_backup_init( ); p = 0; }else { - /* Allocate space for a new sqlite3_backup object */ + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); if( !p ){ sqlite3Error(pDestDb, SQLITE_NOMEM, 0); @@ -51955,6 +53321,15 @@ static int backupOnePage(sqlite3_backup *p, Pgno iSrcPg, const u8 *zSrcData){ rc = SQLITE_READONLY; } +#ifdef SQLITE_HAS_CODEC + /* Backup is not possible if the page size of the destination is changing + ** a a codec is in use. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ + rc = SQLITE_READONLY; + } +#endif + /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset ** of the destination page. @@ -52258,6 +53633,9 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ } sqlite3BtreeLeave(p->pSrc); if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ sqlite3_free(p); } sqlite3_mutex_leave(mutex); @@ -52509,6 +53887,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; pMem->flags |= MEM_Term; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; +#endif } return SQLITE_OK; @@ -52629,7 +54010,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ ctx.s.db = pMem->db; ctx.pMem = pMem; ctx.pFunc = pFunc; - pFunc->xFinalize(&ctx); + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); sqlite3DbFree(pMem->db, pMem->zMalloc); memcpy(pMem, &ctx.s, sizeof(ctx.s)); @@ -52742,13 +54123,9 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ return doubleToInt64(pMem->r); }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value; - pMem->flags |= MEM_Str; - if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8) - || sqlite3VdbeMemNulTerminate(pMem) ){ - return 0; - } - assert( pMem->z ); - sqlite3Atoi64(pMem->z, &value); + assert( pMem->z || pMem->n==0 ); + testcase( pMem->z==0 ); + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; }else{ return 0; @@ -52771,14 +54148,7 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ double val = (double)0; - pMem->flags |= MEM_Str; - if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8) - || sqlite3VdbeMemNulTerminate(pMem) ){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - return (double)0; - } - assert( pMem->z ); - sqlite3AtoF(pMem->z, &val); + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); return val; }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ @@ -52851,21 +54221,19 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ ** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - int rc; - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ); - assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8); - if( rc ) return rc; - rc = sqlite3VdbeMemNulTerminate(pMem); - if( rc ) return rc; - if( sqlite3Atoi64(pMem->z, &pMem->u.i) ){ - MemSetTypeFlag(pMem, MEM_Int); - }else{ - pMem->r = sqlite3VdbeRealValue(pMem); - MemSetTypeFlag(pMem, MEM_Real); - sqlite3VdbeIntegerAffinity(pMem); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ + MemSetTypeFlag(pMem, MEM_Int); + }else{ + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + sqlite3VdbeIntegerAffinity(pMem); + } } + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob); return SQLITE_OK; } @@ -52970,6 +54338,28 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ return 0; } +#ifdef SQLITE_DEBUG +/* +** This routine prepares a memory cell for modication by breaking +** its link to a shallow copy and by marking any current shallow +** copies of this cell as invalid. +** +** This is used for testing and debugging only - to make sure shallow +** copies are not misused. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ + int i; + Mem *pX; + for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + pX->flags |= MEM_Invalid; + pX->pScopyFrom = 0; + } + } + pMem->pScopyFrom = 0; +} +#endif /* SQLITE_DEBUG */ + /* ** Size of struct Mem not including the Mem.zMalloc member. */ @@ -53338,7 +54728,7 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return 0; } } - sqlite3VdbeMemNulTerminate(pVal); + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-59893-45467 */ }else{ assert( (pVal->flags&MEM_Blob)==0 ); sqlite3VdbeMemStringify(pVal, enc); @@ -53386,6 +54776,8 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( int op; char *zVal = 0; sqlite3_value *pVal = 0; + int negInt = 1; + const char *zNeg = ""; if( !pExpr ){ *ppVal = 0; @@ -53393,7 +54785,7 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( } op = pExpr->op; - /* op can only be TK_REGISTER is we have compiled with SQLITE_ENABLE_STAT2. + /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2. ** The ifdef here is to enable us to achieve 100% branch test coverage even ** when SQLITE_ENABLE_STAT2 is omitted. */ @@ -53403,13 +54795,24 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; #endif + /* Handle negative integers in a single step. This is needed in the + ** case when the value is -9223372036854775808. + */ + if( op==TK_UMINUS + && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ + pExpr = pExpr->pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ pVal = sqlite3ValueNew(db); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ - sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue); + sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3DbStrDup(db, pExpr->u.zToken); + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT; @@ -53419,14 +54822,18 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } + if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; if( enc!=SQLITE_UTF8 ){ sqlite3VdbeChangeEncoding(pVal, enc); } }else if( op==TK_UMINUS ) { + /* This branch happens for multiple negative signs. Ex: -(-5) */ if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){ + sqlite3VdbeMemNumerify(pVal); pVal->u.i = -1 * pVal->u.i; /* (double)-1 In case of SQLITE_OMIT_FLOATING_POINT... */ pVal->r = (double)-1 * pVal->r; + sqlite3ValueApplyAffinity(pVal, affinity, enc); } } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -54071,15 +55478,17 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ } } +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + /* ** Delete a P4 value if necessary. */ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ + assert( db ); switch( p4type ){ case P4_REAL: case P4_INT64: - case P4_MPRINTF: case P4_DYNAMIC: case P4_KEYINFO: case P4_INTARRAY: @@ -54087,10 +55496,14 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ sqlite3DbFree(db, p4); break; } + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; + } case P4_VDBEFUNC: { VdbeFunc *pVdbeFunc = (VdbeFunc *)p4; freeEphemeralFunction(db, pVdbeFunc->pFunc); - sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); + if( db->pnBytesFreed==0 ) sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); sqlite3DbFree(db, pVdbeFunc); break; } @@ -54099,15 +55512,17 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ break; } case P4_MEM: { - sqlite3ValueFree((sqlite3_value*)p4); + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + Mem *p = (Mem*)p4; + sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); + } break; } case P4_VTAB : { - sqlite3VtabUnlock((VTable *)p4); - break; - } - case P4_SUBPROGRAM : { - sqlite3VdbeProgramDelete(db, (SubProgram *)p4, 1); + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); break; } } @@ -54133,35 +55548,15 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ } /* -** Decrement the ref-count on the SubProgram structure passed as the -** second argument. If the ref-count reaches zero, free the structure. -** -** The array of VDBE opcodes stored as SubProgram.aOp is freed if -** either the ref-count reaches zero or parameter freeop is non-zero. -** -** Since the array of opcodes pointed to by SubProgram.aOp may directly -** or indirectly contain a reference to the SubProgram structure itself. -** By passing a non-zero freeop parameter, the caller may ensure that all -** SubProgram structures and their aOp arrays are freed, even when there -** are such circular references. +** Link the SubProgram object passed as the second argument into the linked +** list at Vdbe.pSubProgram. This list is used to delete all sub-program +** objects when the VM is no longer required. */ -SQLITE_PRIVATE void sqlite3VdbeProgramDelete(sqlite3 *db, SubProgram *p, int freeop){ - if( p ){ - assert( p->nRef>0 ); - if( freeop || p->nRef==1 ){ - Op *aOp = p->aOp; - p->aOp = 0; - vdbeFreeOpArray(db, aOp, p->nOp); - p->nOp = 0; - } - p->nRef--; - if( p->nRef==0 ){ - sqlite3DbFree(db, p); - } - } +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ + p->pNext = pVdbe->pProgram; + pVdbe->pProgram = p; } - /* ** Change N opcodes starting at addr to No-ops. */ @@ -54237,7 +55632,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int nField = ((KeyInfo*)zP4)->nField; nByte = sizeof(*pKeyInfo) + (nField-1)*sizeof(pKeyInfo->aColl[0]) + nField; - pKeyInfo = sqlite3Malloc( nByte ); + pKeyInfo = sqlite3DbMallocRaw(0, nByte); pOp->p4.pKeyInfo = pKeyInfo; if( pKeyInfo ){ u8 *aSortOrder; @@ -54501,6 +55896,12 @@ static void releaseMemArray(Mem *p, int N){ Mem *pEnd; sqlite3 *db = p->db; u8 malloc_failed = db->mallocFailed; + if( db->pnBytesFreed ){ + for(pEnd=&p[N]; pzMalloc); + } + return; + } for(pEnd=&p[N]; pzErrMsg); - if( rc!=SQLITE_OK ){ - return rc; - } /* This loop determines (a) if the commit hook should be invoked and ** (b) how many database files have open write transactions, not @@ -55160,13 +56558,17 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** one database file has an open write transaction, a master journal ** file is required for an atomic commit. */ - for(i=0; inDb; i++){ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); } } + if( rc!=SQLITE_OK ){ + return rc; + } /* If there are any write-transactions at all, invoke the commit hook */ if( needXcommit && db->xCommitCallback ){ @@ -55258,9 +56660,10 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ char const *zFile = sqlite3BtreeGetJournalname(pBt); - if( zFile==0 || zFile[0]==0 ){ + if( zFile==0 ){ continue; /* Ignore TEMP and :memory: databases */ } + assert( zFile[0]!=0 ); if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ needSync = 1; } @@ -55305,6 +56708,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); if( rc!=SQLITE_OK ){ sqlite3DbFree(db, zMaster); return rc; @@ -55563,8 +56967,17 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; if( isSpecialError ){ - /* If the query was read-only, we need do no rollback at all. Otherwise, - ** proceed with the special handling. + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a + ** consistent state. + ** + ** Even if the statement is read-only, it is important to perform + ** a statement or transaction rollback operation. If the error + ** occured while writing to the journal, sub-journal or database + ** file as part of an effort to free up cache space (see function + ** pagerStress() in pager.c), the rollback is required to restore + ** the pager to a consistent state. */ if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ @@ -55833,6 +57246,30 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ } } +/* +** Free all memory associated with the Vdbe passed as the second argument. +** The difference between this function and sqlite3VdbeDelete() is that +** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with +** the database connection. +*/ +SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ + SubProgram *pSub, *pNext; + assert( p->db==0 || p->db==db ); + releaseMemArray(p->aVar, p->nVar); + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + for(pSub=p->pProgram; pSub; pSub=pNext){ + pNext = pSub->pNext; + vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); + sqlite3DbFree(db, pSub); + } + vdbeFreeOpArray(db, p->aOp, p->nOp); + sqlite3DbFree(db, p->aLabel); + sqlite3DbFree(db, p->aColName); + sqlite3DbFree(db, p->zSql); + sqlite3DbFree(db, p->pFree); + sqlite3DbFree(db, p); +} + /* ** Delete an entire VDBE. */ @@ -55850,16 +57287,9 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ if( p->pNext ){ p->pNext->pPrev = p->pPrev; } - releaseMemArray(p->aVar, p->nVar); - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->aLabel); - sqlite3DbFree(db, p->aColName); - sqlite3DbFree(db, p->zSql); p->magic = VDBE_MAGIC_DEAD; - sqlite3DbFree(db, p->pFree); p->db = 0; - sqlite3DbFree(db, p); + sqlite3VdbeDeleteObject(db, p); } /* @@ -55885,11 +57315,8 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); if( rc ) return rc; p->lastRowid = p->movetoTarget; - p->rowidIsValid = ALWAYS(res==0) ?1:0; - if( NEVER(res<0) ){ - rc = sqlite3BtreeNext(p->pCursor, &res); - if( rc ) return rc; - } + if( res!=0 ) return SQLITE_CORRUPT_BKPT; + p->rowidIsValid = 1; #ifdef SQLITE_TEST sqlite3_search_count++; #endif @@ -56700,6 +58127,8 @@ static int vdbeSafetyNotNull(Vdbe *p){ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ + /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL + ** pointer is a harmless no-op. */ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; @@ -56776,7 +58205,7 @@ SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ sqlite3VdbeMemExpandBlob(p); p->flags &= ~MEM_Str; p->flags |= MEM_Blob; - return p->z; + return p->n ? p->z : 0; }else{ return sqlite3_value_text(pVal); } @@ -57033,7 +58462,7 @@ static int sqlite3Step(Vdbe *p){ if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ sqlite3_int64 iNow; sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, iNow - p->startTime); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); } #endif @@ -57130,6 +58559,12 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ /* ** Extract the user data from a sqlite3_context structure and return a ** pointer to it. +** +** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface +** returns a copy of the pointer to the database connection (the 1st +** parameter) of the sqlite3_create_function() and +** sqlite3_create_function16() routines that originally registered the +** application defined function. */ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ assert( p && p->pFunc ); @@ -57339,8 +58774,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ ** sqlite3_column_real() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() -** -** But not for sqlite3_column_blob(), which never calls malloc(). +** sqiite3_column_blob() */ static void columnMallocFailure(sqlite3_stmt *pStmt) { @@ -57608,6 +59042,12 @@ static int vdbeUnbind(Vdbe *p, int i){ /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. + ** + ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call + ** following any change to the bindings of that parameter. */ if( p->isPrepareV2 && ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) @@ -58105,6 +59545,17 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( ** commenting and indentation practices when changing or adding code. */ +/* +** Invoke this macro on memory cells just prior to changing the +** value of the cell. This macro verifies that shallow copies are +** not misused. +*/ +#ifdef SQLITE_DEBUG +# define memAboutToChange(P,M) sqlite3VdbeMemPrepareToChange(P,M) +#else +# define memAboutToChange(P,M) +#endif + /* ** The following global variable is incremented every time a cursor ** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test @@ -58297,31 +59748,17 @@ static VdbeCursor *allocateCursor( */ static void applyNumericAffinity(Mem *pRec){ if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ - int realnum; + double rValue; + i64 iValue; u8 enc = pRec->enc; - sqlite3VdbeMemNulTerminate(pRec); - if( (pRec->flags&MEM_Str) && sqlite3IsNumber(pRec->z, &realnum, enc) ){ - i64 value; - char *zUtf8 = pRec->z; -#ifndef SQLITE_OMIT_UTF16 - if( enc!=SQLITE_UTF8 ){ - assert( pRec->db ); - zUtf8 = sqlite3Utf16to8(pRec->db, pRec->z, pRec->n, enc); - if( !zUtf8 ) return; - } -#endif - if( !realnum && sqlite3Atoi64(zUtf8, &value) ){ - pRec->u.i = value; - MemSetTypeFlag(pRec, MEM_Int); - }else{ - sqlite3AtoF(zUtf8, &pRec->r); - MemSetTypeFlag(pRec, MEM_Real); - } -#ifndef SQLITE_OMIT_UTF16 - if( enc!=SQLITE_UTF8 ){ - sqlite3DbFree(pRec->db, zUtf8); - } -#endif + if( (pRec->flags&MEM_Str)==0 ) return; + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->r = rValue; + pRec->flags |= MEM_Real; } } } @@ -58649,6 +60086,20 @@ static int checkSavepointCount(sqlite3 *db){ } #endif +/* +** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored +** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored +** in memory obtained from sqlite3DbMalloc). +*/ +static void importVtabErrMsg(Vdbe *p, sqlite3_vtab *pVtab){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; +} + + /* ** Execute as much of a VDBE program as we can then return. ** @@ -59200,6 +60651,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); sqlite3VdbeMemReleaseExternal(pOut); pOut->flags = MEM_Int; } @@ -59209,25 +60661,30 @@ SQLITE_PRIVATE int sqlite3VdbeExec( if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); assert( pOp->p1<=p->nMem ); + assert( memIsValid(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); + assert( memIsValid(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=p->nMem ); + assert( memIsValid(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_OUT3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p3]); } #endif @@ -59289,6 +60746,7 @@ case OP_Goto: { /* jump */ case OP_Gosub: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); + memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; pIn1->u.i = pc; REGISTER_TRACE(pOp->p1, pIn1); @@ -59496,11 +60954,7 @@ case OP_Null: { /* out2-prerelease */ /* Opcode: Blob P1 P2 * P4 ** ** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. This instruction is not coded directly -** by the compiler. Instead, the compiler layer specifies -** an OP_HexBlob opcode, with the hex string representation of -** the blob as P4. This opcode is transformed to an OP_Blob -** the first time it is executed. +** blob in register P2. */ case OP_Blob: { /* out2-prerelease */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); @@ -59558,6 +61012,8 @@ case OP_Move: { while( u.ac.n-- ){ assert( pOut<=&aMem[p->nMem] ); assert( pIn1<=&aMem[p->nMem] ); + assert( memIsValid(pIn1) ); + memAboutToChange(p, pOut); u.ac.zMalloc = pOut->zMalloc; pOut->zMalloc = 0; sqlite3VdbeMemMove(pOut, pIn1); @@ -59603,6 +61059,9 @@ case OP_SCopy: { /* in1, out2 */ pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); +#ifdef SQLITE_DEBUG + if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; +#endif REGISTER_TRACE(pOp->p2, pOut); break; } @@ -59663,6 +61122,10 @@ case OP_ResultRow: { */ u.ad.pMem = p->pResultSet = &aMem[pOp->p1]; for(u.ad.i=0; u.ad.ip2; u.ad.i++){ + assert( memIsValid(&u.ad.pMem[u.ad.i]) ); + Deephemeralize(&u.ad.pMem[u.ad.i]); + assert( (u.ad.pMem[u.ad.i].flags & MEM_Ephem)==0 + || (u.ad.pMem[u.ad.i].flags & (MEM_Str|MEM_Blob))==0 ); sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]); sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]); REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]); @@ -59894,12 +61357,17 @@ case OP_Function: { u.ag.n = pOp->p5; u.ag.apVal = p->apArg; assert( u.ag.apVal || u.ag.n==0 ); + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+u.ag.n ); u.ag.pArg = &aMem[pOp->p2]; for(u.ag.i=0; u.ag.ip2+u.ag.i, u.ag.pArg); } @@ -59913,8 +61381,6 @@ case OP_Function: { u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc; } - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pOut = &aMem[pOp->p3]; u.ag.ctx.s.flags = MEM_Null; u.ag.ctx.s.db = db; u.ag.ctx.s.xDel = 0; @@ -59934,7 +61400,7 @@ case OP_Function: { assert( pOp[-1].opcode==OP_CollSeq ); u.ag.ctx.pColl = pOp[-1].p4.pColl; } - (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); + (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ if( db->mallocFailed ){ /* Even though a malloc() has failed, the implementation of the ** user function may have called an sqlite3_result_XXX() function @@ -59986,7 +61452,7 @@ case OP_Function: { /* Opcode: ShiftLeft P1 P2 P3 * * ** ** Shift the integer value in register P2 to the left by the -** number of bits specified by the integer in regiser P1. +** number of bits specified by the integer in register P1. ** Store the result in register P3. ** If either input is NULL, the result is NULL. */ @@ -60036,6 +61502,7 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ */ case OP_AddImm: { /* in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); sqlite3VdbeMemIntegerify(pIn1); pIn1->u.i += pOp->p2; break; @@ -60050,6 +61517,7 @@ case OP_AddImm: { /* in1 */ */ case OP_MustBeInt: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); if( (pIn1->flags & MEM_Int)==0 ){ if( pOp->p2==0 ){ @@ -60095,6 +61563,7 @@ case OP_RealAffinity: { /* in1 */ */ case OP_ToText: { /* same as TK_TO_TEXT, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); if( pIn1->flags & MEM_Null ) break; assert( MEM_Str==(MEM_Blob>>3) ); pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; @@ -60141,16 +61610,14 @@ case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ */ case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){ - sqlite3VdbeMemNumerify(pIn1); - } + sqlite3VdbeMemNumerify(pIn1); break; } #endif /* SQLITE_OMIT_CAST */ /* Opcode: ToInt P1 * * * * ** -** Force the value in register P1 be an integer. If +** Force the value in register P1 to be an integer. If ** The value is currently a real number, drop its fractional part. ** If the value is text or blob, try to convert it to an integer using the ** equivalent of atoi() and store 0 if no such conversion is possible. @@ -60177,6 +61644,7 @@ case OP_ToInt: { /* same as TK_TO_INT, in1 */ */ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); if( (pIn1->flags & MEM_Null)==0 ){ sqlite3VdbeMemRealify(pIn1); } @@ -60191,7 +61659,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** ** If the SQLITE_JUMPIFNULL bit of P5 is set and either reg(P1) or ** reg(P3) is NULL then take the jump. If the SQLITE_JUMPIFNULL -** bit is clear then fall thru if either operand is NULL. +** bit is clear then fall through if either operand is NULL. ** ** The SQLITE_AFF_MASK portion of P5 must be an affinity character - ** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made @@ -60321,6 +61789,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = u.ai.res; REGISTER_TRACE(pOp->p2, pOut); @@ -60352,8 +61821,8 @@ case OP_Permutation: { /* Opcode: Compare P1 P2 P3 P4 * ** -** Compare to vectors of registers in reg(P1)..reg(P1+P3-1) (all this -** one "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of +** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this +** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of ** the comparison for use by the next OP_Jump instruct. ** ** P4 is a KeyInfo structure that defines collating sequences and sort @@ -60395,6 +61864,8 @@ case OP_Compare: { #endif /* SQLITE_DEBUG */ for(u.aj.i=0; u.aj.inField ); @@ -60626,6 +62097,7 @@ case OP_Column: { assert( u.am.p1nCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.am.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.am.pDest); MemSetTypeFlag(u.am.pDest, MEM_Null); u.am.zRec = 0; @@ -60673,6 +62145,7 @@ case OP_Column: { }else if( u.am.pC->pseudoTableReg>0 ){ u.am.pReg = &aMem[u.am.pC->pseudoTableReg]; assert( u.am.pReg->flags & MEM_Blob ); + assert( memIsValid(u.am.pReg) ); u.am.payloadSize = u.am.pReg->n; u.am.zRec = u.am.pReg->z; u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr; @@ -60897,6 +62370,7 @@ case OP_Affinity: { pIn1 = &aMem[pOp->p1]; while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){ assert( pIn1 <= &p->aMem[p->nMem] ); + assert( memIsValid(pIn1) ); ExpandBlob(pIn1); applyAffinity(pIn1, u.an.cAff, encoding); pIn1++; @@ -60906,12 +62380,9 @@ case OP_Affinity: { /* Opcode: MakeRecord P1 P2 P3 P4 * ** -** Convert P2 registers beginning with P1 into a single entry -** suitable for use as a data record in a database table or as a key -** in an index. The details of the format are irrelevant as long as -** the OP_Column opcode can decode the record later. -** Refer to source code comments for the details of the record -** format. +** Convert P2 registers beginning with P1 into the [record format] +** use as a data record in a database table or as a key +** in an index. The OP_Column opcode can decode the record later. ** ** P4 may be a string that is P2 characters long. The nth character of the ** string indicates the column affinity that should be used for the nth @@ -60968,10 +62439,16 @@ case OP_MakeRecord: { u.ao.pLast = &u.ao.pData0[u.ao.nField-1]; u.ao.file_format = p->minWriteFileFormat; + /* Identify the output register */ + assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); + /* Loop through the elements that will make up the record to figure ** out how much space is required for the new record. */ for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ + assert( memIsValid(u.ao.pRec) ); if( u.ao.zAffinity ){ applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding); } @@ -61006,8 +62483,6 @@ case OP_MakeRecord: { ** be one of the input registers (because the following call to ** sqlite3VdbeMemGrow() could clobber the value before it is used). */ - assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); - pOut = &aMem[pOp->p3]; if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){ goto no_mem; } @@ -61180,6 +62655,7 @@ case OP_Savepoint: { if( u.aq.p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetInternalSchema(db, 0); + db->flags = (db->flags | SQLITE_InternChanges); } } @@ -61570,6 +63046,8 @@ case OP_OpenWrite: { assert( u.aw.p2>0 ); assert( u.aw.p2<=p->nMem ); pIn2 = &aMem[u.aw.p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); sqlite3VdbeMemIntegerify(pIn2); u.aw.p2 = (int)pIn2->u.i; /* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and @@ -61592,6 +63070,7 @@ case OP_OpenWrite: { u.aw.pCur = allocateCursor(p, pOp->p1, u.aw.nField, u.aw.iDb, 1); if( u.aw.pCur==0 ) goto no_mem; u.aw.pCur->nullRow = 1; + u.aw.pCur->isOrdered = 1; rc = sqlite3BtreeCursor(u.aw.pX, u.aw.p2, u.aw.wrFlag, u.aw.pKeyInfo, u.aw.pCur->pCursor); u.aw.pCur->pKeyInfo = u.aw.pKeyInfo; @@ -61644,7 +63123,7 @@ case OP_OpenEphemeral: { #if 0 /* local variables moved into u.ax */ VdbeCursor *pCx; #endif /* local variables moved into u.ax */ - static const int openFlags = + static const int vfsFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | @@ -61655,21 +63134,21 @@ case OP_OpenEphemeral: { u.ax.pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( u.ax.pCx==0 ) goto no_mem; u.ax.pCx->nullRow = 1; - rc = sqlite3BtreeFactory(db, 0, 1, SQLITE_DEFAULT_TEMP_CACHE_SIZE, openFlags, - &u.ax.pCx->pBt); + rc = sqlite3BtreeOpen(0, db, &u.ax.pCx->pBt, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(u.ax.pCx->pBt, 1); } if( rc==SQLITE_OK ){ /* If a transient index is required, create it by calling - ** sqlite3BtreeCreateTable() with the BTREE_ZERODATA flag before + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before ** opening it. If a transient table is required, just use the - ** automatically created table with root-page 1 (an INTKEY table). + ** automatically created table with root-page 1 (an BLOB_INTKEY table). */ if( pOp->p4.pKeyInfo ){ int pgno; assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_ZERODATA); + rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY); if( rc==SQLITE_OK ){ assert( pgno==MASTER_ROOT+1 ); rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1, @@ -61683,6 +63162,7 @@ case OP_OpenEphemeral: { u.ax.pCx->isTable = 1; } } + u.ax.pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); u.ax.pCx->isIndex = !u.ax.pCx->isTable; break; } @@ -61802,6 +63282,7 @@ case OP_SeekGt: { /* jump, in3 */ assert( OP_SeekLe == OP_SeekLt+1 ); assert( OP_SeekGe == OP_SeekLt+2 ); assert( OP_SeekGt == OP_SeekLt+3 ); + assert( u.az.pC->isOrdered ); if( u.az.pC->pCursor!=0 ){ u.az.oc = pOp->opcode; u.az.pC->nullRow = 0; @@ -61884,6 +63365,9 @@ case OP_SeekGt: { /* jump, in3 */ assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 ); u.az.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ipCursor, &u.az.r, 0, 0, &u.az.res); if( rc!=SQLITE_OK ){ @@ -62012,11 +63496,14 @@ case OP_Found: { /* jump, in3 */ u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo; u.bb.r.nField = (u16)pOp->p4.i; u.bb.r.aMem = pIn3; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iflags & MEM_Blob ); - ExpandBlob(pIn3); + assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z, u.bb.aTempRec, sizeof(u.bb.aTempRec)); if( u.bb.pIdxKey==0 ){ @@ -62111,6 +63598,9 @@ case OP_IsUnique: { /* jump, in3 */ u.bc.r.nField = u.bc.nField + 1; u.bc.r.flags = UNPACKED_PREFIX_SEARCH; u.bc.r.aMem = u.bc.aMx; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ip3<=p->nMem ); u.be.pMem = &aMem[pOp->p3]; + memAboutToChange(p, u.be.pMem); } + assert( memIsValid(u.be.pMem) ); REGISTER_TRACE(pOp->p3, u.be.pMem); sqlite3VdbeMemIntegerify(u.be.pMem); @@ -62310,29 +63802,36 @@ case OP_NewRowid: { /* out2-prerelease */ sqlite3BtreeSetCachedRowid(u.be.pC->pCursor, u.be.vuseRandomRowid ){ - /* IMPLEMENTATION-OF: R-48598-02938 If the largest ROWID is equal to the + /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the ** largest possible integer (9223372036854775807) then the database - ** engine starts picking candidate ROWIDs at random until it finds one - ** that is not previously used. - */ + ** engine starts picking positive candidate ROWIDs at random until + ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ + /* on the first attempt, simply do one more than previous */ u.be.v = db->lastRowid; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + u.be.v++; /* ensure non-zero */ u.be.cnt = 0; - do{ - if( u.be.cnt==0 && (u.be.v&0xffffff)==u.be.v ){ - u.be.v++; + while( ((rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, + 0, &u.be.res))==SQLITE_OK) + && (u.be.res==0) + && (++u.be.cnt<100)){ + /* collision - try another random rowid */ + sqlite3_randomness(sizeof(u.be.v), &u.be.v); + if( u.be.cnt<5 ){ + /* try "small" random rowids for the initial attempts */ + u.be.v &= 0xffffff; }else{ - sqlite3_randomness(sizeof(u.be.v), &u.be.v); - if( u.be.cnt<5 ) u.be.v &= 0xffffff; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ } - rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, 0, &u.be.res); - u.be.cnt++; - }while( u.be.cnt<100 && rc==SQLITE_OK && u.be.res==0 ); + u.be.v++; /* ensure non-zero */ + } if( rc==SQLITE_OK && u.be.res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } + assert( u.be.v>0 ); /* EV: R-40812-03570 */ } u.be.pC->rowidIsValid = 0; u.be.pC->deferredMoveto = 0; @@ -62402,6 +63901,7 @@ case OP_InsertInt: { u.bf.pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( memIsValid(u.bf.pData) ); u.bf.pC = p->apCsr[pOp->p1]; assert( u.bf.pC!=0 ); assert( u.bf.pC->pCursor!=0 ); @@ -62412,6 +63912,7 @@ case OP_InsertInt: { if( pOp->opcode==OP_Insert ){ u.bf.pKey = &aMem[pOp->p3]; assert( u.bf.pKey->flags & MEM_Int ); + assert( memIsValid(u.bf.pKey) ); REGISTER_TRACE(pOp->p3, u.bf.pKey); u.bf.iKey = u.bf.pKey->u.i; }else{ @@ -62563,6 +64064,7 @@ case OP_RowData: { #endif /* local variables moved into u.bh */ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -62647,9 +64149,7 @@ case OP_Rowid: { /* out2-prerelease */ u.bi.pModule = u.bi.pVtab->pModule; assert( u.bi.pModule->xRowid ); rc = u.bi.pModule->xRowid(u.bi.pC->pVtabCursor, &u.bi.v); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.bi.pVtab->zErrMsg; - u.bi.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.bi.pVtab); #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ assert( u.bi.pC->pCursor!=0 ); @@ -62907,6 +64407,9 @@ case OP_IdxDelete: { u.bo.r.nField = (u16)pOp->p3; u.bo.r.flags = 0; u.bo.r.aMem = &aMem[pOp->p2]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ip1>=0 && pOp->p1nCursor ); u.bq.pC = p->apCsr[pOp->p1]; assert( u.bq.pC!=0 ); + assert( u.bq.pC->isOrdered ); if( ALWAYS(u.bq.pC->pCursor!=0) ){ assert( u.bq.pC->deferredMoveto==0 ); assert( pOp->p5==0 || pOp->p5==1 ); @@ -63003,6 +64507,9 @@ case OP_IdxGE: { /* jump */ u.bq.r.flags = UNPACKED_IGNORE_ROWID; } u.bq.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iopcode==OP_IdxLT ){ u.bq.res = -u.bq.res; @@ -63106,6 +64613,8 @@ case OP_Clear: { if( pOp->p3 ){ p->nChange += u.bs.nChange; if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); aMem[pOp->p3].u.i += u.bs.nChange; } } @@ -63149,9 +64658,9 @@ case OP_CreateTable: { /* out2-prerelease */ assert( u.bt.pDb->pBt!=0 ); if( pOp->opcode==OP_CreateTable ){ /* u.bt.flags = BTREE_INTKEY; */ - u.bt.flags = BTREE_LEAFDATA|BTREE_INTKEY; + u.bt.flags = BTREE_INTKEY; }else{ - u.bt.flags = BTREE_ZERODATA; + u.bt.flags = BTREE_BLOBKEY; } rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags); pOut->u.i = u.bt.pgno; @@ -63480,6 +64989,7 @@ case OP_Program: { /* jump */ u.by.pProgram = pOp->p4.pProgram; u.by.pRt = &aMem[pOp->p3]; + assert( memIsValid(u.by.pRt) ); assert( u.by.pProgram->nOp>0 ); /* If the p5 flag is clear, then recursive invocation of triggers is @@ -63653,6 +65163,7 @@ case OP_MemMax: { /* in2 */ }else{ u.ca.pIn1 = &aMem[pOp->p1]; } + assert( memIsValid(u.ca.pIn1) ); sqlite3VdbeMemIntegerify(u.ca.pIn1); pIn2 = &aMem[pOp->p2]; sqlite3VdbeMemIntegerify(pIn2); @@ -63739,7 +65250,9 @@ case OP_AggStep: { u.cb.apVal = p->apArg; assert( u.cb.apVal || u.cb.n==0 ); for(u.cb.i=0; u.cb.ip4.pFunc; @@ -63759,7 +65272,7 @@ case OP_AggStep: { assert( pOp[-1].opcode==OP_CollSeq ); u.cb.ctx.pColl = pOp[-1].p4.pColl; } - (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); + (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); /* IMP: R-24505-23230 */ if( u.cb.ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s)); rc = u.cb.ctx.isError; @@ -64038,11 +65551,7 @@ case OP_VBegin: { #endif /* local variables moved into u.cf */ u.cf.pVTab = pOp->p4.pVtab; rc = sqlite3VtabBegin(db, u.cf.pVTab); - if( u.cf.pVTab ){ - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cf.pVTab->pVtab->zErrMsg; - u.cf.pVTab->pVtab->zErrMsg = 0; - } + if( u.cf.pVTab ) importVtabErrMsg(p, u.cf.pVTab->pVtab); break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -64094,9 +65603,7 @@ case OP_VOpen: { u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule; assert(u.cg.pVtab && u.cg.pModule); rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cg.pVtab->zErrMsg; - u.cg.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.cg.pVtab); if( SQLITE_OK==rc ){ /* Initialize sqlite3_vtab_cursor base class */ u.cg.pVtabCursor->pVtab = u.cg.pVtab; @@ -64152,6 +65659,7 @@ case OP_VFilter: { /* jump */ u.ch.pQuery = &aMem[pOp->p3]; u.ch.pArgc = &u.ch.pQuery[1]; u.ch.pCur = p->apCsr[pOp->p1]; + assert( memIsValid(u.ch.pQuery) ); REGISTER_TRACE(pOp->p3, u.ch.pQuery); assert( u.ch.pCur->pVtabCursor ); u.ch.pVtabCursor = u.ch.pCur->pVtabCursor; @@ -64175,9 +65683,7 @@ case OP_VFilter: { /* jump */ p->inVtabMethod = 1; rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg); p->inVtabMethod = 0; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.ch.pVtab->zErrMsg; - u.ch.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.ch.pVtab); if( rc==SQLITE_OK ){ u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor); } @@ -64211,6 +65717,7 @@ case OP_VColumn: { assert( pCur->pVtabCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.ci.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.ci.pDest); if( pCur->nullRow ){ sqlite3VdbeMemSetNull(u.ci.pDest); break; @@ -64229,9 +65736,7 @@ case OP_VColumn: { MemSetTypeFlag(&u.ci.sContext.s, MEM_Null); rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.ci.pVtab->zErrMsg; - u.ci.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.ci.pVtab); if( u.ci.sContext.isError ){ rc = u.ci.sContext.isError; } @@ -64286,9 +65791,7 @@ case OP_VNext: { /* jump */ p->inVtabMethod = 1; rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor); p->inVtabMethod = 0; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cj.pVtab->zErrMsg; - u.cj.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.cj.pVtab); if( rc==SQLITE_OK ){ u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor); } @@ -64317,12 +65820,12 @@ case OP_VRename: { u.ck.pVtab = pOp->p4.pVtab->pVtab; u.ck.pName = &aMem[pOp->p1]; assert( u.ck.pVtab->pModule->xRename ); + assert( memIsValid(u.ck.pName) ); REGISTER_TRACE(pOp->p1, u.ck.pName); assert( u.ck.pName->flags & MEM_Str ); rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.ck.pVtab->zErrMsg; - u.ck.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.ck.pVtab); + p->expired = 0; break; } @@ -64371,14 +65874,14 @@ case OP_VUpdate: { u.cl.apArg = p->apArg; u.cl.pX = &aMem[pOp->p3]; for(u.cl.i=0; u.cl.ixUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cl.pVtab->zErrMsg; - u.cl.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.cl.pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) ); db->lastRowid = u.cl.rowid; @@ -65427,8 +66930,7 @@ SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ } /* -** Return the number of bytes required to store a MemJournal that uses vfs -** pVfs to create the underlying on-disk files. +** Return the number of bytes required to store a MemJournal file descriptor. */ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ return sizeof(MemJournal); @@ -66832,6 +68334,18 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ return pExpr->affinity; } +/* +** Set the explicit collating sequence for an expression to the +** collating sequence supplied in the second argument. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ + if( pExpr && pColl ){ + pExpr->pColl = pColl; + pExpr->flags |= EP_ExpCollate; + } + return pExpr; +} + /* ** Set the collating sequence for expression pExpr to be the collating ** sequence named by pToken. Return a pointer to the revised expression. @@ -66839,18 +68353,13 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ ** flag. An explicit collating sequence will override implicit ** collating sequences. */ -SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pCollName){ +SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ char *zColl = 0; /* Dequoted name of collation sequence */ CollSeq *pColl; sqlite3 *db = pParse->db; zColl = sqlite3NameFromToken(db, pCollName); - if( pExpr && zColl ){ - pColl = sqlite3LocateCollSeq(pParse, zColl); - if( pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; - } - } + pColl = sqlite3LocateCollSeq(pParse, zColl); + sqlite3ExprSetColl(pExpr, pColl); sqlite3DbFree(db, zColl); return pExpr; } @@ -67324,18 +68833,19 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ }else if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ - int i = atoi((char*)&z[1]); + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); pExpr->iColumn = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); } if( i>pParse->nVar ){ - pParse->nVar = i; + pParse->nVar = (int)i; } }else{ /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable @@ -68304,8 +69814,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ #endif /* -** Generate code for scalar subqueries used as an expression -** and IN operators. Examples: +** Generate code for scalar subqueries used as a subquery expression, EXISTS, +** or IN operators. Examples: ** ** (SELECT a FROM b) -- subquery ** EXISTS (SELECT a FROM b) -- EXISTS subquery @@ -68368,10 +69878,10 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { - char affinity; - KeyInfo keyInfo; - int addr; /* Address of OP_OpenEphemeral instruction */ - Expr *pLeft = pExpr->pLeft; + char affinity; /* Affinity of the LHS of the IN */ + KeyInfo keyInfo; /* Keyinfo for the generated table */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ if( rMayHaveNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); @@ -68394,6 +69904,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( */ pExpr->iTable = pParse->nTab++; addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); + if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; @@ -68686,7 +70197,7 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; char *zV; - sqlite3AtoF(z, &value); + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; zV = dup8bytes(v, (char*)&value); @@ -68700,9 +70211,7 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ ** Generate an instruction that will put the integer describe by ** text z[0..n-1] into register iMem. ** -** The z[] string will probably not be zero-terminated. But the -** z[n] character is guaranteed to be something that does not look -** like the continuation of the number. +** Expr.u.zToken is always UTF8 and zero-terminated. */ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ Vdbe *v = pParse->pVdbe; @@ -68711,13 +70220,14 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ if( negFlag ) i = -i; sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); }else{ + int c; + i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - if( sqlite3FitsIn64Bits(z, negFlag) ){ - i64 value; + c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + if( c==0 || (c==2 && negFlag) ){ char *zV; - sqlite3Atoi64(z, &value); - if( negFlag ) value = -value; + if( negFlag ){ value = -value; } zV = dup8bytes(v, (char*)&value); sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); }else{ @@ -69002,73 +70512,6 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ -/* -** If the last instruction coded is an ephemeral copy of any of -** the registers in the nReg registers beginning with iReg, then -** convert the last instruction from OP_SCopy to OP_Copy. -*/ -SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse *pParse, int iReg, int nReg){ - VdbeOp *pOp; - Vdbe *v; - - assert( pParse->db->mallocFailed==0 ); - v = pParse->pVdbe; - assert( v!=0 ); - pOp = sqlite3VdbeGetOp(v, -1); - assert( pOp!=0 ); - if( pOp->opcode==OP_SCopy && pOp->p1>=iReg && pOp->p1opcode = OP_Copy; - } -} - -/* -** Generate code to store the value of the iAlias-th alias in register -** target. The first time this is called, pExpr is evaluated to compute -** the value of the alias. The value is stored in an auxiliary register -** and the number of that register is returned. On subsequent calls, -** the register number is returned without generating any code. -** -** Note that in order for this to work, code must be generated in the -** same order that it is executed. -** -** Aliases are numbered starting with 1. So iAlias is in the range -** of 1 to pParse->nAlias inclusive. -** -** pParse->aAlias[iAlias-1] records the register number where the value -** of the iAlias-th alias is stored. If zero, that means that the -** alias has not yet been computed. -*/ -static int codeAlias(Parse *pParse, int iAlias, Expr *pExpr, int target){ -#if 0 - sqlite3 *db = pParse->db; - int iReg; - if( pParse->nAliasAllocnAlias ){ - pParse->aAlias = sqlite3DbReallocOrFree(db, pParse->aAlias, - sizeof(pParse->aAlias[0])*pParse->nAlias ); - testcase( db->mallocFailed && pParse->nAliasAlloc>0 ); - if( db->mallocFailed ) return 0; - memset(&pParse->aAlias[pParse->nAliasAlloc], 0, - (pParse->nAlias-pParse->nAliasAlloc)*sizeof(pParse->aAlias[0])); - pParse->nAliasAlloc = pParse->nAlias; - } - assert( iAlias>0 && iAlias<=pParse->nAlias ); - iReg = pParse->aAlias[iAlias-1]; - if( iReg==0 ){ - if( pParse->iCacheLevel>0 ){ - iReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - }else{ - iReg = ++pParse->nMem; - sqlite3ExprCode(pParse, pExpr, iReg); - pParse->aAlias[iAlias-1] = iReg; - } - } - return iReg; -#else - UNUSED_PARAMETER(iAlias); - return sqlite3ExprCodeTarget(pParse, pExpr, target); -#endif -} - /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -69177,7 +70620,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } case TK_AS: { - inReg = codeAlias(pParse, pExpr->iTable, pExpr->pLeft, target); + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; } #ifndef SQLITE_OMIT_CAST @@ -69609,6 +71052,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) opCompare.op = TK_EQ; opCompare.pLeft = &cacheX; pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; } for(i=0; i0 && target<=pParse->nMem ); - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + if( pExpr && pExpr->op==TK_REGISTER ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); + }else{ + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + } } return target; } @@ -69878,19 +71330,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( int i, n; assert( pList!=0 ); assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ n = pList->nExpr; for(pItem=pList->a, i=0; iiAlias ){ - int iReg = codeAlias(pParse, pItem->iAlias, pItem->pExpr, target+i); - Vdbe *v = sqlite3GetVdbe(pParse); - if( iReg!=target+i ){ - sqlite3VdbeAddOp2(v, OP_SCopy, iReg, target+i); - } - }else{ - sqlite3ExprCode(pParse, pItem->pExpr, target+i); - } - if( doHardCopy && !pParse->db->mallocFailed ){ - sqlite3ExprHardCopy(pParse, target, n); + Expr *pExpr = pItem->pExpr; + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + sqlite3VdbeAddOp2(pParse->pVdbe, doHardCopy ? OP_Copy : OP_SCopy, + inReg, target+i); } } return n; @@ -70872,6 +72319,11 @@ static char *whereTempTriggers(Parse *pParse, Table *pTab){ } } } + if( zWhere ){ + char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); + sqlite3DbFree(pParse->db, zWhere); + zWhere = zNew; + } return zWhere; } @@ -71328,7 +72780,6 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ if( !pNew ) goto exit_begin_add_column; pParse->pNewTable = pNew; pNew->nRef = 1; - pNew->dbMem = pTab->dbMem; pNew->nCol = pTab->nCol; assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; @@ -71480,7 +72931,8 @@ static void analyzeOneTable( int i; /* Loop counter */ int topOfLoop; /* The top of the loop */ int endOfLoop; /* The end of the loop */ - int addr; /* The address of an instruction */ + int addr = 0; /* The address of an instruction */ + int jZeroRows = 0; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ @@ -71499,8 +72951,15 @@ static void analyzeOneTable( #endif v = sqlite3GetVdbe(pParse); - if( v==0 || NEVER(pTab==0) || pTab->pIndex==0 ){ - /* Do no analysis for tables that have no indices */ + if( v==0 || NEVER(pTab==0) ){ + return; + } + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ + return; + } + if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + /* Do not gather statistics on system tables */ return; } assert( sqlite3BtreeHoldsAllMutexes(db) ); @@ -71517,6 +72976,7 @@ static void analyzeOneTable( sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol = pIdx->nColumn; KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); @@ -71531,10 +72991,7 @@ static void analyzeOneTable( (char *)pKey, P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - /* Populate the registers containing the table and index names. */ - if( pTab->pIndex==pIdx ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); - } + /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); #ifdef SQLITE_ENABLE_STAT2 @@ -71669,8 +73126,10 @@ static void analyzeOneTable( ** If K>0 then it is always the case the D>0 so division by zero ** is never possible. */ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); + if( jZeroRows==0 ){ + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); + } for(i=0; ipIndex==0 ){ + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); + VdbeComment((v, "%s", pTab->zName)); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); + sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); + }else{ + assert( jZeroRows>0 ); + addr = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, jZeroRows); + } + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + if( pParse->nMemnMem = regRec; + if( jZeroRows ){ sqlite3VdbeJumpHere(v, addr); } } /* ** Generate code that will cause the most recent index analysis to -** be laoded into internal hash tables where is can be used. +** be loaded into internal hash tables where is can be used. */ static void loadAnalysis(Parse *pParse, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); @@ -71820,33 +73301,46 @@ struct analysisInfo { ** This callback is invoked once for each index when reading the ** sqlite_stat1 table. ** -** argv[0] = name of the index -** argv[1] = results of analysis - on integer for each column +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. */ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ analysisInfo *pInfo = (analysisInfo*)pData; Index *pIndex; - int i, c; + Table *pTable; + int i, c, n; unsigned int v; const char *z; - assert( argc==2 ); + assert( argc==3 ); UNUSED_PARAMETER2(NotUsed, argc); - if( argv==0 || argv[0]==0 || argv[1]==0 ){ + if( argv==0 || argv[0]==0 || argv[2]==0 ){ return 0; } - pIndex = sqlite3FindIndex(pInfo->db, argv[0], pInfo->zDatabase); - if( pIndex==0 ){ + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ return 0; } - z = argv[1]; - for(i=0; *z && i<=pIndex->nColumn; i++){ + if( argv[1] ){ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + }else{ + pIndex = 0; + } + n = pIndex ? pIndex->nColumn : 0; + z = argv[2]; + for(i=0; *z && i<=n; i++){ v = 0; while( (c=z[0])>='0' && c<='9' ){ v = v*10 + c - '0'; z++; } + if( i==0 ) pTable->nRowEst = v; + if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; } @@ -71857,21 +73351,20 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** If the Index.aSample variable is not NULL, delete the aSample[] array ** and its contents. */ -SQLITE_PRIVATE void sqlite3DeleteIndexSamples(Index *pIdx){ +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #ifdef SQLITE_ENABLE_STAT2 if( pIdx->aSample ){ int j; - sqlite3 *dbMem = pIdx->pTable->dbMem; for(j=0; jaSample[j]; if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ - sqlite3DbFree(pIdx->pTable->dbMem, p->u.z); + sqlite3DbFree(db, p->u.z); } } - sqlite3DbFree(dbMem, pIdx->aSample); - pIdx->aSample = 0; + sqlite3DbFree(db, pIdx->aSample); } #else + UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); #endif } @@ -71910,7 +73403,8 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); - sqlite3DeleteIndexSamples(pIdx); + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; } /* Check to make sure the sqlite_stat1 table exists */ @@ -71922,7 +73416,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load new statistics out of the sqlite_stat1 table */ zSql = sqlite3MPrintf(db, - "SELECT idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ @@ -71954,18 +73448,17 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ Index *pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase); if( pIdx ){ int iSample = sqlite3_column_int(pStmt, 1); - sqlite3 *dbMem = pIdx->pTable->dbMem; - assert( dbMem==db || dbMem==0 ); if( iSample=0 ){ int eType = sqlite3_column_type(pStmt, 2); if( pIdx->aSample==0 ){ static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; - pIdx->aSample = (IndexSample *)sqlite3DbMallocZero(dbMem, sz); + pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); if( pIdx->aSample==0 ){ db->mallocFailed = 1; break; } + memset(pIdx->aSample, 0, sz); } assert( pIdx->aSample ); @@ -71988,10 +73481,8 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ if( n < 1){ pSample->u.z = 0; }else{ - pSample->u.z = sqlite3DbMallocRaw(dbMem, n); - if( pSample->u.z ){ - memcpy(pSample->u.z, z, n); - }else{ + pSample->u.z = sqlite3DbStrNDup(0, z, n); + if( pSample->u.z==0 ){ db->mallocFailed = 1; break; } @@ -72142,9 +73633,8 @@ static void attachFunc( ** it to obtain the database schema. At this point the schema may ** or may not be initialised. */ - rc = sqlite3BtreeFactory(db, zFile, 0, SQLITE_DEFAULT_CACHE_SIZE, - db->openFlags | SQLITE_OPEN_MAIN_DB, - &aNew->pBt); + rc = sqlite3BtreeOpen(zFile, db, &aNew->pBt, 0, + db->openFlags | SQLITE_OPEN_MAIN_DB); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; @@ -72385,7 +73875,8 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ 0, /* xStep */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - 0 /* pHash */ + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } @@ -72406,7 +73897,8 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p 0, /* xStep */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - 0 /* pHash */ + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } @@ -73154,33 +74646,14 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha /* ** Reclaim the memory used by an index */ -static void freeIndex(Index *p){ - sqlite3 *db = p->pTable->dbMem; +static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE - sqlite3DeleteIndexSamples(p); + sqlite3DeleteIndexSamples(db, p); #endif sqlite3DbFree(db, p->zColAff); sqlite3DbFree(db, p); } -/* -** Remove the given index from the index hash table, and free -** its memory structures. -** -** The index is removed from the database hash tables but -** it is not unlinked from the Table that it indexes. -** Unlinking from the Table must be done by the calling function. -*/ -static void sqlite3DeleteIndex(Index *p){ - Index *pOld; - const char *zName = p->zName; - - pOld = sqlite3HashInsert(&p->pSchema->idxHash, zName, - sqlite3Strlen30(zName), 0); - assert( pOld==0 || pOld==p ); - freeIndex(p); -} - /* ** For the index called zIdxName which is found in the database iDb, ** unlike that index from its Table then remove the index from @@ -73207,7 +74680,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char p->pNext = pIndex->pNext; } } - freeIndex(pIndex); + freeIndex(db, pIndex); } db->flags |= SQLITE_InternChanges; } @@ -73278,13 +74751,12 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ } /* -** Clear the column names from a table or view. +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). */ -static void sqliteResetColumnNames(Table *pTable){ +static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; - sqlite3 *db = pTable->dbMem; - testcase( db==0 ); assert( pTable!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ @@ -73296,8 +74768,6 @@ static void sqliteResetColumnNames(Table *pTable){ } sqlite3DbFree(db, pTable->aCol); } - pTable->aCol = 0; - pTable->nCol = 0; } /* @@ -73309,42 +74779,44 @@ static void sqliteResetColumnNames(Table *pTable){ ** memory structures of the indices and foreign keys associated with ** the table. */ -SQLITE_PRIVATE void sqlite3DeleteTable(Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; - sqlite3 *db; - if( pTable==0 ) return; - db = pTable->dbMem; - testcase( db==0 ); + assert( !pTable || pTable->nRef>0 ); /* Do not delete the table until the reference count reaches zero. */ - pTable->nRef--; - if( pTable->nRef>0 ){ - return; - } - assert( pTable->nRef==0 ); + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - /* Delete all indices associated with this table - */ + /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema ); - sqlite3DeleteIndex(pIndex); + if( !db || db->pnBytesFreed==0 ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + ); + assert( pOld==pIndex || pOld==0 ); + } + freeIndex(db, pIndex); } /* Delete any foreign keys attached to this table. */ - sqlite3FkDelete(pTable); + sqlite3FkDelete(db, pTable); /* Delete the Table structure itself. */ - sqliteResetColumnNames(pTable); + sqliteDeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK sqlite3ExprDelete(db, pTable->pCheck); #endif - sqlite3VtabClear(pTable); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3VtabClear(db, pTable); +#endif sqlite3DbFree(db, pTable); } @@ -73363,7 +74835,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char pDb = &db->aDb[iDb]; p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, sqlite3Strlen30(zTabName),0); - sqlite3DeleteTable(p); + sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } @@ -73555,8 +75027,9 @@ SQLITE_PRIVATE void sqlite3StartTable( */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && iDb>1 ){ - /* If creating a temp table, the name may not be qualified */ + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); return; } @@ -73604,17 +75077,18 @@ SQLITE_PRIVATE void sqlite3StartTable( ** collisions. */ if( !IN_DECLARE_VTAB ){ + char *zDb = db->aDb[iDb].zName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } - pTable = sqlite3FindTable(db, zName, db->aDb[iDb].zName); + pTable = sqlite3FindTable(db, zName, zDb); if( pTable ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "table %T already exists", pName); } goto begin_table_error; } - if( sqlite3FindIndex(db, zName, 0)!=0 && (iDb==0 || !db->init.busy) ){ + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); goto begin_table_error; } @@ -73631,7 +75105,7 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->dbMem = 0; + pTable->nRowEst = 1000000; assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -74183,7 +75657,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zEnd = "\n)"; } n += 35 + 6*p->nCol; - zStmt = sqlite3Malloc( n ); + zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ db->mallocFailed = 1; return 0; @@ -74364,7 +75838,7 @@ SQLITE_PRIVATE void sqlite3EndTable( p->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(pSelTab); + sqlite3DeleteTable(db, pSelTab); } } @@ -74478,12 +75952,10 @@ SQLITE_PRIVATE void sqlite3CreateView( } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 ){ + if( p==0 || pParse->nErr ){ sqlite3SelectDelete(db, pSelect); return; } - assert( pParse->nErr==0 ); /* If sqlite3StartTable return non-NULL then - ** there could not have been an error */ sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) @@ -74608,7 +76080,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(pSelTab); + sqlite3DeleteTable(db, pSelTab); pTable->pSchema->flags |= DB_UnresetViews; }else{ pTable->nCol = 0; @@ -74633,7 +76105,9 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteResetColumnNames(pTab); + sqliteDeleteColumnNames(db, pTab); + pTab->aCol = 0; + pTab->nCol = 0; } } DbClearProperty(db, idx, DB_UnresetViews); @@ -75599,7 +77073,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q'", pIndex->zName), P4_DYNAMIC); + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), + P4_DYNAMIC); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } @@ -75630,7 +77105,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* Clean up before exiting */ exit_create_index: if( pIndex ){ - sqlite3_free(pIndex->zColAff); + sqlite3DbFree(db, pIndex->zColAff); sqlite3DbFree(db, pIndex); } sqlite3ExprListDelete(db, pList); @@ -75660,14 +77135,14 @@ exit_create_index: SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ unsigned *a = pIdx->aiRowEst; int i; + unsigned n; assert( a!=0 ); - a[0] = 1000000; - for(i=pIdx->nColumn; i>=5; i--){ - a[i] = 5; - } - while( i>=1 ){ - a[i] = 11 - i; - i--; + a[0] = pIdx->pTable->nRowEst; + if( a[0]<10 ) a[0] = 10; + n = 10; + for(i=1; i<=pIdx->nColumn; i++){ + a[i] = n; + if( n>5 ) n--; } if( pIdx->onError!=OE_None ){ a[pIdx->nColumn] = 1; @@ -75727,7 +77202,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q", + "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName ); @@ -76009,7 +77484,7 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); sqlite3DbFree(db, pItem->zIndex); - sqlite3DeleteTable(pItem->pTab); + sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); sqlite3IdListDelete(db, pItem->pUsing); @@ -76219,7 +77694,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TEMP_DB; - rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, flags, &pBt); + rc = sqlite3BtreeOpen(0, db, &pBt, 0, flags); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " "file for storing temporary tables"); @@ -76876,7 +78351,7 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** priority to built-in functions. ** ** Except, if createFlag is true, that means that we are trying to - ** install a new function. Whatever FuncDef structure is returned will + ** install a new function. Whatever FuncDef structure is returned it will ** have fields overwritten with new information appropriate for the ** new function. But the FuncDefs for built-in functions are read-only. ** So we must not search for built-ins when creating a new function. @@ -76940,8 +78415,7 @@ SQLITE_PRIVATE void sqlite3SchemaFree(void *p){ sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - assert( pTab->dbMem==0 ); - sqlite3DeleteTable(pTab); + sqlite3DeleteTable(0, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); @@ -76958,7 +78432,7 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ if( pBt ){ p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaFree); }else{ - p = (Schema *)sqlite3MallocZero(sizeof(Schema)); + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ db->mallocFailed = 1; @@ -76999,7 +78473,7 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc==1 ); pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); - sqlite3DeleteTable(pItem->pTab); + sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ pTab->nRef++; @@ -77893,7 +79367,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r); + sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); sqlite3_free(zBuf); } sqlite3_result_double(context, r); @@ -78396,8 +79870,10 @@ static void compileoptionusedFunc( const char *zOptName; assert( argc==1 ); UNUSED_PARAMETER(argc); - /* IMP: R-xxxx This function is an SQL wrapper around the - ** sqlite3_compileoption_used() C interface. */ + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); } @@ -78418,8 +79894,9 @@ static void compileoptiongetFunc( int n; assert( argc==1 ); UNUSED_PARAMETER(argc); - /* IMP: R-xxxx This function is an SQL wrapper around the - ** sqlite3_compileoption_get() C interface. */ + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ n = sqlite3_value_int(argv[0]); sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); } @@ -78618,14 +80095,14 @@ static void replaceFunc( testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); - sqlite3DbFree(db, zOut); + sqlite3_free(zOut); return; } zOld = zOut; zOut = sqlite3_realloc(zOut, (int)nOut); if( zOut==0 ){ sqlite3_result_error_nomem(context); - sqlite3DbFree(db, zOld); + sqlite3_free(zOld); return; } memcpy(&zOut[j], zRep, nRep); @@ -78986,7 +80463,7 @@ static void groupConcatStep( if( pAccum ){ sqlite3 *db = sqlite3_context_db_handle(context); int firstTerm = pAccum->useMalloc==0; - pAccum->useMalloc = 1; + pAccum->useMalloc = 2; pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; if( !firstTerm ){ if( argc==2 ){ @@ -79055,10 +80532,10 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) }else{ pInfo = (struct compareInfo*)&likeInfoNorm; } - sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0); + sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); sqlite3CreateFunc(db, "glob", 2, SQLITE_ANY, - (struct compareInfo*)&globInfo, likeFunc, 0,0); + (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); setLikeOptFlag(db, "like", caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); @@ -79142,10 +80619,10 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), /* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0}, + {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, FUNCTION(hex, 1, 0, 0, hexFunc ), /* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0}, + {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), @@ -79172,7 +80649,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0}, + {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), @@ -79583,7 +81060,7 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); for(i=0; iaiColumn[i]; - pCol = &pIdx->pTable->aCol[iCol]; + pCol = &pTab->aCol[iCol]; + if( pTab->iPKey==iCol ) iCol = -1; pLeft->iTable = regData+iCol+1; pLeft->affinity = pCol->affinity; pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); @@ -80264,11 +81742,7 @@ static Trigger *fkActionTrigger( pWhere = 0; } - /* In the current implementation, pTab->dbMem==0 for all tables except - ** for temporary tables used to describe subqueries. And temporary - ** tables do not have foreign key constraints. Hence, pTab->dbMem - ** should always be 0 there. - */ + /* Disable lookaside memory allocation */ enableLookaside = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; @@ -80358,37 +81832,39 @@ SQLITE_PRIVATE void sqlite3FkActions( ** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash ** hash table. */ -SQLITE_PRIVATE void sqlite3FkDelete(Table *pTab){ +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ - if( pFKey->pPrevTo ){ - pFKey->pPrevTo->pNextTo = pFKey->pNextTo; - }else{ - void *data = (void *)pFKey->pNextTo; - const char *z = (data ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), data); + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; + }else{ + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + } } - if( pFKey->pNextTo ){ - pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; - } - - /* Delete any triggers created to implement actions for this FK. */ -#ifndef SQLITE_OMIT_TRIGGER - fkTriggerDelete(pTab->dbMem, pFKey->apTrigger[0]); - fkTriggerDelete(pTab->dbMem, pFKey->apTrigger[1]); -#endif /* EV: R-30323-21917 Each foreign key constraint in SQLite is ** classified as either immediate or deferred. */ assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); +#endif + pNext = pFKey->pNextFrom; - sqlite3DbFree(pTab->dbMem, pFKey); + sqlite3DbFree(db, pFKey); } } #endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ @@ -80463,7 +81939,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ int n; Table *pTab = pIdx->pTable; sqlite3 *db = sqlite3VdbeDb(v); - pIdx->zColAff = (char *)sqlite3Malloc(pIdx->nColumn+2); + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+2); if( !pIdx->zColAff ){ db->mallocFailed = 1; return 0; @@ -80505,7 +81981,7 @@ SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ int i; sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3Malloc(pTab->nCol+1); + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ db->mallocFailed = 1; return; @@ -81616,6 +83092,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( onError==OE_Ignore ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ sqlite3HaltConstraint(pParse, onError, 0, 0); } sqlite3VdbeResolveLabel(v, allOk); @@ -83130,13 +84607,11 @@ static int sqlite3LoadExtension( handle = sqlite3OsDlOpen(pVfs, zFile); if( handle==0 ){ if( pzErrMsg ){ - zErrmsg = sqlite3StackAllocZero(db, nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "unable to open shared library [%s]", zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - *pzErrMsg = sqlite3DbStrDup(0, zErrmsg); - sqlite3StackFree(db, zErrmsg); } } return SQLITE_ERROR; @@ -83145,13 +84620,11 @@ static int sqlite3LoadExtension( sqlite3OsDlSym(pVfs, handle, zProc); if( xInit==0 ){ if( pzErrMsg ){ - zErrmsg = sqlite3StackAllocZero(db, nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zProc,zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - *pzErrMsg = sqlite3DbStrDup(0, zErrmsg); - sqlite3StackFree(db, zErrmsg); } sqlite3OsDlClose(pVfs, handle); } @@ -83952,7 +85425,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit); + sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); @@ -84130,7 +85603,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3_free(sqlite3_temp_directory); if( zRight[0] ){ - sqlite3_temp_directory = sqlite3DbStrDup(0, zRight); + sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); }else{ sqlite3_temp_directory = 0; } @@ -85551,7 +87024,6 @@ static int sqlite3Prepare( while( pParse->pTriggerPrg ){ TriggerPrg *pT = pParse->pTriggerPrg; pParse->pTriggerPrg = pT->pNext; - sqlite3VdbeProgramDelete(db, pT->pProgram, 0); sqlite3DbFree(db, pT); } @@ -86186,7 +87658,6 @@ static void pushOntoSorter( sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); sqlite3VdbeJumpHere(v, addr2); - pSelect->iLimit = 0; } } @@ -86235,11 +87706,13 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate an error message when a SELECT is used within a subexpression ** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error occurs in multiple -** places. +** column. We do this in a subroutine because the error used to occur +** in multiple places. (The error only occurs in one place now, but we +** retain the subroutine to minimize code disruption.) */ static int checkForMultiColumnSelectError( Parse *pParse, /* Parse context. */ @@ -86255,6 +87728,7 @@ static int checkForMultiColumnSelectError( return 0; } } +#endif /* ** This routine generates the code for the inside of the inner loop @@ -86334,10 +87808,6 @@ static void selectInnerLoop( } } - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - return; - } - switch( eDest ){ /* In this mode, write each query result to the key of the temporary ** table iParm. @@ -86466,11 +87936,11 @@ static void selectInnerLoop( #endif } - /* Jump to the end of the loop if the LIMIT is reached. + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. */ - if( p->iLimit ){ - assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to - ** pushOntoSorter() would have cleared p->iLimit */ + if( pOrderBy==0 && p->iLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } } @@ -86605,10 +88075,6 @@ static void generateSortTail( sqlite3ReleaseTempReg(pParse, regRow); sqlite3ReleaseTempReg(pParse, regRowid); - /* LIMIT has been implemented by the pushOntoSorter() routine. - */ - assert( p->iLimit==0 ); - /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -87042,16 +88508,15 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ return 0; } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled, so we might as well hard-code pTab->dbMem to NULL. */ + ** is disabled */ assert( db->lookaside.bEnabled==0 ); - pTab->dbMem = 0; pTab->nRef = 1; pTab->zName = 0; selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ - sqlite3DeleteTable(pTab); + sqlite3DeleteTable(db, pTab); return 0; } return pTab; @@ -87248,6 +88713,7 @@ static int multiSelect( if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -87543,7 +89009,7 @@ multi_select_end: ** regReturn is the number of the register holding the subroutine ** return address. ** -** If regPrev>0 then it is a the first register in a vector that +** If regPrev>0 then it is the first register in a vector that ** records the previous output. mem[regPrev] is a flag that is false ** if there has been no previous output. If regPrev>0 then code is ** generated to suppress duplicates. pKeyInfo is used for comparing @@ -87926,7 +89392,6 @@ static int multiSelectOrderBy( /* Separate the left and the right query from one another */ p->pPrior = 0; - pPrior->pRightmost = 0; sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); if( pPrior->pPrior==0 ){ sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); @@ -88240,12 +89705,13 @@ static void substSelect( ** (2) The subquery is not an aggregate or the outer query is not a join. ** ** (3) The subquery is not the right operand of a left outer join -** (Originally ticket #306. Strenghtened by ticket #3300) +** (Originally ticket #306. Strengthened by ticket #3300) ** -** (4) The subquery is not DISTINCT or the outer query is not a join. +** (4) The subquery is not DISTINCT. ** -** (5) The subquery is not DISTINCT or the outer query does not use -** aggregates. +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. ** ** (6) The subquery does not use aggregates or the outer query is not ** DISTINCT. @@ -88265,9 +89731,9 @@ static void substSelect( ** (**) Not implemented. Subsumed into restriction (3). Was previously ** a separate restriction deriving from ticket #350. ** -** (13) The subquery and outer query do not both use LIMIT +** (13) The subquery and outer query do not both use LIMIT. ** -** (14) The subquery does not use OFFSET +** (14) The subquery does not use OFFSET. ** ** (15) The outer query is not part of a compound select or the ** subquery does not have a LIMIT clause. @@ -88358,9 +89824,9 @@ static int flattenSubquery( return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ - if( ((pSub->selFlags & SF_Distinct)!=0 || pSub->pLimit) - && (pSrc->nSrc>1 || isAgg) ){ /* Restrictions (4)(5)(8)(9) */ - return 0; + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ } if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ return 0; /* Restriction (6) */ @@ -88840,7 +90306,6 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3WalkSelect(pWalker, pSel); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; - pTab->dbMem = db->lookaside.bEnabled ? db : 0; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } @@ -89210,7 +90675,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, 0); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 1); }else{ nArg = 0; regAgg = 0; @@ -89370,6 +90835,15 @@ SQLITE_PRIVATE int sqlite3Select( v = sqlite3GetVdbe(pParse); if( v==0 ) goto select_end; + /* If writing to memory or generating a set + ** only a single column may be output. + */ +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; + } +#endif + /* Generate code for all sub-queries in the FROM clause */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) @@ -89443,15 +90917,6 @@ SQLITE_PRIVATE int sqlite3Select( } #endif - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - goto select_end; - } -#endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. ** GROUP BY might use an index, DISTINCT never does. */ @@ -89514,6 +90979,7 @@ SQLITE_PRIVATE int sqlite3Select( pKeyInfo = keyInfoFromExprList(pParse, p->pEList); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ distinct = -1; } @@ -91054,6 +92520,7 @@ static TriggerPrg *codeRowTrigger( int iEndTrigger = 0; /* Label to jump to if WHEN is false */ assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + assert( pTop->pVdbe ); /* Allocate the TriggerPrg and SubProgram objects. To ensure that they ** are freed if an error occurs, link them into the Parse.pTriggerPrg @@ -91064,7 +92531,7 @@ static TriggerPrg *codeRowTrigger( pTop->pTriggerPrg = pPrg; pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); if( !pProgram ) return 0; - pProgram->nRef = 1; + sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); pPrg->pTrigger = pTrigger; pPrg->orconf = orconf; pPrg->aColmask[0] = 0xffffffff; @@ -91198,8 +92665,9 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program ** is a pointer to the sub-vdbe containing the trigger program. */ if( pPrg ){ + int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); + sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - pPrg->pProgram->nRef++; sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); @@ -91209,7 +92677,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( ** invocation is disallowed if (a) the sub-program is really a trigger, ** not a foreign key action, and (b) the flag to enable recursive triggers ** is clear. */ - sqlite3VdbeChangeP5(v, (u8)(p->zName && !(pParse->db->flags&SQLITE_RecTriggers))); + sqlite3VdbeChangeP5(v, (u8)bRecursive); } } @@ -91358,7 +92826,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -sqlite************************************************************************* +************************************************************************* ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ @@ -91987,6 +93455,7 @@ static void updateVirtualTable( assert( v ); ephemTab = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); /* fill the ephemeral table */ @@ -92126,6 +93595,10 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); return SQLITE_ERROR; } + if( db->activeVdbeCnt>1 ){ + sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); + return SQLITE_ERROR; + } /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and @@ -92577,14 +94050,14 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ ** in the list are moved to the sqlite3.pDisconnect list of the associated ** database connection. */ -SQLITE_PRIVATE void sqlite3VtabClear(Table *p){ - vtabDisconnectAll(0, p); +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ + if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->azModuleArg ){ int i; for(i=0; inModuleArg; i++){ - sqlite3DbFree(p->dbMem, p->azModuleArg[i]); + sqlite3DbFree(db, p->azModuleArg[i]); } - sqlite3DbFree(p->dbMem, p->azModuleArg); + sqlite3DbFree(db, p->azModuleArg); } } @@ -92727,7 +94200,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); - zWhere = sqlite3MPrintf(db, "name='%q'", pTab->zName); + zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 1, 0, zWhere, P4_DYNAMIC); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); @@ -92749,7 +94222,6 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } - pSchema->db = pParse->db; pParse->pNewTable = 0; } } @@ -92823,7 +94295,7 @@ static int vtabCallConstructor( *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); }else { *pzErr = sqlite3MPrintf(db, "%s", zErr); - sqlite3DbFree(db, zErr); + sqlite3_free(zErr); } sqlite3DbFree(db, pVTable); }else if( ALWAYS(pVTable->pVtab) ){ @@ -93038,7 +94510,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ if( pParse->pVdbe ){ sqlite3VdbeFinalize(pParse->pVdbe); } - sqlite3DeleteTable(pParse->pNewTable); + sqlite3DeleteTable(db, pParse->pNewTable); sqlite3StackFree(db, pParse); } @@ -93125,8 +94597,8 @@ SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ rc = x(pVtab); sqlite3DbFree(db, *pzErrmsg); - *pzErrmsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); } } db->aVTrans = aVTrans; @@ -93626,6 +95098,7 @@ static void whereClauseClear(WhereClause *pWC){ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ WhereTerm *pTerm; int idx; + testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pParse->db; @@ -93771,6 +95244,13 @@ static Bitmask exprSelectTableUsage(WhereMaskSet *pMaskSet, Select *pS){ ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are ** "=", "<", ">", "<=", ">=", and "IN". +** +** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be +** of one of the following forms: column = expression column > expression +** column >= expression column < expression column <= expression +** expression = column expression > column expression >= column +** expression < column expression <= column column IN +** (expression-list) column IN (subquery) column IS NULL */ static int allowedOp(int op){ assert( TK_GT>TK_EQ && TK_GTdb; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ @@ -93953,19 +95432,6 @@ static int isLikeOrGlob( return 0; } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pColl = sqlite3ExprCollSeq(pParse, pLeft); - if( pColl==0 ) return 0; /* Happens when LHS has an undefined collation */ - if( (pColl->type!=SQLITE_COLL_BINARY || *pnoCase) && - (pColl->type!=SQLITE_COLL_NOCASE || !*pnoCase) ){ - /* IMP: R-09003-32046 For the GLOB operator, the column must use the - ** default BINARY collating sequence. - ** IMP: R-41408-28306 For the LIKE operator, if case_sensitive_like mode - ** is enabled then the column must use the default BINARY collating - ** sequence, or if case_sensitive_like mode is disabled then the column - ** must use the built-in NOCASE collating sequence. - */ - return 0; - } pRight = pList->a[0].pExpr; op = pRight->op; @@ -93974,11 +95440,12 @@ static int isLikeOrGlob( } if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; - pVal = sqlite3VdbeGetValue(pReprepare, pRight->iColumn, SQLITE_AFF_NONE); + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = (char *)sqlite3_value_text(pVal); } - sqlite3VdbeSetVarmask(pParse->pVdbe, pRight->iColumn); + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ z = pRight->u.zToken; @@ -93988,15 +95455,15 @@ static int isLikeOrGlob( while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; } - if( cnt!=0 && c!=0 && 255!=(u8)z[cnt-1] ){ + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ Expr *pPrefix; - *pisComplete = z[cnt]==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0; pPrefix = sqlite3Expr(db, TK_STRING, z); if( pPrefix ) pPrefix->u.zToken[cnt] = 0; *ppPrefix = pPrefix; if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); + sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ** value of the variable means there is no need to invoke the LIKE @@ -94329,6 +95796,8 @@ static void exprAnalyzeOrTerm( /* At this point, okToChngToIN is true if original pTerm satisfies ** case 1. In that case, construct a new virtual term that is ** pTerm converted into an IN operator. + ** + ** EV: R-00211-15100 */ if( okToChngToIN ){ Expr *pDup; /* A transient duplicate expression */ @@ -94545,6 +96014,7 @@ static void exprAnalyze( Expr *pNewExpr2; int idxNew1; int idxNew2; + CollSeq *pColl; /* Collating sequence to use */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); @@ -94559,17 +96029,23 @@ static void exprAnalyze( ** inequality. To avoid this, make sure to also run the full ** LIKE on all candidate expressions by clearing the isComplete flag */ - if( c=='A'-1 ) isComplete = 0; + if( c=='A'-1 ) isComplete = 0; /* EV: R-64339-08207 */ + c = sqlite3UpperToLower[c]; } *pC = c + 1; } - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprDup(db,pLeft,0),pStr1,0); + pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr1, 0); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprDup(db,pLeft,0),pStr2,0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr2, 0); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); @@ -94851,7 +96327,8 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ ** Required because bestIndex() is called by bestOrClauseIndex() */ static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*); + Parse*, WhereClause*, struct SrcList_item*, + Bitmask, Bitmask, ExprList*, WhereCost*); /* ** This routine attempts to find an scanning strategy that can be used @@ -94864,7 +96341,8 @@ static void bestOrClauseIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ @@ -94900,7 +96378,7 @@ static void bestOrClauseIndex( )); if( pOrTerm->eOperator==WO_AND ){ WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost); + bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); }else if( pOrTerm->leftCursor==iCur ){ WhereClause tempWC; tempWC.pParse = pWC->pParse; @@ -94908,7 +96386,7 @@ static void bestOrClauseIndex( tempWC.op = TK_AND; tempWC.a = pOrTerm; tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost); + bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); }else{ continue; } @@ -95001,7 +96479,7 @@ static void bestAutomaticIndex( assert( pParse->nQueryLoop >= (double)1 ); pTable = pSrc->pTab; - nTableRow = pTable->pIndex ? pTable->pIndex->aiRowEst[0] : 1000000; + nTableRow = pTable->nRowEst; logN = estLog(nTableRow); costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); if( costTempIdx>=pCost->rCost ){ @@ -95321,7 +96799,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } - sqlite3DbFree(pParse->db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; for(i=0; inConstraint; i++){ @@ -95355,7 +96833,8 @@ static void bestVirtualIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for index */ + Bitmask notValid, /* Cursors not valid for any purpose */ ExprList *pOrderBy, /* The order by clause */ WhereCost *pCost, /* Lowest cost query plan */ sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ @@ -95485,7 +96964,7 @@ static void bestVirtualIndex( /* Try to find a more efficient access pattern by using multiple indexes ** to optimize an OR expression within the WHERE clause. */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -95611,7 +97090,7 @@ static int valueFromExpr( assert( pExpr->op!=TK_VARIABLE ); if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){ int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); return SQLITE_OK; } @@ -95766,7 +97245,8 @@ static void bestBtreeIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ @@ -95808,23 +97288,14 @@ static void bestBtreeIndex( sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; sPk.aiRowEst = aiRowEstPk; - aiRowEstPk[1] = 1; sPk.onError = OE_Replace; sPk.pTable = pSrc->pTab; + aiRowEstPk[0] = pSrc->pTab->nRowEst; + aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ sPk.pNext = pFirst; } - /* The aiRowEstPk[0] is an estimate of the total number of rows in the - ** table. Get this information from the ANALYZE information if it is - ** available. If not available, assume the table 1 million rows in size. - */ - if( pFirst ){ - assert( pFirst->aiRowEst!=0 ); /* Allocated together with pFirst */ - aiRowEstPk[0] = pFirst->aiRowEst[0]; - }else{ - aiRowEstPk[0] = 1000000; - } pProbe = &sPk; wsFlagMask = ~( WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE @@ -96037,16 +97508,16 @@ static void bestBtreeIndex( ** with this step if we already know this index will not be chosen. ** Also, never reduce the output row count below 2 using this step. ** - ** Do not reduce the output row count if pSrc is the only table that - ** is notReady; if notReady is a power of two. This will be the case - ** when the main sqlite3WhereBegin() loop is scanning for a table with - ** and "optimal" index, and on such a scan the output row count - ** reduction is not valid because it does not update the "pCost->used" - ** bitmap. The notReady bitmap will also be a power of two when we - ** are scanning for the last table in a 64-way join. We are willing - ** to bypass this optimization in that corner case. + ** It is critical that the notValid mask be used here instead of + ** the notReady mask. When computing an "optimal" index, the notReady + ** mask will only have one bit set - the bit for the current table. + ** The notValid mask, on the other hand, always has all bits set for + ** tables that are not in outer loops. If notReady is used here instead + ** of notValid, then a optimal index that depends on inner joins loops + ** might be selected even when there exists an optimal index that has + ** no such dependency. */ - if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){ + if( nRow>2 && cost<=pCost->rCost ){ int k; /* Loop counter */ int nSkipEq = nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ @@ -96055,7 +97526,7 @@ static void bestBtreeIndex( thisTab = getMask(pWC->pMaskSet, iCur); for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notReady)!=thisTab ) continue; + if( (pTerm->prereqAll & notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ /* Ignore the first nEq equality matches since the index @@ -96137,7 +97608,7 @@ static void bestBtreeIndex( pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") )); - bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); pCost->plan.wsFlags |= eqTermMask; } @@ -96152,14 +97623,15 @@ static void bestIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pSrc->pTab) ){ sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p); + bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); if( p->needToFreeIdxStr ){ sqlite3_free(p->idxStr); } @@ -96167,7 +97639,7 @@ static void bestIndex( }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); } } @@ -96186,6 +97658,9 @@ static void bestIndex( ** in the ON clause. The term is disabled in (3) because it is not part ** of a LEFT OUTER JOIN. In (1), the term is not disabled. ** +** IMPLEMENTATION-OF: R-24597-58655 No tests are done for terms that are +** completely satisfied by indices. +** ** Disabling a term causes that term to not be tested in the inner loop ** of the join. Disabling is an optimization. When terms are satisfied ** by indices, we disable them to prevent redundant tests in the inner @@ -96397,6 +97872,7 @@ static int codeAllEqualityTerms( /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ @@ -96540,6 +98016,7 @@ static Bitmask codeOneLoopStart( assert( pTerm->pExpr!=0 ); assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); @@ -96580,6 +98057,7 @@ static Bitmask codeOneLoopStart( assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ pX = pStart->pExpr; assert( pX!=0 ); assert( pStart->leftCursor==iCur ); @@ -96597,6 +98075,7 @@ static Bitmask codeOneLoopStart( pX = pEnd->pExpr; assert( pX!=0 ); assert( pEnd->leftCursor==iCur ); + testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ memEndValue = ++pParse->nMem; sqlite3ExprCode(pParse, pX->pRight, memEndValue); if( pX->op==TK_LT || pX->op==TK_GT ){ @@ -96654,7 +98133,7 @@ static Bitmask codeOneLoopStart( ** constraints but an index is selected anyway, in order ** to force the output order to conform to an ORDER BY. */ - int aStartOp[] = { + static const u8 aStartOp[] = { 0, 0, OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ @@ -96664,12 +98143,12 @@ static Bitmask codeOneLoopStart( OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ }; - int aEndOp[] = { + static const u8 aEndOp[] = { OP_Noop, /* 0: (!end_constraints) */ OP_IdxGE, /* 1: (end_constraints && !bRev) */ OP_IdxLT /* 2: (end_constraints && bRev) */ }; - int nEq = pLevel->plan.nEq; + int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ @@ -96679,10 +98158,10 @@ static Bitmask codeOneLoopStart( int endEq; /* True if range end uses ==, >= or <= */ int start_constraints; /* Start of range is constrained */ int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ char *zStartAff; /* Affinity for start of range constraint */ char *zEndAff; /* Affinity for end of range constraint */ @@ -96764,6 +98243,7 @@ static Bitmask codeOneLoopStart( } } nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ }else if( isMinQuery ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); nConstraint++; @@ -96803,6 +98283,7 @@ static Bitmask codeOneLoopStart( } codeApplyAffinity(pParse, regBase, nEq+1, zEndAff); nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ } sqlite3DbFree(pParse->db, zStartAff); sqlite3DbFree(pParse->db, zEndAff); @@ -97009,11 +98490,15 @@ static Bitmask codeOneLoopStart( /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. + ** + ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through + ** the use of indices become tests that are evaluated against each row of + ** the relevant input tables. */ k = 0; for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->prereqAll & notReady)!=0 ){ @@ -97041,7 +98526,7 @@ static Bitmask codeOneLoopStart( VdbeComment((v, "record LEFT JOIN hit")); sqlite3ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->prereqAll & notReady)!=0 ){ @@ -97259,7 +98744,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( initMaskSet(pMaskSet); whereClauseInit(pWC, pParse, pMaskSet); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); + whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -97347,6 +98832,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( int bestJ = -1; /* The value of j */ Bitmask m; /* Bitmask value for j or bestJ */ int isOptimal; /* Iterator for optimal/non-optimal search */ + int nUnconstrained; /* Number tables without INDEXED BY */ + Bitmask notIndexed; /* Mask of tables that cannot use an index */ memset(&bestPlan, 0, sizeof(bestPlan)); bestPlan.rCost = SQLITE_BIG_DBL; @@ -97368,9 +98855,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** other FROM clause terms that are notReady. If no notReady terms are ** used then the "optimal" query plan works. ** + ** Note that the WhereCost.nRow parameter for an optimal scan might + ** not be as small as it would be if the table really were the innermost + ** join. The nRow value can be reduced by WHERE clause constraints + ** that do not use indices. But this nRow reduction only happens if the + ** table really is the innermost join. + ** ** The second loop iteration is only performed if no optimal scan - ** strategies were found by the first loop. This 2nd iteration is used to - ** search for the lowest cost scan overall. + ** strategies were found by the first iteration. This second iteration + ** is used to search for the lowest cost scan overall. ** ** Previous versions of SQLite performed only the second iteration - ** the next outermost loop was always that with the lowest overall @@ -97383,13 +98876,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** ** The best strategy is to iterate through table t1 first. However it ** is not possible to determine this with a simple greedy algorithm. - ** However, since the cost of a linear scan through table t2 is the same + ** Since the cost of a linear scan through table t2 is the same ** as the cost of a linear scan through table t1, a simple greedy ** algorithm may choose to use t2 for the outer loop, which is a much ** costlier approach. */ - for(isOptimal=(iFrom=0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ + nUnconstrained = 0; + notIndexed = 0; + for(isOptimal=(iFrom=0 && bestJ<0; isOptimal--){ + Bitmask mask; /* Mask of tables not yet ready */ for(j=iFrom, pTabItem=&pTabList->a[j]; jpIndex==0 ) nUnconstrained++; assert( pTabItem->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTabItem->pTab) ){ sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost, pp); + bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost, pp); }else #endif { - bestBtreeIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost); + bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); - if( (sCost.used¬Ready)==0 - && (bestJ<0 || sCost.rCostpIndex==0 + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sCost.plan.u.pIdx==pTabItem->pIndex ); + + if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + notIndexed |= m; + } + + /* Conditions under which this table becomes the best so far: + ** + ** (1) The table must not depend on other tables that have not + ** yet run. + ** + ** (2) A full-table-scan plan cannot supercede another plan unless + ** it is an "optimal" plan as defined above. + ** + ** (3) All tables have an INDEXED BY clause or this table lacks an + ** INDEXED BY clause or this table uses the specific + ** index specified by its INDEXED BY clause. This rule ensures + ** that a best-so-far is always selected even if an impossible + ** combination of INDEXED BY clauses are given. The error + ** will be detected and relayed back to the application later. + ** The NEVER() comes about because rule (2) above prevents + ** An indexable full-table-scan from reaching rule (3). + ** + ** (4) The plan cost must be lower than prior plans or else the + ** cost must be the same and the number of rows must be lower. + */ + if( (sCost.used¬Ready)==0 /* (1) */ + && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) + && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ + || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || sCost.rCost0 ){ p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetColl(pParse, p, &yymsp[-1].minor.yy0); + sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); } yygotominor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, p); sqlite3ExprListSetName(pParse,yygotominor.yy322,&yymsp[-2].minor.yy0,1); @@ -100703,7 +102235,7 @@ static void yy_reduce( Expr *p = 0; if( yymsp[-1].minor.yy0.n>0 ){ p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetColl(pParse, p, &yymsp[-1].minor.yy0); + sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); } yygotominor.yy322 = sqlite3ExprListAppend(pParse,0, p); sqlite3ExprListSetName(pParse, yygotominor.yy322, &yymsp[-2].minor.yy0, 1); @@ -101983,7 +103515,7 @@ abort_parse: } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3DbFree(db, pParse->apVtabLock); + sqlite3_free(pParse->apVtabLock); #endif if( !IN_DECLARE_VTAB ){ @@ -101991,7 +103523,7 @@ abort_parse: ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ** will take responsibility for freeing the Table structure. */ - sqlite3DeleteTable(pParse->pNewTable); + sqlite3DeleteTable(db, pParse->pNewTable); } sqlite3DeleteTrigger(db, pParse->pNewTrigger); @@ -102005,7 +103537,7 @@ abort_parse: while( pParse->pZombieTab ){ Table *p = pParse->pZombieTab; pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(p); + sqlite3DeleteTable(db, p); } if( nErr>0 && pParse->rc==SQLITE_OK ){ pParse->rc = SQLITE_ERROR; @@ -102415,15 +103947,33 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); /************** Continuing where we left off in main.c ***********************/ #endif -/* -** The version of the library -*/ #ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; #endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } #if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) @@ -102544,6 +104094,13 @@ SQLITE_API int sqlite3_initialize(void){ ** sqlite3_initialize(). The recursive calls normally come through ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). */ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ @@ -102824,12 +104381,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ sz = 0; pStart = 0; }else if( pBuf==0 ){ - sz = ROUND8(sz); + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); }else{ - sz = ROUNDDOWN8(sz); + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ pStart = pBuf; } db->lookaside.pStart = pStart; @@ -102872,14 +104429,14 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_start(ap, op); switch( op ){ case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); - int sz = va_arg(ap, int); - int cnt = va_arg(ap, int); + void *pBuf = va_arg(ap, void*); /* IMP: R-21112-12275 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ rc = setupLookaside(db, pBuf, sz, cnt); break; } default: { - rc = SQLITE_ERROR; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ break; } } @@ -102984,11 +104541,28 @@ SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ db->isTransactionSavepoint = 0; } +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + /* ** Close an existing SQLite database */ SQLITE_API int sqlite3_close(sqlite3 *db){ - HashElem *i; + HashElem *i; /* Hash table iterator */ int j; if( !db ){ @@ -103056,6 +104630,7 @@ SQLITE_API int sqlite3_close(sqlite3 *db){ for(p=db->aFunc.a[j]; p; p=pHash){ pHash = p->pHash; while( p ){ + functionDestroy(db, p); pNext = p->pNext; sqlite3DbFree(db, p); p = pNext; @@ -103330,7 +104905,8 @@ SQLITE_PRIVATE int sqlite3CreateFunc( void *pUserData, void (*xFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor ){ FuncDef *p; int nName; @@ -103358,10 +104934,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, - pUserData, xFunc, xStep, xFinal); + pUserData, xFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, - pUserData, xFunc, xStep, xFinal); + pUserData, xFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ return rc; @@ -103394,6 +104970,15 @@ SQLITE_PRIVATE int sqlite3CreateFunc( if( !p ){ return SQLITE_NOMEM; } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; p->flags = 0; p->xFunc = xFunc; p->xStep = xStep; @@ -103408,7 +104993,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc( */ SQLITE_API int sqlite3_create_function( sqlite3 *db, - const char *zFunctionName, + const char *zFunc, int nArg, int enc, void *p, @@ -103416,9 +105001,41 @@ SQLITE_API int sqlite3_create_function( void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*) ){ - int rc; + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; sqlite3_mutex_enter(db->mutex); - rc = sqlite3CreateFunc(db, zFunctionName, nArg, enc, p, xFunc, xStep, xFinal); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } + + out: rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -103440,7 +105057,7 @@ SQLITE_API int sqlite3_create_function16( sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -103471,7 +105088,7 @@ SQLITE_API int sqlite3_overload_function( sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0); + 0, sqlite3InvalidFunction, 0, 0, 0); } rc = sqlite3ApiExit(db, SQLITE_OK); sqlite3_mutex_leave(db->mutex); @@ -103609,7 +105226,10 @@ SQLITE_PRIVATE int sqlite3WalDefaultHook( ** configured by this function. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifndef SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -103697,16 +105317,7 @@ SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb){ for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( sqlite3BtreeIsInReadTrans(pBt) ){ - rc = SQLITE_LOCKED; - }else{ - sqlite3BtreeEnter(pBt); - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); - sqlite3BtreeLeave(pBt); - } - } + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt); } } @@ -103748,60 +105359,6 @@ SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ #endif } -/* -** This routine is called to create a connection to a database BTree -** driver. If zFilename is the name of a file, then that file is -** opened and used. If zFilename is the magic name ":memory:" then -** the database is stored in memory (and is thus forgotten as soon as -** the connection is closed.) If zFilename is NULL then the database -** is a "virtual" database for transient use only and is deleted as -** soon as the connection is closed. -** -** A virtual database can be either a disk file (that is automatically -** deleted when the file is closed) or it an be held entirely in memory. -** The sqlite3TempInMemory() function is used to determine which. -*/ -SQLITE_PRIVATE int sqlite3BtreeFactory( - sqlite3 *db, /* Main database when opening aux otherwise 0 */ - const char *zFilename, /* Name of the file containing the BTree database */ - int omitJournal, /* if TRUE then do not journal this file */ - int nCache, /* How many pages in the page cache */ - int vfsFlags, /* Flags passed through to vfsOpen */ - Btree **ppBtree /* Pointer to new Btree object written here */ -){ - int btFlags = 0; - int rc; - - assert( sqlite3_mutex_held(db->mutex) ); - assert( ppBtree != 0); - if( omitJournal ){ - btFlags |= BTREE_OMIT_JOURNAL; - } - if( db->flags & SQLITE_NoReadlock ){ - btFlags |= BTREE_NO_READLOCK; - } -#ifndef SQLITE_OMIT_MEMORYDB - if( zFilename==0 && sqlite3TempInMemory(db) ){ - zFilename = ":memory:"; - } -#endif - - if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (zFilename==0 || *zFilename==0) ){ - vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; - } - rc = sqlite3BtreeOpen(zFilename, (sqlite3 *)db, ppBtree, btFlags, vfsFlags); - - /* If the B-Tree was successfully opened, set the pager-cache size to the - ** default value. Except, if the call to BtreeOpen() returned a handle - ** open on an existing shared pager-cache, do not change the pager-cache - ** size. - */ - if( rc==SQLITE_OK && 0==sqlite3BtreeSchema(*ppBtree, 0, 0) ){ - sqlite3BtreeSetCacheSize(*ppBtree, nCache); - } - return rc; -} - /* ** Return UTF-8 encoded English language explanation of the most recent ** error. @@ -104044,17 +105601,39 @@ static const int aHardLimit[] = { */ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; + + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ return -1; } oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ } db->aLimit[limitId] = newLimit; } - return oldLimit; + return oldLimit; /* IMP: R-53341-35419 */ } /* @@ -104078,6 +105657,24 @@ static int openDatabase( if( rc ) return rc; #endif + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; }else if( flags & SQLITE_OPEN_NOMUTEX ){ @@ -104111,7 +105708,8 @@ static int openDatabase( SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_MASTER_JOURNAL | SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL ); /* Allocate the sqlite data structure */ @@ -104183,9 +105781,8 @@ static int openDatabase( /* Open the backend database driver */ db->openFlags = flags; - rc = sqlite3BtreeFactory(db, zFilename, 0, SQLITE_DEFAULT_CACHE_SIZE, - flags | SQLITE_OPEN_MAIN_DB, - &db->aDb[0].pBt); + rc = sqlite3BtreeOpen(zFilename, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM; @@ -104892,6 +106489,22 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -105155,7 +106768,7 @@ SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3DbMallocSize(db, aDyn)/sizeof(void*))) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) ){ /* The aArg[] array needs to grow. */ void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); @@ -106080,6 +107693,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -108029,6 +109643,9 @@ static int fts3FilterMethod( return rc; } + rc = sqlite3Fts3ReadLock(p); + if( rc!=SQLITE_OK ) return rc; + rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; @@ -108407,11 +110024,14 @@ static int fts3RenameMethod( const char *zName /* New name of table */ ){ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db; /* Database connection */ + sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ - - db = p->db; - rc = SQLITE_OK; + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ){ + return rc; + } + fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", p->zDb, p->zName, zName @@ -108479,7 +110099,7 @@ static void hashDestroy(void *p){ ** used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point a the "simple" tokenizer implementation. +** to by the argument to point to the "simple" tokenizer implementation. ** Function ...PorterTokenizerModule() sets *pModule to point to the ** porter tokenizer/stemmer implementation. */ @@ -108681,7 +110301,7 @@ struct ParseContext { ** negative values). */ static int fts3isspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } /* @@ -111068,6 +112688,9 @@ typedef struct simple_tokenizer_cursor { static int simpleDelim(simple_tokenizer *t, unsigned char c){ return c<0x80 && t->delim[c]; } +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} /* ** Create a new tokenizer instance. @@ -111102,7 +112725,7 @@ static int simpleCreate( /* Mark non-alphanumeric ASCII characters as delimiters */ int i; for(i=1; i<0x80; i++){ - t->delim[i] = !isalnum(i) ? -1 : 0; + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; } } @@ -111208,7 +112831,7 @@ static int simpleNext( ** case-insensitivity. */ unsigned char ch = p[iStartOffset+i]; - c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch); + c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); } *ppToken = c->pToken; *pnBytes = n; @@ -111568,6 +113191,36 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( return SQLITE_OK; } +/* +** This function ensures that the caller has obtained a shared-cache +** table-lock on the %_content table. This is required before reading +** data from the fts3 table. If this lock is not acquired first, then +** the caller may end up holding read-locks on the %_segments and %_segdir +** tables, but no read-lock on the %_content table. If this happens +** a second connection will be able to write to the fts3 table, but +** attempting to commit those writes might return SQLITE_LOCKED or +** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain +** write-locks on the %_segments and %_segdir ** tables). +** +** We try to avoid this because if FTS3 returns any error when committing +** a transaction, the whole transaction will be rolled back. And this is +** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can +** still happen if the user reads data directly from the %_segments or +** %_segdir tables instead of going through FTS3 though. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ + int rc; /* Return code */ + sqlite3_stmt *pStmt; /* Statement used to obtain lock */ + + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; +} + /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -115059,6 +116712,45 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor * ** algorithms packaged as an SQLite virtual table module. */ +/* +** Database Format of R-Tree Tables +** -------------------------------- +** +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. +** +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) +** +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. +** +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: +** +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. +** +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. +** +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. +*/ + #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) /* @@ -115099,6 +116791,9 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor * #define AssignCells splitNodeStartree #endif +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 @@ -115107,6 +116802,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor * #ifndef SQLITE_AMALGAMATION +#include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; typedef unsigned int u32; @@ -115117,6 +116813,8 @@ typedef struct RtreeCursor RtreeCursor; typedef struct RtreeNode RtreeNode; typedef struct RtreeCell RtreeCell; typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ @@ -115186,6 +116884,15 @@ struct Rtree { #define RTREE_REINSERT(p) RTREE_MINCELLS(p) #define RTREE_MAXCELLS 51 +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 + /* ** An rtree cursor object. */ @@ -115218,35 +116925,23 @@ union RtreeCoord { ** A search constraint. */ struct RtreeConstraint { - int iCoord; /* Index of constrained coordinate */ - int op; /* Constraining operation */ - double rValue; /* Constraint value. */ + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + double rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 +#define RTREE_EQ 0x41 +#define RTREE_LE 0x42 +#define RTREE_LT 0x43 +#define RTREE_GE 0x44 +#define RTREE_GT 0x45 +#define RTREE_MATCH 0x46 /* ** An rtree structure node. -** -** Data format (RtreeNode.zData): -** -** 1. If the node is the root node (node 1), then the first 2 bytes -** of the node contain the tree depth as a big-endian integer. -** For non-root nodes, the first 2 bytes are left unused. -** -** 2. The next 2 bytes contain the number of entries currently -** stored in the node. -** -** 3. The remainder of the node contains the node entries. Each entry -** consists of a single 8-byte integer followed by an even number -** of 4-byte coordinates. For leaf nodes the integer is the rowid -** of a record. For internal nodes it is the node number of a -** child page. */ struct RtreeNode { RtreeNode *pParent; /* Parent node */ @@ -115266,6 +116961,40 @@ struct RtreeCell { RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; }; + +/* +** Value for the first field of every RtreeMatchArg object. The MATCH +** operator tests that the first field of a blob operand matches this +** value to avoid operating on invalid blobs (which could cause a segfault). +*/ +#define RTREE_GEOMETRY_MAGIC 0x891245AB + +/* +** An instance of this structure must be supplied as a blob argument to +** the right-hand-side of an SQL MATCH operator used to constrain an +** r-tree query. +*/ +struct RtreeMatchArg { + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + void *pContext; + int nParam; + double aParam[1]; +}; + +/* +** When a geometry callback is created (see sqlite3_rtree_geometry_callback), +** a single instance of the following structure is allocated. It is used +** as the context for the user-function created by by s_r_g_c(). The object +** is eventually deleted by the destructor mechanism provided by +** sqlite3_create_function_v2() (which is called by s_r_g_c() to create +** the geometry callback function). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + void *pContext; +}; + #ifndef MAX # define MAX(x,y) ((x) < (y) ? (y) : (x)) #endif @@ -115348,10 +117077,8 @@ static void nodeReference(RtreeNode *p){ ** Clear the content of node p (set all bytes to 0x00). */ static void nodeZero(Rtree *pRtree, RtreeNode *p){ - if( p ){ - memset(&p->zData[2], 0, pRtree->iNodeSize-2); - p->isDirty = 1; - } + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; } /* @@ -115371,7 +117098,6 @@ static int nodeHash(i64 iNode){ */ static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ RtreeNode *p; - assert( iNode!=0 ); for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); return p; } @@ -115380,13 +117106,11 @@ static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ ** Add node pNode to the node hash table. */ static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ - if( pNode ){ - int iHash; - assert( pNode->pNext==0 ); - iHash = nodeHash(pNode->iNode); - pNode->pNext = pRtree->aHash[iHash]; - pRtree->aHash[iHash] = pNode; - } + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; } /* @@ -115408,11 +117132,11 @@ static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ ** assigned a node number when nodeWrite() is called to write the ** node contents out to the database. */ -static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent, int zero){ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ RtreeNode *pNode; pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); if( pNode ){ - memset(pNode, 0, sizeof(RtreeNode) + (zero?pRtree->iNodeSize:0)); + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); pNode->zData = (u8 *)&pNode[1]; pNode->nRef = 1; pNode->pParent = pParent; @@ -115433,6 +117157,7 @@ nodeAcquire( RtreeNode **ppNode /* OUT: Acquired node */ ){ int rc; + int rc2 = SQLITE_OK; RtreeNode *pNode; /* Check if the requested node is already in the hash table. If so, @@ -115449,40 +117174,64 @@ nodeAcquire( return SQLITE_OK; } - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); - if( !pNode ){ - *ppNode = 0; - return SQLITE_NOMEM; - } - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); rc = sqlite3_step(pRtree->pReadNode); if( rc==SQLITE_ROW ){ const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - assert( sqlite3_column_bytes(pRtree->pReadNode, 0)==pRtree->iNodeSize ); - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); + if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc2 = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + memcpy(pNode->zData, zBlob, pRtree->iNodeSize); + nodeReference(pParent); + } + } + } + rc = sqlite3_reset(pRtree->pReadNode); + if( rc==SQLITE_OK ) rc = rc2; + + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT; + } + } + + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT; + } + } + + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT; + } + *ppNode = pNode; }else{ sqlite3_free(pNode); - pNode = 0; + *ppNode = 0; } - *ppNode = pNode; - rc = sqlite3_reset(pRtree->pReadNode); - - if( rc==SQLITE_OK && iNode==1 ){ - pRtree->iDepth = readInt16(pNode->zData); - } - - assert( (rc==SQLITE_OK && pNode) || (pNode==0 && rc!=SQLITE_OK) ); - nodeHashInsert(pRtree, pNode); - return rc; } @@ -115534,8 +117283,7 @@ nodeInsertCell( nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; nCell = NCELL(pNode); - assert(nCell<=nMaxCell); - + assert( nCell<=nMaxCell ); if( nCellzData[2], nCell+1); @@ -115755,6 +117503,25 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return rc; } + +/* +** Free the RtreeCursor.aConstraint[] array and its contents. +*/ +static void freeCursorConstraints(RtreeCursor *pCsr){ + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; + if( pGeom ){ + if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); + sqlite3_free(pGeom); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } +} + /* ** Rtree virtual table module xClose method. */ @@ -115762,7 +117529,7 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); int rc; RtreeCursor *pCsr = (RtreeCursor *)cur; - sqlite3_free(pCsr->aConstraint); + freeCursorConstraints(pCsr); rc = nodeRelease(pRtree, pCsr->pNode); sqlite3_free(pCsr); return rc; @@ -115779,13 +117546,39 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ return (pCsr->pNode==0); } +/* +** The r-tree constraint passed as the second argument to this function is +** guaranteed to be a MATCH constraint. +*/ +static int testRtreeGeom( + Rtree *pRtree, /* R-Tree object */ + RtreeConstraint *pConstraint, /* MATCH constraint to test */ + RtreeCell *pCell, /* Cell to test */ + int *pbRes /* OUT: Test result */ +){ + int i; + double aCoord[RTREE_MAX_DIMENSIONS*2]; + int nCoord = pRtree->nDim*2; + + assert( pConstraint->op==RTREE_MATCH ); + assert( pConstraint->pGeom ); + + for(i=0; iaCoord[i]); + } + return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); +} + /* ** Cursor pCursor currently points to a cell in a non-leaf page. -** Return true if the sub-tree headed by the cell is filtered +** Set *pbEof to true if the sub-tree headed by the cell is filtered ** (excluded) by the constraints in the pCursor->aConstraint[] ** array, or false otherwise. +** +** Return SQLITE_OK if successful or an SQLite error code if an error +** occurs within a geometry callback. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){ +static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; int bRes = 0; @@ -115797,31 +117590,55 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){ double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH ); switch( p->op ){ - case RTREE_LE: case RTREE_LT: bRes = p->rValuerValue>cell_max; break; - case RTREE_EQ: + case RTREE_LE: case RTREE_LT: + bRes = p->rValuerValue>cell_max; + break; + + case RTREE_EQ: bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); + rc = testRtreeGeom(pRtree, p, &cell, &bRes); + if( rc!=SQLITE_OK ){ + return rc; + } + bRes = !bRes; + break; + } } } - return bRes; + *pbEof = bRes; + return SQLITE_OK; } /* -** Return true if the cell that cursor pCursor currently points to +** Test if the cell that cursor pCursor currently points to ** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array, or false otherwise. +** pCursor->aConstraint[] array. If so, set *pbEof to true before +** returning. If the cell is not filtered (excluded) by the constraints, +** set pbEof to zero. +** +** Return SQLITE_OK if successful or an SQLite error code if an error +** occurs within a geometry callback. ** ** This function assumes that the cell is part of a leaf node. */ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ +static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; + *pbEof = 0; nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; iinConstraint; ii++){ @@ -115829,7 +117646,7 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ double coord = DCOORD(cell.aCoord[p->iCoord]); int res; assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH ); switch( p->op ){ case RTREE_LE: res = (coord<=p->rValue); break; @@ -115837,12 +117654,24 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ case RTREE_GE: res = (coord>=p->rValue); break; case RTREE_GT: res = (coord>p->rValue); break; case RTREE_EQ: res = (coord==p->rValue); break; + default: { + int rc; + assert( p->op==RTREE_MATCH ); + rc = testRtreeGeom(pRtree, p, &cell, &res); + if( rc!=SQLITE_OK ){ + return rc; + } + break; + } } - if( !res ) return 1; + if( !res ){ + *pbEof = 1; + return SQLITE_OK; + } } - return 0; + return SQLITE_OK; } /* @@ -115869,13 +117698,13 @@ static int descendToCell( assert( iHeight>=0 ); if( iHeight==0 ){ - isEof = testRtreeEntry(pRtree, pCursor); + rc = testRtreeEntry(pRtree, pCursor, &isEof); }else{ - isEof = testRtreeCell(pRtree, pCursor); + rc = testRtreeCell(pRtree, pCursor, &isEof); } - if( isEof || iHeight==0 ){ + if( rc!=SQLITE_OK || isEof || iHeight==0 ){ *pEof = isEof; - return SQLITE_OK; + return rc; } iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); @@ -115911,24 +117740,34 @@ static int descendToCell( ** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. */ -static int nodeRowidIndex(Rtree *pRtree, RtreeNode *pNode, i64 iRowid){ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ int ii; - for(ii=0; nodeGetRowid(pRtree, pNode, ii)!=iRowid; ii++){ - assert( ii<(NCELL(pNode)-1) ); + int nCell = NCELL(pNode); + for(ii=0; iipParent; if( pParent ){ - return nodeRowidIndex(pRtree, pParent, pNode->iNode); + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); } - return -1; + *piIndex = -1; + return SQLITE_OK; } /* @@ -115939,13 +117778,17 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; int rc = SQLITE_OK; + /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is + ** already at EOF. It is against the rules to call the xNext() method of + ** a cursor that has already reached EOF. + */ + assert( pCsr->pNode ); + if( pCsr->iStrategy==1 ){ /* This "scan" is a direct lookup by rowid. There is no next entry. */ nodeRelease(pRtree, pCsr->pNode); pCsr->pNode = 0; - } - - else if( pCsr->pNode ){ + }else{ /* Move to the next entry that matches the configured constraints. */ int iHeight = 0; while( pCsr->pNode ){ @@ -115959,7 +117802,10 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ } } pCsr->pNode = pNode->pParent; - pCsr->iCell = nodeParentIndex(pRtree, pNode); + rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); + if( rc!=SQLITE_OK ){ + return rc; + } nodeReference(pCsr->pNode); nodeRelease(pRtree, pNode); iHeight++; @@ -116027,6 +117873,51 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ return rc; } +/* +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. +*/ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *p; + sqlite3_rtree_geometry *pGeom; + int nBlob; + + /* Check that value is actually a blob. */ + if( !sqlite3_value_type(pValue)==SQLITE_BLOB ) return SQLITE_ERROR; + + /* Check that the blob is roughly the right size. */ + nBlob = sqlite3_value_bytes(pValue); + if( nBlobmagic!=RTREE_GEOMETRY_MAGIC + || nBlob!=(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) + ){ + sqlite3_free(pGeom); + return SQLITE_ERROR; + } + + pGeom->pContext = p->pContext; + pGeom->nParam = p->nParam; + pGeom->aParam = p->aParam; + + pCons->xGeom = p->xGeom; + pCons->pGeom = pGeom; + return SQLITE_OK; +} /* ** Rtree virtual table module xFilter method. @@ -116045,8 +117936,7 @@ static int rtreeFilter( rtreeReference(pRtree); - sqlite3_free(pCsr->aConstraint); - pCsr->aConstraint = 0; + freeCursorConstraints(pCsr); pCsr->iStrategy = idxNum; if( idxNum==1 ){ @@ -116055,8 +117945,9 @@ static int rtreeFilter( i64 iRowid = sqlite3_value_int64(argv[0]); rc = findLeafNode(pRtree, iRowid, &pLeaf); pCsr->pNode = pLeaf; - if( pLeaf && rc==SQLITE_OK ){ - pCsr->iCell = nodeRowidIndex(pRtree, pLeaf, iRowid); + if( pLeaf ){ + assert( rc==SQLITE_OK ); + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array @@ -116068,12 +117959,24 @@ static int rtreeFilter( if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); assert( (idxStr==0 && argc==0) || strlen(idxStr)==argc*2 ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; p->iCoord = idxStr[ii*2+1]-'a'; - p->rValue = sqlite3_value_double(argv[ii]); + if( p->op==RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + }else{ + p->rValue = sqlite3_value_double(argv[ii]); + } } } } @@ -116114,11 +118017,10 @@ static int rtreeFilter( ** idxNum idxStr Strategy ** ------------------------------------------------ ** 1 Unused Direct lookup by rowid. -** 2 See below R-tree query. -** 3 Unused Full table scan. +** 2 See below R-tree query or full-table scan. ** ------------------------------------------------ ** -** If strategy 1 or 3 is used, then idxStr is not meaningful. If strategy +** If strategy 1 is used, then idxStr is not meaningful. If strategy ** 2 is used, idxStr is formatted to contain 2 bytes for each ** constraint used. The first two bytes of idxStr correspond to ** the constraint in sqlite3_index_info.aConstraintUsage[] with @@ -116134,6 +118036,7 @@ static int rtreeFilter( ** < 0x43 ('C') ** >= 0x44 ('D') ** > 0x45 ('E') +** MATCH 0x46 ('F') ** ---------------------- ** ** The second of each pair of bytes identifies the coordinate column @@ -116172,7 +118075,9 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ return SQLITE_OK; } - if( p->usable && p->iColumn>0 ){ + if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + int j, opmsk; + static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; u8 op = 0; switch( p->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; @@ -116180,31 +118085,33 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + default: + assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); + op = RTREE_MATCH; + break; } - if( op ){ - /* Make sure this particular constraint has not been used before. - ** If it has been used before, ignore it. - ** - ** A <= or < can be used if there is a prior >= or >. - ** A >= or > can be used if there is a prior < or <=. - ** A <= or < is disqualified if there is a prior <=, <, or ==. - ** A >= or > is disqualified if there is a prior >=, >, or ==. - ** A == is disqualifed if there is any prior constraint. - */ - int j, opmsk; - static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; - assert( compatible[RTREE_EQ & 7]==0 ); - assert( compatible[RTREE_LT & 7]==1 ); - assert( compatible[RTREE_LE & 7]==1 ); - assert( compatible[RTREE_GT & 7]==2 ); - assert( compatible[RTREE_GE & 7]==2 ); - cCol = p->iColumn - 1 + 'a'; - opmsk = compatible[op & 7]; - for(j=0; j= or >. + ** A >= or > can be used if there is a prior < or <=. + ** A <= or < is disqualified if there is a prior <=, <, or ==. + ** A >= or > is disqualified if there is a prior >=, >, or ==. + ** A == is disqualifed if there is any prior constraint. + */ + assert( compatible[RTREE_EQ & 7]==0 ); + assert( compatible[RTREE_LT & 7]==1 ); + assert( compatible[RTREE_LE & 7]==1 ); + assert( compatible[RTREE_GT & 7]==2 ); + assert( compatible[RTREE_GE & 7]==2 ); + cCol = p->iColumn - 1 + 'a'; + opmsk = compatible[op & 7]; + for(j=0; jnDim*2); jj+=2){ @@ -116405,22 +118317,31 @@ static int ChooseLeaf( ** the smallest area. */ for(iCell=0; iCelliDepth-1) ){ overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); } -#endif if( (iCell==0) || (overlappParent ){ - RtreeCell cell; RtreeNode *pParent = p->pParent; - int iCell = nodeParentIndex(pRtree, p); + RtreeCell cell; + int iCell; + + if( nodeParentIndex(pRtree, p, &iCell) ){ + return SQLITE_CORRUPT; + } nodeGetCell(pRtree, pParent, iCell, &cell); if( !cellContains(pRtree, &cell, pCell) ){ @@ -116462,6 +118387,7 @@ static void AdjustTree( p = pParent; } + return SQLITE_OK; } /* @@ -116990,14 +118916,14 @@ static int SplitNode( nCell++; if( pNode->iNode==1 ){ - pRight = nodeNew(pRtree, pNode, 1); - pLeft = nodeNew(pRtree, pNode, 1); + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); pRtree->iDepth++; pNode->isDirty = 1; writeInt16(pNode->zData, pRtree->iDepth); }else{ pLeft = pNode; - pRight = nodeNew(pRtree, pLeft->pParent, 1); + pRight = nodeNew(pRtree, pLeft->pParent); nodeReference(pLeft); } @@ -117014,8 +118940,12 @@ static int SplitNode( goto splitnode_out; } - /* Ensure both child nodes have node numbers assigned to them. */ - if( (0==pRight->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pRight))) + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) ){ goto splitnode_out; @@ -117031,9 +118961,15 @@ static int SplitNode( } }else{ RtreeNode *pParent = pLeft->pParent; - int iCell = nodeParentIndex(pRtree, pLeft); - nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); - AdjustTree(pRtree, pParent, &leftbbox); + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } } if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ goto splitnode_out; @@ -117077,20 +119013,43 @@ splitnode_out: return rc; } +/* +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. +*/ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ int rc = SQLITE_OK; - if( pLeaf->iNode!=1 && pLeaf->pParent==0 ){ - sqlite3_bind_int64(pRtree->pReadParent, 1, pLeaf->iNode); - if( sqlite3_step(pRtree->pReadParent)==SQLITE_ROW ){ - i64 iNode = sqlite3_column_int64(pRtree->pReadParent, 0); - rc = nodeAcquire(pRtree, iNode, 0, &pLeaf->pParent); - }else{ - rc = SQLITE_ERROR; - } - sqlite3_reset(pRtree->pReadParent); - if( rc==SQLITE_OK ){ - rc = fixLeafParent(pRtree, pLeaf->pParent); + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT; + pChild = pChild->pParent; } return rc; } @@ -117099,18 +119058,24 @@ static int deleteCell(Rtree *, RtreeNode *, int, int); static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ int rc; + int rc2; RtreeNode *pParent; int iCell; assert( pNode->nRef==1 ); /* Remove the entry in the parent cell. */ - iCell = nodeParentIndex(pRtree, pNode); - pParent = pNode->pParent; - pNode->pParent = 0; - if( SQLITE_OK!=(rc = deleteCell(pRtree, pParent, iCell, iHeight+1)) - || SQLITE_OK!=(rc = nodeRelease(pRtree, pParent)) - ){ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ return rc; } @@ -117140,8 +119105,9 @@ static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ return SQLITE_OK; } -static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; if( pParent ){ int ii; int nCell = NCELL(pNode); @@ -117153,10 +119119,13 @@ static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ cellUnion(pRtree, &box, &cell); } box.iRowid = pNode->iNode; - ii = nodeParentIndex(pRtree, pNode); - nodeOverwriteCell(pRtree, pParent, &box, ii); - fixBoundingBox(pRtree, pParent); + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } } + return rc; } /* @@ -117164,6 +119133,7 @@ static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ ** cell, adjust the r-tree data structure if required. */ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; int rc; if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ @@ -117180,14 +119150,13 @@ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ ** cell in the parent node so that it tightly contains the updated ** node. */ - if( pNode->iNode!=1 ){ - RtreeNode *pParent = pNode->pParent; - if( (pParent->iNode!=1 || NCELL(pParent)!=1) - && (NCELL(pNode)pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)iNode currently contains @@ -117324,11 +119293,13 @@ static int rtreeInsertCell( rc = SplitNode(pRtree, pNode, pCell, iHeight); #endif }else{ - AdjustTree(pRtree, pNode, pCell); - if( iHeight==0 ){ - rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + rc = AdjustTree(pRtree, pNode, pCell); + if( rc==SQLITE_OK ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } } } return rc; @@ -117398,7 +119369,6 @@ static int rtreeUpdate( rtreeReference(pRtree); assert(nData>=1); - assert(hashIsEmpty(pRtree)); /* If azData[0] is not an SQL NULL value, it is the rowid of a ** record to delete from the r-tree table. The following block does @@ -117424,8 +119394,10 @@ static int rtreeUpdate( /* Delete the cell in question from the leaf node. */ if( rc==SQLITE_OK ){ int rc2; - iCell = nodeRowidIndex(pRtree, pLeaf, iDelete); - rc = deleteCell(pRtree, pLeaf, iCell, 0); + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ rc = rc2; @@ -117447,19 +119419,20 @@ static int rtreeUpdate( ** the root node (the operation that Gutman's paper says to perform ** in this scenario). */ - if( rc==SQLITE_OK && pRtree->iDepth>0 ){ - if( rc==SQLITE_OK && NCELL(pRoot)==1 ){ - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; } } @@ -117749,7 +119722,7 @@ static int rtreeInit( Rtree *pRtree; int nDb; /* Length of string argv[1] */ int nName; /* Length of string argv[2] */ - int eCoordType = (int)pAux; + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); const char *aErrMsg[] = { 0, /* 0 */ @@ -117895,12 +119868,10 @@ static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ ** function "rtreenode". */ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ - int rc = SQLITE_OK; + const int utf8 = SQLITE_UTF8; + int rc; - if( rc==SQLITE_OK ){ - int utf8 = SQLITE_UTF8; - rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); - } + rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); if( rc==SQLITE_OK ){ int utf8 = SQLITE_UTF8; rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); @@ -117917,6 +119888,70 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ return rc; } +/* +** A version of sqlite3_free() that can be used as a callback. This is used +** in two places - as the destructor for the blob value returned by the +** invocation of a geometry function, and as the destructor for the geometry +** functions themselves. +*/ +static void doSqlite3Free(void *p){ + sqlite3_free(p); +} + +/* +** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite +** scalar user function. This C function is the callback used for all such +** registered SQL functions. +** +** The scalar user functions return a blob that is interpreted by r-tree +** table MATCH operators. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + int nBlob; + + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double); + pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->xGeom = pGeomCtx->xGeom; + pBlob->pContext = pGeomCtx->pContext; + pBlob->nParam = nArg; + for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); + } + sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + } +} + +/* +** Register a new geometry function for use with the r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *), + void *pContext +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->pContext = pContext; + + /* Create the new user-function. Register a destructor function to delete + ** the context object when it is no longer required. */ + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + ); +} + #if !SQLITE_CORE SQLITE_API int sqlite3_extension_init( sqlite3 *db, diff --git a/ext/sqlite3/libsqlite/sqlite3.h b/ext/sqlite3/libsqlite/sqlite3.h index 3d02c7cca68..d78ae39b880 100644 --- a/ext/sqlite3/libsqlite/sqlite3.h +++ b/ext/sqlite3/libsqlite/sqlite3.h @@ -97,7 +97,7 @@ extern "C" { ** ** Since version 3.6.18, SQLite source code has been stored in the ** Fossil configuration management -** system. ^The SQLITE_SOURCE_ID macro evalutes to +** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID ** string contains the date and time of the check-in (UTC) and an SHA1 @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.0.1" -#define SQLITE_VERSION_NUMBER 3007000 -#define SQLITE_SOURCE_ID "2010-08-04 12:31:11 042a1abb030a0711386add7eb6e10832cc8b0f57" +#define SQLITE_VERSION "3.7.3" +#define SQLITE_VERSION_NUMBER 3007003 +#define SQLITE_SOURCE_ID "2010-10-08 02:34:02 2677848087c9c090efb17c1893e77d6136a9111d" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -154,7 +154,7 @@ SQLITE_API int sqlite3_libversion_number(void); ** compile time. ^The SQLITE_ prefix may be omitted from the ** option name passed to sqlite3_compileoption_used(). ** -** ^The sqlite3_compileoption_get() function allows interating +** ^The sqlite3_compileoption_get() function allows iterating ** over the list of options that were defined at compile time by ** returning the N-th compile time option string. ^If N is out of range, ** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ @@ -162,7 +162,7 @@ SQLITE_API int sqlite3_libversion_number(void); ** sqlite3_compileoption_get(). ** ** ^Support for the diagnostic functions sqlite3_compileoption_used() -** and sqlite3_compileoption_get() may be omitted by specifing the +** and sqlite3_compileoption_get() may be omitted by specifying the ** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. ** ** See also: SQL functions [sqlite_compileoption_used()] and @@ -268,7 +268,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** ** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. ** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfullly destroyed and all associated resources are deallocated. +** successfully destroyed and all associated resources are deallocated. ** ** Applications must [sqlite3_finalize | finalize] all [prepared statements] ** and [sqlite3_blob_close | close] all [BLOB handles] associated with @@ -695,12 +695,21 @@ struct sqlite3_io_methods { ** is often close. The underlying VFS might choose to preallocate database ** file space based on this hint in order to help writes to the database ** file run faster. +** +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 /* ** CAPI3REF: Mutex Handle @@ -748,15 +757,19 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** -** SQLite will guarantee that the zFilename parameter to xOpen +** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained -** from xFullPathname(). SQLite further guarantees that +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 10 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. -** If the zFilename parameter is xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. Whenever the +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** @@ -767,7 +780,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** -** SQLite will also add one of the following flags to the xOpen() +** ^(SQLite will also add one of the following flags to the xOpen() ** call, depending on the object being opened: ** **
      @@ -778,7 +791,8 @@ typedef struct sqlite3_mutex sqlite3_mutex; **
    • [SQLITE_OPEN_TRANSIENT_DB] **
    • [SQLITE_OPEN_SUBJOURNAL] **
    • [SQLITE_OPEN_MASTER_JOURNAL] -**
    +**
  • [SQLITE_OPEN_WAL] +** )^ ** ** The file I/O implementation can use the object type flags to ** change the way it deals with files. For example, an application @@ -797,10 +811,11 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** ** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be -** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] -** will be set for TEMP databases, journals and for subjournals. +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. ** -** The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the @@ -809,7 +824,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** It is not used to indicate the file should be opened ** for exclusive access. ** -** At least szOsFile bytes of memory are allocated by SQLite +** ^At least szOsFile bytes of memory are allocated by SQLite ** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that @@ -819,13 +834,13 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** -** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] ** to test whether a file is at least readable. The file can be a ** directory. ** -** SQLite will always allocate at least mxPathname+1 bytes for the +** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer ** is also passed as a parameter to both methods. If the output buffer ** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is @@ -839,10 +854,10 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** of good-quality randomness into zOut. The return value is ** the actual number of bytes of randomness obtained. ** The xSleep() method causes the calling thread to sleep for at -** least the number of microseconds given. The xCurrentTime() +** least the number of microseconds given. ^The xCurrentTime() ** method returns a Julian Day Number for the current date and time as ** a floating point value. -** The xCurrentTimeInt64() method returns, as an integer, the Julian +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian ** Day Number multipled by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current @@ -1239,7 +1254,7 @@ struct sqlite3_mem_methods { ** )^ ** ^Memory allocation statistics are enabled by default unless SQLite is @@ -1253,15 +1268,14 @@ struct sqlite3_mem_methods { ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. The sz parameter should be a few bytes -** larger than the actual scratch space required due to internal overhead. +** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than one scratch buffer per thread. So -** N should be set to the expected maximum number of threads. ^SQLite will -** never require a scratch buffer that is more than 6 times the database -** page size. ^If SQLite needs needs additional scratch memory beyond -** what is provided by this configuration option, then +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed. ** **
    SQLITE_CONFIG_PAGECACHE
    @@ -1281,8 +1295,7 @@ struct sqlite3_mem_methods { ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** ^The implementation might use one or more of the N buffers to hold -** memory accounting information. The pointer in the first argument must +** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined. ** @@ -1411,8 +1424,14 @@ struct sqlite3_mem_methods { ** or equal to the product of the second and third arguments. The buffer ** must be aligned to an 8-byte boundary. ^If the second argument to ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller -** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE] +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^ ** ** */ @@ -1717,6 +1736,9 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries ** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. @@ -1737,7 +1759,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** It is not safe to pass a result table directly to [sqlite3_free()]. ** A result table should be deallocated using [sqlite3_free_table()]. ** -** As an example of the result table format, suppose a query result +** ^(As an example of the result table format, suppose a query result ** is as follows: ** **
    @@ -1761,7 +1783,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms);
     **        azResult[5] = "28";
     **        azResult[6] = "Cindy";
     **        azResult[7] = "21";
    -** 
    +** )^ ** ** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 @@ -1769,19 +1791,19 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** pointer given in its 3rd parameter. ** ** After the application has finished with the result from sqlite3_get_table(), -** it should pass the result table pointer to sqlite3_free_table() in order to +** it must pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** ^(The sqlite3_get_table() interface is implemented as a wrapper around +** The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not ** reflected in subsequent calls to [sqlite3_errcode()] or -** [sqlite3_errmsg()].)^ +** [sqlite3_errmsg()]. */ SQLITE_API int sqlite3_get_table( sqlite3 *db, /* An open database */ @@ -1933,7 +1955,9 @@ SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); ** is not freed. ** ** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() -** is always aligned to at least an 8 byte boundary. +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. ** ** In SQLite version 3.5.0 and 3.5.1, it was possible to define ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in @@ -2176,7 +2200,13 @@ SQLITE_API int sqlite3_set_authorizer( ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time -** of how long that statement took to run. +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. */ SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, @@ -2185,17 +2215,28 @@ SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, /* ** CAPI3REF: Query Progress Callbacks ** -** ^This routine configures a callback function - the -** progress callback - that is invoked periodically during long -** running calls to [sqlite3_exec()], [sqlite3_step()] and -** [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** ** ^If the progress callback returns non-zero, the operation is ** interrupted. This feature can be used to implement a ** "Cancel" button on a GUI progress dialog box. ** -** The progress handler must not do anything that will modify +** The progress handler callback must not do anything that will modify ** the database connection that invoked the progress handler. ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. @@ -2254,7 +2295,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** If the 3rd parameter to sqlite3_open_v2() is not one of the ** combinations shown above or one of the combinations shown above combined ** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags, +** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2379,17 +2420,22 @@ typedef struct sqlite3_stmt sqlite3_stmt; ** [database connection] whose limit is to be set or queried. The ** second parameter is one of the [limit categories] that define a ** class of constructs to be size limited. The third parameter is the -** new limit for that construct. The function returns the old limit.)^ +** new limit for that construct.)^ ** ** ^If the new limit is a negative number, the limit is unchanged. -** ^(For the limit category of SQLITE_LIMIT_XYZ there is a +** ^(For each limit category SQLITE_LIMIT_NAME there is a ** [limits | hard upper bound] -** set by a compile-time C preprocessor macro named -** [limits | SQLITE_MAX_XYZ]. +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. ** (The "_LIMIT_" in the name is changed to "_MAX_".))^ ** ^Attempts to increase a limit above its hard upper bound are ** silently truncated to the hard upper bound. ** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** ** Run-time limits are intended for use in applications that manage ** both their own internal database and also databases that are controlled ** by untrusted external sources. An example application might be a @@ -2418,7 +2464,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** **
    ** ^(
    SQLITE_LIMIT_LENGTH
    -**
    The maximum size of any string or BLOB or table row.
    )^ +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_SQL_LENGTH
    **
    The maximum length of an SQL statement, in bytes.
    )^ @@ -2436,7 +2482,9 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement.
    )^ +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.)^ ** ** ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ @@ -2449,8 +2497,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** [GLOB] operators.)^ ** ** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    -**
    The maximum number of variables in an SQL statement that can -** be bound.
    )^ +**
    The maximum index number of any [parameter] in an SQL statement.)^ ** ** ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ @@ -2522,12 +2569,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. ^If the schema has changed in -** a way that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text -** of the parsing error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. **
  • ** **
  • @@ -2540,11 +2582,16 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** **
  • -** ^If the value of a [parameter | host parameter] in the WHERE clause might -** change the query plan for a statement, then the statement may be -** automatically recompiled (as if there had been a schema change) on the first -** [sqlite3_step()] call following any change to the -** [sqlite3_bind_text | bindings] of the [parameter]. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** the **
  • ** */ @@ -2611,7 +2658,7 @@ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); ** then there is no distinction between protected and unprotected ** sqlite3_value objects and they can be used interchangeably. However, ** for maximum code portability it is recommended that applications -** still make the distinction between between protected and unprotected +** still make the distinction between protected and unprotected ** sqlite3_value objects even when not strictly required. ** ** ^The sqlite3_value objects that are passed as parameters into the @@ -2657,7 +2704,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** ** In the templates above, NNN represents an integer literal, -** and VVV represents an alphanumeric identifer.)^ ^The values of these +** and VVV represents an alphanumeric identifier.)^ ^The values of these ** parameters (also called "host parameter names" or "SQL parameters") ** can be set using the sqlite3_bind_*() routines defined here. ** @@ -2806,6 +2853,8 @@ SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL ** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); @@ -2996,8 +3045,14 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set ** -** ^The sqlite3_data_count(P) the number of columns in the -** of the result set of [prepared statement] P. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** +** See also: [sqlite3_column_count()] */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); @@ -3077,18 +3132,26 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** ^The value returned does not include the zero terminator at the end -** of the string. ^For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero terminated. ^The return -** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary -** pointer, possibly even a NULL pointer. -** -** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 in native byte order instead of UTF-8. -** ^The zero terminator is not included in this count. +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. An unprotected sqlite3_value object @@ -3133,10 +3196,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** ^Note that when type conversions occur, pointers returned by prior +** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** ^(Type conversions and pointer invalidations might occur +** Type conversions and pointer invalidations might occur ** in the following cases: ** ** )^ +** ** ** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds +** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** ^(The safest and easiest to remember policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** ** )^ +** ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result @@ -3202,17 +3265,26 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. ^If execution of the statement failed then an -** [error code] or [extended error code] is returned. +** ^If the most recent evaluation of the statement encountered no errors or +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** ^This routine can be called at any point during the execution of the -** [prepared statement]. ^If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an [sqlite3_interrupt | interrupt]. -** ^Incomplete updates may be rolled back and transactions canceled, -** depending on the circumstances, and the -** [error code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); @@ -3248,23 +3320,25 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** ^These two functions (collectively known as "function creation routines") +** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only difference between the -** two is that the second parameter, the name of the (scalar) function or -** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 -** for sqlite3_create_function16(). +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database ** connection then application-defined SQL functions must be added ** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created or -** redefined. ^The length of the name is limited to 255 bytes, exclusive of -** the zero-terminator. Note that the name length limit is in bytes, not -** characters. ^Any attempt to create a function with a longer name -** will result in [SQLITE_ERROR] being returned. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** ** ^The third parameter (nArg) ** is the number of arguments that the SQL function or @@ -3274,10 +3348,10 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** parameter is less than -1 or greater than 127 then the behavior is ** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be +** its parameters. Every SQL function implementation must be able to work +** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. @@ -3289,13 +3363,21 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc -** callback only; NULL pointers should be passed as the xStep and xFinal +** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL for all three function callbacks. +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL poiners for all three function +** callbacks. +** +** ^If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is invoked when the function is deleted, either by being +** overloaded or when the database connection closes. +** ^When the destructure callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the pointer which was +** the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -3311,11 +3393,6 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** between UTF8 and UTF16. ** ** ^Built-in functions may be overloaded by new application-defined functions. -** ^The first application-defined function with a given name overrides all -** built-in functions in the same [database connection] with the same name. -** ^Subsequent application-defined functions of the same name only override -** prior application-defined functions that are an exact match for the -** number of parameters and preferred encoding. ** ** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not @@ -3342,6 +3419,17 @@ SQLITE_API int sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -3436,7 +3524,7 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context ** -** Implementions of aggregate SQL functions use this +** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. ** ** ^The first time the sqlite3_aggregate_context(C,N) routine is called @@ -3688,46 +3776,70 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences ** -** These functions are used to add new collation sequences to the -** [database connection] specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** ^The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases -** the name is passed as the second function argument. +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. ** -** ^The third argument may be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The -** third argument might also be [SQLITE_UTF16] to indicate that the routine -** expects pointers to be UTF-16 strings in the native byte order, or the -** argument can be [SQLITE_UTF16_ALIGNED] if the -** the routine expects pointers to 16-bit word aligned strings -** of UTF-16 in the native byte order. +** ^(The third argument (eTextRep) must be one of the constants: +** )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. ** -** A pointer to the user supplied routine must be passed as the fifth -** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). -** ^Each time the application supplied function is invoked, it is passed -** as its first parameter a copy of the void* passed as the fourth argument -** to sqlite3_create_collation() or sqlite3_create_collation16(). +** ^The fourth argument, pArg, is a application data pointer that is passed +** through as the first argument to the collating function callback. ** -** ^The remaining arguments to the application-supplied routine are two strings, -** each represented by a (length, data) pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The application defined collation routine should -** return negative, zero or positive if the first string is less than, -** equal to, or greater than the second string. i.e. (STRING1 - STRING2). +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must alway return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** except that it takes an extra argument which is a destructor for -** the collation. ^The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). -** ^Collations are destroyed when they are overridden by later calls to the -** collation creation functions or when the [database connection] is closed -** using [sqlite3_close()]. +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ @@ -3735,14 +3847,14 @@ SQLITE_API int sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); SQLITE_API int sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); @@ -3750,7 +3862,7 @@ SQLITE_API int sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); @@ -3839,16 +3951,19 @@ SQLITE_API void sqlite3_activate_cerod( /* ** CAPI3REF: Suspend Execution For A Short Time ** -** ^The sqlite3_sleep() function causes the current thread to suspend execution +** The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** ^If the operating system does not support sleep requests with +** If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. ^The number of milliseconds of sleep actually +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. ** ** ^SQLite implements this interface by calling the xSleep() -** method of the default [sqlite3_vfs] object. +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. */ SQLITE_API int sqlite3_sleep(int); @@ -4070,40 +4185,73 @@ SQLITE_API int sqlite3_enable_shared_cache(int); ** pages to improve performance is an example of non-essential memory. ** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. */ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Impose A Limit On Heap Size ** -** ^The sqlite3_soft_heap_limit() interface places a "soft" limit -** on the amount of heap memory that may be allocated by SQLite. -** ^If an internal allocation is requested that would exceed the -** soft heap limit, [sqlite3_release_memory()] is invoked one or -** more times to free up some space before the allocation is performed. +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. ** -** ^The limit is called "soft" because if [sqlite3_release_memory()] -** cannot free sufficient memory to prevent the limit from being exceeded, -** the memory is allocated anyway and the current operation proceeds. +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. ** -** ^A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhausted. -** ^The default value for the soft heap limit is zero. +** ^If the argument N is zero then the soft heap limit is disabled. ** -** ^(SQLite makes a best effort to honor the soft heap limit. -** But if the soft heap limit cannot be honored, execution will -** continue without error or notification.)^ This is why the limit is -** called a "soft" limit. It is advisory only. +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: ** -** Prior to SQLite version 3.5.0, this routine only constrained the memory -** allocated by a single thread - the same thread in which this routine -** runs. Beginning with SQLite version 3.5.0, the soft heap limit is -** applied to all threads. The value specified for the soft heap limit -** is an upper bound on the total memory allocation for all threads. In -** version 3.5.0 there is no mechanism for limiting the heap usage for -** individual threads. +** )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. */ -SQLITE_API void sqlite3_soft_heap_limit(int); +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + /* ** CAPI3REF: Extract Metadata About A Column Of A Table @@ -4227,34 +4375,47 @@ SQLITE_API int sqlite3_load_extension( SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions +** CAPI3REF: Automatically Load Statically Linked Extensions ** -** ^This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new [database connections]. +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked SQLite extension +** that is to be automatically loaded into all new database connections. ** -** ^(This routine stores a pointer to the extension entry point -** in an array that is obtained from [sqlite3_malloc()]. That memory -** is deallocated by [sqlite3_reset_auto_extension()].)^ +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: ** -** ^This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** ^Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** ^Automatic extensions apply across all threads. +**
    +**    int xEntryPoint(
    +**      sqlite3 *db,
    +**      const char **pzErrMsg,
    +**      const struct sqlite3_api_routines *pThunk
    +**    );
    +** 
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()]. */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** ^(This function disables all previously registered automatic -** extensions. It undoes the effect of all prior -** [sqlite3_auto_extension()] calls.)^ -** -** ^This function disables automatic extensions in all threads. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ SQLITE_API void sqlite3_reset_auto_extension(void); @@ -4325,7 +4486,8 @@ struct sqlite3_module { ** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info ** -** The sqlite3_index_info structure and its substructures is used to +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to ** pass information into and receive the reply from the [xBestIndex] ** method of a [virtual table module]. The fields under **Inputs** are the ** inputs to xBestIndex and are read-only. xBestIndex inserts its @@ -4333,10 +4495,12 @@ struct sqlite3_module { ** ** ^(The aConstraint[] array records WHERE clause constraints of the form: ** -**
    column OP expr
    +**
    column OP expr
    ** ** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is -** stored in aConstraint[].op.)^ ^(The index of the column is stored in +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in ** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint ** is usable) and false if it cannot.)^ @@ -4396,6 +4560,15 @@ struct sqlite3_index_info { int orderByConsumed; /* True if output is already ordered */ double estimatedCost; /* Estimated cost of using this index */ }; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 #define SQLITE_INDEX_CONSTRAINT_LE 8 @@ -4881,7 +5054,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** ** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** ^The xMutexInit routine is calle by SQLite exactly once for each +** ^The xMutexInit routine is called by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** ** ^The xMutexEnd method defined by this structure is invoked as @@ -4914,7 +5087,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** it is passed a NULL pointer). ** ** The xMutexInit() method must be threadsafe. ^It must be harmless to -** invoke xMutexInit() mutiple times within the same process and without +** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** @@ -5078,13 +5251,14 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_LAST 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 +#define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information -** about the preformance of SQLite, and optionally to reset various +** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes ** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ @@ -5097,7 +5271,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** ^The sqlite3_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be @@ -5136,6 +5310,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.)^ ** +** ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations.
    )^ +** ** ^(
    SQLITE_STATUS_PAGECACHE_USED
    **
    This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using @@ -5144,7 +5321,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache -** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to @@ -5167,7 +5344,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** ** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory -** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values ** returned include overflows because the requested allocation was too ** larger (that is, because the requested allocation was larger than the @@ -5197,6 +5374,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 #define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status @@ -5206,7 +5384,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that -** determiness the parameter to interrogate. The set of +** determines the parameter to interrogate. The set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely ** to grow in future releases of SQLite. ** @@ -5215,6 +5393,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. ** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); @@ -5236,16 +5417,33 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r **
    This parameter returns the number of lookaside memory slots currently ** checked out.
    )^ ** -**
    SQLITE_DBSTATUS_CACHE_USED
    -**
    ^This parameter returns the approximate number of of bytes of heap -** memory used by all pager caches associated with the database connection. +** ^(
    SQLITE_DBSTATUS_CACHE_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** ^(
    SQLITE_DBSTATUS_STMT_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
    **
    */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 #define SQLITE_DBSTATUS_CACHE_USED 1 -#define SQLITE_DBSTATUS_MAX 1 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */ /* @@ -5324,32 +5522,42 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ The majority of the -** heap memory used by SQLite is used by the page cache to cache data read -** from, or ready to be written to, the database file. By implementing a -** custom page cache using this API, an application can control more -** precisely the amount of memory consumed by SQLite, the way in which +** instance of the sqlite3_pcache_methods structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which ** that memory is allocated and released, and the policies used to ** determine exactly which parts of a database file are cached and for ** how long. ** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** ** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** -** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ -** ^The xInit() method can set up up global structures and/or any mutexes +** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ ** -** ^The xShutdown() method is called from within [sqlite3_shutdown()], -** if the application invokes this API. It can be used to clean up +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up ** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. ** -** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. ^The +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. @@ -5357,47 +5565,50 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** ^The xCreate() method is used to construct a new cache instance. SQLite -** will typically create one cache instance for each open database file, +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. ^SQLite will use the +** increment (here called "R") of about 100 or 200. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. ** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. ^The cache implementation +** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** ^In other words, a cache created with bPurgeable set to false will +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** ^The xPagecount() method should return the number of pages currently -** stored in the cache. +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. ** -** ^The xFetch() method is used to fetch a page and return a pointer to it. -** ^A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. ^The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** The xFetch() method locates a page in the cache and returns a pointer to +** the page, or a NULL pointer. +** A "page", in this context, means a buffer of szPage bytes aligned at an +** 8-byte boundary. The page to be fetched is determined by the key. ^The +** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** ^If the requested page is already in the page cache, then the page cache +** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. ^(If the requested page is not already in the cache, then the -** behavior of the cache implementation is determined by the value of the -** createFlag parameter passed to xFetch, according to the following table: +** intact. If the requested page is not already in the cache, then the +** behavior of the cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: ** ** **
    createFlag Behaviour when page is not already in cache @@ -5406,36 +5617,35 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** Otherwise return NULL. **
    2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -**
    )^ +** ** -** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If -** a call to xFetch() with createFlag==1 returns NULL, then SQLite will +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of -** pinned pages to disk and synching the operating system disk cache. After -** attempting to unpin pages, the xFetch() method will be invoked again with -** a createFlag of 2. +** pinned pages to disk and synching the operating system disk cache. ** ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. ^(If the third parameter, discard, is non-zero, -** then the page should be evicted from the cache. In this case SQLite -** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is -** zero, then the page is considered to be unpinned. ^The cache implementation +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** ^(The cache is not required to perform any reference counting. A single +** The cache must not perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch().)^ +** to xFetch(). ** -** ^The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. ^If the cache -** previously contains an entry associated with newKey, it should be +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be ** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** ^When SQLite calls the xTruncate() method, the cache must discard all +** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). ^If any +** to the value of the iLimit parameter passed to xTruncate(). If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** @@ -5611,7 +5821,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source databae file. +** up and the total number of pages in the source database file. ** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces ** retrieve these two values, respectively. ** @@ -5707,7 +5917,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is cancelled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -5789,7 +5999,7 @@ SQLITE_API int sqlite3_unlock_notify( ** ** ^The [sqlite3_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a -** case-indendent fashion, using the same definition of case independence +** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); @@ -5913,3 +6123,59 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); #endif #endif +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + double *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ +