diff options
492 files changed, 23999 insertions, 6200 deletions
diff --git a/Makefile.in b/Makefile.in index cb894666d..b8149f846 100644 --- a/Makefile.in +++ b/Makefile.in @@ -117,13 +117,14 @@ HAVE_TCL = @HAVE_TCL@ # TCLSH_CMD = @TCLSH_CMD@ -# Where do we want to install the tcl plugin +# Additional options when running tests using testrunner.tcl +# This is usually either blank, or else --status # -TCLLIBDIR = @TCLLIBDIR@ +TSTRNNR_OPTS = @TSTRNNR_OPTS@ -# The suffix used on shared libraries. Ex: ".dll", ".so", ".dylib" +# Where do we want to install the tcl plugin # -SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ +TCLLIBDIR = @TCLLIBDIR@ # If gcov support was enabled by the configure script, add the appropriate # flags here. It's not always as easy as just having the user add the right @@ -418,6 +419,8 @@ TESTSRC = \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/test_recover.c \ + $(TOP)/ext/intck/test_intck.c \ + $(TOP)/ext/intck/sqlite3intck.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions @@ -452,6 +455,7 @@ TESTSRC += \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ + $(TOP)/ext/misc/stmtrand.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c \ @@ -797,6 +801,10 @@ has_tclsh85: sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD) touch has_tclsh85 +has_tclconfig: + @ if test x"$(HAVE_TCL)" != "x1"; then echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi + touch has_tclconfig + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to @@ -815,7 +823,7 @@ has_tclsh85: touch .target_source sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84 - $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) + $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC) cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . @@ -826,7 +834,7 @@ sqlite3r.c: sqlite3.c sqlite3r.h has_tclsh84 cp $(TOP)/ext/recover/sqlite3recover.c tsrc/ cp $(TOP)/ext/recover/sqlite3recover.h tsrc/ cp $(TOP)/ext/recover/dbdata.c tsrc/ - $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) + $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC) sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . @@ -1119,7 +1127,7 @@ tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c -tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la +tclsqlite3$(TEXE): has_tclconfig tclsqlite-shell.lo libsqlite3.la $(LTLINK) -o $@ tclsqlite-shell.lo \ libsqlite3.la $(LIBTCL) @@ -1152,35 +1160,37 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c ./mkkeywordhash$(BEXE) >keywordhash.h -# Source files that go into making shell.c -SHELL_SRC = \ - $(TOP)/src/shell.c.in \ - $(TOP)/ext/consio/console_io.c \ - $(TOP)/ext/consio/console_io.h \ - $(TOP)/ext/misc/appendvfs.c \ - $(TOP)/ext/misc/completion.c \ - $(TOP)/ext/misc/decimal.c \ - $(TOP)/ext/misc/basexx.c \ - $(TOP)/ext/misc/base64.c \ - $(TOP)/ext/misc/base85.c \ - $(TOP)/ext/misc/fileio.c \ - $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/regexp.c \ - $(TOP)/ext/misc/series.c \ - $(TOP)/ext/misc/shathree.c \ - $(TOP)/ext/misc/sqlar.c \ - $(TOP)/ext/misc/uint.c \ - $(TOP)/ext/expert/sqlite3expert.c \ - $(TOP)/ext/expert/sqlite3expert.h \ - $(TOP)/ext/misc/zipfile.c \ - $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/misc/pcachetrace.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/sqlite3recover.h \ - $(TOP)/src/test_windirent.c - -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84 +# Source and header files that shell.c depends on +SHELL_DEP = \ + $(TOP)/src/shell.c.in \ + $(TOP)/ext/consio/console_io.c \ + $(TOP)/ext/consio/console_io.h \ + $(TOP)/ext/expert/sqlite3expert.c \ + $(TOP)/ext/expert/sqlite3expert.h \ + $(TOP)/ext/intck/sqlite3intck.c \ + $(TOP)/ext/intck/sqlite3intck.h \ + $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ + $(TOP)/ext/misc/completion.c \ + $(TOP)/ext/misc/decimal.c \ + $(TOP)/ext/misc/fileio.c \ + $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/pcachetrace.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/shathree.c \ + $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/zipfile.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ + $(TOP)/src/test_windirent.c \ + $(TOP)/src/test_windirent.h + +shell.c: $(SHELL_DEP) $(TOP)/tool/mkshellc.tcl has_tclsh84 $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c @@ -1304,13 +1314,13 @@ TESTFIXTURE_SRC1 = sqlite3.c TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) -testfixture$(TEXE): has_tclsh85 $(TESTFIXTURE_SRC) +testfixture$(TEXE): has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) -coretestprogs: $(TESTPROGS) +coretestprogs: testfixture$(BEXE) sqlite3$(BEXE) -testprogs: coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE) +testprogs: $(TESTPROGS) srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE) # A very detailed test running most or all test cases fulltest: alltest fuzztest @@ -1358,10 +1368,10 @@ testrunner: testfixture$(TEXE) devtest: srctree-check testfixture$(TEXE) fuzztest testrunner mdevtest: srctree-check has_tclsh85 - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) sdevtest: has_tclsh85 - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) # Validate that various generated files in the source tree # are up-to-date. @@ -1372,7 +1382,7 @@ srctree-check: $(TOP)/tool/srctree-check.tcl # Testing for a release # releasetest: srctree-check testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release + ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) # Minimal testing that runs in less than 3 minutes # @@ -1403,13 +1413,13 @@ shelltest: $(TESTPROGS) sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c -sqlite3_analyzer$(TEXE): sqlite3_analyzer.c +sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c -sqltclsh$(TEXE): sqltclsh.c +sqltclsh$(TEXE): has_tclconfig sqltclsh.c $(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS) sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c @@ -1428,7 +1438,7 @@ CHECKER_DEPS =\ sqlite3_checker.c: $(CHECKER_DEPS) has_tclsh85 $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ -sqlite3_checker$(TEXE): sqlite3_checker.c +sqlite3_checker$(TEXE): has_tclconfig sqlite3_checker.c $(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS) dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo @@ -1553,12 +1563,37 @@ install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_instal pkgIndex.tcl: echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@ + tcl_install: lib_install libtclsqlite3.la pkgIndex.tcl $(INSTALL) -d $(DESTDIR)$(TCLLIBDIR) $(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR) rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a $(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR) +# Build the SQLite TCL extension in a way that make it compatible +# with whatever version of TCL is running as $TCLSH_CMD, possibly defined +# by --with-tclsh= +# +tclextension: tclsqlite3.c + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --build-only --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) + +# Install the SQLite TCL extension in a way that is appropriate for $TCLSH_CMD +# to find it. +# +tclextension-install: tclsqlite3.c + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) + +# Install the SQLite TCL extension that is used by $TCLSH_CMD +# +tclextension-uninstall: + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --uninstall + +# List all installed the SQLite TCL extension that is are accessible +# by $TCLSH_CMD, included prior versions. +# +tclextension-list: + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --info + clean: rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la rm -f sqlite3.h opcodes.* diff --git a/Makefile.msc b/Makefile.msc index 19bfe2f38..e31aee089 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -18,6 +18,13 @@ USE_AMALGAMATION = 1 !ENDIF # <</mark>> +# Optionally set EXTRA_SRC to a list of C files to append to +# the generated sqlite3.c. +# +!IFNDEF EXTRA_SRC +EXTRA_SRC = +!ENDIF + # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN @@ -1589,12 +1596,15 @@ TESTEXT = \ $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ + $(TOP)\ext\misc\stmtrand.c \ $(TOP)\ext\misc\totype.c \ $(TOP)\ext\misc\unionvtab.c \ $(TOP)\ext\misc\wholenumber.c \ $(TOP)\ext\rtree\test_rtreedoc.c \ $(TOP)\ext\recover\sqlite3recover.c \ $(TOP)\ext\recover\test_recover.c \ + $(TOP)\ext\intck\test_intck.c \ + $(TOP)\ext\intck\sqlite3intck.c \ $(TOP)\ext\recover\dbdata.c # If use of zlib is enabled, add the "zipfile.c" source file. @@ -1810,6 +1820,19 @@ pkgIndex.tcl: $(TOP)\VERSION $(SQLITE3TCLDLL): libtclsqlite3.lib $(LIBRESOBJS) tclsqlite3.def pkgIndex.tcl $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:tclsqlite3.def /OUT:$@ libtclsqlite3.lib $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + +tclextension: $(SQLITE3TCLDLL) + +tclextension-install: $(SQLITE3TCLDLL) + $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --install-only + +tclextension-uninstall: + $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --uninstall + +tclextension-list: + $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --info + + # <</mark>> $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) @@ -1913,7 +1936,7 @@ mptest: mptester.exe echo > .target_source sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-verify.exe - $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) + $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) $(EXTRA_SRC) sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl @@ -2263,39 +2286,44 @@ mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h -# Source files that go into making shell.c -SHELL_SRC = \ - $(TOP)\src\shell.c.in \ - $(TOP)\ext\consio\console_io.c \ - $(TOP)\ext\consio\console_io.h \ - $(TOP)\ext\misc\appendvfs.c \ - $(TOP)\ext\misc\completion.c \ - $(TOP)\ext\misc\base64.c \ - $(TOP)\ext\misc\base85.c \ - $(TOP)\ext\misc\decimal.c \ - $(TOP)\ext\misc\fileio.c \ - $(TOP)\ext\misc\ieee754.c \ - $(TOP)\ext\misc\regexp.c \ - $(TOP)\ext\misc\series.c \ - $(TOP)\ext\misc\shathree.c \ - $(TOP)\ext\misc\uint.c \ - $(TOP)\ext\expert\sqlite3expert.c \ - $(TOP)\ext\expert\sqlite3expert.h \ - $(TOP)\ext\misc\memtrace.c \ - $(TOP)\ext\misc\pcachetrace.c \ - $(TOP)\ext\recover\dbdata.c \ - $(TOP)\ext\recover\sqlite3recover.c \ - $(TOP)\ext\recover\sqlite3recover.h \ - $(TOP)\src\test_windirent.c +# Source and header files that shell.c depends on +SHELL_DEP = \ + $(TOP)\src\shell.c.in \ + $(TOP)\ext\consio\console_io.c \ + $(TOP)\ext\consio\console_io.h \ + $(TOP)\ext\expert\sqlite3expert.c \ + $(TOP)\ext\expert\sqlite3expert.h \ + $(TOP)\ext\intck\sqlite3intck.c \ + $(TOP)\ext\intck\sqlite3intck.h \ + $(TOP)\ext\misc\appendvfs.c \ + $(TOP)\ext\misc\base64.c \ + $(TOP)\ext\misc\base85.c \ + $(TOP)\ext\misc\completion.c \ + $(TOP)\ext\misc\decimal.c \ + $(TOP)\ext\misc\fileio.c \ + $(TOP)\ext\misc\ieee754.c \ + $(TOP)\ext\misc\memtrace.c \ + $(TOP)\ext\misc\pcachetrace.c \ + $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\series.c \ + $(TOP)\ext\misc\shathree.c \ + $(TOP)\ext\misc\sqlar.c \ + $(TOP)\ext\misc\uint.c \ + $(TOP)\ext\misc\zipfile.c \ + $(TOP)\ext\recover\dbdata.c \ + $(TOP)\ext\recover\sqlite3recover.c \ + $(TOP)\ext\recover\sqlite3recover.h \ + $(TOP)\src\test_windirent.c \ + $(TOP)\src\test_windirent.h # If use of zlib is enabled, add the "zipfile.c" source file. # !IF $(USE_ZLIB)!=0 -SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c -SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c +SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\sqlar.c +SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\zipfile.c !ENDIF -shell.c: $(SHELL_SRC) $(TOP)\tool\mkshellc.tcl +shell.c: $(SHELL_DEP) $(TOP)\tool\mkshellc.tcl $(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c zlib: @@ -2482,9 +2510,9 @@ extensiontest: testfixture.exe testloadext.dll tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe $(TOP)\tool\mktoolzip.tcl .\testfixture.exe $(TOP)\tool\mktoolzip.tcl -coretestprogs: $(TESTPROGS) +coretestprogs: testfixture.exe sqlite3.exe -testprogs: coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe +testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe fulltest: alltest fuzztest @@ -2539,7 +2567,7 @@ mdevtest: # Testing for a release # -releasetest: testfixture.exe fuzztest +releasetest: testfixture.exe testfixture.exe $(TOP)\test\testrunner.tcl release @@ -1,14 +1,19 @@ <h1 align="center">SQLite Source Repository</h1> This repository contains the complete source code for the -[SQLite database engine](https://sqlite.org/). Some test scripts -are also included. However, many other test scripts +[SQLite database engine](https://sqlite.org/), including +many test scripts. However, other test scripts and most of the documentation are managed separately. +See the [on-line documentation](https://sqlite.org/) for more information +about what SQLite is and how it works from a user's perspective. This +README file is about the source code that goes into building SQLite, +not about how SQLite is used. + ## Version Control SQLite sources are managed using -[Fossil](https://www.fossil-scm.org/), a distributed version control system +[Fossil](https://fossil-scm.org/), a distributed version control system that was specifically designed and written to support SQLite development. The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext. @@ -68,12 +73,11 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: then click on the "Tarball" or "ZIP Archive" links on the information page. -If you do want to use Fossil to check out the source tree, -first install Fossil version 2.0 or later. -(Source tarballs and precompiled binaries available -[here](https://www.fossil-scm.org/fossil/uv/download.html). Fossil is +To access sources directly using Fossil, first install Fossil version 2.0 or later. +Source tarballs and precompiled binaries available +[here](https://www.fossil-scm.org/home/uv/download.html). Fossil is a stand-alone program. To install, simply download or build the single -executable file and put that file someplace on your $PATH.) +executable file and put that file someplace on your $PATH. Then run commands like this: mkdir -p ~/sqlite ~/Fossils @@ -81,8 +85,8 @@ Then run commands like this: fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil fossil open ~/Fossils/sqlite.fossil -After setting up a repository using the steps above, you can always -update to the latest version using: +After setting up a repository using the steps above, you can do +bandwidth-efficient updates to the latest version using: fossil update trunk ;# latest trunk check-in fossil update release ;# latest official release @@ -99,15 +103,30 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Builds the "sqlite3" command-line tool - make sqlite3.c ;# Build the "amalgamation" source file - make devtest ;# Run some tests (requires Tcl) + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make sqlite3 ;# Builds the "sqlite3" command-line tool + make sqlite3.c ;# Build the "amalgamation" source file + make mdevtest ;# Run development tests (requires tcl-dev) + make releasetest ;# Run full release tests (requires tcl-dev) + make sqldiff ;# Builds the "sqldiff" command-line tool + make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool (requires tcl-dev) + make tclextension-install ;# Build and install the SQLite TCL extension + +See the makefile for additional targets. For debugging builds, the +core developers typically run "configure" with options like this: + + ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g' + +For release builds, the core developers usually do: -See the makefile for additional targets. + ../sqlite/configure --enable-all + +Almost all makefile targets require a "tclsh" TCL interpreter +version 8.6 or later. The targets marked with "(requires tcl-dev)" also require +the TCL development libraries. The configure script uses autoconf 2.61 and libtool. If the configure script does not work out for you, there is a generic makefile named @@ -128,48 +147,61 @@ TCL library, using a command like this: set TCLDIR=c:\Tcl -SQLite uses "tclsh.exe" as part of the build process, and so that utility -program will need to be somewhere on your %PATH%. The finished SQLite library -does not contain any TCL code, but it does use TCL to help with the build process -and to run tests. +SQLite uses "tclsh.exe" as part of the build process, and so that +program will need to be somewhere on your %PATH%. SQLite itself +does not contain any TCL code, but it does use TCL to help with the +build process and to run tests. Build using Makefile.msc. Example: - nmake /f Makefile.msc + nmake /f Makefile.msc sqlite3.exe nmake /f Makefile.msc sqlite3.c - nmake /f Makefile.msc devtest + nmake /f Makefile.msc mdevtest nmake /f Makefile.msc releasetest + nmake /f Makefile.msc tclextension-install There are many other makefile targets. See comments in Makefile.msc for details. -## Source Code Tour - -Most of the core source files are in the **src/** subdirectory. The -**src/** folder also contains files used to build the "testfixture" test -harness. The names of the source files used by "testfixture" all begin -with "test". -The **src/** also contains the "shell.c" file -which is the main program for the "sqlite3.exe" -[command-line shell](https://sqlite.org/cli.html) and -the "tclsqlite.c" file which implements the -[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite. -(Historical note: SQLite began as a Tcl -extension and only later escaped to the wild as an independent library.) - -Test scripts and programs are found in the **test/** subdirectory. -Additional test code is found in other source repositories. -See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for -additional information. - -The **ext/** subdirectory contains code for extensions. The -Full-text search engine is in **ext/fts3**. The R-Tree engine is in -**ext/rtree**. The **ext/misc** subdirectory contains a number of -smaller, single-file extensions, such as a REGEXP operator. - -The **tool/** subdirectory contains various scripts and programs used -for building generated source code files or for testing or for generating -accessory programs such as "sqlite3_analyzer(.exe)". +## Source Tree Map + + * **src/** - This directory contains the primary source code for the + SQLite core. For historical reasons, C-code used for testing is + also found here. Source files intended for testing begin with "`test`". + The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface + for SQLite and are also not part of the core. + + * **test/** - This directory and its subdirectories contains code used + for testing. Files that end in "`.test`" are TCL scripts that run + tests using an augmented TCL interpreter named "testfixture". Use + a command like "`make testfixture`" (unix) or + "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that + augmented TCL interpreter, then run individual tests using commands like + "`testfixture test/main.test`". This test/ subdirectory also contains + additional C code modules and scripts for other kinds of testing. + + * **tool/** - This directory contains programs and scripts used to + build some of the machine-generated code that goes into the SQLite + core, as well as to build and run tests and perform diagnostics. + The source code to [the Lemon parser generator](./doc/lemon.html) is + found here. There are also TCL scripts used to build and/or transform + source code files. For example, the tool/mksqlite3h.tcl script reads + the src/sqlite.h.in file and uses it as a template to construct + the deliverable "sqlite3.h" file that defines the SQLite interface. + + * **ext/** - Various extensions to SQLite are found under this + directory. For example, the FTS5 subsystem is in "ext/fts5/". + Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built + into the SQLite amalgamation, but not all of them. The + "ext/misc/" subdirectory contains an assortment of one-file extensions, + many of which are omitted from the SQLite core, but which are included + in the [SQLite CLI](https://sqlite.org/cli.html). + + * **doc/** - Some documentation files about SQLite internals are found + here. Note, however, that the primary documentation designed for + application developers and users of SQLite is in a completely separate + repository. Note also that the primary API documentation is derived + from specially constructed comments in the src/sqlite.h.in file. ### Generated Source Code Files @@ -252,31 +284,37 @@ individual source file exceeds 32K lines in length. SQLite is modular in design. See the [architectural description](https://www.sqlite.org/arch.html) for details. Other documents that are useful in -(helping to understand how SQLite works include the +helping to understand how SQLite works include the [file format](https://www.sqlite.org/fileformat2.html) description, the [virtual machine](https://www.sqlite.org/opcode.html) that runs prepared statements, the description of [how transactions work](https://www.sqlite.org/atomiccommit.html), and the [overview of the query planner](https://www.sqlite.org/optoverview.html). -Years of effort have gone into optimizing SQLite, both +Decades of effort have gone into optimizing SQLite, both for small size and high performance. And optimizations tend to result in complex code. So there is a lot of complexity in the current SQLite implementation. It will not be the easiest library in the world to hack. -Key files: +### Key source code files * **sqlite.h.in** - This file defines the public interface to the SQLite library. Readers will need to be familiar with this interface before - trying to understand how the library works internally. + trying to understand how the library works internally. This file is + really a template that is transformed into the "sqlite3.h" deliverable + using a script invoked by the makefile. * **sqliteInt.h** - this header file defines many of the data objects used internally by SQLite. In addition to "sqliteInt.h", some - subsystems have their own header files. + subsystems inside of sQLite have their own header files. These internal + interfaces are not for use by applications. They can and do change + from one release of SQLite to the next. * **parse.y** - This file describes the LALR(1) grammar that SQLite uses to parse SQL statements, and the actions that are taken at each step - in the parsing process. + in the parsing process. The file is processed by the + [Lemon Parser Generator](./doc/lemon.html) to produce the actual C code + used for parsing. * **vdbe.c** - This file implements the virtual machine that runs prepared statements. There are various helper files whose names @@ -319,6 +357,11 @@ Key files: (and some other test programs too) is built and run when you type "make test". + * **VERSION**, **manifest**, and **manifest.uuid** - These files define + the current SQLite version number. The "VERSION" file is human generated, + but the "manifest" and "manifest.uuid" files are automatically generated + by the [Fossil version control system](https://fossil-scm/). + There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. @@ -336,7 +379,7 @@ The `manifest.uuid` file should contain the SHA3-256 hash of the `manifest` file. If all of the above hash comparisons are correct, then you can be confident that your source tree is authentic and unadulterated. Details on the format for the `manifest` files are available -[on the Fossil website](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest). +[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest). The process of checking source code authenticity is automated by the makefile: @@ -357,3 +400,7 @@ The main SQLite website is [https://sqlite.org/](https://sqlite.org/) with geographically distributed backups at [https://www2.sqlite.org/](https://www2.sqlite.org) and [https://www3.sqlite.org/](https://www3.sqlite.org). + +Contact the SQLite developers through the +[SQLite Forum](https://sqlite.org/forum/). In an emergency, you +can send private email to the lead developer at drh at sqlite dot org. @@ -1 +1 @@ -3.45.0 +3.47.0 diff --git a/art/icon-243x273.gif b/art/icon-243x273.gif Binary files differnew file mode 100644 index 000000000..e1cdfd0b5 --- /dev/null +++ b/art/icon-243x273.gif diff --git a/art/icon-80x90.gif b/art/icon-80x90.gif Binary files differnew file mode 100644 index 000000000..ebb239000 --- /dev/null +++ b/art/icon-80x90.gif diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 45a07a9f3..a4270fb2a 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -18,6 +18,13 @@ TOP = . +# Optionally set EXTRA_SRC to a list of C files to append to +# the generated sqlite3.c. +# +!IFNDEF EXTRA_SRC +EXTRA_SRC = +!ENDIF + # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index 4df57344b..4fe1a8396 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -19,7 +19,7 @@ dnl to configure the system for the local environment. # so that we create the export library with the dll. #----------------------------------------------------------------------- -AC_INIT([sqlite],[3.45.0]) +AC_INIT([sqlite],[3.47.0]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. diff --git a/autoconf/tea/tclconfig/tcl.m4 b/autoconf/tea/tclconfig/tcl.m4 index c83d660e1..455a7964d 100644 --- a/autoconf/tea/tclconfig/tcl.m4 +++ b/autoconf/tea/tclconfig/tcl.m4 @@ -4064,4 +4064,4 @@ AC_DEFUN([TEA_ZIPFS_SUPPORT], [ # Local Variables: # mode: autoconf -# End:
\ No newline at end of file +# End: diff --git a/autoconf/tea/win/rules.vc b/autoconf/tea/win/rules.vc index 99471053c..f09e2ea48 100644 --- a/autoconf/tea/win/rules.vc +++ b/autoconf/tea/win/rules.vc @@ -708,4 +708,3 @@ TK_INCLUDES = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib" !message *** Link options '$(LINKERFLAGS)' !endif - @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.45.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.47.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.45.0' -PACKAGE_STRING='sqlite 3.45.0' +PACKAGE_VERSION='3.47.0' +PACKAGE_STRING='sqlite 3.47.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -781,18 +781,6 @@ TARGET_HAVE_EDITLINE TARGET_HAVE_READLINE TARGET_READLINE_INC TARGET_READLINE_LIBS -HAVE_TCL -TCL_SHLIB_SUFFIX -TCL_STUB_LIB_SPEC -TCL_STUB_LIB_FLAG -TCL_STUB_LIB_FILE -TCL_LIB_SPEC -TCL_LIB_FLAG -TCL_LIB_FILE -TCL_INCLUDE_SPEC -TCL_SRC_DIR -TCL_BIN_DIR -TCL_VERSION TARGET_EXEEXT SQLITE_OS_WIN SQLITE_OS_UNIX @@ -805,7 +793,12 @@ HAVE_WASI_SDK RELEASE VERSION program_prefix +TSTRNNR_OPTS TCLLIBDIR +HAVE_TCL +TCL_STUB_LIB_SPEC +TCL_LIB_SPEC +TCL_INCLUDE_SPEC TCLSH_CMD INSTALL_DATA INSTALL_SCRIPT @@ -894,12 +887,14 @@ enable_fast_install with_gnu_ld enable_libtool_lock enable_largefile +with_tclsh +with_tcl +enable_tcl +enable_test_status with_wasi_sdk enable_threadsafe enable_releasemode enable_tempstore -enable_tcl -with_tcl enable_editline enable_readline with_readline_lib @@ -1472,7 +1467,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.45.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1537,7 +1532,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.45.0:";; + short | recursive ) echo "Configuration of sqlite 3.47.0:";; esac cat <<\_ACEOF @@ -1551,11 +1546,13 @@ Optional Features: optimize for fast installation [default=yes] --disable-libtool-lock avoid locking (might break parallel builds) --disable-largefile omit support for large files + --disable-tcl omit building accessory programs that require + TCL-dev + --enable-test-status Full-screen status of tests --disable-threadsafe Disable mutexing --enable-releasemode Support libtool link to release mode --enable-tempstore Use an in-ram database for temporary tables (never,no,yes,always) - --disable-tcl do not build TCL extension --enable-editline enable BSD editline support --disable-readline disable readline support --enable-debug enable debugging & verbose explain @@ -1583,10 +1580,10 @@ Optional Packages: --with-pic try to use only PIC/non-PIC objects [default=use both] --with-gnu-ld assume the C compiler uses GNU ld [default=no] + --with-tclsh=PATHNAME full pathname of a tclsh to use + --with-tcl=DIR directory containing (tclConfig.sh) --with-wasi-sdk=DIR directory containing the WASI SDK. Triggers cross-compile to WASM. - --with-tcl=DIR directory containing tcl configuration - (tclConfig.sh) --with-readline-lib specify readline library --with-readline-inc specify readline include paths --with-linenoise=DIR source directory for linenoise library @@ -1668,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.45.0 +sqlite configure 3.47.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2087,7 +2084,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.45.0, which was +It was created by sqlite $as_me 3.47.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3945,13 +3942,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3948: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3945: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3951: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3954: output\"" >&5) + (eval echo "\"\$as_me:3951: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5157,7 +5154,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5160 "configure"' > conftest.$ac_ext + echo '#line 5157 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6682,11 +6679,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6685: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6682: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6689: \$? = $ac_status" >&5 + echo "$as_me:6686: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7021,11 +7018,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7024: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7021: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7028: \$? = $ac_status" >&5 + echo "$as_me:7025: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7126,11 +7123,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7129: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7126: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7133: \$? = $ac_status" >&5 + echo "$as_me:7130: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7181,11 +7178,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7184: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7181: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7188: \$? = $ac_status" >&5 + echo "$as_me:7185: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9561,7 +9558,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9564 "configure" +#line 9561 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9657,7 +9654,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9660 "configure" +#line 9657 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10314,11 +10311,31 @@ done USE_AMALGAMATION=1 ######### -# See whether we can run specific tclsh versions known to work well; -# if not, then we fall back to plain tclsh. -# TODO: try other versions before falling back? +# Figure out all the name of a working tclsh and parameters needed to compile against Tcl. +# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this. # -for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh + +# Check whether --with-tclsh was given. +if test "${with_tclsh+set}" = set; then : + withval=$with_tclsh; +fi + + +# Check whether --with-tcl was given. +if test "${with_tcl+set}" = set; then : + withval=$with_tcl; +fi + +# Check whether --enable-tcl was given. +if test "${enable_tcl+set}" = set; then : + enableval=$enable_tcl; use_tcl=$enableval +else + use_tcl=yes +fi + +original_use_tcl=${use_tcl} +if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then + for ac_prog in tclsh8.6 tclsh tclsh9.0 do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -10361,10 +10378,99 @@ fi done test -n "$TCLSH_CMD" || TCLSH_CMD="none" + with_tclsh=${TCLSH_CMD} +fi +if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then + TCLSH_CMD=${with_tclsh} + { $as_echo "$as_me:${as_lineno-$LINENO}: result: using tclsh at \"$TCLSH_CMD\"" >&5 +$as_echo "using tclsh at \"$TCLSH_CMD\"" >&6; } + if test x"${use_tcl}" = "xyes"; then + with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl` + if test x"${with_tcl}" != x; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5 +$as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5 +$as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;} + use_tcl=no + fi + fi +fi +if test x"${use_tcl}" = "xyes"; then + if test x"${with_tcl}" != x; then + if test -r ${with_tcl}/tclConfig.sh; then + tclconfig="${with_tcl}/tclConfig.sh" + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${with_tcl}/$i/tclConfig.sh; then + tclconfig=${with_tcl}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5 + fi + else + # If we have not yet found a tclConfig.sh file, look in $libdir whic is + # set automatically by autoconf or by the --prefix command-line option. + # See https://sqlite.org/forum/forumpost/e04e693439a22457 + libdir=${prefix}/lib + if test -r ${libdir}/tclConfig.sh; then + tclconfig=${libdir}/tclConfig.sh + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${libdir}/$i/tclConfig.sh; then + tclconfig=${libdir}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + as_fn_error $? "cannot find a usable tclConfig.sh file. + Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found. + SQLite does not use TCL internally, but TCL is required to build SQLite + from canonical sources and TCL is required for testing." "$LINENO" 5 + fi + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5 +$as_echo "loading TCL configuration from ${tclconfig}" >&6; } + . ${tclconfig} + + + + # There are lots of other configuration variables that are provided by the + # tclConfig.sh file and that could be included here. But as of right now, + # TCL_LIB_SPEC is the only what that the Makefile uses. + HAVE_TCL=1 +elif test x"${original_use_tcl}" = "xno"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5 +$as_echo "unable to run tests because of --disable-tcl" >&6; } + HAVE_TCL=0 +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5 +$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; } + HAVE_TCL=0 +fi + +if test x"$TCLSH_CMD" == x; then + TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION} + if test ! -x ${TCLSH_CMD}; then + TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh + if test ! -x ${TCLSH_CMD_2}; then + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&5 +$as_echo "$as_me: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&2;} + TCLSH_CMD=none + else + TCLSH_CMD=${TCLSH_CMD_2} + fi + fi +fi if test "$TCLSH_CMD" = "none"; then # If we can't find a local tclsh, then building the amalgamation will fail. # We act as though --disable-amalgamation has been used. - echo "Warning: can't find tclsh - defaulting to non-amalgamation build." + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&5 +$as_echo "$as_me: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&2;} USE_AMALGAMATION=0 TCLSH_CMD="tclsh" fi @@ -10372,7 +10478,6 @@ fi if test "x${TCLLIBDIR+set}" != "xset" ; then - TCLLIBDIR='$(libdir)' for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do if test -d $i ; then TCLLIBDIR=$i @@ -10382,6 +10487,23 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then TCLLIBDIR="${TCLLIBDIR}/sqlite3" fi +######### +# Set up options for running tests. +# +# Check whether --enable-test-status was given. +if test "${enable_test_status+set}" = set; then : + enableval=$enable_test_status; use_vt100=$enableval +else + use_vt100=no +fi + +if test $use_vt100 != no; then + TSTRNNR_OPTS=--status +else + TSTRNNR_OPTS= +fi + + ######### # Set up an appropriate program prefix @@ -10770,213 +10892,6 @@ fi ########## -# Figure out all the parameters needed to compile against Tcl. -# -# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG -# macros in the in the tcl.m4 file of the standard TCL distribution. -# Those macros could not be used directly since we have to make some -# minor changes to accomodate systems that do not have TCL installed. -# -# Check whether --enable-tcl was given. -if test "${enable_tcl+set}" = set; then : - enableval=$enable_tcl; use_tcl=$enableval -else - use_tcl=yes -fi - -if test "${use_tcl}" = "yes" ; then - -# Check whether --with-tcl was given. -if test "${with_tcl+set}" = set; then : - withval=$with_tcl; with_tclconfig=${withval} -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Tcl configuration" >&5 -$as_echo_n "checking for Tcl configuration... " >&6; } - if ${ac_cv_c_tclconfig+:} false; then : - $as_echo_n "(cached) " >&6 -else - - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)` - else - as_fn_error $? "${with_tclconfig} directory doesn't contain tclConfig.sh" "$LINENO" 5 - fi - fi - - # Start autosearch by asking tclsh - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # On ubuntu 14.10, $auto_path on tclsh is not quite correct. - # So try again after applying corrections. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # Recent versions of Xcode on Macs hid the tclConfig.sh file - # in a strange place. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ../tcl[8-9].[0-9]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ../../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ../../tcl[8-9].[0-9]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ../../../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ../../../tcl[8-9].[0-9]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - `ls -d ${libdir} 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i; pwd)` - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[8-9].[0-9]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - -fi - - - if test x"${ac_cv_c_tclconfig}" = x ; then - use_tcl=no - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Can't find Tcl configuration definitions" >&5 -$as_echo "$as_me: WARNING: Can't find Tcl configuration definitions" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&5 -$as_echo "$as_me: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&5 -$as_echo "$as_me: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&2;} - else - TCL_BIN_DIR=${ac_cv_c_tclconfig} - { $as_echo "$as_me:${as_lineno-$LINENO}: result: found $TCL_BIN_DIR/tclConfig.sh" >&5 -$as_echo "found $TCL_BIN_DIR/tclConfig.sh" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for existence of $TCL_BIN_DIR/tclConfig.sh" >&5 -$as_echo_n "checking for existence of $TCL_BIN_DIR/tclConfig.sh... " >&6; } - if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading" >&5 -$as_echo "loading" >&6; } - . $TCL_BIN_DIR/tclConfig.sh - else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: file not found" >&5 -$as_echo "file not found" >&6; } - fi - - # - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - # - - if test -f $TCL_BIN_DIR/Makefile ; then - TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC} - TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC} - TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH} - fi - - # - # eval is required to do the TCL_DBGX substitution - # - - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - - - - - - - - - - - - - - - fi -fi -if test "${use_tcl}" = "no" ; then - HAVE_TCL="" -else - HAVE_TCL=1 -fi - - -########## # Figure out what C libraries are required to compile programs # that use "readline()" library. # @@ -12481,7 +12396,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.45.0, which was +This file was extended by sqlite $as_me 3.47.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12547,7 +12462,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.45.0 +sqlite config.status 3.47.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 54f985fd7..b3a0dd299 100644 --- a/configure.ac +++ b/configure.ac @@ -116,15 +116,101 @@ AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_s USE_AMALGAMATION=1 ######### -# See whether we can run specific tclsh versions known to work well; -# if not, then we fall back to plain tclsh. -# TODO: try other versions before falling back? -# -AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none) +# Figure out all the name of a working tclsh and parameters needed to compile against Tcl. +# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this. +# +AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use])) +AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)])) +AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]), + [use_tcl=$enableval],[use_tcl=yes]) +original_use_tcl=${use_tcl} +if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then + AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none) + with_tclsh=${TCLSH_CMD} +fi +if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then + TCLSH_CMD=${with_tclsh} + AC_MSG_RESULT([using tclsh at "$TCLSH_CMD"]) + if test x"${use_tcl}" = "xyes"; then + with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl` + if test x"${with_tcl}" != x; then + AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}]) + else + AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh]) + use_tcl=no + fi + fi +fi +if test x"${use_tcl}" = "xyes"; then + if test x"${with_tcl}" != x; then + if test -r ${with_tcl}/tclConfig.sh; then + tclconfig="${with_tcl}/tclConfig.sh" + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${with_tcl}/$i/tclConfig.sh; then + tclconfig=${with_tcl}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}]) + fi + else + # If we have not yet found a tclConfig.sh file, look in $libdir whic is + # set automatically by autoconf or by the --prefix command-line option. + # See https://sqlite.org/forum/forumpost/e04e693439a22457 + libdir=${prefix}/lib + if test -r ${libdir}/tclConfig.sh; then + tclconfig=${libdir}/tclConfig.sh + else + for i in tcl8.6 tcl9.0 lib; do + if test -r ${libdir}/$i/tclConfig.sh; then + tclconfig=${libdir}/$i/tclConfig.sh + break + fi + done + fi + if test ! -r "${tclconfig}"; then + AC_MSG_ERROR([cannot find a usable tclConfig.sh file. + Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found. + SQLite does not use TCL internally, but TCL is required to build SQLite + from canonical sources and TCL is required for testing.]) + fi + fi + AC_MSG_RESULT([loading TCL configuration from ${tclconfig}]) + . ${tclconfig} + AC_SUBST(TCL_INCLUDE_SPEC) + AC_SUBST(TCL_LIB_SPEC) + AC_SUBST(TCL_STUB_LIB_SPEC) + # There are lots of other configuration variables that are provided by the + # tclConfig.sh file and that could be included here. But as of right now, + # TCL_LIB_SPEC is the only what that the Makefile uses. + HAVE_TCL=1 +elif test x"${original_use_tcl}" = "xno"; then + AC_MSG_RESULT([unable to run tests because of --disable-tcl]) + HAVE_TCL=0 +else + AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located]) + HAVE_TCL=0 +fi +AC_SUBST(HAVE_TCL) +if test x"$TCLSH_CMD" == x; then + TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION} + if test ! -x ${TCLSH_CMD}; then + TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh + if test ! -x ${TCLSH_CMD_2}; then + AC_MSG_WARN([cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}]) + TCLSH_CMD=none + else + TCLSH_CMD=${TCLSH_CMD_2} + fi + fi +fi if test "$TCLSH_CMD" = "none"; then # If we can't find a local tclsh, then building the amalgamation will fail. # We act as though --disable-amalgamation has been used. - echo "Warning: can't find tclsh - defaulting to non-amalgamation build." + AC_MSG_WARN([Warning: can't find tclsh - defaulting to non-amalgamation build.]) USE_AMALGAMATION=0 TCLSH_CMD="tclsh" fi @@ -132,7 +218,6 @@ AC_SUBST(TCLSH_CMD) AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin]) if test "x${TCLLIBDIR+set}" != "xset" ; then - TCLLIBDIR='$(libdir)' for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do if test -d $i ; then TCLLIBDIR=$i @@ -142,6 +227,18 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then TCLLIBDIR="${TCLLIBDIR}/sqlite3" fi +######### +# Set up options for running tests. +# +AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[Full-screen status of tests]), + [use_vt100=$enableval],[use_vt100=no]) +if test $use_vt100 != no; then + TSTRNNR_OPTS=--status +else + TSTRNNR_OPTS= +fi +AC_SUBST(TSTRNNR_OPTS) + ######### # Set up an appropriate program prefix @@ -340,190 +437,6 @@ AC_SUBST(SQLITE_OS_WIN) AC_SUBST(TARGET_EXEEXT) ########## -# Figure out all the parameters needed to compile against Tcl. -# -# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG -# macros in the in the tcl.m4 file of the standard TCL distribution. -# Those macros could not be used directly since we have to make some -# minor changes to accomodate systems that do not have TCL installed. -# -AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]), - [use_tcl=$enableval],[use_tcl=yes]) -if test "${use_tcl}" = "yes" ; then - AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval}) - AC_MSG_CHECKING([for Tcl configuration]) - AC_CACHE_VAL(ac_cv_c_tclconfig,[ - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)` - else - AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh]) - fi - fi - - # Start autosearch by asking tclsh - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # On ubuntu 14.10, $auto_path on tclsh is not quite correct. - # So try again after applying corrections. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # Recent versions of Xcode on Macs hid the tclConfig.sh file - # in a strange place. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - `ls -d ${libdir} 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i; pwd)` - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tclconfig}" = x ; then - use_tcl=no - AC_MSG_WARN(Can't find Tcl configuration definitions) - AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***) - AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***) - else - TCL_BIN_DIR=${ac_cv_c_tclconfig} - AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh) - - AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh]) - if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then - AC_MSG_RESULT([loading]) - . $TCL_BIN_DIR/tclConfig.sh - else - AC_MSG_RESULT([file not found]) - fi - - # - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - # - - if test -f $TCL_BIN_DIR/Makefile ; then - TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC} - TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC} - TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH} - fi - - # - # eval is required to do the TCL_DBGX substitution - # - - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - - AC_SUBST(TCL_VERSION) - AC_SUBST(TCL_BIN_DIR) - AC_SUBST(TCL_SRC_DIR) - AC_SUBST(TCL_INCLUDE_SPEC) - - AC_SUBST(TCL_LIB_FILE) - AC_SUBST(TCL_LIB_FLAG) - AC_SUBST(TCL_LIB_SPEC) - - AC_SUBST(TCL_STUB_LIB_FILE) - AC_SUBST(TCL_STUB_LIB_FLAG) - AC_SUBST(TCL_STUB_LIB_SPEC) - AC_SUBST(TCL_SHLIB_SUFFIX) - fi -fi -if test "${use_tcl}" = "no" ; then - HAVE_TCL="" -else - HAVE_TCL=1 -fi -AC_SUBST(HAVE_TCL) - -########## # Figure out what C libraries are required to compile programs # that use "readline()" library. # diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md index b8a50afb3..5e6a2d1a2 100644 --- a/doc/compile-for-windows.md +++ b/doc/compile-for-windows.md @@ -59,8 +59,16 @@ canonical source on a new Windows 11 PC, as of 2023-11-01: <li> `nmake /f makefile.msc sqlite3.c` <li> `nmake /f makefile.msc devtest` <li> `nmake /f makefile.msc releasetest` + <li> `nmake /f makefile.msc sqlite3.exe` </ul> + 7. For a debugging build of the CLI, where the ".treetrace" and ".wheretrace" + commands work, add the DEBUG=3 argument to nmake. Like this: + <ul> + <li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe` + </ul> + + ## 32-bit Builds Doing a 32-bit build is just like doing a 64-bit build with the diff --git a/doc/lemon.html b/doc/lemon.html index 66665f46f..4147d9b31 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -683,6 +683,7 @@ other than that, the order of directives in Lemon is arbitrary.</p> <li><tt><a href='#pifdef'>%endif</a></tt> <li><tt><a href='#extraarg'>%extra_argument</a></tt> <li><tt><a href='#pfallback'>%fallback</a></tt> +<li><tt><a href='#reallc'>%free</a></tt> <li><tt><a href='#pifdef'>%if</a></tt> <li><tt><a href='#pifdef'>%ifdef</a></tt> <li><tt><a href='#pifdef'>%ifndef</a></tt> @@ -693,6 +694,7 @@ other than that, the order of directives in Lemon is arbitrary.</p> <li><tt><a href='#parse_accept'>%parse_accept</a></tt> <li><tt><a href='#parse_failure'>%parse_failure</a></tt> <li><tt><a href='#pright'>%right</a></tt> +<li><tt><a href='#reallc'>%realloc</a></tt> <li><tt><a href='#stack_overflow'>%stack_overflow</a></tt> <li><tt><a href='#stack_size'>%stack_size</a></tt> <li><tt><a href='#start_symbol'>%start_symbol</a></tt> @@ -1200,6 +1202,21 @@ match any input token.</p> the wildcard token and some other token, the other token is always used. The wildcard token is only matched if there are no alternatives.</p> +<a id='reallc'></a> +<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4> + +<p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function +that allocate and free heap memory. The signatures of these functions +should be the same as the realloc() and free() functions from the standard +C library. + +<p>If both of these functions are defined +then these functions are used to allocate and free +memory for supplemental parser stack space, if the initial +parse stack space is exceeded. The initial parser stack size +is specified by either <tt>%stack_size</tt> or the +-DYYSTACKDEPTH compile-time flag. + <a id='errors'></a> <h2>5.0 Error Processing</h2> @@ -1224,6 +1241,7 @@ to begin parsing a new file. This is what will happen at the very first syntax error, of course, if there are no instances of the "error" non-terminal in your grammar.</p> + <a id='history'></a> <h2>6.0 History of Lemon</h2> diff --git a/doc/testrunner.md b/doc/testrunner.md index d420076c4..d0248573e 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -17,7 +17,6 @@ <li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a> </ul> <li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a> -# 4. Extra testrunner.tcl Options <li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a> </ul> @@ -29,12 +28,18 @@ multiple jobs. It supports the following types of tests: * Tcl test scripts. - * Tests run with [make] commands. Specifically, at time of writing, - [make fuzztest], [make mptest], [make sourcetest] and [make threadtest]. + * Tests run with `make` commands. Examples: + - `make mdevtest` + - `make releasetest` + - `make sdevtest` + - `make testrunner` testrunner.tcl pipes the output of all tests and builds run into log file -**testrunner.log**, created in the cwd directory. Searching this file for -"failed" is a good way to find the output of a failed test. +**testrunner.log**, created in the current working directory. Search this +file to find details of errors. Suggested search commands: + + * `grep "^!" testrunner.log` + * `grep failed testrunner.log` testrunner.tcl also populates SQLite database **testrunner.db**. This database contains details of all tests run, running and to be run. A useful query @@ -60,7 +65,7 @@ Running: in another terminal is a good way to keep an eye on a long running test. -Sometimes testrunner.tcl uses the [testfixture] binary that it is run with +Sometimes testrunner.tcl uses the `testfixture` binary that it is run with to run tests (see "Binary Tests" below). Sometimes it builds testfixture and other binaries in specific configurations to test (see "Source Tests"). @@ -68,9 +73,9 @@ other binaries in specific configurations to test (see "Source Tests"). # 2. Binary Tests The commands described in this section all run various combinations of the Tcl -test scripts using the [testfixture] binary used to run the testrunner.tcl +test scripts using the `testfixture` binary used to run the testrunner.tcl script (i.e. they do not invoke the compiler to build new binaries, or the -[make] command to run tests that are not Tcl scripts). The procedure to run +`make` command to run tests that are not Tcl scripts). The procedure to run these tests is therefore: 1. Build the "testfixture" (or "testfixture.exe" for windows) binary using @@ -193,7 +198,7 @@ TODO: ./configure + Makefile.msc build systems. ## 3.1. Commands to Run SQLite Tests The **mdevtest** command is equivalent to running the veryquick tests and -the [make fuzztest] target once for each of two --enable-all builds - one +the `make fuzztest` target once for each of two --enable-all builds - one with debugging enabled and one without: ``` @@ -283,7 +288,7 @@ a dos \*.bat file on windows. For example: ``` The generated bash or \*.bat file script accepts a single argument - a makefile -target to build. This may be used either to run a [make] command test directly, +target to build. This may be used either to run a `make` command test directly, or else to build a testfixture (or testfixture.exe) binary with which to run a Tcl test script, as <a href=#binary_test_failures>described above</a>. @@ -310,6 +315,16 @@ would normally execute into the testrunner.log file. Example: tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" ``` +The **--explain** option is similar to --dryrun in that it prevents testrunner.tcl +from building any binaries or running any tests. The difference is that --explain +prints on standard output a human-readable summary of all the builds and tests that +would have been run. + +``` + # Show what builds and tests would have been run + tclsh $TESTDIR/testrunner.tcl --explain mdevtest +``` + <a name=cpu_cores></a> # 5. Controlling CPU Core Utilization @@ -339,6 +354,3 @@ testrunner.log and testrunner.db files: ``` $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS ``` - - - diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index 8ecd97f0b..40989e77b 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -29,6 +29,9 @@ #ifndef HAVE_CONSOLE_IO_H # include "console_io.h" #endif +#if defined(_MSC_VER) +# pragma warning(disable : 4204) +#endif #ifndef SQLITE_CIO_NO_TRANSLATE # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT @@ -51,11 +54,6 @@ #endif #if CIO_WIN_WC_XLATE -/* Character used to represent a known-incomplete UTF-8 char group (�) */ -static WCHAR cBadGroup = 0xfffd; -#endif - -#if CIO_WIN_WC_XLATE static HANDLE handleOfFile(FILE *pf){ int fileDesc = _fileno(pf); union { intptr_t osfh; HANDLE fh; } fid = { @@ -127,6 +125,10 @@ static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){ # endif } +# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +# define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) +# endif + # if CIO_WIN_WC_XLATE /* Define console modes for use with the Windows Console API. */ # define SHELL_CONI_MODE \ @@ -755,4 +757,8 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ +#if defined(_MSC_VER) +# pragma warning(default : 4204) +#endif + #undef SHELL_INVALID_FILE_PTR diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index 453334234..72c4fd72c 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. Specifically, # the ".recommend" command. @@ -464,6 +465,13 @@ do_execsql_test 5.3 { t2 t2_idx_0001295b {100 20 5} } +do_catchsql_test 5.4 { + SELECT sqlite_expert_rem(123, 123); +} {1 {no such function: sqlite_expert_rem}} +do_catchsql_test 5.5 { + SELECT sqlite_expert_sample(); +} {1 {no such function: sqlite_expert_sample}} + if 0 { do_test expert1-6.0 { catchcmd :memory: { diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index 33d62226f..b59a59728 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -626,7 +626,7 @@ static int expertFilter( pCsr->pData = 0; if( rc==SQLITE_OK ){ rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, - "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName ); } @@ -1500,7 +1500,7 @@ struct IdxRemCtx { }; /* -** Implementation of scalar function rem(). +** Implementation of scalar function sqlite_expert_rem(). */ static void idxRemFunc( sqlite3_context *pCtx, @@ -1513,7 +1513,7 @@ static void idxRemFunc( assert( argc==2 ); iSlot = sqlite3_value_int(argv[0]); - assert( iSlot<=p->nSlot ); + assert( iSlot<p->nSlot ); pSlot = &p->aSlot[iSlot]; switch( pSlot->eType ){ @@ -1624,7 +1624,8 @@ static int idxPopulateOneStat1( const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); zCols = idxAppendText(&rc, zCols, - "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl ); zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); } @@ -1757,13 +1758,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ if( rc==SQLITE_OK ){ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = sqlite3_create_function( - dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 ); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 ); } @@ -1815,6 +1816,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); } + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); return rc; } @@ -1948,7 +1952,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_stmt *pSql = 0; rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" - " AND sql NOT LIKE 'CREATE VIRTUAL %%'" + " AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid" ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c index 064c1908a..cae5d0f25 100644 --- a/ext/expert/test_expert.c +++ b/ext/expert/test_expert.c @@ -16,15 +16,7 @@ #include "sqlite3expert.h" #include <assert.h> #include <string.h> - -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 65852804a..f977aabfb 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4006,7 +4006,7 @@ static int fts3ShadowName(const char *zName){ ** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual ** table. */ -static int fts3Integrity( +static int fts3IntegrityMethod( sqlite3_vtab *pVtab, /* The virtual table to be checked */ const char *zSchema, /* Name of schema in which pVtab lives */ const char *zTabname, /* Name of the pVTab table */ @@ -4014,31 +4014,24 @@ static int fts3Integrity( char **pzErr /* Write error message here */ ){ Fts3Table *p = (Fts3Table*)pVtab; - char *zSql; - int rc; - char *zErr = 0; + int rc = SQLITE_OK; + int bOk = 0; - assert( pzErr!=0 ); - assert( *pzErr==0 ); UNUSED_PARAMETER(isQuick); - zSql = sqlite3_mprintf( - "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');", - zSchema, zTabname, zTabname); - if( zSql==0 ){ - return SQLITE_NOMEM; - } - rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr); - sqlite3_free(zSql); - if( (rc&0xff)==SQLITE_CORRUPT ){ - *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", - p->bFts4 ? 4 : 3, zSchema, zTabname); - }else if( rc!=SQLITE_OK ){ + rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( rc!=SQLITE_CORRUPT_VTAB ); + if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" " FTS%d table %s.%s: %s", - p->bFts4 ? 4 : 3, zSchema, zTabname, zErr); + p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", + p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; } - sqlite3_free(zErr); - return SQLITE_OK; + sqlite3Fts3SegmentsClose(p); + return rc; } @@ -4068,7 +4061,7 @@ static const sqlite3_module fts3Module = { /* xRelease */ fts3ReleaseMethod, /* xRollbackTo */ fts3RollbackToMethod, /* xShadowName */ fts3ShadowName, - /* xIntegrity */ fts3Integrity, + /* xIntegrity */ fts3IntegrityMethod, }; /* diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 5a5b123b4..3b236faf4 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -653,5 +653,7 @@ int sqlite3FtsUnicodeIsdiacritic(int); int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); + #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 227c5f00f..80f62eb3b 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -398,6 +398,7 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){ return 1; } + assert( pIter->nSnippet>=0 ); pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; for(i=0; i<pIter->nPhrase; i++){ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; @@ -446,7 +447,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; j<pPhrase->nToken; j++){ + for(j=0; j<pPhrase->nToken && j<pIter->nSnippet; j++){ mHighlight |= (mPos>>j); } diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index f3a9746a0..655dd9f35 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -78,6 +78,8 @@ static int fts3termConnectMethod( iIndex = atoi(argv[4]); argc--; } + + *ppVtab = 0; /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ @@ -95,12 +97,17 @@ static int fts3termConnectMethod( rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); if( rc!=SQLITE_OK ) return rc; - nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3termTable *)sqlite3_malloc64(nByte); + nByte = sizeof(Fts3termTable); + p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, (size_t)nByte); - p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero( + sizeof(Fts3Table) + nDb + nFts3 + 2 + ); + if( p->pFts3Tab==0 ){ + sqlite3_free(p); + return SQLITE_NOMEM; + } p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; @@ -130,6 +137,7 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_finalize(pFts3->aStmt[i]); } sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(pFts3); sqlite3_free(p); return SQLITE_OK; } diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 49a8476bf..3c42a7bf0 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -18,14 +18,7 @@ ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. */ -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #include <string.h> #include <assert.h> @@ -167,7 +160,8 @@ static int SQLITE_TCLAPI fts3_near_match_cmd( Tcl_Obj *pPhrasecount = 0; Tcl_Obj **apExprToken; - int nExprToken; + Tcl_Size nExprToken; + Tcl_Size nn; UNUSED_PARAMETER(clientData); @@ -201,23 +195,25 @@ static int SQLITE_TCLAPI fts3_near_match_cmd( } } - rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken); + rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken); + doc.nToken = (int)nn; if( rc!=TCL_OK ) goto near_match_out; doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken)); for(ii=0; ii<doc.nToken; ii++){ - doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n); + doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn); + doc.aToken[ii].n = (int)nn; } rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken); if( rc!=TCL_OK ) goto near_match_out; - nPhrase = (nExprToken + 1) / 2; + nPhrase = (int)(nExprToken + 1) / 2; aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase)); memset(aPhrase, 0, nPhrase * sizeof(NearPhrase)); for(ii=0; ii<nPhrase; ii++){ Tcl_Obj *pPhrase = apExprToken[ii*2]; Tcl_Obj **apToken; - int nToken; + Tcl_Size nToken; int jj; rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken); @@ -227,11 +223,12 @@ static int SQLITE_TCLAPI fts3_near_match_cmd( rc = TCL_ERROR; goto near_match_out; } - for(jj=0; jj<nToken; jj++){ + for(jj=0; jj<(int)nToken; jj++){ NearToken *pT = &aPhrase[ii].aToken[jj]; - pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n); + pT->z = Tcl_GetStringFromObj(apToken[jj], &nn); + pT->n = (int)nn; } - aPhrase[ii].nToken = nToken; + aPhrase[ii].nToken = (int)nToken; } for(ii=1; ii<nPhrase; ii++){ Tcl_Obj *pNear = apExprToken[2*ii-1]; diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index eab3f513e..24c237a89 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -226,11 +226,7 @@ int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <string.h> /* diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 0f894943d..5a449dec1 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -5294,7 +5294,7 @@ static u64 fts3ChecksumIndex( ** If an error occurs (e.g. an OOM or IO error), return an SQLite error ** code. The final value of *pbOk is undefined in this case. */ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int rc = SQLITE_OK; /* Return code */ u64 cksum1 = 0; /* Checksum based on FTS index contents */ u64 cksum2 = 0; /* Checksum based on %_content contents */ @@ -5372,7 +5372,12 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -5412,7 +5417,7 @@ static int fts3DoIntegrityCheck( ){ int rc; int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 58d90c68c..1306629da 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -628,6 +628,9 @@ proc print_categories {lMap} { $caseP $caseS $caseZ + + default: + return 1; } return 0; } diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index 6762a036d..6ee71c262 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -82,7 +82,7 @@ proc get_struct_docs {data names} { set current_doc "" } set subject n/a - regexp {^ *([[:alpha:]]*)} $line -> subject + regexp {^ *([[:alnum:]_]*)} $line -> subject if {[lsearch $names $subject]>=0} { set current_header $subject } else { @@ -108,8 +108,11 @@ proc get_tokenizer_docs {data} { append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n" continue } + if {[regexp {FTS5_TOKENIZER} $line]} { + set line </dl><p> + } if {[regexp {SYNONYM SUPPORT} $line]} { - set line "</dl><h3>Synonym Support</h3>" + set line "<h3>Synonym Support</h3>" } if {[string trim $line] == ""} { append res "<p>\n" diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 250d2ee7e..ea69f8f02 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -55,8 +55,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -238,6 +238,10 @@ struct Fts5PhraseIter { ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -304,9 +308,32 @@ struct Fts5PhraseIter { ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -348,6 +375,15 @@ struct Fts5ExtensionApi { const char **ppToken, int *pnToken ); int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; /* @@ -360,6 +396,7 @@ struct Fts5ExtensionApi { ** Applications may also register custom tokenizer types. A tokenizer ** is registered by providing fts5 with a populated instance of the ** following structure. All structure methods must be defined, setting +** ** any member of the fts5_tokenizer struct to NULL leads to undefined ** behaviour. The structure methods are expected to function as follows: ** @@ -368,7 +405,7 @@ struct Fts5ExtensionApi { ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -392,7 +429,7 @@ struct Fts5ExtensionApi { ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -416,6 +453,13 @@ struct Fts5ExtensionApi { ** on a columnsize=0 database. ** </ul> ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -439,6 +483,30 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +** <ul> +** <li> There is no "iVersion" field, and +** <li> The xTokenize() method does not take a locale argument. +** </ul> +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -547,6 +615,33 @@ struct Fts5ExtensionApi { ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ typedef struct fts5_tokenizer fts5_tokenizer; struct fts5_tokenizer { int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); @@ -566,6 +661,7 @@ struct fts5_tokenizer { ); }; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -585,7 +681,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( @@ -612,6 +708,25 @@ struct fts5_api { fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 9beb26e05..7e4111957 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -59,6 +59,22 @@ typedef sqlite3_uint64 u64; # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +/* The uptr type is an unsigned integer large enough to hold a pointer +*/ +#if defined(HAVE_STDINT_H) + typedef uintptr_t uptr; +#elif SQLITE_PTRSIZE==4 + typedef u32 uptr; +#else + typedef u64 uptr; +#endif + +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) +#else +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) +#endif + #endif /* Truncate very long tokens to this many bytes. Hard limit is @@ -142,6 +158,18 @@ struct Fts5Colset { */ typedef struct Fts5Config Fts5Config; +typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; + +struct Fts5TokenizerConfig { + Fts5Tokenizer *pTok; + fts5_tokenizer_v2 *pApi2; + fts5_tokenizer *pApi1; + const char **azArg; + int nArg; + int ePattern; /* FTS_PATTERN_XXX constant */ + const char *pLocale; /* Current locale to use */ + int nLocale; /* Size of pLocale in bytes */ +}; /* ** An instance of the following structure encodes all information that can @@ -181,9 +209,12 @@ typedef struct Fts5Config Fts5Config; ** ** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); ** +** bLocale: +** Set to true if locale=1 was specified when the table was created. */ struct Fts5Config { sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* Global fts5 object for handle db */ char *zDb; /* Database holding FTS index (e.g. "main") */ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ @@ -197,12 +228,12 @@ struct Fts5Config { char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ int bTokendata; /* "tokendata=" option value (dflt==0) */ + int bLocale; /* "locale=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; - Fts5Tokenizer *pTok; - fts5_tokenizer *pTokApi; + Fts5TokenizerConfig t; int bLock; /* True when table is preparing statement */ - int ePattern; /* FTS_PATTERN_XXX constant */ + /* Values loaded from the %_config table */ int iVersion; /* fts5 file format 'version' */ @@ -268,6 +299,8 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); int sqlite3Fts5ConfigParseRank(const char*, char**, char**); +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...); + /* ** End of interface to code in fts5_config.c. **************************************************************************/ @@ -312,7 +345,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); void sqlite3Fts5Put32(u8*, int); int sqlite3Fts5Get32(const u8*); -#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) +#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF) #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) typedef struct Fts5PoslistReader Fts5PoslistReader; @@ -597,18 +630,23 @@ struct Fts5Table { Fts5Index *pIndex; /* Full-text index */ }; -int sqlite3Fts5GetTokenizer( - Fts5Global*, - const char **azArg, - int nArg, - Fts5Config*, - char **pzErr -); +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig); Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); int sqlite3Fts5FlushToDisk(Fts5Table*); +int sqlite3Fts5ExtractText( + Fts5Config *pConfig, + sqlite3_value *pVal, /* Value to extract text from */ + int bContent, /* Loaded from content table */ + int *pbResetTokenizer, /* OUT: True if ClearLocale() required */ + const char **ppText, /* OUT: Pointer to text buffer */ + int *pnText /* OUT: Size of (*ppText) in bytes */ +); + +void sqlite3Fts5ClearLocale(Fts5Config *pConfig); + /* ** End of interface to code in fts5.c. **************************************************************************/ @@ -688,7 +726,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int); int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); @@ -714,6 +752,9 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); int sqlite3Fts5StorageReset(Fts5Storage *p); +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*); +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ @@ -866,6 +907,7 @@ int sqlite3Fts5TokenizerPattern( int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), Fts5Tokenizer *pTok ); +int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*); /* ** End of interface to code in fts5_tokenizer.c. **************************************************************************/ diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 30101fbe2..ad578156d 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -226,6 +226,7 @@ static int fts5HighlightCb( return rc; } + /* ** Implementation of highlight() function. */ @@ -256,12 +257,19 @@ static void fts5HighlightFunction( sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); rc = SQLITE_OK; }else if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb + ); } if( ctx.bOpen ){ fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); @@ -458,6 +466,8 @@ static void fts5SnippetFunction( memset(&sFinder, 0, sizeof(Fts5SFinder)); for(i=0; i<nCol; i++){ if( iCol<0 || iCol==i ){ + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ int nDoc; int nDocsize; int ii; @@ -465,8 +475,10 @@ static void fts5SnippetFunction( sFinder.nFirst = 0; rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc); if( rc!=SQLITE_OK ) break; - rc = pApi->xTokenize(pFts, - sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize_v2(pFts, + sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb ); if( rc!=SQLITE_OK ) break; rc = pApi->xColumnSize(pFts, i, &nDocsize); @@ -524,6 +536,9 @@ static void fts5SnippetFunction( rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); } if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iBestCol */ + int nLoc = 0; /* Bytes in pLoc */ + if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } @@ -542,7 +557,12 @@ static void fts5SnippetFunction( } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb + ); } if( ctx.bOpen ){ fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); @@ -726,6 +746,53 @@ static void fts5Bm25Function( } } +/* +** Implementation of fts5_get_locale() function. +*/ +static void fts5GetLocaleFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + int iCol = 0; + int eType = 0; + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + /* xColumnLocale() must be available */ + assert( pApi->iVersion>=4 ); + + if( nVal!=1 ){ + const char *z = "wrong number of arguments to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + eType = sqlite3_value_numeric_type(apVal[0]); + if( eType!=SQLITE_INTEGER ){ + const char *z = "non-integer argument passed to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ + sqlite3_result_error_code(pCtx, SQLITE_RANGE); + return; + } + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + return; + } + + sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT); +} + int sqlite3Fts5AuxInit(fts5_api *pApi){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ @@ -733,9 +800,10 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { - { "snippet", 0, fts5SnippetFunction, 0 }, - { "highlight", 0, fts5HighlightFunction, 0 }, - { "bm25", 0, fts5Bm25Function, 0 }, + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index d2e8309cd..3cb1bd3be 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -234,7 +234,6 @@ static int fts5ConfigSetEnum( ** eventually free any such error message using sqlite3_free(). */ static int fts5ConfigParseSpecial( - Fts5Global *pGlobal, Fts5Config *pConfig, /* Configuration object to update */ const char *zCmd, /* Special command to parse */ const char *zArg, /* Argument to parse */ @@ -298,12 +297,11 @@ static int fts5ConfigParseSpecial( if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ const char *p = (const char*)zArg; sqlite3_int64 nArg = strlen(zArg) + 1; - char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); - char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); - char *pSpace = pDel; + char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg); - if( azArg && pSpace ){ - if( pConfig->pTok ){ + if( azArg ){ + char *pSpace = (char*)&azArg[nArg]; + if( pConfig->t.azArg ){ *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); rc = SQLITE_ERROR; }else{ @@ -326,16 +324,14 @@ static int fts5ConfigParseSpecial( *pzErr = sqlite3_mprintf("parse error in tokenize directive"); rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, (int)nArg, pConfig, - pzErr - ); + pConfig->t.azArg = (const char**)azArg; + pConfig->t.nArg = nArg; + azArg = 0; } } } - sqlite3_free(azArg); - sqlite3_free(pDel); + return rc; } @@ -384,6 +380,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed locale=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bLocale = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, @@ -413,16 +419,6 @@ static int fts5ConfigParseSpecial( } /* -** Allocate an instance of the default tokenizer ("simple") at -** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error -** code if an error occurs. -*/ -static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ - assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); - return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0); -} - -/* ** Gobble up the first bareword or quoted word from the input buffer zIn. ** Return a pointer to the character immediately following the last in ** the gobbled word if successful, or a NULL pointer otherwise (failed @@ -554,6 +550,7 @@ int sqlite3Fts5ConfigParse( *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); + pRet->pGlobal = pGlobal; pRet->db = db; pRet->iCookie = -1; @@ -602,7 +599,7 @@ int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; }else{ if( bOption ){ - rc = fts5ConfigParseSpecial(pGlobal, pRet, + rc = fts5ConfigParseSpecial(pRet, ALWAYS(zOne)?zOne:"", zTwo?zTwo:"", pzErr @@ -640,13 +637,6 @@ int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; } - /* If a tokenizer= option was successfully parsed, the tokenizer has - ** already been allocated. Otherwise, allocate an instance of the default - ** tokenizer (unicode61) now. */ - if( rc==SQLITE_OK && pRet->pTok==0 ){ - rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); - } - /* If no zContent option was specified, fill in the default values. */ if( rc==SQLITE_OK && pRet->zContent==0 ){ const char *zTail = 0; @@ -688,9 +678,14 @@ int sqlite3Fts5ConfigParse( void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ if( pConfig ){ int i; - if( pConfig->pTok ){ - pConfig->pTokApi->xDelete(pConfig->pTok); + if( pConfig->t.pTok ){ + if( pConfig->t.pApi1 ){ + pConfig->t.pApi1->xDelete(pConfig->t.pTok); + }else{ + pConfig->t.pApi2->xDelete(pConfig->t.pTok); + } } + sqlite3_free((char*)pConfig->t.azArg); sqlite3_free(pConfig->zDb); sqlite3_free(pConfig->zName); for(i=0; i<pConfig->nCol; i++){ @@ -765,10 +760,24 @@ int sqlite3Fts5Tokenize( void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ - if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize( - pConfig->pTok, pCtx, flags, pText, nText, xToken - ); + int rc = SQLITE_OK; + if( pText ){ + if( pConfig->t.pTok==0 ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } + if( rc==SQLITE_OK ){ + if( pConfig->t.pApi1 ){ + rc = pConfig->t.pApi1->xTokenize( + pConfig->t.pTok, pCtx, flags, pText, nText, xToken + ); + }else{ + rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, + pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken + ); + } + } + } + return rc; } /* @@ -1022,13 +1031,10 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE ){ rc = SQLITE_ERROR; - if( pConfig->pzErrmsg ){ - assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " - "(found %d, expected %d or %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE - ); - } + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE + ); }else{ pConfig->iVersion = iVersion; } @@ -1038,3 +1044,26 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ } return rc; } + +/* +** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer +** containing the error message created using printf() style formatting +** string zFmt and its trailing arguments. +*/ +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ + va_list ap; /* ... printf arguments */ + char *zMsg = 0; + + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + if( pConfig->pzErrmsg ){ + assert( *pConfig->pzErrmsg==0 ); + *pConfig->pzErrmsg = zMsg; + }else{ + sqlite3_free(zMsg); + } + + va_end(ap); +} + + diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 05c1b59c1..cd44b96bd 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -54,7 +54,7 @@ struct Fts5Expr { /* ** eType: -** Expression node type. Always one of: +** Expression node type. Usually one of: ** ** FTS5_AND (nChild, apChild valid) ** FTS5_OR (nChild, apChild valid) @@ -62,6 +62,10 @@ struct Fts5Expr { ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) ** +** An expression node with eType==0 may also exist. It always matches zero +** rows. This is created when a phrase containing no tokens is parsed. +** e.g. "". +** ** iHeight: ** Distance from this node to furthest leaf. This is always 0 for nodes ** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one @@ -282,11 +286,12 @@ int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert( sParse.pExpr || sParse.rc!=SQLITE_OK ); assert_expr_depth_ok(sParse.rc, sParse.pExpr); /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ - if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ + if( sParse.rc==SQLITE_OK && iCol<pConfig->nCol ){ int n = sizeof(Fts5Colset); Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); if( pColset ){ @@ -303,15 +308,7 @@ int sqlite3Fts5ExprNew( sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ - if( !sParse.pExpr ){ - const int nByte = sizeof(Fts5ExprNode); - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); - if( pNew->pRoot ){ - pNew->pRoot->bEof = 1; - } - }else{ - pNew->pRoot = sParse.pExpr; - } + pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; @@ -324,7 +321,11 @@ int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } @@ -1125,7 +1126,7 @@ static int fts5ExprNodeTest_STRING( } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - if( pIter->iRowid==iLast || pIter->bEof ) continue; + if( pIter->iRowid==iLast ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; @@ -1647,9 +1648,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( Fts5ExprNearset *pRet = 0; if( pParse->rc==SQLITE_OK ){ - if( pPhrase==0 ){ - return pNear; - } if( pNear==0 ){ sqlite3_int64 nByte; nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); @@ -1871,6 +1869,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( }else if( sCtx.pPhrase->nTerm ){ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; } + assert( pParse->apPhrase!=0 ); pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; } @@ -1890,7 +1889,7 @@ int sqlite3Fts5ExprClonePhrase( Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ - if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){ rc = SQLITE_RANGE; }else{ pOrig = pExpr->apExprPhrase[iPhrase]; @@ -2258,6 +2257,9 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } } +/* +** Add pSub as a child of p. +*/ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ @@ -2402,19 +2404,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode( "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); - sqlite3_free(pRet); + sqlite3Fts5ParseNodeFree(pRet); pRet = 0; + pNear = 0; + assert( pLeft==0 && pRight==0 ); } } }else{ + assert( pNear==0 ); fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); + pLeft = pRight = 0; if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ sqlite3Fts5ParseError(pParse, "fts5 expression tree is too large (maximum depth %d)", SQLITE_FTS5_MAX_EXPR_DEPTH ); - sqlite3_free(pRet); + sqlite3Fts5ParseNodeFree(pRet); pRet = 0; } } @@ -2452,6 +2458,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -2465,6 +2472,8 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( ); if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>0 ); assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); sqlite3Fts5ParseNodeFree(pRight); pRet = pLeft; @@ -3097,6 +3106,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ + case 0: case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 8eb8f328f..1f0a68d3e 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -831,11 +831,12 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ int nByte = sqlite3_blob_bytes(p->pReader); - sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + int szData = (sizeof(Fts5Data) + 7) & ~7; + sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); if( pRet ){ pRet->nn = nByte; - aOut = pRet->p = (u8*)&pRet[1]; + aOut = pRet->p = (u8*)pRet + szData; }else{ rc = SQLITE_NOMEM; } @@ -858,6 +859,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ } assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) ); return pRet; } @@ -6837,23 +6839,26 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ int ii; Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; for(ii=0; ii<pT->nIter; ii++){ Fts5Iter *p = pT->apIter[ii]; if( p->base.bEof==0 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowid<iFrom)) ){ - fts5MultiIterNext(p->pIndex, p, bFrom, iFrom); + fts5MultiIterNext(pIndex, p, bFrom, iFrom); while( bFrom && p->base.bEof==0 && p->base.iRowid<iFrom - && p->pIndex->rc==SQLITE_OK + && pIndex->rc==SQLITE_OK ){ - fts5MultiIterNext(p->pIndex, p, 0, 0); + fts5MultiIterNext(pIndex, p, 0, 0); } } } - fts5IterSetOutputsTokendata(pIter); + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } } /* diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 45dee44b1..4addc0729 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -103,11 +103,28 @@ struct Fts5Auxiliary { ** Each tokenizer module registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part ** of the Fts5Global.pTok list. +** +** bV2Native: +** True if the tokenizer was registered using xCreateTokenizer_v2(), false +** for xCreateTokenizer(). If this variable is true, then x2 is populated +** with the routines as supplied by the caller and x1 contains synthesized +** wrapper routines. In this case the user-data pointer passed to +** x1.xCreate should be a pointer to the Fts5TokenizerModule structure, +** not a copy of pUserData. +** +** Of course, if bV2Native is false, then x1 contains the real routines and +** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule +** object should be passed to x2.xCreate. +** +** The synthesized wrapper routines are necessary for xFindTokenizer(_v2) +** calls. */ struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ - fts5_tokenizer x; /* Tokenizer functions */ + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; @@ -118,7 +135,7 @@ struct Fts5FullTable { Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ int iSavepoint; /* Successful xSavepoint()+1 */ - + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -195,7 +212,7 @@ struct Fts5Cursor { Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ - /* Cache used by auxiliary functions xInst() and xInstCount() */ + /* Cache used by auxiliary API functions xInst() and xInstCount() */ Fts5PoslistReader *aInstIter; /* One for each phrase */ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ @@ -230,6 +247,12 @@ struct Fts5Cursor { #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) +/* +** The subtype value and header bytes used by fts5_locale(). +*/ +#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') +#define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB" + /* ** Macros to Set(), Clear() and Test() cursor flags. @@ -377,8 +400,12 @@ static int fts5InitVtab( assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); } if( rc==SQLITE_OK ){ + pConfig->pzErrmsg = pzErr; pTab->p.pConfig = pConfig; pTab->pGlobal = pGlobal; + if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } } /* Open the index sub-system */ @@ -400,11 +427,8 @@ static int fts5InitVtab( /* Load the initial configuration */ if( rc==SQLITE_OK ){ - assert( pConfig->pzErrmsg==0 ); - pConfig->pzErrmsg = pzErr; rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); sqlite3Fts5IndexRollback(pTab->p.pIndex); - pConfig->pzErrmsg = 0; } if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ @@ -414,6 +438,7 @@ static int fts5InitVtab( rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); } + if( pConfig ) pConfig->pzErrmsg = 0; if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab); pTab = 0; @@ -481,10 +506,10 @@ static int fts5UsePatternMatch( ){ assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB ); assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE ); - if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ + if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ return 1; } - if( pConfig->ePattern==FTS5_PATTERN_LIKE + if( pConfig->t.ePattern==FTS5_PATTERN_LIKE && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB) ){ return 1; @@ -531,10 +556,10 @@ static int fts5UsePatternMatch( ** This function ensures that there is at most one "r" or "=". And that if ** there exists an "=" then there is no "<" or ">". ** -** Costs are assigned as follows: +** If an unusable MATCH operator is present in the WHERE clause, then +** SQLITE_CONSTRAINT is returned. ** -** a) If an unusable MATCH operator is present in the WHERE clause, the -** cost is unconditionally set to 1e50 (a really big number). +** Costs are assigned as follows: ** ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: @@ -567,7 +592,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ int bSeenEq = 0; int bSeenGt = 0; int bSeenLt = 0; - int bSeenMatch = 0; + int nSeenMatch = 0; int bSeenRank = 0; @@ -598,18 +623,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* A MATCH operator or equivalent */ if( p->usable==0 || iCol<0 ){ /* As there exists an unusable MATCH constraint this is an - ** unusable plan. Set a prohibitively high cost. */ - pInfo->estimatedCost = 1e50; - assert( iIdxStr < pInfo->nConstraint*6 + 1 ); - idxStr[iIdxStr] = 0; - return SQLITE_OK; + ** unusable plan. Return SQLITE_CONSTRAINT. */ + return SQLITE_CONSTRAINT; }else{ if( iCol==nCol+1 ){ if( bSeenRank ) continue; idxStr[iIdxStr++] = 'r'; bSeenRank = 1; - }else if( iCol>=0 ){ - bSeenMatch = 1; + }else{ + nSeenMatch++; idxStr[iIdxStr++] = 'M'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); idxStr += strlen(&idxStr[iIdxStr]); @@ -626,6 +648,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ idxStr += strlen(&idxStr[iIdxStr]); pInfo->aConstraintUsage[i].argvIndex = ++iCons; assert( idxStr[iIdxStr]=='\0' ); + nSeenMatch++; }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){ idxStr[iIdxStr++] = '='; bSeenEq = 1; @@ -662,7 +685,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; - if( iSort==(pConfig->nCol+1) && bSeenMatch ){ + if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){ idxFlags |= FTS5_BI_ORDER_RANK; }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ idxFlags |= FTS5_BI_ORDER_ROWID; @@ -677,14 +700,17 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0; - if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo); + pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0; + if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo); }else if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0; + pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0; }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0; + pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0; }else{ - pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0; + pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0; + } + for(i=1; i<nSeenMatch; i++){ + pInfo->estimatedCost *= 0.4; } pInfo->idxNum = idxFlags; @@ -960,6 +986,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ } }else{ rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE); } break; } @@ -989,7 +1016,7 @@ static int fts5PrepareStatement( rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, SQLITE_PREPARE_PERSISTENT, &pRet, 0); if( rc!=SQLITE_OK ){ - *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } @@ -1214,6 +1241,200 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ } /* +** Set the error message on the virtual table passed as the first argument. +*/ +static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + sqlite3_free(p->p.base.zErrMsg); + p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale +** specified by pLocale/nLocale. The buffer indicated by pLocale must remain +** valid until after the final call to sqlite3Fts5Tokenize() that will use +** the locale. +*/ +static void fts5SetLocale( + Fts5Config *pConfig, + const char *zLocale, + int nLocale +){ + Fts5TokenizerConfig *pT = &pConfig->t; + pT->pLocale = zLocale; + pT->nLocale = nLocale; +} + +/* +** Clear any locale configured by an earlier call to fts5SetLocale() or +** sqlite3Fts5ExtractText(). +*/ +void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ + fts5SetLocale(pConfig, 0, 0); +} + +/* +** This function is used to extract utf-8 text from an sqlite3_value. This +** is usually done in order to tokenize it. For example, when: +** +** * a value is written to an fts5 table, +** * a value is deleted from an FTS5_CONTENT_NORMAL table, +** * a value containing a query expression is passed to xFilter() +** +** and so on. +** +** This function handles 2 cases: +** +** 1) Ordinary values. The text can be extracted from these using +** sqlite3_value_text(). +** +** 2) Combination text/locale blobs created by fts5_locale(). There +** are several cases for these: +** +** * Blobs tagged with FTS5_LOCALE_SUBTYPE. +** * Blobs read from the content table of a locale=1 external-content +** table, and +** * Blobs read from the content table of a locale=1 regular +** content table. +** +** The first two cases above should have the 4 byte FTS5_LOCALE_HEADER +** header. It is an error if a blob with the subtype or a blob read +** from the content table of an external content table does not have +** the required header. A blob read from the content table of a regular +** locale=1 table does not have the header. This is to save space. +** +** If successful, SQLITE_OK is returned and output parameters (*ppText) +** and (*pnText) are set to point to a buffer containing the extracted utf-8 +** text and its length in bytes, respectively. The buffer is not +** nul-terminated. It has the same lifetime as the sqlite3_value object +** from which it is extracted. +** +** Parameter bContent must be true if the value was read from an indexed +** column (i.e. not UNINDEXED) of the on disk content. +** +** If pbResetTokenizer is not NULL and if case (2) is used, then +** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls +** use the locale. In this case (*pbResetTokenizer) is set to true before +** returning, to indicate that the caller must call sqlite3Fts5ClearLocale() +** to clear the locale after tokenizing the text. +*/ +int sqlite3Fts5ExtractText( + Fts5Config *pConfig, + sqlite3_value *pVal, /* Value to extract text from */ + int bContent, /* True if indexed table content */ + int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ + const char **ppText, /* OUT: Pointer to text buffer */ + int *pnText /* OUT: Size of (*ppText) in bytes */ +){ + const char *pText = 0; + int nText = 0; + int rc = SQLITE_OK; + int bDecodeBlob = 0; + + assert( pbResetTokenizer==0 || *pbResetTokenizer==0 ); + assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE ); + assert( bContent==0 || sqlite3_value_subtype(pVal)==0 ); + + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE + || (bContent && pConfig->bLocale) + ){ + bDecodeBlob = 1; + } + } + + if( bDecodeBlob ){ + const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + + /* Unless this blob was read from the %_content table of an + ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() + ** header. Check for this. If it is not found, return an error. */ + if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){ + if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){ + rc = SQLITE_ERROR; + }else{ + pBlob += 4; + nBlob -= 4; + } + } + + if( rc==SQLITE_OK ){ + int nLocale = 0; + + for(nLocale=0; nLocale<nBlob; nLocale++){ + if( pBlob[nLocale]==0x00 ) break; + } + if( nLocale==nBlob || nLocale==0 ){ + rc = SQLITE_ERROR; + }else{ + pText = (const char*)&pBlob[nLocale+1]; + nText = nBlob-nLocale-1; + + if( pbResetTokenizer ){ + fts5SetLocale(pConfig, (const char*)pBlob, nLocale); + *pbResetTokenizer = 1; + } + } + } + + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + *ppText = pText; + *pnText = nText; + return rc; +} + +/* +** Argument pVal is the text of a full-text search expression. It may or +** may not have been wrapped by fts5_locale(). This function extracts +** the text of the expression, and sets output variable (*pzText) to +** point to a nul-terminated buffer containing the expression. +** +** If pVal was an fts5_locale() value, then fts5SetLocale() is called to +** set the tokenizer to use the specified locale. +** +** If output variable (*pbFreeAndReset) is set to true, then the caller +** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer +** locale, and (b) call sqlite3_free() to free (*pzText). +*/ +static int fts5ExtractExprText( + Fts5Config *pConfig, /* Fts5 configuration */ + sqlite3_value *pVal, /* Value to extract expression text from */ + char **pzText, /* OUT: nul-terminated buffer of text */ + int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ +){ + const char *zText = 0; + int nText = 0; + int rc = SQLITE_OK; + int bReset = 0; + + *pbFreeAndReset = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, &bReset, &zText, &nText); + if( rc==SQLITE_OK ){ + if( bReset ){ + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText); + if( rc!=SQLITE_OK ){ + sqlite3Fts5ClearLocale(pConfig); + }else{ + *pbFreeAndReset = 1; + } + }else{ + *pzText = (char*)zText; + } + } + + return rc; +} + + +/* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. @@ -1247,13 +1468,7 @@ static int fts5FilterMethod( int iIdxStr = 0; Fts5Expr *pExpr = 0; - if( pConfig->bLock ){ - pTab->p.base.zErrMsg = sqlite3_mprintf( - "recursively defined fts5 content table" - ); - return SQLITE_ERROR; - } - + assert( pConfig->bLock==0 ); if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); @@ -1277,8 +1492,14 @@ static int fts5FilterMethod( pRank = apVal[i]; break; case 'M': { - const char *zText = (const char*)sqlite3_value_text(apVal[i]); + char *zText = 0; + int bFreeAndReset = 0; + int bInternal = 0; + + rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset); + if( rc!=SQLITE_OK ) goto filter_out; if( zText==0 ) zText = ""; + iCol = 0; do{ iCol = iCol*10 + (idxStr[iIdxStr]-'0'); @@ -1290,7 +1511,7 @@ static int fts5FilterMethod( ** indicates that the MATCH expression is not a full text query, ** but a request for an internal parameter. */ rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); - goto filter_out; + bInternal = 1; }else{ char **pzErr = &pTab->p.base.zErrMsg; rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); @@ -1298,9 +1519,15 @@ static int fts5FilterMethod( rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); pExpr = 0; } - if( rc!=SQLITE_OK ) goto filter_out; } + if( bFreeAndReset ){ + sqlite3_free(zText); + sqlite3Fts5ClearLocale(pConfig); + } + + if( bInternal || rc!=SQLITE_OK ) goto filter_out; + break; } case 'L': @@ -1388,9 +1615,7 @@ static int fts5FilterMethod( } } }else if( pConfig->zContent==0 ){ - *pConfig->pzErrmsg = sqlite3_mprintf( - "%s: table does not support scanning", pConfig->zName - ); + fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName); rc = SQLITE_ERROR; }else{ /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup @@ -1433,9 +1658,13 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ assert( pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE + || pCsr->ePlan==FTS5_PLAN_SCAN + || pCsr->ePlan==FTS5_PLAN_ROWID ); if( pCsr->pSorter ){ return pCsr->pSorter->iRowid; + }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){ + return sqlite3_column_int64(pCsr->pStmt, 0); }else{ return sqlite3Fts5ExprRowid(pCsr->pExpr); } @@ -1452,25 +1681,16 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ int ePlan = pCsr->ePlan; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); - switch( ePlan ){ - case FTS5_PLAN_SPECIAL: - *pRowid = 0; - break; - - case FTS5_PLAN_SOURCE: - case FTS5_PLAN_MATCH: - case FTS5_PLAN_SORTED_MATCH: - *pRowid = fts5CursorRowid(pCsr); - break; - - default: - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - break; + if( ePlan==FTS5_PLAN_SPECIAL ){ + *pRowid = 0; + }else{ + *pRowid = fts5CursorRowid(pCsr); } return SQLITE_OK; } + /* ** If the cursor requires seeking (bSeekRequired flag is set), seek it. ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. @@ -1507,8 +1727,13 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK ){ rc = FTS5_CORRUPT; + fts5SetVtabError((Fts5FullTable*)pTab, + "fts5: missing row %lld from content table %s", + fts5CursorRowid(pCsr), + pTab->pConfig->zContent + ); }else if( pTab->pConfig->pzErrmsg ){ - *pTab->pConfig->pzErrmsg = sqlite3_mprintf( + fts5SetVtabError((Fts5FullTable*)pTab, "%s", sqlite3_errmsg(pTab->pConfig->db) ); } @@ -1517,14 +1742,6 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ return rc; } -static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ - va_list ap; /* ... printf arguments */ - va_start(ap, zFormat); - assert( p->p.base.zErrMsg==0 ); - p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); - va_end(ap); -} - /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: @@ -1586,7 +1803,10 @@ static int fts5SpecialInsert( }else if( 0==sqlite3_stricmp("flush", zCmd) ){ rc = sqlite3Fts5FlushToDisk(&pTab->p); }else{ - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } if( rc==SQLITE_OK ){ rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError); } @@ -1615,7 +1835,7 @@ static int fts5SpecialDelete( int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0); } return rc; } @@ -1697,6 +1917,7 @@ static int fts5UpdateMethod( rc = SQLITE_ERROR; }else{ rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); @@ -1738,7 +1959,7 @@ static int fts5UpdateMethod( /* DELETE */ else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0); bUpdateOrDelete = 1; } @@ -1746,16 +1967,31 @@ static int fts5UpdateMethod( else{ int eType1 = sqlite3_value_numeric_type(apVal[1]); - if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ - rc = SQLITE_MISMATCH; + /* Ensure that no fts5_locale() values are written to locale=0 tables. + ** And that no blobs except fts5_locale() blobs are written to indexed + ** (i.e. not UNINDEXED) columns of locale=1 tables. */ + int ii; + for(ii=0; ii<pConfig->nCol; ii++){ + if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){ + int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE); + if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0) + || (pConfig->bLocale==0 && bSub) + ){ + if( pConfig->bLocale==0 ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + } + rc = SQLITE_MISMATCH; + goto update_out; + } + } } - else if( eType0!=SQLITE_INTEGER ){ + if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); @@ -1765,28 +2001,35 @@ static int fts5UpdateMethod( else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ - if( eType1==SQLITE_INTEGER && iOld!=iNew ){ + if( eType1!=SQLITE_INTEGER ){ + rc = SQLITE_MISMATCH; + }else if( iOld!=iNew ){ if( eConflict==SQLITE_REPLACE ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + rc = sqlite3Fts5StorageFindDeleteRow(pTab->pStorage, iOld); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage,apVal,pRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal,*pRowid); } } }else{ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0, 1); fts5StorageInsert(&rc, pTab, apVal, pRowid); } bUpdateOrDelete = 1; + sqlite3Fts5StorageReleaseDeleteRow(pTab->pStorage); } + } } @@ -1803,6 +2046,7 @@ static int fts5UpdateMethod( } } + update_out: pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -1880,17 +2124,40 @@ static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } -static int fts5ApiTokenize( +/* +** Implementation of xTokenize_v2() API. +*/ +static int fts5ApiTokenize_v2( Fts5Context *pCtx, const char *pText, int nText, + const char *pLoc, int nLoc, void *pUserData, int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize( - pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + int rc = SQLITE_OK; + + fts5SetLocale(pTab->pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pTab->pConfig, + FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken ); + fts5SetLocale(pTab->pConfig, 0, 0); + + return rc; +} + +/* +** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0 +** passed as the locale. +*/ +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -1912,28 +2179,37 @@ static int fts5ApiColumnText( int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); if( iCol<0 || iCol>=pTab->pConfig->nCol ){ rc = SQLITE_RANGE; - }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) - || pCsr->ePlan==FTS5_PLAN_SPECIAL - ){ + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + Fts5Config *pConfig = pTab->pConfig; + int bContent = (pConfig->abUnindexed[iCol]==0); + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn); } } return rc; } +/* +** This is called by various API functions - xInst, xPhraseFirst, +** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase +** of the current row. This function works for both detail=full tables (in +** which case the position-list was read from the fts index) or for other +** detail= modes if the row content is available. +*/ static int fts5CsrPoslist( - Fts5Cursor *pCsr, - int iPhrase, - const u8 **pa, - int *pn + Fts5Cursor *pCsr, /* Fts5 cursor object */ + int iPhrase, /* Phrase to find position list for */ + const u8 **pa, /* OUT: Pointer to position list buffer */ + int *pn /* OUT: Size of (*pa) in bytes */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; @@ -1941,20 +2217,34 @@ static int fts5CsrPoslist( if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ rc = SQLITE_RANGE; + }else if( pConfig->eDetail!=FTS5_DETAIL_FULL + && pConfig->eContent==FTS5_CONTENT_NONE + ){ + *pa = 0; + *pn = 0; + return SQLITE_OK; }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = fts5SeekCursor(pCsr, 0); + } for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ - int n; const char *z; - rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); + const char *z = 0; + int n = 0; + int bReset = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } + if( bReset ) sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -1979,7 +2269,6 @@ static int fts5CsrPoslist( *pn = 0; } - return rc; } @@ -2048,7 +2337,8 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); - if( aInst[1]<0 || aInst[1]>=nCol ){ + assert( aInst[1]>=0 ); + if( aInst[1]>=nCol ){ rc = FTS5_CORRUPT; break; } @@ -2135,16 +2425,21 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ } }else{ int i; + rc = fts5SeekCursor(pCsr, 0); for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ if( pConfig->abUnindexed[i]==0 ){ - const char *z; int n; - void *p = (void*)(&pCsr->aColumnSize[i]); + const char *z = 0; + int n = 0; + int bReset = 0; + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); + pCsr->aColumnSize[i] = 0; - rc = fts5ApiColumnText(pCtx, i, &z, &n); + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize( - pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, + z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); } } } @@ -2225,11 +2520,10 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ } static void fts5ApiPhraseNext( - Fts5Context *pUnused, + Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ - UNUSED_PARAM(pUnused); if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; @@ -2237,8 +2531,12 @@ static void fts5ApiPhraseNext( int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ + /* Avoid returning a (*piCol) value that is too large for the table, + ** even if the position-list is corrupt. The caller might not be + ** expecting it. */ + int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol; pIter->a += fts5GetVarint32(pIter->a, iVal); - *piCol = iVal; + *piCol = (iVal>=nCol ? nCol-1 : iVal); *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } @@ -2388,8 +2686,71 @@ static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); +/* +** The xColumnLocale() API. +*/ +static int fts5ApiColumnLocale( + Fts5Context *pCtx, + int iCol, + const char **pzLocale, + int *pnLocale +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + *pzLocale = 0; + *pnLocale = 0; + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( + pConfig->abUnindexed[iCol]==0 + && pConfig->eContent!=FTS5_CONTENT_NONE + && pConfig->bLocale + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + /* Load the value into pVal. pVal is a locale/text pair iff: + ** + ** 1) It is an SQLITE_BLOB, and + ** 2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the + ** value was loaded from an FTS5_CONTENT_NORMAL table, and + ** 3) It does not begin with an 0x00 byte. + */ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ + const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){ + rc = SQLITE_ERROR; + } + pBlob += 4; + nBlob -= 4; + } + if( rc==SQLITE_OK ){ + int nLocale = 0; + for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++); + if( nLocale==nBlob || nLocale==0 ){ + rc = SQLITE_ERROR; + }else{ + /* A locale/text pair */ + *pzLocale = (const char*)pBlob; + *pnLocale = nLocale; + } + } + } + } + } + + return rc; +} + static const Fts5ExtensionApi sFts5Api = { - 3, /* iVersion */ + 4, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -2410,7 +2771,9 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, fts5ApiQueryToken, - fts5ApiInstToken + fts5ApiInstToken, + fts5ApiColumnLocale, + fts5ApiTokenize_v2 }; /* @@ -2461,6 +2824,7 @@ static void fts5ApiInvoke( sqlite3_value **argv ){ assert( pCsr->pAux==0 ); + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; @@ -2474,6 +2838,21 @@ static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ return pCsr; } +/* +** Parameter zFmt is a printf() style formatting string. This function +** formats it using the trailing arguments and returns the result as +** an error message to the context passed as the first argument. +*/ +static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + va_end(ap); +} + static void fts5ApiCallback( sqlite3_context *context, int argc, @@ -2489,12 +2868,13 @@ static void fts5ApiCallback( iCsrId = sqlite3_value_int64(argv[0]); pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); - if( pCsr==0 || pCsr->ePlan==0 ){ - char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){ + fts5ResultError(context, "no such cursor: %lld", iCsrId); }else{ + sqlite3_vtab *pTab = pCsr->base.pVtab; fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + sqlite3_free(pTab->zErrMsg); + pTab->zErrMsg = 0; } } @@ -2583,6 +2963,57 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ return rc; } +/* +** Value pVal was read from column iCol of the FTS5 table. This function +** returns it to the owner of pCtx via a call to an sqlite3_result_xxx() +** function. This function deals with the same cases as +** sqlite3Fts5ExtractText(): +** +** 1) Ordinary values. These can be returned using sqlite3_result_value(). +** +** 2) Blobs from fts5_locale(). The text is extracted from these and +** returned via sqlite3_result_text(). The locale is discarded. +*/ +static void fts5ExtractValueFromColumn( + sqlite3_context *pCtx, + Fts5Config *pConfig, + int iCol, + sqlite3_value *pVal +){ + assert( pConfig->eContent!=FTS5_CONTENT_NONE ); + + if( pConfig->bLocale + && sqlite3_value_type(pVal)==SQLITE_BLOB + && pConfig->abUnindexed[iCol]==0 + ){ + const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + int ii; + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ + if( nBlob<SZHDR || memcmp(pBlob, FTS5_LOCALE_HEADER, SZHDR) ){ + sqlite3_result_error_code(pCtx, SQLITE_ERROR); + return; + }else{ + pBlob += 4; + nBlob -= 4; + } + } + + for(ii=0; ii<nBlob && pBlob[ii]; ii++); + if( ii==0 || ii==nBlob ){ + sqlite3_result_error_code(pCtx, SQLITE_ERROR); + }else{ + const char *pText = (const char*)&pBlob[ii+1]; + sqlite3_result_text(pCtx, pText, nBlob-ii-1, SQLITE_TRANSIENT); + } + return; + } + + sqlite3_result_value(pCtx, pVal); +} + /* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. @@ -2612,8 +3043,8 @@ static int fts5ColumnMethod( ** auxiliary function. */ sqlite3_result_int64(pCtx, pCsr->iCsrId); }else if( iCol==pConfig->nCol+1 ){ - /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); }else if( @@ -2624,20 +3055,27 @@ static int fts5ColumnMethod( fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); } } - }else if( !fts5IsContentless(pTab) ){ - pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - rc = fts5SeekCursor(pCsr, 1); - if( rc==SQLITE_OK ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + }else{ + /* A column created by the user containing values. */ + int bNochange = sqlite3_vtab_nochange(pCtx); + + if( fts5IsContentless(pTab) ){ + if( bNochange && pConfig->bContentlessDelete ){ + fts5ResultError(pCtx, "cannot UPDATE a subset of " + "columns on fts5 contentless-delete table: %s", pConfig->zName + ); + } + }else if( bNochange==0 || pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal); + } + pConfig->pzErrmsg = 0; } - pConfig->pzErrmsg = 0; - }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){ - char *zErr = sqlite3_mprintf("cannot UPDATE a subset of " - "columns on fts5 contentless-delete table: %s", pConfig->zName - ); - sqlite3_result_error(pCtx, zErr, -1); - sqlite3_free(zErr); } + return rc; } @@ -2777,47 +3215,209 @@ static int fts5CreateAux( } /* -** Register a new tokenizer. This is the implementation of the -** fts5_api.xCreateTokenizer() method. +** This function is used by xCreateTokenizer_v2() and xCreateTokenizer(). +** It allocates and partially populates a new Fts5TokenizerModule object. +** The new object is already linked into the Fts5Global context before +** returning. +** +** If successful, SQLITE_OK is returned and a pointer to the new +** Fts5TokenizerModule object returned via output parameter (*ppNew). All +** that is required is for the caller to fill in the methods in +** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native +** as appropriate. +** +** If an error occurs, an SQLite error code is returned and the final value +** of (*ppNew) undefined. */ -static int fts5CreateTokenizer( - fts5_api *pApi, /* Global context (one per db handle) */ +static int fts5NewTokenizerModule( + Fts5Global *pGlobal, /* Global context (one per db handle) */ const char *zName, /* Name of new function */ void *pUserData, /* User data for aux. function */ - fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ - void(*xDestroy)(void*) /* Destructor for pUserData */ + void(*xDestroy)(void*), /* Destructor for pUserData */ + Fts5TokenizerModule **ppNew ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; - Fts5TokenizerModule *pNew; - sqlite3_int64 nName; /* Size of zName and its \0 terminator */ - sqlite3_int64 nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ nName = strlen(zName) + 1; nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); + *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); if( pNew ){ - memset(pNew, 0, (size_t)nByte); pNew->zName = (char*)&pNew[1]; memcpy(pNew->zName, zName, nName); pNew->pUserData = pUserData; - pNew->x = *pTokenizer; pNew->xDestroy = xDestroy; pNew->pNext = pGlobal->pTok; pGlobal->pTok = pNew; if( pNew->pNext==0 ){ pGlobal->pDfltTok = pNew; } + } + + return rc; +} + +/* +** An instance of this type is used as the Fts5Tokenizer object for +** wrapper tokenizers - those that provide access to a v1 tokenizer via +** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer +** via the fts5_tokenizer API. +*/ +typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; +struct Fts5VtoVTokenizer { + Fts5TokenizerModule *pMod; + Fts5Tokenizer *pReal; +}; + +/* +** Create a wrapper tokenizer. The context argument pCtx points to the +** Fts5TokenizerModule object. +*/ +static int fts5VtoVCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx; + Fts5VtoVTokenizer *pNew = 0; + int rc = SQLITE_OK; + + pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + if( rc==SQLITE_OK ){ + pNew->pMod = pMod; + if( pMod->bV2Native ){ + rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + }else{ + rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + } + } + + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Delete an Fts5VtoVTokenizer wrapper tokenizer. +*/ +static void fts5VtoVDelete(Fts5Tokenizer *pTok){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + if( p ){ + Fts5TokenizerModule *pMod = p->pMod; + if( pMod->bV2Native ){ + pMod->x2.xDelete(p->pReal); + }else{ + pMod->x1.xDelete(p->pReal); + } + sqlite3_free(p); + } +} + + +/* +** xTokenizer method for a wrapper tokenizer that offers the v1 interface +** (no support for locales). +*/ +static int fts5V1toV2Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + Fts5TokenizerModule *pMod = p->pMod; + assert( pMod->bV2Native ); + return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** xTokenizer method for a wrapper tokenizer that offers the v2 interface +** (with locale support). +*/ +static int fts5V2toV1Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + Fts5TokenizerModule *pMod = p->pMod; + assert( pMod->bV2Native==0 ); + UNUSED_PARAM2(pLocale,nLocale); + return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer_v2() method. +*/ +static int fts5CreateTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + + if( pTokenizer->iVersion>2 ){ + rc = SQLITE_ERROR; }else{ - rc = SQLITE_NOMEM; + Fts5TokenizerModule *pNew = 0; + rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew); + if( pNew ){ + pNew->x2 = *pTokenizer; + pNew->bV2Native = 1; + pNew->x1.xCreate = fts5VtoVCreate; + pNew->x1.xTokenize = fts5V1toV2Tokenize; + pNew->x1.xDelete = fts5VtoVDelete; + } } return rc; } +/* +** The fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5TokenizerModule *pNew = 0; + int rc = SQLITE_OK; + + rc = fts5NewTokenizerModule( + (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew + ); + if( pNew ){ + pNew->x1 = *pTokenizer; + pNew->x2.xCreate = fts5VtoVCreate; + pNew->x2.xTokenize = fts5V2toV1Tokenize; + pNew->x2.xDelete = fts5VtoVDelete; + } + return rc; +} + +/* +** Search the global context passed as the first argument for a tokenizer +** module named zName. If found, return a pointer to the Fts5TokenizerModule +** object. Otherwise, return NULL. +*/ static Fts5TokenizerModule *fts5LocateTokenizer( - Fts5Global *pGlobal, - const char *zName + Fts5Global *pGlobal, /* Global (one per db handle) object */ + const char *zName /* Name of tokenizer module to find */ ){ Fts5TokenizerModule *pMod = 0; @@ -2834,6 +3434,36 @@ static Fts5TokenizerModule *fts5LocateTokenizer( /* ** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer_v2() method. +*/ +static int fts5FindTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of tokenizer */ + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *ppTokenizer = &pMod->x2; + }else{ + *ppTokenizer = 0; + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. */ static int fts5FindTokenizer( @@ -2847,53 +3477,75 @@ static int fts5FindTokenizer( pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); if( pMod ){ - *pTokenizer = pMod->x; - *ppUserData = pMod->pUserData; + if( pMod->bV2Native==0 ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *pTokenizer = pMod->x1; }else{ - memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + memset(pTokenizer, 0, sizeof(*pTokenizer)); + *ppUserData = 0; rc = SQLITE_ERROR; } return rc; } -int sqlite3Fts5GetTokenizer( - Fts5Global *pGlobal, - const char **azArg, - int nArg, - Fts5Config *pConfig, - char **pzErr -){ - Fts5TokenizerModule *pMod; +/* +** Attempt to instantiate the tokenizer. +*/ +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ + const char **azArg = pConfig->t.azArg; + const int nArg = pConfig->t.nArg; + Fts5TokenizerModule *pMod = 0; int rc = SQLITE_OK; - pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]); }else{ - rc = pMod->x.xCreate( - pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; + if( pMod->bV2Native ){ + xCreate = pMod->x2.xCreate; + pConfig->t.pApi2 = &pMod->x2; + }else{ + pConfig->t.pApi1 = &pMod->x1; + xCreate = pMod->x1.xCreate; + } + + rc = xCreate(pMod->pUserData, + (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok ); - pConfig->pTokApi = &pMod->x; + if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); - }else{ - pConfig->ePattern = sqlite3Fts5TokenizerPattern( - pMod->x.xCreate, pConfig->pTok + if( rc!=SQLITE_NOMEM ){ + sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor"); + } + }else if( pMod->bV2Native==0 ){ + pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( + pMod->x1.xCreate, pConfig->t.pTok ); } } if( rc!=SQLITE_OK ){ - pConfig->pTokApi = 0; - pConfig->pTok = 0; + pConfig->t.pApi1 = 0; + pConfig->t.pApi2 = 0; + pConfig->t.pTok = 0; } return rc; } + +/* +** xDestroy callback passed to sqlite3_create_module(). This is invoked +** when the db handle is being closed. Free memory associated with +** tokenizers and aux functions registered with this db handle. +*/ static void fts5ModuleDestroy(void *pCtx){ Fts5TokenizerModule *pTok, *pNextTok; Fts5Auxiliary *pAux, *pNextAux; @@ -2914,6 +3566,10 @@ static void fts5ModuleDestroy(void *pCtx){ sqlite3_free(pGlobal); } +/* +** Implementation of the fts5() function used by clients to obtain the +** API pointer. +*/ static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ @@ -2941,6 +3597,69 @@ static void fts5SourceIdFunc( } /* +** Implementation of fts5_locale(LOCALE, TEXT) function. +** +** If parameter LOCALE is NULL, or a zero-length string, then a copy of +** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as +** text, and the value returned is a blob consisting of: +** +** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER). +** * The LOCALE, as utf-8 text, followed by +** * 0x00, followed by +** * The TEXT, as utf-8 text. +** +** There is no final nul-terminator following the TEXT value. +*/ +static void fts5LocaleFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + const char *zLocale = 0; + int nLocale = 0; + const char *zText = 0; + int nText = 0; + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + + zLocale = (const char*)sqlite3_value_text(apArg[0]); + nLocale = sqlite3_value_bytes(apArg[0]); + + zText = (const char*)sqlite3_value_text(apArg[1]); + nText = sqlite3_value_bytes(apArg[1]); + + if( zLocale==0 || zLocale[0]=='\0' ){ + sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT); + }else{ + u8 *pBlob = 0; + u8 *pCsr = 0; + int nBlob = 0; + const int nHdr = 4; + assert( sizeof(FTS5_LOCALE_HEADER)==nHdr+1 ); + + nBlob = nHdr + nLocale + 1 + nText; + pBlob = (u8*)sqlite3_malloc(nBlob); + if( pBlob==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + + pCsr = pBlob; + memcpy(pCsr, FTS5_LOCALE_HEADER, nHdr); + pCsr += nHdr; + memcpy(pCsr, zLocale, nLocale); + pCsr += nLocale; + (*pCsr++) = 0x00; + if( zText ) memcpy(pCsr, zText, nText); + assert( &pCsr[nText]==&pBlob[nBlob] ); + + sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); + sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE); + } +} + +/* ** Return true if zName is the extension on one of the shadow tables used ** by this module. */ @@ -2968,28 +3687,29 @@ static int fts5IntegrityMethod( char **pzErr /* Write error message here */ ){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - Fts5Config *pConfig = pTab->p.pConfig; - char *zSql; - char *zErr = 0; int rc; + assert( pzErr!=0 && *pzErr==0 ); UNUSED_PARAM(isQuick); - zSql = sqlite3_mprintf( - "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');", - zSchema, zTabname, pConfig->zName); - if( zSql==0 ) return SQLITE_NOMEM; - rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr); - sqlite3_free(zSql); - if( (rc&0xff)==SQLITE_CORRUPT ){ - *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", - zSchema, zTabname); - }else if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unable to validate the inverted index for" - " FTS5 table %s.%s: %s", - zSchema, zTabname, zErr); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } } - sqlite3_free(zErr); - return SQLITE_OK; + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; + + return rc; } static int fts5Init(sqlite3 *db){ @@ -3031,10 +3751,12 @@ static int fts5Init(sqlite3 *db){ void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 2; + pGlobal->api.iVersion = 3; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; + pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; + pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); @@ -3053,6 +3775,13 @@ static int fts5Init(sqlite3 *db){ p, fts5SourceIdFunc, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_locale", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5LocaleFunc, 0, 0 + ); + } } /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index a04b152fb..cf25eb361 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -16,13 +16,40 @@ #include "fts5Int.h" +/* +** pSavedRow: +** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it +** does a by-rowid lookup to retrieve a single row from the %_content +** table or equivalent external-content table/view. +** +** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original +** values for a row being UPDATEd. In that case, the SQL statement is +** not reset and pSavedRow is set to point at it. This is so that the +** insert operation that follows the delete may access the original +** row values for any new values for which sqlite3_value_nochange() returns +** true. i.e. if the user executes: +** +** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1); +** ... +** UPDATE fts SET a=?, b=? WHERE rowid=?; +** +** then the value passed to the xUpdate() method of this table as the +** new.c value is an sqlite3_value_nochange() value. So in this case it +** must be read from the saved row stored in Fts5Storage.pSavedRow. +** +** This is necessary - using sqlite3_value_nochange() instead of just having +** SQLite pass the original value back via xUpdate() - so as not to discard +** any locale information associated with such values. +** +*/ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ i64 nTotalRow; /* Total number of rows in FTS table */ i64 *aTotalSize; /* Total sizes of each column */ - sqlite3_stmt *aStmt[11]; + sqlite3_stmt *pSavedRow; + sqlite3_stmt *aStmt[12]; }; @@ -36,14 +63,15 @@ struct Fts5Storage { # error "FTS5_STMT_LOOKUP mismatch" #endif -#define FTS5_STMT_INSERT_CONTENT 3 -#define FTS5_STMT_REPLACE_CONTENT 4 -#define FTS5_STMT_DELETE_CONTENT 5 -#define FTS5_STMT_REPLACE_DOCSIZE 6 -#define FTS5_STMT_DELETE_DOCSIZE 7 -#define FTS5_STMT_LOOKUP_DOCSIZE 8 -#define FTS5_STMT_REPLACE_CONFIG 9 -#define FTS5_STMT_SCAN 10 +#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_INSERT_CONTENT 4 +#define FTS5_STMT_REPLACE_CONTENT 5 +#define FTS5_STMT_DELETE_CONTENT 6 +#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_DELETE_DOCSIZE 8 +#define FTS5_STMT_LOOKUP_DOCSIZE 9 +#define FTS5_STMT_REPLACE_CONFIG 10 +#define FTS5_STMT_SCAN 11 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -73,6 +101,7 @@ static int fts5StorageGetStmt( "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ @@ -88,6 +117,8 @@ static int fts5StorageGetStmt( Fts5Config *pC = p->pConfig; char *zSql = 0; + assert( ArraySize(azStmt)==ArraySize(p->aStmt) ); + switch( eStmt ){ case FTS5_STMT_SCAN: zSql = sqlite3_mprintf(azStmt[eStmt], @@ -104,6 +135,7 @@ static int fts5StorageGetStmt( break; case FTS5_STMT_LOOKUP: + case FTS5_STMT_LOOKUP2: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent, pC->zContentRowid ); @@ -150,7 +182,7 @@ static int fts5StorageGetStmt( rc = SQLITE_NOMEM; }else{ int f = SQLITE_PREPARE_PERSISTENT; - if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB; + if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB; p->pConfig->bLock++; rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); p->pConfig->bLock--; @@ -400,14 +432,48 @@ static int fts5StorageInsertCallback( } /* +** This function is used as part of an UPDATE statement that modifies the +** rowid of a row. In that case, this function is called first to set +** Fts5Storage.pSavedRow to point to a statement that may be used to +** access the original values of the row being deleted - iDel. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if row iDel does not exist. In this case +** pSavedRow is not set and SQLITE_OK returned. +*/ +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ + int rc = SQLITE_OK; + sqlite3_stmt *pSeek = 0; + + assert( p->pSavedRow==0 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + rc = sqlite3_reset(pSeek); + }else{ + p->pSavedRow = pSeek; + } + } + + return rc; +} + +/* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. +** +** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left +** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access +** the original values of the row being deleted. This is used by UPDATE +** statements. */ static int fts5StorageDeleteFromIndex( Fts5Storage *p, i64 iDel, - sqlite3_value **apVal + sqlite3_value **apVal, + int bSaveRow /* True to set pSavedRow */ ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ @@ -416,12 +482,21 @@ static int fts5StorageDeleteFromIndex( int iCol; Fts5InsertCtx ctx; + assert( bSaveRow==0 || apVal==0 ); + assert( bSaveRow==0 || bSaveRow==1 ); + assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 ); + if( apVal==0 ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pSeek, 1, iDel); - if( sqlite3_step(pSeek)!=SQLITE_ROW ){ - return sqlite3_reset(pSeek); + if( p->pSavedRow && bSaveRow ){ + pSeek = p->pSavedRow; + p->pSavedRow = 0; + }else{ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } } } @@ -429,26 +504,32 @@ static int fts5StorageDeleteFromIndex( ctx.iCol = -1; for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ - const char *zText; - int nText; + sqlite3_value *pVal = 0; + const char *pText = 0; + int nText = 0; + int bReset = 0; + assert( pSeek==0 || apVal==0 ); assert( pSeek!=0 || apVal!=0 ); if( pSeek ){ - zText = (const char*)sqlite3_column_text(pSeek, iCol); - nText = sqlite3_column_bytes(pSeek, iCol); - }else if( ALWAYS(apVal) ){ - zText = (const char*)sqlite3_value_text(apVal[iCol-1]); - nText = sqlite3_value_bytes(apVal[iCol-1]); + pVal = sqlite3_column_value(pSeek, iCol); }else{ - continue; + pVal = apVal[iCol-1]; } - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, - zText, nText, (void*)&ctx, fts5StorageInsertCallback + + rc = sqlite3Fts5ExtractText( + pConfig, pVal, pSeek!=0, &bReset, &pText, &nText ); - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; - if( p->aTotalSize[iCol-1]<0 ){ - rc = FTS5_CORRUPT; + if( rc==SQLITE_OK ){ + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + pText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ + rc = FTS5_CORRUPT; + } + if( bReset ) sqlite3Fts5ClearLocale(pConfig); } } } @@ -458,12 +539,30 @@ static int fts5StorageDeleteFromIndex( p->nTotalRow--; } - rc2 = sqlite3_reset(pSeek); - if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bSaveRow ){ + assert( p->pSavedRow==0 ); + p->pSavedRow = pSeek; + }else{ + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + } return rc; } /* +** Reset any saved statement pSavedRow. Zero pSavedRow as well. This +** should be called by the xUpdate() method of the fts5 table before +** returning from any operation that may have set Fts5Storage.pSavedRow. +*/ +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + assert( pStorage->pSavedRow==0 + || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] + ); + sqlite3_reset(pStorage->pSavedRow); + pStorage->pSavedRow = 0; +} + +/* ** This function is called to process a DELETE on a contentless_delete=1 ** table. It adds the tombstone required to delete the entry with rowid ** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, @@ -519,12 +618,12 @@ static int fts5StorageInsertDocsize( rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); sqlite3_bind_int64(pReplace, 3, iOrigin); } - if( rc==SQLITE_OK ){ - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); - } + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } } return rc; @@ -578,7 +677,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){ /* ** Remove a row from the FTS table. */ -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ +int sqlite3Fts5StorageDelete( + Fts5Storage *p, /* Storage object */ + i64 iDel, /* Rowid to delete from table */ + sqlite3_value **apVal, /* Optional - values to remove from index */ + int bSaveRow /* If true, set pSavedRow for deleted row */ +){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; @@ -595,7 +699,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ if( p->pConfig->bContentlessDelete ){ rc = fts5StorageContentlessDelete(p, iDel); }else{ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow); } } @@ -684,14 +788,21 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1); - int nText = sqlite3_column_bytes(pScan, ctx.iCol+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback - ); + int bReset = 0; /* True if tokenizer locale must be reset */ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + + rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -775,7 +886,31 @@ int sqlite3Fts5StorageContentInsert( int i; /* Counter variable */ rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); + sqlite3_value *pVal = apVal[i]; + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + /* This is an UPDATE statement, and column (i-2) was not modified. + ** Retrieve the value from Fts5Storage.pSavedRow instead. */ + pVal = sqlite3_column_value(p->pSavedRow, i-1); + }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ + assert( pConfig->bLocale ); + assert( i>1 ); + if( pConfig->abUnindexed[i-2] ){ + /* At attempt to insert an fts5_locale() value into an UNINDEXED + ** column. Strip the locale away and just bind the text. */ + const char *pText = 0; + int nText = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + }else{ + const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + assert( nBlob>4 ); + sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT); + } + continue; + } + + rc = sqlite3_bind_value(pInsert, i, pVal); } if( rc==SQLITE_OK ){ sqlite3_step(pInsert); @@ -810,14 +945,24 @@ int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]); - int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback - ); + int bReset = 0; /* True if tokenizer locale must be reset */ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + sqlite3_value *pVal = apVal[ctx.iCol+2]; + int bDisk = 0; + if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ + pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); + bDisk = 1; + } + rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText); + if( rc==SQLITE_OK ){ + assert( bReset==0 || pConfig->bLocale ); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -988,14 +1133,22 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ - const char *zText = (const char*)sqlite3_column_text(pScan, i+1); - int nText = sqlite3_column_bytes(pScan, i+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageIntegrityCallback + int bReset = 0; /* True if tokenizer locale must be reset */ + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + + rc = sqlite3Fts5ExtractText(pConfig, + sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + if( bReset ) sqlite3Fts5ClearLocale(pConfig); + } } if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 853a41865..1e9f7bbb6 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -14,14 +14,7 @@ #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #ifdef SQLITE_ENABLE_FTS5 @@ -247,6 +240,7 @@ static int SQLITE_TCLAPI xF5tApi( { "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */ { "xInstToken", 2, "IDX ITERM" }, /* 19 */ + { "xColumnLocale", 1, "COL" }, /* 20 */ { 0, 0, 0} }; @@ -297,12 +291,12 @@ static int SQLITE_TCLAPI xF5tApi( break; } CASE(3, "xTokenize") { - int nText; + Tcl_Size nText; char *zText = Tcl_GetStringFromObj(objv[2], &nText); F5tFunction ctx; ctx.interp = interp; ctx.pScript = objv[3]; - rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb); + rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb); if( rc==SQLITE_OK ){ Tcl_ResetResult(interp); } @@ -535,6 +529,20 @@ static int SQLITE_TCLAPI xF5tApi( break; } + CASE(20, "xColumnLocale") { + const char *z = 0; + int n = 0; + int iCol; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ + return TCL_ERROR; + } + rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n); + if( rc==SQLITE_OK && z ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); + } + break; + } + default: assert( 0 ); break; @@ -605,15 +613,16 @@ static void xF5tFunction( sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1); }else{ Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); - int n; const char *zType = (pVar->typePtr ? pVar->typePtr->name : ""); char c = zType[0]; if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ /* Only return a BLOB type if the Tcl variable is a bytearray and ** has no string representation. */ - unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT); + Tcl_Size nn; + unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn); + sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT); }else if( c=='b' && strcmp(zType,"boolean")==0 ){ + int n; Tcl_GetIntFromObj(0, pVar, &n); sqlite3_result_int(pCtx, n); }else if( c=='d' && strcmp(zType,"double")==0 ){ @@ -626,8 +635,9 @@ static void xF5tFunction( Tcl_GetWideIntFromObj(0, pVar, &v); sqlite3_result_int64(pCtx, v); }else{ - unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT); + Tcl_Size nn; + unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn); + sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT); } } } @@ -720,7 +730,7 @@ static int SQLITE_TCLAPI f5tTokenize( Tcl_Obj *CONST objv[] ){ char *zText; - int nText; + Tcl_Size nText; sqlite3 *db = 0; fts5_api *pApi = 0; Fts5Tokenizer *pTok = 0; @@ -729,7 +739,7 @@ static int SQLITE_TCLAPI f5tTokenize( void *pUserdata; int rc; - int nArg; + Tcl_Size nArg; const char **azArg; F5tTokenizeCtx ctx; @@ -761,7 +771,7 @@ static int SQLITE_TCLAPI f5tTokenize( return TCL_ERROR; } - rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok); + rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0); return TCL_ERROR; @@ -773,7 +783,7 @@ static int SQLITE_TCLAPI f5tTokenize( ctx.pRet = pRet; ctx.zInput = zText; rc = tokenizer.xTokenize( - pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2 + pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText,(int)nText, xTokenizeCb2 ); tokenizer.xDelete(pTok); if( rc!=SQLITE_OK ){ @@ -801,18 +811,32 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; int (*xToken)(void*, int, const char*, int, int, int); + F5tTokenizerInstance *pInst; }; struct F5tTokenizerModule { Tcl_Interp *interp; Tcl_Obj *pScript; + void *pParentCtx; + fts5_tokenizer_v2 parent_v2; + fts5_tokenizer parent; F5tTokenizerContext *pContext; }; +/* +** zLocale: +** Within a call to xTokenize_v2(), pLocale/nLocale store the locale +** passed to the call by fts5. This can be retrieved by a Tcl tokenize +** script using [sqlite3_fts5_locale]. +*/ struct F5tTokenizerInstance { Tcl_Interp *interp; Tcl_Obj *pScript; + F5tTokenizerModule *pModule; + Fts5Tokenizer *pParent; F5tTokenizerContext *pContext; + const char *pLocale; + int nLocale; }; static int f5tTokenizerCreate( @@ -821,11 +845,20 @@ static int f5tTokenizerCreate( int nArg, Fts5Tokenizer **ppOut ){ + Fts5Tokenizer *pParent = 0; F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; Tcl_Obj *pEval; int rc = TCL_OK; int i; + assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 ); + if( pMod->parent_v2.xCreate ){ + rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + if( pMod->parent.xCreate ){ + rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + pEval = Tcl_DuplicateObj(pMod->pScript); Tcl_IncrRefCount(pEval); for(i=0; rc==TCL_OK && i<nArg; i++){ @@ -845,6 +878,8 @@ static int f5tTokenizerCreate( pInst->interp = pMod->interp; pInst->pScript = Tcl_GetObjResult(pMod->interp); pInst->pContext = pMod->pContext; + pInst->pParent = pParent; + pInst->pModule = pMod; Tcl_IncrRefCount(pInst->pScript); *ppOut = (Fts5Tokenizer*)pInst; } @@ -855,11 +890,21 @@ static int f5tTokenizerCreate( static void f5tTokenizerDelete(Fts5Tokenizer *p){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; - Tcl_DecrRefCount(pInst->pScript); - ckfree((char *)pInst); + if( pInst ){ + if( pInst->pParent ){ + if( pInst->pModule->parent_v2.xDelete ){ + pInst->pModule->parent_v2.xDelete(pInst->pParent); + }else{ + pInst->pModule->parent.xDelete(pInst->pParent); + } + } + Tcl_DecrRefCount(pInst->pScript); + ckfree((char *)pInst); + } } -static int f5tTokenizerTokenize( + +static int f5tTokenizerReallyTokenize( Fts5Tokenizer *p, void *pCtx, int flags, @@ -867,6 +912,7 @@ static int f5tTokenizerTokenize( int (*xToken)(void*, int, const char*, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + F5tTokenizerInstance *pOldInst = 0; void *pOldCtx; int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; @@ -875,9 +921,11 @@ static int f5tTokenizerTokenize( pOldCtx = pInst->pContext->pCtx; xOldToken = pInst->pContext->xToken; + pOldInst = pInst->pContext->pInst; pInst->pContext->pCtx = pCtx; pInst->pContext->xToken = xToken; + pInst->pContext->pInst = pInst; assert( flags==FTS5_TOKENIZE_DOCUMENT @@ -913,9 +961,105 @@ static int f5tTokenizerTokenize( pInst->pContext->pCtx = pOldCtx; pInst->pContext->xToken = xOldToken; + pInst->pContext->pInst = pOldInst; return rc; } +typedef struct CallbackCtx CallbackCtx; +struct CallbackCtx { + Fts5Tokenizer *p; + void *pCtx; + int flags; + int (*xToken)(void*, int, const char*, int, int, int); +}; + +static int f5tTokenizeCallback( + void *pCtx, + int tflags, + const char *z, int n, + int iStart, int iEnd +){ + CallbackCtx *p = (CallbackCtx*)pCtx; + return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken); +} + +static int f5tTokenizerTokenize_v2( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + const char *pLoc, int nLoc, + int (*xToken)(void*, int, const char*, int, int, int) +){ + int rc = SQLITE_OK; + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + + pInst->pLocale = pLoc; + pInst->nLocale = nLoc; + + if( pInst->pParent ){ + CallbackCtx ctx; + ctx.p = p; + ctx.pCtx = pCtx; + ctx.flags = flags; + ctx.xToken = xToken; + if( pInst->pModule->parent_v2.xTokenize ){ + rc = pInst->pModule->parent_v2.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, + pLoc, nLoc, f5tTokenizeCallback + ); + }else{ + rc = pInst->pModule->parent.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback + ); + } + }else{ + rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); + } + + pInst->pLocale = 0; + pInst->nLocale = 0; + return rc; +} +static int f5tTokenizerTokenize( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** sqlite3_fts5_locale +*/ +static int SQLITE_TCLAPI f5tTokenizerLocale( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + if( p->xToken==0 ){ + Tcl_AppendResult(interp, + "sqlite3_fts5_locale may only be used by tokenizer callback", 0 + ); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, + Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale) + ); + return TCL_OK; +} + /* ** sqlite3_fts5_token ?-colocated? TEXT START END */ @@ -928,13 +1072,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn( F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; int iStart; int iEnd; - int nToken; + Tcl_Size nToken; int tflags = 0; char *zToken; int rc; if( objc==5 ){ - int nArg; + Tcl_Size nArg; char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ tflags |= FTS5_TOKEN_COLOCATED; @@ -959,7 +1103,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn( return TCL_ERROR; } - rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return rc==SQLITE_OK ? TCL_OK : TCL_ERROR; @@ -1001,32 +1145,112 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( fts5_api *pApi; char *zName; Tcl_Obj *pScript; - fts5_tokenizer t; F5tTokenizerModule *pMod; - int rc; + int rc = SQLITE_OK; + int bV2 = 0; /* True to use _v2 API */ + int iVersion = 2; /* Value for _v2.iVersion */ + const char *zParent = 0; /* Name of parent tokenizer, if any */ + int ii = 0; - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT"); + if( objc<4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT"); return TCL_ERROR; } - if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ - return TCL_ERROR; + + /* Parse any options. Set stack variables bV2 and zParent. */ + for(ii=1; ii<objc-3; ii++){ + int iOpt = 0; + const char *azOpt[] = { "-v2", "-parent", "-version", 0 }; + if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){ + return TCL_ERROR; + } + switch( iOpt ){ + case 0: /* -v2 */ { + bV2 = 1; + break; + } + case 1: /* -parent */ { + ii++; + if( ii==objc-3 ){ + Tcl_AppendResult( + interp, "option requires an argument: -parent", (char*)0 + ); + return TCL_ERROR; + } + zParent = Tcl_GetString(objv[ii]); + break; + } + case 2: /* -version */ { + ii++; + if( ii==objc-3 ){ + Tcl_AppendResult( + interp, "option requires an argument: -version", (char*)0 + ); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){ + return TCL_ERROR; + } + break; + } + default: + assert( 0 ); + break; + } } - zName = Tcl_GetString(objv[2]); - pScript = objv[3]; - t.xCreate = f5tTokenizerCreate; - t.xTokenize = f5tTokenizerTokenize; - t.xDelete = f5tTokenizerDelete; + if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){ + return TCL_ERROR; + } + zName = Tcl_GetString(objv[objc-2]); + pScript = objv[objc-1]; pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule)); + memset(pMod, 0, sizeof(F5tTokenizerModule)); pMod->interp = interp; pMod->pScript = pScript; - pMod->pContext = pContext; Tcl_IncrRefCount(pScript); - rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer); + pMod->pContext = pContext; + if( zParent ){ + if( bV2 ){ + fts5_tokenizer_v2 *pParent = 0; + rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent); + if( rc==SQLITE_OK ){ + memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2)); + pMod->parent_v2.xDelete(0); + } + }else{ + rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent); + if( rc==SQLITE_OK ){ + pMod->parent.xDelete(0); + } + } + } + + if( rc==SQLITE_OK ){ + void *pModCtx = (void*)pMod; + if( bV2==0 ){ + fts5_tokenizer t; + t.xCreate = f5tTokenizerCreate; + t.xTokenize = f5tTokenizerTokenize; + t.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer); + }else{ + fts5_tokenizer_v2 t2; + memset(&t2, 0, sizeof(t2)); + t2.iVersion = iVersion; + t2.xCreate = f5tTokenizerCreate; + t2.xTokenize = f5tTokenizerTokenize_v2; + t2.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); + } + } + if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0); + Tcl_AppendResult(interp, ( + bV2 ? "error in fts5_api.xCreateTokenizer_v2()" + : "error in fts5_api.xCreateTokenizer()" + ), 0); return TCL_ERROR; } @@ -1083,7 +1307,7 @@ static int SQLITE_TCLAPI f5tTokenHash( Tcl_Obj *CONST objv[] ){ char *z; - int n; + Tcl_Size n; unsigned int iVal; int nSlot; @@ -1096,7 +1320,7 @@ static int SQLITE_TCLAPI f5tTokenHash( } z = Tcl_GetStringFromObj(objv[2], &n); - iVal = f5t_fts5HashKey(nSlot, z, n); + iVal = f5t_fts5HashKey(nSlot, z, (int)n); Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); return TCL_OK; } @@ -1169,7 +1393,7 @@ struct OriginTextTokenizer { */ static void f5tOrigintextTokenizerDelete(void *pCtx){ OriginTextCtx *p = (OriginTextCtx*)pCtx; - ckfree(p); + ckfree((char*)p); } static int f5tOrigintextCreate( @@ -1333,6 +1557,7 @@ int Fts5tcl_Init(Tcl_Interp *interp){ } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, + { "sqlite3_fts5_locale", f5tTokenizerLocale, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index f12056170..f92529b84 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -386,7 +386,6 @@ static int fts5UnicodeCreate( zCat = azArg[i+1]; } } - if( rc==SQLITE_OK ){ rc = unicodeSetCategories(p, zCat); } @@ -416,7 +415,6 @@ static int fts5UnicodeCreate( rc = SQLITE_ERROR; } } - }else{ rc = SQLITE_NOMEM; } @@ -555,7 +553,7 @@ static int fts5UnicodeTokenize( typedef struct PorterTokenizer PorterTokenizer; struct PorterTokenizer { - fts5_tokenizer tokenizer; /* Parent tokenizer module */ + fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */ Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; }; @@ -567,7 +565,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){ if( pTok ){ PorterTokenizer *p = (PorterTokenizer*)pTok; if( p->pTokenizer ){ - p->tokenizer.xDelete(p->pTokenizer); + p->tokenizer_v2.xDelete(p->pTokenizer); } sqlite3_free(p); } @@ -586,6 +584,7 @@ static int fts5PorterCreate( PorterTokenizer *pRet; void *pUserdata = 0; const char *zBase = "unicode61"; + fts5_tokenizer_v2 *pV2 = 0; if( nArg>0 ){ zBase = azArg[0]; @@ -594,14 +593,15 @@ static int fts5PorterCreate( pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); }else{ rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ int nArg2 = (nArg>0 ? nArg-1 : 0); - const char **azArg2 = (nArg2 ? &azArg[1] : 0); - rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + const char **az2 = (nArg2 ? &azArg[1] : 0); + memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2)); + rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer); } if( rc!=SQLITE_OK ){ @@ -1252,6 +1252,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, + const char *pLoc, int nLoc, int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; @@ -1259,8 +1260,8 @@ static int fts5PorterTokenize( sCtx.xToken = xToken; sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; - return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + return p->tokenizer_v2.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb ); } @@ -1290,40 +1291,46 @@ static int fts5TriCreate( Fts5Tokenizer **ppOut ){ int rc = SQLITE_OK; - TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + TrigramTokenizer *pNew = 0; UNUSED_PARAM(pUnused); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + if( nArg%2 ){ + rc = SQLITE_ERROR; }else{ int i; - pNew->bFold = 1; - pNew->iFoldParam = 0; - for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ - const char *zArg = azArg[i+1]; - if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){ - if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){ - rc = SQLITE_ERROR; + pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->bFold = 1; + pNew->iFoldParam = 0; + + for(i=0; rc==SQLITE_OK && i<nArg; i+=2){ + const char *zArg = azArg[i+1]; + if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->bFold = (zArg[0]=='0'); + } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } }else{ - pNew->bFold = (zArg[0]=='0'); - } - }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ - if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ rc = SQLITE_ERROR; - }else{ - pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; } - }else{ + } + + if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ rc = SQLITE_ERROR; } - } - - if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ - rc = SQLITE_ERROR; - } - - if( rc!=SQLITE_OK ){ - fts5TriDelete((Fts5Tokenizer*)pNew); - pNew = 0; + + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } } } *ppOut = (Fts5Tokenizer*)pNew; @@ -1429,6 +1436,16 @@ int sqlite3Fts5TokenizerPattern( } /* +** Return true if the tokenizer described by p->azArg[] is the trigram +** tokenizer. This tokenizer needs to be loaded before xBestIndex is +** called for the first time in order to correctly handle LIKE/GLOB. +*/ +int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){ + return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram")); +} + + +/* ** Register all built-in tokenizers with FTS5. */ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ @@ -1438,7 +1455,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ } aBuiltin[] = { { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, - { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, }; @@ -1453,6 +1469,19 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ 0 ); } - + if( rc==SQLITE_OK ){ + fts5_tokenizer_v2 sPorter = { + 2, + fts5PorterCreate, + fts5PorterDelete, + fts5PorterTokenize + }; + rc = pApi->xCreateTokenizer_v2(pApi, + "porter", + (void*)pApi, + &sPorter, + 0 + ); + } return rc; } diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 3e97264fa..cc164a456 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -364,6 +364,9 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ default: return 1; } break; + + default: + return 1; } return 0; } diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 4782d0fb9..fb280567f 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -64,6 +64,7 @@ struct Fts5VocabCursor { int nLeTerm; /* Size of zLeTerm in bytes */ char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + int colUsed; /* Copy of sqlite3_index_info.colUsed */ /* These are used by 'col' tables only */ int iCol; @@ -90,9 +91,11 @@ struct Fts5VocabCursor { /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. */ -#define FTS5_VOCAB_TERM_EQ 0x01 -#define FTS5_VOCAB_TERM_GE 0x02 -#define FTS5_VOCAB_TERM_LE 0x04 +#define FTS5_VOCAB_TERM_EQ 0x0100 +#define FTS5_VOCAB_TERM_GE 0x0200 +#define FTS5_VOCAB_TERM_LE 0x0400 + +#define FTS5_VOCAB_COLUSED_MASK 0xFF /* @@ -269,11 +272,13 @@ static int fts5VocabBestIndexMethod( int iTermEq = -1; int iTermGe = -1; int iTermLe = -1; - int idxNum = 0; + int idxNum = (int)pInfo->colUsed; int nArg = 0; UNUSED_PARAM(pUnused); + assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed ); + for(i=0; i<pInfo->nConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; if( p->usable==0 ) continue; @@ -365,7 +370,7 @@ static int fts5VocabOpenMethod( if( rc==SQLITE_OK ){ pVTab->zErrMsg = sqlite3_mprintf( "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl - ); + ); rc = SQLITE_ERROR; } }else{ @@ -525,9 +530,19 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ switch( pTab->eType ){ case FTS5_VOCAB_ROW: - if( eDetail==FTS5_DETAIL_FULL ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; + /* Do not bother counting the number of instances if the "cnt" + ** column is not being read (according to colUsed). */ + if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){ + while( iPos<nPos ){ + u32 ii; + fts5FastGetVarint32(pPos, iPos, ii); + if( ii==1 ){ + /* New column in the position list */ + fts5FastGetVarint32(pPos, iPos, ii); + }else{ + /* An instance - increment pCsr->aCnt[] */ + pCsr->aCnt[0]++; + } } } pCsr->aDoc[0]++; @@ -625,6 +640,7 @@ static int fts5VocabFilterMethod( if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK); if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index fda388a6f..8ea87dbdd 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -51,6 +51,10 @@ proc fts5_test_poslist2 {cmd} { sort_poslist $res } +proc fts5_test_insttoken {cmd iInst iToken} { + $cmd xInstToken $iInst $iToken +} + proc fts5_test_collist {cmd} { set res [list] @@ -78,6 +82,9 @@ proc fts5_test_columnsize {cmd} { proc fts5_columntext {cmd iCol} { $cmd xColumnText $iCol } +proc fts5_columnlocale {cmd iCol} { + $cmd xColumnLocale $iCol +} proc fts5_test_columntext {cmd} { set res [list] @@ -87,6 +94,14 @@ proc fts5_test_columntext {cmd} { set res } +proc fts5_test_columnlocale {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnLocale $i] + } + set res +} + proc fts5_test_columntotalsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -114,6 +129,10 @@ proc fts5_test_rowcount {cmd} { $cmd xRowCount } +proc fts5_test_rowid {cmd} { + $cmd xRowid +} + proc test_queryphrase_cb {cnt cmd} { upvar $cnt L for {set i 0} {$i < [$cmd xInstCount]} {incr i} { @@ -161,17 +180,21 @@ proc fts5_aux_test_functions {db} { foreach f { fts5_test_columnsize fts5_test_columntext + fts5_test_columnlocale fts5_test_columntotalsize fts5_test_poslist fts5_test_poslist2 fts5_test_collist + fts5_test_insttoken fts5_test_tokenize fts5_test_rowcount + fts5_test_rowid fts5_test_all fts5_test_queryphrase fts5_test_phrasecount fts5_columntext + fts5_columnlocale fts5_queryphrase fts5_collist } { diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index a80a307a4..bcad9e724 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -440,7 +440,7 @@ do_execsql_test 16.1 { proc funk {} { db eval { UPDATE n1_config SET v=50 WHERE k='version' } set fd [db incrblob main n1_data block 10] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary # puts -nonewline $fd "\x44\x45" close $fd } @@ -453,7 +453,7 @@ db func funk funk # statement no longer fails. # do_catchsql_test 16.2 { - SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' + SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' } {0 {{} -1e-06 {}}} # {1 {SQL logic error}} diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 5aa745658..7e312286f 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ab -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index f3a914653..4628e909c 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ac -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index 524da6dea..27806a4c0 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -18,7 +18,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ad -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test index d9f132ca9..205a59a69 100644 --- a/ext/fts5/test/fts5ae.test +++ b/ext/fts5/test/fts5ae.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ae -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index 3d7929509..9c95ef2da 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -18,7 +18,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5af -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ag.test b/ext/fts5/test/fts5ag.test index 9ead957c9..42cd91378 100644 --- a/ext/fts5/test/fts5ag.test +++ b/ext/fts5/test/fts5ag.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ag -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 000435137..bf9c9e9db 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ah -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -163,6 +163,17 @@ do_execsql_test 1.8.2 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; } {10000} +do_execsql_test 1.8.3 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text'; +} {4999} +do_execsql_test 1.8.4 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text'; +} {0} + +do_catchsql_test 1.9 { + SELECT * FROM t1('*xy'); +} {1 {unknown special query: xy}} + } ;# foreach_detail_mode #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5ai.test b/ext/fts5/test/fts5ai.test index 20e106939..a6576d3af 100644 --- a/ext/fts5/test/fts5ai.test +++ b/ext/fts5/test/fts5ai.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ai -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5aj.test b/ext/fts5/test/fts5aj.test index 50dae2016..e802306b3 100644 --- a/ext/fts5/test/fts5aj.test +++ b/ext/fts5/test/fts5aj.test @@ -19,7 +19,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5aj -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test index e248f2a32..253f14fc7 100644 --- a/ext/fts5/test/fts5ak.test +++ b/ext/fts5/test/fts5ak.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ak -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index 842d991a3..7187ad67c 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5al -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -293,6 +293,16 @@ do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} +# Check that the second and subsequent rank= constraints are ignored. +# +do_catchsql_test 4.3.3 { + SELECT *, rank FROM t3 + WHERE t3 MATCH 'a' AND + rank MATCH 'nosuch()' AND + rank MATCH 'rowidmod(3)' + ORDER BY rank ASC +} {1 {unable to use function MATCH in the requested context}} + } ;# foreach_detail_mode diff --git a/ext/fts5/test/fts5alter.test b/ext/fts5/test/fts5alter.test index 67f948cbb..bb5f78dc8 100644 --- a/ext/fts5/test/fts5alter.test +++ b/ext/fts5/test/fts5alter.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5alter -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test index 79d432b81..b771af912 100644 --- a/ext/fts5/test/fts5auto.test +++ b/ext/fts5/test/fts5auto.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5auto -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test index 5569f48cf..960dbc511 100644 --- a/ext/fts5/test/fts5aux.test +++ b/ext/fts5/test/fts5aux.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5aux -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -377,4 +377,28 @@ do_catchsql_test 12.3.3 { SELECT fts5_collist(t1, 1) FROM t1('one AND two'); } {0 1} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +do_catchsql_test 13.2 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {0 {{a b c} {d e f}}} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +do_catchsql_test 13.4 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {1 {SQL logic error}} + finish_test diff --git a/ext/fts5/test/fts5aux2.test b/ext/fts5/test/fts5aux2.test new file mode 100644 index 000000000..2352970ec --- /dev/null +++ b/ext/fts5/test/fts5aux2.test @@ -0,0 +1,71 @@ +# 2024 June 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the auxiliary function APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aux + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('a b', 'c d'); + INSERT INTO x1 VALUES('d e', 'a b'); + INSERT INTO x1 VALUES('a b', 'e f'); + INSERT INTO x1 VALUES('d e', 'c d'); +} + +fts5_aux_test_functions db +do_execsql_test 1.1 { + SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2 +} [list [list {*}{ + columnsize {2 2} + columntext {{d e} {a b}} + columntotalsize {8 8} + poslist {} + tokenize {{d e} {a b}} + rowcount 4 +}]] + +do_execsql_test 1.2 { + SELECT fts5_test_columntext(x1) FROM x1 +} { + {{a b} {c d}} + {{d e} {a b}} + {{a b} {e f}} + {{d e} {c d}} +} + +do_execsql_test 1.3 { + SELECT fts5_test_rowid(x1) FROM x1 +} { + 1 2 3 4 +} +do_execsql_test 1.4 { + SELECT fts5_test_phrasecount(x1) FROM x1 +} { + 0 0 0 0 +} +do_catchsql_test 1.5 { + SELECT fts5_queryphrase(x1, 0) FROM x1 +} {1 SQLITE_RANGE} +do_execsql_test 1.6 { + SELECT fts5_test_rowcount(x1) FROM x1 +} {4 4 4 4} + + +finish_test diff --git a/ext/fts5/test/fts5auxdata.test b/ext/fts5/test/fts5auxdata.test index a2a41704c..7f99fed31 100644 --- a/ext/fts5/test/fts5auxdata.test +++ b/ext/fts5/test/fts5auxdata.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5auxdata -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5bigpl.test b/ext/fts5/test/fts5bigpl.test index 2c9df11b1..9e3d86c0e 100644 --- a/ext/fts5/test/fts5bigpl.test +++ b/ext/fts5/test/fts5bigpl.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5bigpl -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test new file mode 100644 index 000000000..4233719fb --- /dev/null +++ b/ext/fts5/test/fts5blob.test @@ -0,0 +1,166 @@ +# 2024 July 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file verifies that: +# +# * blob values may be written to locale=0 tables. +# +# * blob values - other than fts5_locale() values - may not be written +# to locale=0 tables. This is an SQLITE_MISMATCH error +# +# * blob values may be returned by queries on the external-content table +# of a locale=0 table. +# +# * blob values not may be returned by queries on the external-content +# table of a locale=1 table, apart from fts5_locale() blobs. This is an +# SQLITE_MISMATCH error. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5blob + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Test that blobs may be stored in normal locale=0 tables. +# +foreach {tn enc} { + 1 utf8 + 2 utf16 +} { + reset_db + fts5_aux_test_functions db + + execsql "PRAGMA encoding = $enc" + + execsql " + CREATE VIRTUAL TABLE t1 USING fts5(x, y); + " + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546'); + INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546'); + INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz'); + } + + do_execsql_test 1.$tn.1 { + SELECT rowid, quote(x), quote(y) FROM t1 + } { + 1 555 X'0000000041424320444546' + 2 666 X'41424300444546' + 3 777 'xyz' + } + + do_execsql_test 1.$tn.2 { + DELETE FROM t1 WHERE rowid=2; + DELETE FROM t1 WHERE rowid=1; + } + + do_execsql_test 1.$tn.3 { + PRAGMA integrity_check; + } {ok} +} + +#-------------------------------------------------------------------------- +# Test that a blob may be stored and retrieved in an unindexed column of +# a regular table with locale=1. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1); + INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546'); +} + +do_execsql_test 2.1 { + select rowid, x, quote(y) FROM t1 +} { + 12 twelve X'0000000041424320444546' +} + +#-------------------------------------------------------------------------- +# Test that blobs may not be written to any type of table with locale=1 +# set. Except, they may be written to UNINDEXED columns. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {1 {datatype mismatch}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {1 {datatype mismatch}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {1 {datatype mismatch}} + + +#-------------------------------------------------------------------------- +# Test that fts5_locale() values may not be written to any type of table +# without locale=1 set. Even to an UNINDEXED column. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=); + + CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.4 { + INSERT INTO x4(rowid, a, b, c) + VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +foreach {tn sql} { + 1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') } + 2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') } + 3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') } +} { + do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}} +} + + +finish_test + + diff --git a/ext/fts5/test/fts5cat.test b/ext/fts5/test/fts5cat.test index 483f64bfe..71e2abe3a 100644 --- a/ext/fts5/test/fts5cat.test +++ b/ext/fts5/test/fts5cat.test @@ -55,5 +55,22 @@ do_execsql_test 1.5 { SELECT * FROM t4t } {สนามกีฬา 1 1} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 " + CREATE VIRTUAL TABLE x1 USING fts5(c, + tokenize=\"unicode61 categories ' \t'\"); +" + +do_catchsql_test 2.1 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tMYZ'\"); +" {1 {error in tokenizer constructor}} + +do_catchsql_test 2.2 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tXYZ'\"); +" {1 {error in tokenizer constructor}} + finish_test diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test index 7243743b5..e5429572c 100644 --- a/ext/fts5/test/fts5colset.test +++ b/ext/fts5/test/fts5colset.test @@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 4.1 { SELECT * FROM t1 WHERE rowid MATCH 'a' - } {1 {unable to use function MATCH in the requested context}} + } {1 {no query solution}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test index 2b03d575a..7af49184b 100644 --- a/ext/fts5/test/fts5columnsize.test +++ b/ext/fts5/test/fts5columnsize.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5columnsize -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5config.test b/ext/fts5/test/fts5config.test index 35894c6bb..28f3146ea 100644 --- a/ext/fts5/test/fts5config.test +++ b/ext/fts5/test/fts5config.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5config -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index 986d7f331..05b5cc611 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5content -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -328,4 +328,41 @@ do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') } do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one} do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one two three'); + INSERT INTO t1 VALUES(2, 'one two three'); + + CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a); + INSERT INTO ft(ft) VALUES('rebuild'); +} + +do_execsql_test 9.2 { + SELECT rowid, b FROM ft('two'); +} { + 1 {one two three} + 2 {one two three} +} + +do_execsql_test 9.3 { + DELETE FROM t1 WHERE a=2; +} + +do_catchsql_test 9.4 { + SELECT rowid FROM ft('two'); +} {0 {1 2}} + +do_catchsql_test 9.5 { + SELECT * FROM ft('two'); +} {1 {fts5: missing row 2 from content table 'main'.'t1'}} + +fts5_aux_test_functions db + +do_catchsql_test 9.6 { + SELECT rowid, fts5_columntext(ft, 0) FROM ft('two'); +} {1 SQLITE_CORRUPT_VTAB} + finish_test + diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test index 48cfd10ff..991e9888f 100644 --- a/ext/fts5/test/fts5contentless.test +++ b/ext/fts5/test/fts5contentless.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5contentless -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -267,4 +267,24 @@ do_execsql_test 8.2 { SELECT rowid FROM ft('four'); } {} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 9.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {0 {}} + +do_catchsql_test 9.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2); +} {1 {malformed contentless_delete=... directive}} + +do_catchsql_test 9.3 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11); +} {1 {malformed contentless_delete=... directive}} + finish_test diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test index fdd7a60fc..248534bce 100644 --- a/ext/fts5/test/fts5contentless2.test +++ b/ext/fts5/test/fts5contentless2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5contentless2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test index 76119dc59..693840da8 100644 --- a/ext/fts5/test/fts5contentless3.test +++ b/ext/fts5/test/fts5contentless3.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5contentless3 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test index 702b33a9d..7fdf8c4b0 100644 --- a/ext/fts5/test/fts5contentless4.test +++ b/ext/fts5/test/fts5contentless4.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5contentless4 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5contentless5.test b/ext/fts5/test/fts5contentless5.test index a20134d1e..356367886 100644 --- a/ext/fts5/test/fts5contentless5.test +++ b/ext/fts5/test/fts5contentless5.test @@ -15,11 +15,12 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5contentless5 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return } +unset -nocomplain res do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1); diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 9aa84a0ef..0abd8b86d 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -99,6 +99,27 @@ sqlite3_db_config db DEFENSIVE 0 do_catchsql_test 3.1 { DELETE FROM t3_content WHERE rowid = 3; SELECT * FROM t3 WHERE t3 MATCH 'o'; +} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}} + +#-------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('one two three'); + INSERT INTO t2 VALUES('four five six'); + INSERT INTO t2 VALUES('seven eight nine'); + INSERT INTO t2 VALUES('ten eleven twelve'); +} +do_execsql_test 4.1 { + SELECT hex(block) FROM t2_data WHERE id=1; +} {040C} +do_execsql_test 4.2 { + UPDATE t2_data SET block = X'0402' WHERE id=1 +} +breakpoint +do_catchsql_test 4.3 { + DELETE FROM t2 WHERE rowid=3 } {1 {database disk image is malformed}} finish_test diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 06e2e7425..6b4d6d411 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -100,6 +100,7 @@ set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}] set all [db eval {SELECT rowid FROM t1}] sqlite3_db_config db DEFENSIVE 0 +unset -nocomplain res for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { do_execsql_test 2.$i.1 { BEGIN; @@ -152,7 +153,7 @@ foreach {tn hdr} { execsql BEGIN set fd [db incrblob main x3_data block $rowid] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary set existing [read $fd [string length $hdr]] seek $fd 0 puts -nonewline $fd $hdr @@ -238,7 +239,7 @@ foreach {tn hdr} { execsql BEGIN set fd [db incrblob main x5_data block $rowid] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary puts -nonewline $fd $hdr close $fd diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index f9a95665c..c5faaa87b 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt3 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -4267,7 +4267,7 @@ do_test 35.0 { do_catchsql_test 35.1 { SELECT * FROM t1 WHERE t1 MATCH 'e*'; -} {1 {database disk image is malformed}} +} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}} #------------------------------------------------------------------------- reset_db @@ -8958,7 +8958,6 @@ do_catchsql_test 61.2 { SELECT * FROM t3 ORDER BY rowid; } {/*malformed database schema*/} -breakpoint #------------------------------------------------------------------------- do_test 62.0 { sqlite3 db {} @@ -10768,6 +10767,7 @@ do_catchsql_test 73.1 { reset_db do_test 74.0 { sqlite3 db {} + sqlite3_fts5_register_matchinfo db db deserialize [decode_hexdb { | size 106496 pagesize 4096 filename x.db | page 1 offset 0 @@ -14587,14 +14587,19 @@ do_test 74.0 { | end x.db }]} {} -do_catchsql_test 74.1 { - SELECT rowid, quote(matchinfo(t1,'p�xyb<s')) FROM t1 WHERE t1 MATCH 'e*'; +do_catchsql_test 74.0.5 { + SELECT matchinfo(1,2); } {1 {unable to use function matchinfo in the requested context}} +do_catchsql_test 74.1 { + SELECT rowid, quote(matchinfo(t1,'pxyb<s')) FROM t1 WHERE t1 MATCH 'e*'; +} {1 {unrecognized matchinfo flag: <}} + #------------------------------------------------------------------------- reset_db do_test 75.0 { sqlite3 db {} + sqlite3_fts5_register_matchinfo db db deserialize [decode_hexdb { | size 32768 pagesize 4096 filename crash-033d665d5caa8d.db | page 1 offset 0 @@ -14791,7 +14796,7 @@ do_test 75.0 { do_catchsql_test 75.1 { SELECT rowid, quote(matchinfo(t1,'pcxybs')) FROM t1 WHERE t1 MATCH 'e*'; -} {1 {unable to use function matchinfo in the requested context}} +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- reset_db @@ -15519,6 +15524,160 @@ do_catchsql_test 80.1 { SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 81.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-44e8035a976422.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c ..........n....L +| 112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50 .....$...r.....P +| 128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00 .....n.......... +| 2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d ..............C. +| 2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43 .....utablet4t4C +| 2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73 BLE t4 USING fts +| 2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f 5vocab('t2', 'ro +| 2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65 w')C......utable +| 3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55 t3t3CREATE VIRTU +| 3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e AL TABLE t3 USIN +| 3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27 G fts5vocab('t1' +| 3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d , 'row')V....... +| 3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32 tablet2_configt2 +| 3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54 _config.CREATE T +| 3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27 ABLE 't2_config' +| 3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 (k PRIMARY KEY, +| 3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 v) WITHOUT ROWID +| 3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32 ^...!!...tablet2 +| 3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65 _contentt2_conte +| 3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20 nt.CREATE TABLE +| 3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20 't2_content'(id +| 3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 INTEGER PRIMARY +| 3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29 KEY, c0, c1, c2) +| 3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32 i.......-tablet2 +| 3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54 _idxt2_idx.CREAT +| 3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27 E TABLE 't2_idx' +| 3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67 (segid, term, pg +| 3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28 no, PRIMARY KEY( +| 3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49 segid, term)) WI +| 3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b THOUT ROWIDU.... +| 3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61 ....tablet2_data +| 3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54 t2_data.CREATE T +| 3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69 ABLE 't2_data'(i +| 3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 d INTEGER PRIMAR +| 3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f Y KEY, block BLO +| 3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65 B)X........table +| 3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55 t2t2CREATE VIRTU +| 3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e AL TABLE t2 USIN +| 3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22 G fts5('a',[b],. +| 3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63 c.,detail=none,c +| 3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17 olumnsize=0)V... +| 3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66 ....tablet1_conf +| 3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41 igt1_config.CREA +| 3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e TE TABLE 't1_con +| 3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62 OWID[...!!...tab +| 3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64 let1_docsizet1_d +| 3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41 ocsize.CREATE TA +| 3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27 BLE 't1_docsize' +| 3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42 ARY KEY, sz BLOB +| 3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74 )^...!!...tablet +| 3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74 1_contentt1_cont +| 3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45 ent.CREATE TABLE +| 3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64 't1_content'(id +| 3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 INTEGER PRIMARY +| 3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 KEY, c0, c1, c2 +| 3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74 )i.......-tablet +| 3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41 1_idxt1_idx.CREA +| 3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78 TE TABLE 't1_idx +| 3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 '(segid, term, p +| 3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 gno, PRIMARY KEY +| 3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 (segid, term)) W +| 3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17 ITHOUT ROWIDU... +| 3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74 .....tablet1_dat +| 3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20 at1_data.CREATE +| 3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28 TABLE 't1_data'( +| 3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 id INTEGER PRIMA +| 3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c RY KEY, block BL +| 4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c OB)[.......#tabl +| 4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e NG fts5(a,b unin +| 4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65 dexed,c,tokenize +| 4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29 =.porter ascii.) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42 .....h........jB +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69 h!i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-44e8035a976422.db +}]} {} + +do_catchsql_test 81.2 { + UPDATE t1 SET b=zeroblob(299); +} {1 {database disk image is malformed}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5corrupt4.test b/ext/fts5/test/fts5corrupt4.test index b31f4d96e..0505250c2 100644 --- a/ext/fts5/test/fts5corrupt4.test +++ b/ext/fts5/test/fts5corrupt4.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt4 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index a56467376..19f053841 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt5 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -1222,6 +1222,235 @@ do_catchsql_test 8.1 { SELECT rowid FROM t1('a* NOT ý') ; } {0 {1 2 3 4 5 6 7}} +#------------------------------------------------------------------------- +reset_db +do_test 9.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-c76a16c24c8ba6.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56 ..threadsafe.W4V +| 3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02 .d.F...vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 10 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 31 46 45 3d ..%..THREADS1FE= +| 3152: 30 58 52 64 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRdRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4b 45 20 4d 45 4d 53 59 53 35 58 42 49 NABKE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 B`-EMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 39 0f 19 45 4e 41 42 4c NARY....9..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d TAT VT$.HNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3c 67 ...C..COMPILER<g +| 4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30 cc-5.4.0 2016060 +| 4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43 9XNOCASE&...C..C +| 4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e OMPILER=gcc-5.4. +| 4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d 0 20160609XRTRIM +| page 6 offset 20480 +| 0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0 ....$........... +| 16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0 ................ +| 32: 1f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60 .........x.p.h.` +| 48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20 .X.P.H.@.8.0.(. +| 64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0 ................ +| 3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01 .$.......#...... +| 3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01 .........!...... +| 3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01 . .............. +| 3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01 ................ +| 3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01 ................ +| 3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01 ................ +| 3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01 ................ +| 3920: 06 15 f3 00 12 02 01 01 06 15 03 00 12 02 01 01 ................ +| 3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01 ................ +| 3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01 ................ +| 3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01 ................ +| 3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01 ................ +| 4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01 ................ +| 4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01 ................ +| 4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01 ................ +| 4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01 ................ +| 4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01 ................ +| 4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 8 offset 28672 +| 0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00 ................ +| 4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72 .........+integr +| 4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62 ity-check....reb +| 4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 00 00 00 00 uild....opti.... +| end crash-c76a16c24c8ba6.db +}]} {} + +#.testctrl prng_seed 1 db +#.testctrl internal_functions +#.testctrl json_selfcheck on +# + +do_execsql_test 9.1 { + UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*'; + SAVEPOINT a; + UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*'; + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); +} +do_catchsql_test 9.2 { + DELETE FROM t1; +} {1 {database disk image is malformed}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5corrupt6.test b/ext/fts5/test/fts5corrupt6.test index 6403d3a40..6570871e3 100644 --- a/ext/fts5/test/fts5corrupt6.test +++ b/ext/fts5/test/fts5corrupt6.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt6 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5corrupt7.test b/ext/fts5/test/fts5corrupt7.test index ae7f9da7d..41e359f42 100644 --- a/ext/fts5/test/fts5corrupt7.test +++ b/ext/fts5/test/fts5corrupt7.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt7 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 1fb95a900..db82db1c2 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5dlidx -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5doclist.test b/ext/fts5/test/fts5doclist.test index 08b773f6f..5b1becb51 100644 --- a/ext/fts5/test/fts5doclist.test +++ b/ext/fts5/test/fts5doclist.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5doclist -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test index 3ccbd7d7a..49c2f2753 100644 --- a/ext/fts5/test/fts5ea.test +++ b/ext/fts5/test/fts5ea.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ea -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 0c775791f..bee9683c3 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5eb -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -86,13 +86,13 @@ do_execsql_test 3.0 { INSERT INTO e1 VALUES ('just a few words with a / inside'); } do_execsql_test 3.1 { - SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; + SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; } {1 -1e-06} do_execsql_test 3.2 { SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"' } 1 do_execsql_test 3.3 { - SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; + SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; } {1 -1e-06} do_execsql_test 3.4 " diff --git a/ext/fts5/test/fts5expr.test b/ext/fts5/test/fts5expr.test new file mode 100644 index 000000000..49be61d9c --- /dev/null +++ b/ext/fts5/test/fts5expr.test @@ -0,0 +1,52 @@ +# 2024 August 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5expr + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test'); +} + +do_execsql_test 1.1 { + SELECT rowid FROM x1('expr'); +} {113} + +for {set ii 0} {$ii < 300} {incr ii} { + set expr "expr " + append expr [string repeat "NOT abcd " $ii] + + if {$ii<257} { + set res {0 113} + } else { + set res {1 {fts5 expression tree is too large (maximum depth 256)}} + } + do_catchsql_test 1.1.$ii { + SELECT rowid FROM x1($expr) + } $res +} + +do_execsql_test 1.2 { + SELECT rowid FROM x1 WHERE a MATCH '"..."' +} {} + +finish_test + diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index a39063a35..1aacddce9 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5fault6 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test index 5afab7754..dc060a159 100644 --- a/ext/fts5/test/fts5fault8.test +++ b/ext/fts5/test/fts5fault8.test @@ -57,7 +57,6 @@ foreach_detail_mode $testprefix { } ;# foreach_detail_mode... - do_execsql_test 4.0 { CREATE VIRTUAL TABLE x2 USING fts5(a); INSERT INTO x2(x2, rank) VALUES('crisismerge', 2); @@ -80,5 +79,18 @@ do_faultsim_test 4 -faults oom-* -prep { faultsim_test_result {0 {}} {1 SQLITE_NOMEM} } +set TMPDBERROR {1 {unable to open a temporary database file for storing temporary tables}} + +do_faultsim_test 5 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA temp_store = memory } +} -body { + execsql { PRAGMA integrity_check } +} -test { + if {[string match {*error code=7*} $testresult]==0} { + faultsim_test_result {0 ok} {1 SQLITE_NOMEM} $::TMPDBERROR + } +} + finish_test diff --git a/ext/fts5/test/fts5faultG.test b/ext/fts5/test/fts5faultG.test index 33a41dcb3..9110c6336 100644 --- a/ext/fts5/test/fts5faultG.test +++ b/ext/fts5/test/fts5faultG.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5faultG -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5faultH.test b/ext/fts5/test/fts5faultH.test index 540b889f3..0cbbf7f5e 100644 --- a/ext/fts5/test/fts5faultH.test +++ b/ext/fts5/test/fts5faultH.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5faultG -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -127,7 +127,7 @@ do_execsql_test 3.0 { COMMIT; } -do_faultsim_test 3 -faults oom* -prep { +do_faultsim_test 3.1 -faults oom* -prep { } -body { execsql { SELECT rowid FROM t1('BBB AND AAA'); @@ -136,6 +136,15 @@ do_faultsim_test 3 -faults oom* -prep { faultsim_integrity_check faultsim_test_result {0 {10 35}} } +do_faultsim_test 3.2 -faults oom* -prep { +} -body { + execsql { + SELECT count(*) FROM t1('BBB'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 27} +} finish_test diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test new file mode 100644 index 000000000..63bfdd68a --- /dev/null +++ b/ext/fts5/test/fts5faultI.test @@ -0,0 +1,261 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultI + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultI + +do_execsql_test 1.0 { + PRAGMA encoding = utf16; + CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1); + INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram'); +} + +faultsim_save_and_close +faultsim_restore_and_reopen + +do_faultsim_test 1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1(fts5_locale('en_US', 'origintext')); + } +} -test { + faultsim_test_result {0 1} +} + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + SELECT * FROM t1('ascii'); + } +} -body { + execsql { + UPDATE t1 SET rowid=rowid+1; + } +} -test { + faultsim_test_result {0 {}} +} + +fts5_aux_test_functions db +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*'); + } +} -test { + faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('token token token'); + } +} -body { + execsql { + INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('one'); + SAVEPOINT one; + INSERT INTO w1 VALUES('two'); + ROLLBACK TO one; + } + +} -body { + execsql { + INSERT INTO w1 VALUES('string'); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db + +do_faultsim_test 5 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two'); + } +} -test { + faultsim_test_result {0 two} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db +faultsim_save_and_close + +do_faultsim_test 6 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO w1 VALUES('four five six'); + SAVEPOINT abc; + INSERT INTO w1 VALUES('seven eight nine'); + SAVEPOINT def; + INSERT INTO w1 VALUES('ten eleven twelve'); + } +} -body { + execsql { + RELEASE abc; + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); + INSERT INTO w1 VALUES('three two one'); + DELETE FROM w1_content WHERE rowid=1; +} + +faultsim_save_and_close + +do_faultsim_test 7 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM w1 } +} -body { + execsql { + PRAGMA integrity_check; + } +} -test { +} + +#------------------------------------------------------------------------- +reset_db +fts5_tclnum_register db +fts5_aux_test_functions db + +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize = "tclnum query", detail=columns + ); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_faultsim_test 8.1 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_collist (ft) FROM ft('one two'); + } +} -test { + faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 8.2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM ft('one two') ORDER BY rank; + } +} -test { + faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +faultsim_save_and_close + +do_faultsim_test 9.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE ft SET rowid=4 WHERE rowid=1 + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 9.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 1} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, locale=1); + INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii')); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_execsql_test 10.1 { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' +} {hello} + +faultsim_save_and_close +do_faultsim_test 10 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 hello} +} + + +finish_test + diff --git a/ext/fts5/test/fts5first.test b/ext/fts5/test/fts5first.test index 357672de6..492681eed 100644 --- a/ext/fts5/test/fts5first.test +++ b/ext/fts5/test/fts5first.test @@ -22,6 +22,7 @@ do_execsql_test 1.0 { CREATE VIRTUAL TABLE x1 USING fts5(a, b); } +unset -nocomplain res foreach {tn expr ok} { 1 {^abc} 1 2 {^abc + def} 1 diff --git a/ext/fts5/test/fts5full.test b/ext/fts5/test/fts5full.test index 751e874c3..76fdc0288 100644 --- a/ext/fts5/test/fts5full.test +++ b/ext/fts5/test/fts5full.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5full -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5hash.test b/ext/fts5/test/fts5hash.test index 5df55f226..b3d8b562c 100644 --- a/ext/fts5/test/fts5hash.test +++ b/ext/fts5/test/fts5hash.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5hash -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index f9851dc15..5c4002180 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5integrity -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -153,6 +153,7 @@ do_execsql_test 5.3 { INSERT INTO gg(gg) VALUES('integrity-check'); } +unset -nocomplain res do_test 5.4.1 { set ok 0 for {set i 0} {$i < 10000} {incr i} { @@ -355,4 +356,57 @@ do_execsql_test 11.4 { PRAGMA integrity_check(t2); } {ok} +#------------------------------------------------------------------- +reset_db + +do_execsql_test 12.1 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('one', 'two'); + INSERT INTO x1 VALUES('three', 'four'); + INSERT INTO x1 VALUES('five', 'six'); +} + +do_execsql_test 12.2 { + PRAGMA integrity_check +} {ok} + +db close +sqlite3 db test.db -readonly 1 + +explain_i { + PRAGMA integrity_check + } +do_execsql_test 12.3 { + PRAGMA integrity_check +} {ok} + + +#------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +db close +sqlite3 db test.db +do_catchsql_test 13.2 { + PRAGMA integrity_check; +} {0 ok} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +breakpoint +do_catchsql_test 13.4 { + PRAGMA integrity_check; +} {1 {SQL logic error}} + + finish_test diff --git a/ext/fts5/test/fts5interrupt.test b/ext/fts5/test/fts5interrupt.test index ca682852c..67ef5f7e9 100644 --- a/ext/fts5/test/fts5interrupt.test +++ b/ext/fts5/test/fts5interrupt.test @@ -33,6 +33,7 @@ proc progress_handler {args} { return 0 } +unset -nocomplain res foreach {tn sql} { 1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') } 2 { COMMIT } @@ -64,4 +65,3 @@ foreach {tn sql} { } finish_test - diff --git a/ext/fts5/test/fts5lastrowid.test b/ext/fts5/test/fts5lastrowid.test index d152a8f09..75866139d 100644 --- a/ext/fts5/test/fts5lastrowid.test +++ b/ext/fts5/test/fts5lastrowid.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5lastrowid -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test new file mode 100644 index 000000000..684dcecd8 --- /dev/null +++ b/ext/fts5/test/fts5locale.test @@ -0,0 +1,667 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5locale + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc transform_token {locale token} { + switch -- $locale { + reverse { + set ret "" + foreach c [split $token ""] { + set ret "$c$ret" + } + set token $ret + } + + default { + # no-op + } + } + + set token +} + +proc tcl_create {args} { return "tcl_tokenize" } +proc tcl_tokenize {tflags text} { + set iToken 1 + set bSkip 0 + if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 } + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + incr iToken + if {(($iToken) % ($bSkip + 1))} continue + + set w [transform_token [sqlite3_fts5_locale] $w] + sqlite3_fts5_token $w $iStart $iEnd + } +} + +#------------------------------------------------------------------------- +# Check that queries can have a locale attached to them. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('cba'); +} {} + +do_execsql_test 1.1 { + SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') ); +} {1 abc} + +do_execsql_test 1.2 { + SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); +} {2 cba} + +#------------------------------------------------------------------------- +# Test that the locale= option exists and seems to accept values. And +# that fts5_locale() values may only be inserted into an internal-content +# table if the locale=1 option was specified. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl); + + CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance); +} + +do_catchsql_test 2.2.1 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2); +} {1 {malformed locale=... directive}} +do_catchsql_test 2.2.2 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111); +} {1 {malformed locale=... directive}} + +do_catchsql_test 2.3 { + INSERT INTO b1(b1, rank) VALUES('locale', 0); +} {1 {SQL logic error}} + +do_execsql_test 2.4 { + INSERT INTO b1 VALUES('abc', 'one two three'); + INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); +} + +do_execsql_test 2.5 { + INSERT INTO b2 VALUES('abc', 'one two three'); +} + +do_catchsql_test 2.6 { + INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six')); +} {1 {fts5_locale() requires locale=1}} + +do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1} +do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {} +do_execsql_test 2.9 { SELECT rowid FROM b1('ruof') } 2 +do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2 + +do_execsql_test 2.11 { + SELECT x, quote(y) FROM b1 +} { + abc {'one two three'} + def {'four five six'} +} + +do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { + {'four five six'} +} + +do_execsql_test 2.13 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} +do_execsql_test 2.14 { + INSERT INTO b1(b1) VALUES('rebuild'); +} +do_execsql_test 2.15 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.16 { + DELETE FROM b1 WHERE rowid=2 +} +do_execsql_test 2.17 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.18 { + INSERT INTO b1(rowid, x, y) VALUES( + test_setsubtype(45, 76), 'abc def', 'def abc' + ); + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Test the 'delete' command with contentless tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1); + CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance); + + INSERT INTO c1 VALUES('hello world'); + INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.2 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + eerht eno hello owt world +} + +do_execsql_test 3.3 { + INSERT INTO c1(c1, rowid, x) + VALUES('delete', 2, fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.4 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + hello world +} + +#------------------------------------------------------------------------- +# Test that an UPDATE that updates a subset of the columns does not +# magically discard the locale from those columns not updated. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance); + + INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def'); + INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello')); +} + +do_execsql_test 4.2 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def ghi olleh +} + +do_execsql_test 4.3 { + UPDATE d1 SET x='jkl' WHERE rowid=2; +} + +do_execsql_test 4.4 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def jkl olleh +} + +do_execsql_test 4.5 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 2 jkl hello +} + +do_execsql_test 4.6 { + UPDATE d1 SET rowid=4 WHERE rowid=2 +} + +do_execsql_test 4.7 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 4 jkl hello +} + +fts5_aux_test_functions db + +do_execsql_test 4.8.1 { + SELECT fts5_test_columntext(d1) FROM d1('jkl') +} {{jkl hello}} +do_execsql_test 4.8.2 { + SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{jkl hello}} + +do_execsql_test 4.9 { + SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{{} reverse}} + +do_execsql_test 4.10 { + SELECT fts5_test_columnlocale(d1) FROM d1 +} { + {{} {}} + {{} reverse} +} + +#------------------------------------------------------------------------- +# Test that if an fts5_locale() value is written to an UNINDEXED +# column it is stored as text. This is so that blobs and other values +# can also be stored as is. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, y UNINDEXED, locale=1, tokenize=tcl + ); + + INSERT INTO t1(rowid, x, y) VALUES(111, + fts5_locale('reverse', 'one two three'), + fts5_locale('reverse', 'four five six') + ); +} + +do_execsql_test 5.2 { + SELECT rowid, x, y FROM t1 +} { + 111 {one two three} {four five six} +} + +do_execsql_test 5.3 { + SELECT typeof(c0), typeof(c1) FROM t1_content +} { + blob text +} + +#------------------------------------------------------------------------- + +foreach {tn opt} { + 1 {} + 2 {, columnsize=0} +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 6.$tn.1 " + CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt); + " + + do_execsql_test 6.$tn.2 { + INSERT INTO y1(rowid, t) VALUES + (1, fts5_locale('second', 'the city of London')), + (2, fts5_locale('second', 'shall have all the old')), + (3, fts5_locale('second', 'Liberties and Customs')), + (4, fts5_locale('second', 'which it hath been used to have')); + } + + fts5_aux_test_functions db + + do_execsql_test 5.$tn.3 { + SELECT fts5_test_columnsize(y1) FROM y1 + } { + 2 3 2 4 + } + + do_execsql_test 5.$tn.4 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 5.$tn.5 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 5.$tn.6 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); + } { + 4 4 + } + + do_execsql_test 5.$tn.7 { + SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); + } { + 4 {which it hath been used to [have]} + } + + do_execsql_test 5.$tn.8 { + SELECT rowid, + highlight(y1, 0, '[', ']'), + snippet(y1, 0, '[', ']', '...', 10) + FROM y1('Liberties + Customs'); + } { + 3 {[Liberties and Customs]} + {[Liberties and Customs]} + } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} +do_catchsql_test 6.1 { + INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world')); +} {1 {fts5_locale() requires locale=1}} + +do_execsql_test 6.2 { + SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') ); +} {text text} + +#-------------------------------------------------------------------------- +# Test that fts5_locale() works with external-content tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 7.1 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT); + CREATE VIEW v1 AS + SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1; + + CREATE VIRTUAL TABLE ft USING fts5( + bb, tt, locale=1, tokenize=tcl, content=v1 + ); + + INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL); + INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse'); + INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second'); +} + +do_execsql_test 7.2 { + INSERT INTO ft(ft) VALUES('rebuild'); + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 7.3 { + SELECT rowid, quote(bb), quote(tt) FROM ft +} { + 1 NULL {'one two three'} + 2 '7800616263' {'four five six'} + 3 '000000007800616263' {'seven eight nine'} +} + +do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } +do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2 + +fts5_aux_test_functions db + +do_execsql_test 7.6 { + SELECT fts5_test_columnlocale(ft) FROM ft; +} { + {{} {}} {{} reverse} {{} second} +} + +#------------------------------------------------------------------------- +# Test that the porter tokenizer works with locales. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl"); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + + INSERT INTO ft(rowid, tt) VALUES + (111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')), + (222, fts5_locale('reverse', 'This value may also be set')); +} + +do_execsql_test 8.1 { + SELECT DISTINCT term FROM vocab ORDER BY 1 +} { + a eb eulav osla sihT te the token yam +} + +#------------------------------------------------------------------------- +# Test that position-lists (used by xInst, xPhraseFirst etc.) work with +# locales and modes other than detail=full. +# +foreach {tn detail} { + 1 detail=full + 2 detail=none + 3 detail=column +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 9.$tn.0 " + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail); + " + do_execsql_test 9.$tn.1 { + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + INSERT INTO ft(rowid, tt) VALUES + (-1, fts5_locale('second', 'it is an ancient mariner')); + } + + do_execsql_test 9.$tn.2 { + SELECT DISTINCT term FROM vocab + } {an it mariner} + + do_execsql_test 9.$tn.3 { + SELECT highlight(ft, 0, '[', ']') FROM ft('mariner') + } {{it is an ancient [mariner]}} +} + +#------------------------------------------------------------------------- +# Check some corrupt fts5_locale() blob formats are detected. +# +foreach_detail_mode $::testprefix { + + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + fts5_aux_test_functions db + do_execsql_test 10.1 { + CREATE TABLE x1(ii INTEGER PRIMARY KEY, x); + CREATE VIRTUAL TABLE ft USING fts5(x, + content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0 + ); + + CREATE VIRTUAL TABLE ft2 USING fts5( + x, locale=1, detail=%DETAIL%, columnsize=0 + ); + } + + foreach {tn v} { + 1 X'001122' + 2 X'0011223344' + 3 X'00E0B2EB68656c6c6f' + 4 X'00E0B2EB0068656c6c6f' + } { + do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') } + do_execsql_test 10.2.$tn.1 { DELETE FROM x1; } + do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) " + + do_catchsql_test 10.2.$tn.3 { + INSERT INTO ft(ft) VALUES('rebuild'); + } {1 {SQL logic error}} + + do_catchsql_test 10.2.$tn.4 " + SELECT * FROM ft( test_setsubtype($v, 76) ); + " {1 {SQL logic error}} + + do_execsql_test 10.2.$tn.5 { + INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); + } + + if {"%DETAIL%"!="full"} { + do_catchsql_test 10.2.$tn.6 { + SELECT fts5_test_poslist(ft) FROM ft('world'); + } {1 SQLITE_ERROR} + + do_catchsql_test 10.2.$tn.7 { + SELECT fts5_test_columnsize(ft) FROM ft('world'); + } {1 SQLITE_ERROR} + + do_catchsql_test 10.2.$tn.7 { + SELECT fts5_test_columnlocale(ft) FROM ft('world'); + } {1 SQLITE_ERROR} + } + + do_catchsql_test 10.2.$tn.8 { + SELECT * FROM ft('hello') + } {1 {SQL logic error}} + + do_catchsql_test 10.2.$tn.9 { + PRAGMA integrity_check; + } {0 ok} + + do_execsql_test 10.2.$tn.10 { + DELETE FROM x1; + INSERT INTO x1(ii, x) VALUES(1, 'hello world'); + } + + do_catchsql_test 10.2.$tn.11 " + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) + " {1 {SQL logic error}} + + do_catchsql_test 10.2.$tn.12 " + INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) + " {1 {SQL logic error}} + + do_execsql_test 10.2.$tn.13 { + INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); + } + do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v" + + do_catchsql_test 10.2.$tn.15 { + PRAGMA integrity_check; + } {1 {SQL logic error}} + + do_execsql_test 10.2.$tn.16 { + DELETE FROM ft2_content; + INSERT INTO ft2(ft2) VALUES('rebuild'); + } + } + +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1); + INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three')); +} + +do_catchsql_test 11.1 { + SELECT fts5_columnlocale(x1, -1) FROM x1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 11.2 { + SELECT fts5_columnlocale(x1, 1) FROM x1('two'); +} {1 SQLITE_RANGE} + +#------------------------------------------------------------------------- +# +reset_db +do_test 12.0 { + list [catch { + sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create + } msg] $msg +} {1 {error in fts5_api.xCreateTokenizer_v2()}} + +#------------------------------------------------------------------------- +# Tests for auxiliary function fts5_get_locale(). +# +reset_db + +# Check that if the table does not support locale=1, fts5_get_locale() +# always returns NULL. +do_execsql_test 13.1.0 { + CREATE VIRTUAL TABLE nolocale USING fts5(a, b); + INSERT INTO nolocale VALUES('one two three', 'four five six'); + INSERT INTO nolocale VALUES('three two one', 'seven eight nine'); +} +do_execsql_test 13.1.1 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale; +} {1 1} +do_execsql_test 13.1.2 { + SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two'); +} {1} +do_execsql_test 13.1.3 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two'); +} {1 1} +do_execsql_test 13.1.4 { + SELECT + fts5_get_locale(nolocale, 1) IS NULL + FROM nolocale('three AND two') ORDER BY rank +} {1 1} +do_catchsql_test 13.1.5 { + SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.6 { + SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.7 { + SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.8 { + SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.9 { + SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two'); +} {1 {non-integer argument passed to function fts5_get_locale()}} + + +# Check that if the table does support locale=1, fts5_get_locale() +# returns the locale of the identified row/column. +do_execsql_test 13.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); + INSERT INTO ft VALUES( + fts5_locale('th_TH', 'one two three'), 'four five six seven' + ); + INSERT INTO ft VALUES( + 'three two one', fts5_locale('en_AU', 'seven eight nine') + ); +} + +do_execsql_test 13.2.1 { + SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.2 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.3 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rank +} { NULL 'en_AU' 'th_TH' NULL } +do_execsql_test 13.2.4 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_execsql_test 13.2.5 { + SELECT + quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_catchsql_test 13.2.6 { + SELECT + quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} +do_catchsql_test 13.2.7 { + SELECT + quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 570693373..93361a5fe 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -517,7 +517,7 @@ fts5_aux_test_functions db do_execsql_test 15.3 { SELECT fts5_test_all(t1) FROM t1 LIMIT 1; } { - {columnsize {0 0} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2} + {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2} } finish_test diff --git a/ext/fts5/test/fts5merge.test b/ext/fts5/test/fts5merge.test index 3b86167b0..c57c21ded 100644 --- a/ext/fts5/test/fts5merge.test +++ b/ext/fts5/test/fts5merge.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5merge -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 4e4fe4ba2..534c42fff 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -35,21 +35,21 @@ do_catchsql_test 1.1.2 { do_catchsql_test 1.2.1 { SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id'); -} {0 {{}}} +} {1 {no such cursor: 4}} do_catchsql_test 1.2.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id')); -} {0 {}} +} {1 {no such cursor: 6}} do_catchsql_test 1.3.1 { SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'); -} {1 {no such cursor: 2}} +} {1 {no such cursor: 1}} do_catchsql_test 1.3.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads')); -} {1 {no such cursor: 2}} +} {1 {no such cursor: 1}} db close sqlite3 db test.db @@ -59,6 +59,11 @@ do_catchsql_test 1.3.3 { WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads')); } {1 {no such cursor: 1}} +fts5_aux_test_functions db +do_catchsql_test 1.3.4 { + SELECT fts5_columntext(t1) FROM t1('*reads'); +} {1 {no such cursor: 1}} + #------------------------------------------------------------------------- reset_db do_execsql_test 2.0 { @@ -535,5 +540,130 @@ do_execsql_test 19.0 { COMMIT; } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES + (1, 'a b c d'), + (2, 'x b c d'), + (3, 'x y z d'), + (4, 'a y c x'); +} + +do_execsql_test 20.1 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'b'; +} {1} + +do_execsql_test 20.2 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'y'; +} {4} + +do_execsql_test 20.3 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR x1 MATCH 'y'; +} {1 4 3} + +do_execsql_test 20.4 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR (x1 MATCH 'y' AND x1 MATCH 'd'); +} {1 4 3} + +do_execsql_test 20.5 { + SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd'); +} {3 1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 21.0 { + CREATE TABLE t1(ii INTEGER, x TEXT, y TEXT); + CREATE VIRTUAL TABLE xyz USING fts5(content_rowid=ii, content=t1, x, y); + INSERT INTO t1 VALUES(1, 'one', 'i'); + INSERT INTO t1 VALUES(2, 'two', 'ii'); + INSERT INTO t1 VALUES(3, 'tree', 'iii'); + INSERT INTO xyz(xyz) VALUES('rebuild'); +} + +do_execsql_test 21.1 { + UPDATE xyz SET y='TWO' WHERE rowid=2; + UPDATE t1 SET y='TWO' WHERE ii=2; +} + +do_execsql_test 21.2 { + PRAGMA integrity_check +} {ok} + +breakpoint +sqlite3_db_config db DEFENSIVE 1 +do_execsql_test 21.3 { + CREATE TABLE xyz_notashadow(x, y); + DROP TABLE xyz_notashadow; +} +sqlite3_db_config db DEFENSIVE 0 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 22.0 { + SELECT fts5(NULL); +} {{}} +do_execsql_test 22.1 { + SELECT count(*) FROM ( + SELECT fts5_source_id() + ) +} {1} +execsql_pp { + SELECT fts5_source_id() +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1 VALUES('one + two + three'); + INSERT INTO x1 VALUES('one + xyz + three'); + INSERT INTO x1 VALUES('xyz + two + xyz'); +} +do_execsql_test 23.1 { + SELECT rowid FROM x1('one + two + three'); +} {1} + +do_execsql_test 23.2 { + SELECT rowid FROM x1('^".." AND one'); +} {} + +do_execsql_test 23.3 { + SELECT rowid FROM x1('abc NEAR ".." NEAR def'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 24.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none'); + INSERT INTO t1(a) VALUES('a'); +} + +do_execsql_test 24.2 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ) ORDER BY 1; +} {-1e-06} + +do_execsql_test 24.3 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT � NOT def') ) ORDER BY 1; +} {-1e-06} + +do_execsql_test 24.4 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ); +} {-1e-06} + +#------------------------------------------------------------------------- +reset_db +fts5_aux_test_functions db + +do_execsql_test 25.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none', content=''); + INSERT INTO t1(a) VALUES('a b c'); +} + +do_execsql_test 25.0 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {{}} + finish_test diff --git a/ext/fts5/test/fts5near.test b/ext/fts5/test/fts5near.test index bbe144a89..318a16948 100644 --- a/ext/fts5/test/fts5near.test +++ b/ext/fts5/test/fts5near.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5near -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5optimize.test b/ext/fts5/test/fts5optimize.test index e0f0fd724..610bf439c 100644 --- a/ext/fts5/test/fts5optimize.test +++ b/ext/fts5/test/fts5optimize.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5optimize -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5optimize2.test b/ext/fts5/test/fts5optimize2.test index b0b28874c..57f4e96b9 100644 --- a/ext/fts5/test/fts5optimize2.test +++ b/ext/fts5/test/fts5optimize2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5optimize2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5optimize3.test b/ext/fts5/test/fts5optimize3.test index 7b11b9402..79e62f9f2 100644 --- a/ext/fts5/test/fts5optimize3.test +++ b/ext/fts5/test/fts5optimize3.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5optimize2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 9752f35d3..8e975fa17 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5origintext -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -261,8 +261,8 @@ do_execsql_test 5.3 { reset_db sqlite3_fts5_register_origintext db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE ft USING fts5( - x, y, tokenize='origintext unicode61', detail=%DETAIL% + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0 ); INSERT INTO ft VALUES('One Two', 'Three two'); @@ -291,6 +291,22 @@ do_execsql_test 6.3 { SELECT rowid, tokens(ft) FROM ft('Three*'); } {1 {{}} 2 {{}}} +fts5_aux_test_functions db +do_catchsql_test 6.4 { + SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one'); +} {1 SQLITE_RANGE} + +do_catchsql_test 6.5 { + SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one'); +} {1 SQLITE_RANGE} + +do_catchsql_test 6.6 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2); +} {1 {malformed tokendata=... directive}} +do_catchsql_test 6.7 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11); +} {1 {malformed tokendata=... directive}} + } finish_test diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test index a8c5d4eb5..a8c717234 100644 --- a/ext/fts5/test/fts5origintext2.test +++ b/ext/fts5/test/fts5origintext2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5origintext2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test index 834844595..9dda2a574 100644 --- a/ext/fts5/test/fts5origintext3.test +++ b/ext/fts5/test/fts5origintext3.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5origintext3 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test index c4ae35011..3b907ba2c 100644 --- a/ext/fts5/test/fts5origintext4.test +++ b/ext/fts5/test/fts5origintext4.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5origintext4 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test index 03d5bee21..848cc15b5 100644 --- a/ext/fts5/test/fts5origintext5.test +++ b/ext/fts5/test/fts5origintext5.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5origintext -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5phrase.test b/ext/fts5/test/fts5phrase.test index 10598ccf4..ea425a4dd 100644 --- a/ext/fts5/test/fts5phrase.test +++ b/ext/fts5/test/fts5phrase.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5phrase -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5plan.test b/ext/fts5/test/fts5plan.test index 6862acf17..57d5254a3 100644 --- a/ext/fts5/test/fts5plan.test +++ b/ext/fts5/test/fts5plan.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5plan -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5porter.test b/ext/fts5/test/fts5porter.test index c7b1ce6f3..de1c3e15a 100644 --- a/ext/fts5/test/fts5porter.test +++ b/ext/fts5/test/fts5porter.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5porter -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5porter2.test b/ext/fts5/test/fts5porter2.test index 6e81b2d31..556060baa 100644 --- a/ext/fts5/test/fts5porter2.test +++ b/ext/fts5/test/fts5porter2.test @@ -18,7 +18,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5porter2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5prefix.test b/ext/fts5/test/fts5prefix.test index 279f312f2..7a29628ea 100644 --- a/ext/fts5/test/fts5prefix.test +++ b/ext/fts5/test/fts5prefix.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5prefix -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5prefix2.test b/ext/fts5/test/fts5prefix2.test index 29744c86b..0860b3cdd 100644 --- a/ext/fts5/test/fts5prefix2.test +++ b/ext/fts5/test/fts5prefix2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5prefix2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5query.test b/ext/fts5/test/fts5query.test index 5237e8e25..4e8bab8cf 100644 --- a/ext/fts5/test/fts5query.test +++ b/ext/fts5/test/fts5query.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5query -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5rank.test b/ext/fts5/test/fts5rank.test index 8cf223f44..7a700cb97 100644 --- a/ext/fts5/test/fts5rank.test +++ b/ext/fts5/test/fts5rank.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rank -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test index ae881c02f..d74b148fb 100644 --- a/ext/fts5/test/fts5rebuild.test +++ b/ext/fts5/test/fts5rebuild.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rebuild -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5restart.test b/ext/fts5/test/fts5restart.test index db2c62b67..da58fe3ae 100644 --- a/ext/fts5/test/fts5restart.test +++ b/ext/fts5/test/fts5restart.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5restart -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -29,6 +29,7 @@ do_execsql_test 1.0 { # Run the 'optimize' command. Check that it does not disturb ongoing # full-text queries. # +unset -nocomplain lRowid do_test 1.1 { for {set i 1} {$i < 1000} {incr i} { execsql { INSERT INTO f1 VALUES('a b c d e') } diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test index 8935ecfea..e4e4f6b84 100644 --- a/ext/fts5/test/fts5rowid.test +++ b/ext/fts5/test/fts5rowid.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rowid -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5savepoint.test b/ext/fts5/test/fts5savepoint.test index 112622275..fdb0a25ba 100644 --- a/ext/fts5/test/fts5savepoint.test +++ b/ext/fts5/test/fts5savepoint.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5savepoint -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5secure8.test b/ext/fts5/test/fts5secure8.test index 8ceb9630a..8b65b7c59 100644 --- a/ext/fts5/test/fts5secure8.test +++ b/ext/fts5/test/fts5secure8.test @@ -42,6 +42,26 @@ do_execsql_test 1.2 { PRAGMA integrity_check; } {ok} +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5 ( + name, + content='' + ); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + INSERT INTO xyz (rowid, name) VALUES(1, 'A'); + INSERT INTO xyz (rowid, name) VALUES(2, 'A'); + INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A'); +} + +do_execsql_test 2.1 { + pragma quick_check; +} {ok} + +do_catchsql_test 2.2 { + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 'hello world'); +} {1 {SQL logic error}} + diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test index 63874ece5..2959ab65c 100644 --- a/ext/fts5/test/fts5securefault.test +++ b/ext/fts5/test/fts5securefault.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5securefault -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. return_if_no_fts5 do_execsql_test 1.0 { diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 936bbb254..60ccb5a9c 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -480,4 +480,33 @@ do_execsql_test 22.0 { do_catchsql_test 22.1 {SELECT * FROM x1('')} {1 {fts5: syntax error near ""}} do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.1 { + BEGIN; + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); +} + +do_execsql_test 23.2 { + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.3 { + INSERT INTO x1(x1) VALUES('flush'); + SELECT count(*) FROM x1_data; +} {3} + +do_execsql_test 23.4 { + ROLLBACK; + SELECT count(*) FROM x1_data; +} {2} + + finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index 6c0e0e166..01c590c9f 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test index 0d4972b37..680448081 100644 --- a/ext/fts5/test/fts5simple3.test +++ b/ext/fts5/test/fts5simple3.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple3 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 86610ee9e..55e2f186a 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5synonym -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index 9fdf75769..ec8b750c5 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5synonym2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 27370657e..a828e3a22 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5tokenizer -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -300,5 +300,75 @@ set ::flags [list] do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} do_test 9.5.2 { set ::flags } {query} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61); + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="unicode61 error");' + WHERE name = 'x1'; +} + +db close +sqlite3 db test.db + +do_catchsql_test 10.2 { + SELECT * FROM x1('abc'); +} {1 {error in tokenizer constructor}} + +do_catchsql_test 10.3 { + INSERT INTO x1 VALUES('abc'); +} {1 {error in tokenizer constructor}} + +do_execsql_test 10.4 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="nosuch error");' + WHERE name = 'x1'; +} + +db close +sqlite3 db test.db + +do_catchsql_test 10.5 { + SELECT * FROM x1('abc'); +} {1 {no such tokenizer: nosuch}} +do_catchsql_test 10.6 { + INSERT INTO x1 VALUES('abc'); +} {1 {no such tokenizer: nosuch}} + +do_execsql_test 10.7 { + DROP TABLE x1; + SELECT * FROM sqlite_schema; +} + +reset_db +do_execsql_test 10.8 { + CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61); + INSERT INTO x1 VALUES('a b c'), ('d e f'), ('a b c'); + CREATE VIRTUAL TABLE x1v USING fts5vocab(x1, row); + + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=simplify);' + WHERE name = 'x1'; +} + +do_execsql_test 10.9 { + SELECT * FROM x1v +} { + a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1 +} + +db close +sqlite3 db test.db + +do_execsql_test 10.10 { + SELECT * FROM x1v +} { + a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1 +} finish_test diff --git a/ext/fts5/test/fts5tokenizer2.test b/ext/fts5/test/fts5tokenizer2.test index bdabd5312..4fe31d22c 100644 --- a/ext/fts5/test/fts5tokenizer2.test +++ b/ext/fts5/test/fts5tokenizer2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5tokenizer2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -85,5 +85,25 @@ do_execsql_test 1.7 { SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess'); } {AAdont>BBmess<} +# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd +# Error handling of tokenize= arguments. +# +foreach {n tkz} { + 1 {ascii none} + 2 {unicode61 none} + 3 {porter none} + 4 {trigram none} + 5 {ascii none 0} + 6 {unicode61 none 0} + 7 {porter none 0} + 8 {trigram none 0} +} { + db eval {DROP TABLE IF EXISTS t2;} + do_catchsql_test 2.$n " + DROP TABLE IF EXISTS t2; + CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz'); + " {1 {error in tokenizer constructor}} +} + finish_test diff --git a/ext/fts5/test/fts5tokenizer3.test b/ext/fts5/test/fts5tokenizer3.test new file mode 100644 index 000000000..5cdab743c --- /dev/null +++ b/ext/fts5/test/fts5tokenizer3.test @@ -0,0 +1,77 @@ +# 2024 Aug 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +proc get_sod {args} { return "split_on_dot" } +proc get_lowercase {args} { return "lowercase" } + +proc lowercase {flags txt} { + set n [string length $txt] + sqlite3_fts5_token [string tolower $txt] 0 $n + return 0 +} + +proc split_on_dot {flags txt} { + set iOff 0 + foreach t [split $txt "."] { + set n [string length $txt] + sqlite3_fts5_token $t $iOff [expr $iOff+$n] + incr iOff [expr {$n+1}] + } + return "" +} + +foreach {tn script} { + 1 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 2 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 3 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } + 4 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } +} { + reset_db + eval $script + + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot); + CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance); + INSERT INTO t1 VALUES('ABC.Def.ghi'); + } + + do_execsql_test 1.$tn.1 { + SELECT term FROM t1vocab ORDER BY 1 + } {abc def ghi} +} + + +finish_test diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test index 351c059bf..3742c647f 100644 --- a/ext/fts5/test/fts5trigram.test +++ b/ext/fts5/test/fts5trigram.test @@ -56,6 +56,7 @@ foreach {tn like res} { 7 {ABCDEFG%} 1 8 {%รุงเ%} 2 9 {%งเ%} 2 + 10 {%"งเ"%} {} } { do_execsql_test 1.3.$tn { SELECT rowid FROM t1 WHERE y LIKE $like @@ -69,6 +70,9 @@ do_execsql_test 2.0 { INSERT INTO t1 VALUES('abcdefghijklm'); INSERT INTO t1 VALUES('กรุงเทพมหานคร'); } +do_catchsql_test 2.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram case_sensitive'); +} {1 {error in tokenizer constructor}} foreach {tn s res} { 1 abc "(abc)defghijklm" @@ -197,6 +201,12 @@ do_eqp_test 6.3 { do_eqp_test 6.4 { SELECT * FROM ci1 WHERE x GLOB ? } {VIRTUAL TABLE INDEX 0:G0} +do_eqp_test 6.5 { + SELECT * FROM ci1 WHERE x < ? +} {{SCAN ci1 VIRTUAL TABLE INDEX 0:}} +do_eqp_test 6.6 { + SELECT * FROM ci0 WHERE x < ? +} {{SCAN ci0 VIRTUAL TABLE INDEX 0:}} reset_db do_execsql_test 7.0 { @@ -206,7 +216,7 @@ do_execsql_test 7.0 { (20, "жираф.png"), (30, "cat.png"), (40, "кот.png"), - (50, "misic-🎵-.mp3"); + (50, "misic-ðµ-.mp3"); } do_execsql_test 7.1 { SELECT rowid FROM f WHERE +filename GLOB '*ир*'; @@ -253,4 +263,85 @@ do_execsql_test 8.3 { {[abcde]} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + tokenize=trigram + ); + + INSERT INTO t1(rowid, a12) VALUES(111, 'thats a tricky case though'); + INSERT INTO t1(rowid, a12) VALUES(222, 'the query planner cannot do'); +} + +do_execsql_test 9.1 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' +} {111} + +do_execsql_test 9.2 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' AND a12 LIKE '%case%' +} {111} + +do_execsql_test 9.3 { + SELECT rowid FROM t1 WHERE a12 LIKE NULL +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=trigram); +} + +do_test 10.1 { + foreach {val} { + "abc \UFFjkl\UFF" + "abc \UFFFjkl\UFFF" + "abc \UFFFFjkl\UFFFF" + "abc \UFFFFFjkl\UFFFFF" + "\UFFjkl\UFF abc" + "\UFFFjkl\UFFF abc" + "\UFFFFjkl\UFFFF abc" + "\UFFFFFjkl\UFFFFF abc" + "\U10001jkl\U10001 abc" + } { + execsql { INSERT INTO t1 VALUES( $val ) } + } +} {} + +do_test 10.2 { + foreach {val} { + X'E18000626320646566' + X'61EDA0806320646566' + X'61EDA0806320646566' + X'61EFBFBE6320646566' + X'76686920E18000626320646566' + X'7668692061EDA0806320646566' + X'7668692061EDA0806320646566' + X'7668692061EFBFBE6320646566' + } { + execsql " INSERT INTO t1 VALUES( $val ) " + } +} {} + +do_test 10.3 { + set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] + set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] + set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + execsql { + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + + INSERT INTO t1 VALUES('abcd' || $a); + INSERT INTO t1 VALUES('abcd' || $b); + INSERT INTO t1 VALUES('abcd' || $c); + INSERT INTO t1 VALUES('abcd' || $d); + } +} {} + + + finish_test diff --git a/ext/fts5/test/fts5trigram2.test b/ext/fts5/test/fts5trigram2.test index f5beae5b2..c81684a22 100644 --- a/ext/fts5/test/fts5trigram2.test +++ b/ext/fts5/test/fts5trigram2.test @@ -21,6 +21,9 @@ do_execsql_test 1.0 " INSERT INTO t1 VALUES('abc\u0303defghijklm'); INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm'); " +do_catchsql_test 1.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram remove_diacritics'); +} {1 {error in tokenizer constructor}} do_execsql_test 1.1 { SELECT highlight(t1, 0, '(', ')') FROM t1('abc'); @@ -101,9 +104,34 @@ do_execsql_test 4.0 { CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram); } {} -breakpoint do_execsql_test 4.1 { INSERT INTO t4 VALUES('ABCD'); + INSERT INTO t4 VALUES('DEFG'); +} {} + +db close +sqlite3 db test.db + +do_eqp_test 4.1 { + SELECT rowid FROM t4 WHERE z LIKE '%abc%' +} {VIRTUAL TABLE INDEX 0:L0} + +do_execsql_test 4.2 { + SELECT rowid FROM t4 WHERE z LIKE '%abc%' +} {1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts5( + c1, tokenize='trigram', detail='none' + ); + INSERT INTO t5(rowid, c1) VALUES(1, 'abc_____xyx_yxz'); + INSERT INTO t5(rowid, c1) VALUES(2, 'abc_____xyxz'); + INSERT INTO t5(rowid, c1) VALUES(3, 'ac_____xyxz'); } {} +do_execsql_test 5.1 { + SELECT rowid FROM t5 WHERE c1 LIKE 'abc%xyxz' +} {2} finish_test diff --git a/ext/fts5/test/fts5ubsan.test b/ext/fts5/test/fts5ubsan.test index 2dc0aa7bd..76382a1e1 100644 --- a/ext/fts5/test/fts5ubsan.test +++ b/ext/fts5/test/fts5ubsan.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ubsan -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5unicode.test b/ext/fts5/test/fts5unicode.test index e2d0f6012..f10e0d02d 100644 --- a/ext/fts5/test/fts5unicode.test +++ b/ext/fts5/test/fts5unicode.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unicode -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -60,6 +60,7 @@ do_execsql_test 2.1 " # require 17 or more bits to store). # +unset -nocomplain A B C D set A [db one {SELECT char(0x1F75E)}] ;# Type So set B [db one {SELECT char(0x1F5FD)}] ;# Type So set C [db one {SELECT char(0x2F802)}] ;# Type Lo diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test index 662b9dd87..7a49a1d83 100644 --- a/ext/fts5/test/fts5unicode2.test +++ b/ext/fts5/test/fts5unicode2.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unicode2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -116,6 +116,7 @@ set docs [list { connected by OR. }] +unset -nocomplain map set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS @@ -470,119 +471,23 @@ do_execsql_test 8.2.3 { } {2 4} #------------------------------------------------------------------------- -# -if 0 { -foreach {tn sql} { - 1 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]); - CREATE VIRTUAL TABLE t6 USING fts4( - tokenize=unicode61 [tokenchars=="] "tokenchars=[]"); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]); - } - 2 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= ."); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]"); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4"); - } - 3 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .'); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]'); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4'); - } - 4 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`); - } -} { - do_execsql_test 9.$tn.0 { - DROP TABLE IF EXISTS t5; - DROP TABLE IF EXISTS t5aux; - DROP TABLE IF EXISTS t6; - DROP TABLE IF EXISTS t6aux; - DROP TABLE IF EXISTS t7; - DROP TABLE IF EXISTS t7aux; - } - do_execsql_test 9.$tn.1 $sql - - do_execsql_test 9.$tn.2 { - CREATE VIRTUAL TABLE t5aux USING fts4aux(t5); - INSERT INTO t5 VALUES('one two three/four.five.six'); - SELECT * FROM t5aux; - } { - four.five.six * 1 1 four.five.six 0 1 1 - {one two three} * 1 1 {one two three} 0 1 1 - } - - do_execsql_test 9.$tn.3 { - CREATE VIRTUAL TABLE t6aux USING fts4aux(t6); - INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta'); - SELECT * FROM t6aux; - } { - {alpha=beta"gamma} * 1 1 {alpha=beta"gamma} 0 1 1 - {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1 - } - - do_execsql_test 9.$tn.4 { - CREATE VIRTUAL TABLE t7aux USING fts4aux(t7); - INSERT INTO t7 VALUES('alephxbeth\xC4gimel'); - SELECT * FROM t7aux; - } { - aleph * 1 1 aleph 0 1 1 - beth * 1 1 beth 0 1 1 - gimel * 1 1 gimel 0 1 1 - } -} - -# Check that multiple options are handled correctly. -# -do_execsql_test 10.1 { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61 - "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy" - "separators=a" "separators=a" "tokenchars=a" "tokenchars=a" - ); - - INSERT INTO t1 VALUES('oneatwoxthreeyfour'); - INSERT INTO t1 VALUES('a.single=word'); - CREATE VIRTUAL TABLE t1aux USING fts4aux(t1); - SELECT * FROM t1aux; -} { - .single=word * 1 1 .single=word 0 1 1 - four * 1 1 four 0 1 1 - one * 1 1 one 0 1 1 - three * 1 1 three 0 1 1 - two * 1 1 two 0 1 1 -} - -# Test that case folding happens after tokenization, not before. -# -do_execsql_test 10.2 { - DROP TABLE IF EXISTS t2; - CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB"); - INSERT INTO t2 VALUES('oneatwoBthree'); - INSERT INTO t2 VALUES('onebtwoAthree'); - CREATE VIRTUAL TABLE t2aux USING fts4aux(t2); - SELECT * FROM t2aux; -} { - one * 1 1 one 0 1 1 - onebtwoathree * 1 1 onebtwoathree 0 1 1 - three * 1 1 three 0 1 1 - two * 1 1 two 0 1 1 -} -# Test that the tokenchars and separators options work with the -# fts3tokenize table. -# -do_execsql_test 11.1 { - CREATE VIRTUAL TABLE ft1 USING fts3tokenize( - "unicode61", "tokenchars=@.", "separators=1234567890" - ); - SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road'; +foreach {tn val bErr} { + 1 0 0 + 2 1 0 + 3 2 0 + 4 3 1 + 5 11 1 } { - berlin@street sydney.road -} - + reset_db + set aRes(0) {0 {}} + set aRes(1) {1 {error in tokenizer constructor}} + set res $aRes($bErr) + do_catchsql_test 9.1.$tn " + CREATE VIRTUAL TABLE bl USING fts5( + s, tokenize='trigram remove_diacritics $val' + ); + " $res } finish_test diff --git a/ext/fts5/test/fts5unicode3.test b/ext/fts5/test/fts5unicode3.test index 30eb3c416..ddb61a999 100644 --- a/ext/fts5/test/fts5unicode3.test +++ b/ext/fts5/test/fts5unicode3.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5unicode4.test b/ext/fts5/test/fts5unicode4.test index dfd7f5a25..dc225cb5e 100644 --- a/ext/fts5/test/fts5unicode4.test +++ b/ext/fts5/test/fts5unicode4.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unicode4 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5unindexed.test b/ext/fts5/test/fts5unindexed.test index 8b72c4c77..5099a8969 100644 --- a/ext/fts5/test/fts5unindexed.test +++ b/ext/fts5/test/fts5unindexed.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unindexed -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 79fd94e6b..a92c0dc9f 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5version -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index c457c5c21..b1644527e 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5vocab -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -513,6 +513,7 @@ do_execsql_test 10.5 { INSERT INTO ft(a) VALUES('4 5 6'); } +unset -nocomplain x res do_test 10.6 { set res [list] db eval { SELECT rowid FROM ft('4') } x { diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test index ecacc50da..7b3c3b0d6 100644 --- a/ext/fts5/test/fts5vocab2.test +++ b/ext/fts5/test/fts5vocab2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5vocab2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/icu/icu.c b/ext/icu/icu.c index e745ab025..69867bfa8 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -471,7 +471,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -486,7 +486,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i<sizeof(aStrength)/sizeof(aStrength[0]); i++){ + if( sqlite3_stricmp(zOption,aStrength[i].zName)==0 ){ + ucol_setStrength(pUCollator, aStrength[i].val); + break; + } + } + if( i>=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; i<sizeof(aStrength)/sizeof(aStrength[0]); i++){ + sqlite3_str_appendf(pStr, " %s", aStrength[i].zName); + } + sqlite3_result_error(p, sqlite3_str_value(pStr), -1); + sqlite3_free(sqlite3_str_finish(pStr)); + return; + } + } rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, icuCollationColl, icuCollationDel ); @@ -509,6 +541,7 @@ int sqlite3IcuInit(sqlite3 *db){ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { {"icu_load_collation",2,SQLITE_UTF8|SQLITE_DIRECTONLY,1, icuLoadCollation}, + {"icu_load_collation",3,SQLITE_UTF8|SQLITE_DIRECTONLY,1, icuLoadCollation}, #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) {"regexp", 2, SQLITE_ANY|SQLITEICU_EXTRAFLAGS, 0, icuRegexpFunc}, {"lower", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test new file mode 100644 index 000000000..187132f76 --- /dev/null +++ b/ext/intck/intck1.test @@ -0,0 +1,332 @@ +# 2008 Feb 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the incremental integrity check +# (intck) extension. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intck1 +return_if_no_intck + +foreach {tn sql} { + 1 "CREATE TABLE t1(a PRIMARY KEY, b)" + 2 "CREATE TABLE t2(a PRIMARY KEY, b) WITHOUT ROWID " + 3 "CREATE TABLE t3(a PRIMARY KEY, b) WITHOUT rowID;" + 4 "CREATE TABLE t4(a PRIMARY KEY, ROWID)" + 5 {CREATE TABLE t5(a PRIMARY KEY, ROWID) WITHOUT ROWID + } +} { + do_test 1.1.$tn { + db eval $sql + set {} {} + } {} +} + +set space " \n\v\t\r\f" + +do_execsql_test 1.2 { + SELECT name, (rtrim(sql, $space) LIKE '%rowid') + FROM sqlite_schema WHERE type='table' + ORDER BY 1 +} { + t1 0 + t2 1 + t3 1 + t4 0 + t5 1 +} + +do_execsql_test 1.3 { + CREATE TABLE x1(a COLLATE nocase, b INTEGER, c BLOB); + INSERT INTO x1 VALUES('lEtTeRs', 1234, 1234); +} +do_execsql_test 1.3.1 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c1='letters'; +} {lEtTeRs 1234 1234} +do_execsql_test 1.3.2 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c2='1234'; +} {lEtTeRs 1234 1234} +do_execsql_test 1.3.2 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c3='1234'; +} {} + +do_execsql_test 1.4 { + CREATE TABLE z1(a, b); + CREATE INDEX z1ab ON z1(a+b COLLATE nocase); +} +do_execsql_test 1.4.1 { + SELECT * FROM z1 INDEXED BY z1ab +} + +do_catchsql_test 1.5.1 { + CREATE INDEX z1b ON z1(b ASC NULLS LAST); +} {1 {unsupported use of NULLS LAST}} +do_catchsql_test 1.5.2 { + CREATE INDEX z1b ON z1(b DESC NULLS LAST); +} {1 {unsupported use of NULLS LAST}} +do_catchsql_test 1.5.3 { + CREATE INDEX z1b ON z1(b ASC NULLS FIRST); +} {1 {unsupported use of NULLS FIRST}} +do_catchsql_test 1.5.4 { + CREATE INDEX z1b ON z1(b DESC NULLS FIRST); +} {1 {unsupported use of NULLS FIRST}} + + +reset_db +do_execsql_test 1.6.1 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + ANALYZE; + INSERT INTO sqlite_stat1 VALUES('t1', 'i1', '10000 10000'); + ANALYZE sqlite_schema; +} {} +do_eqp_test 1.6.2 { + SELECT 1 FROM t1 INDEXED BY i1 WHERE (b, i) IS (?, ?); +} {SEARCH} + + + +#------------------------------------------------------------------------- +reset_db + +do_test 2.0 { + set ic [sqlite3_intck db main] + $ic close +} {} + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + + CREATE INDEX i1 ON t1(a COLLATE nocase); + CREATE INDEX i2 ON t1(b, a); + CREATE INDEX i3 ON t1(b + a COLLATE nocase) WHERE a!=1; +} + +do_intck_test 2.2 { +} + +# Delete a row from each of the i1 and i2 indexes using the imposter +# table interface. +# +do_test 2.3 { + db eval {SELECT name, rootpage FROM sqlite_schema} { + set R($name) $rootpage + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i1) + db eval { CREATE TABLE imp1(a PRIMARY KEY, rowid) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i2) + db eval { CREATE TABLE imp2(b, a, rowid, PRIMARY KEY(b, a)) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db eval { + DELETE FROM imp1 WHERE rowid=1; + DELETE FROM imp2 WHERE rowid=2; + } + + db close + sqlite3 db test.db +} {} + +do_intck_test 2.4 { + {entry (1,1) missing from index i1} + {entry (4,3,2) missing from index i2} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(c, b)) WITHOUT ROWID; + CREATE INDEX x1a ON x1(a COLLATE nocase); + + INSERT INTO x1 VALUES(1, 2, 'three'); + INSERT INTO x1 VALUES(4, 5, 'six'); + INSERT INTO x1 VALUES(7, 8, 'nine'); +} + +do_intck_test 3.1 { } + +do_test 3.2 { + db eval {SELECT name, rootpage FROM sqlite_schema} { + set R($name) $rootpage + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(x1a) + db eval { CREATE TABLE imp1(c, b, a, PRIMARY KEY(c, b)) WITHOUT ROWID } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db eval { + DELETE FROM imp1 WHERE a=5; + } + execsql_pp { + } + + db close + sqlite3 db test.db +} {} + +do_intck_test 3.3 { + {entry (4,'six',5) missing from index x1a} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE www(x, y, z); + CREATE INDEX w1 ON www( (x+1), z ); + INSERT INTO www VALUES(1, 1, 1), (2, 2, 2); +} + +do_intck_test 4.1 { } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a COLLATE NOCASE); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); +} + +do_test 5.1 { + set ic [sqlite3_intck db nosuchdb] + $ic step +} {SQLITE_ERROR} + +do_test 5.2 { + $ic close + set ic [sqlite3_intck db {}] + while {[$ic step]=="SQLITE_OK"} {} + set res [$ic error] + $ic close + set res +} {SQLITE_OK {}} + +do_test 5.3 { test_do_intck db "main" } {} + +do_test 5.4 { + set ret {} + set ic [sqlite3_intck db main] + db eval [$ic test_sql t1] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +do_test 5.5 { + set ret {} + set ic [sqlite3_intck db main] + db eval [$ic test_sql {}] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +db cache flush + +do_test 5.6 { + set ret {} + set ic [sqlite3_intck db main] + $ic step + db eval [$ic test_sql {}] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 6.0 { + CREATE TABLE t1(x, y, PRIMARY KEY(x)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(y, x); + INSERT INTO t1 VALUES(X'0000', X'1111'); +} + +do_intck_test 6.1 {} + +do_execsql_test 6.2.1 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1' WHERE name='i1'; +} {} +do_intck_test 6.2.2 {} + +do_execsql_test 6.3.1 { + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1(y' WHERE name='i1'; +} {} +do_intck_test 6.3.2 {} + +do_execsql_test 6.4.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1(y) hello world' + WHERE name='i1'; +} {} +do_intck_test 6.4.2 {} + +do_execsql_test 6.5.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1(y, x) WHERE 1 ' + WHERE name='i1'; +} {} +do_intck_test 6.5.2 {} + +do_execsql_test 6.6.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1( , ) WHERE 1 ' + WHERE name='i1'; +} {} + +do_test 6.7.2 { + set ic [sqlite3_intck db main] + $ic step +} {SQLITE_ERROR} +do_test 6.5.3 { + $ic error +} {SQLITE_ERROR {near "AS": syntax error}} +$ic close + +do_execsql_test 6.6.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1([y' + WHERE name='i1'; +} {} +do_intck_test 6.6.2 {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE x1("1", "22", "3333", four); + CREATE INDEX i1 ON x1( "1" , "22", NULL); + INSERT INTO x1 VALUES(1, 22, 3333, NULL); + INSERT INTO x1 VALUES(1, 22, 3333, NULL); +} +do_execsql_test 7.1 " CREATE INDEX i2 ON x1( \"1\"\r\n\t ) " +do_execsql_test 7.2 { CREATE INDEX i3 ON x1( "22" || 'abc''def' || `1` ) } +do_execsql_test 7.3 { CREATE INDEX i4 ON x1( [22] + [1] ) } +do_execsql_test 7.4 { CREATE INDEX i5 ON x1( four||'hello' ) } + +do_intck_test 7.5 {} + + +finish_test diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test new file mode 100644 index 000000000..23b241b5a --- /dev/null +++ b/ext/intck/intck2.test @@ -0,0 +1,177 @@ +# 2024 Feb 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the incremental integrity check +# (intck) extension. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intck2 +return_if_no_intck + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + CREATE INDEX i1 ON t1(b); +} + +proc imposter_edit {obj create sql} { + sqlite3 xdb test.db + set pgno [xdb one {SELECT rootpage FROM sqlite_schema WHERE name=$obj}] + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 1 $pgno + xdb eval $create + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 0 0 + xdb eval $sql + xdb close +} + +imposter_edit i1 { + CREATE TABLE imp(b, a, PRIMARY KEY(b)) WITHOUT ROWID; +} { + DELETE FROM imp WHERE b='two'; + INSERT INTO imp(b, a) VALUES('four', 4); +} + +do_intck_test 1.1 { + {surplus entry ('four',4) in index i1} + {entry ('two',2) missing from index i1} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(a, b, "c d"); + CREATE INDEX x1a ON x1(a COLLATE nocase DESC , b ASC); + CREATE INDEX x1b ON x1( a || b || ' "''" ' COLLATE binary ASC ); + CREATE INDEX x1c ON x1( format('%s', a)ASC, format('%d', "c d" ) ); + INSERT INTO x1 VALUES('one', 2, 3); + INSERT INTO x1 VALUES('One', 4, 5); + INSERT INTO x1 VALUES('ONE', 6, 7); + INSERT INTO x1 VALUES(NULL, NULL, NULL); +} + +do_intck_test 2.1 {} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE c=7; +} +do_intck_test 2.2 { + {surplus entry ('ONE',6,3) in index x1a} + {surplus entry ('ONE6 "''" ',3) in index x1b} + {surplus entry ('ONE','7',3) in index x1c} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c); + CREATE INDEX x1all ON x1(a DESC, b ASC, c DESC); + INSERT INTO x1 VALUES(2, 1, 2); + INSERT INTO x1 VALUES(2, 1, 1); + INSERT INTO x1 VALUES(2, 2, 2); + INSERT INTO x1 VALUES(2, 2, 1); + INSERT INTO x1 VALUES(1, 1, 2); + INSERT INTO x1 VALUES(1, 1, 1); + INSERT INTO x1 VALUES(1, 2, 2); + INSERT INTO x1 VALUES(1, 2, 1); +} + +do_intck_test 3.1 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; +} + +db close +sqlite3 db test.db + +do_intck_test 3.2 { + {surplus entry (2,1,2,1) in index x1all} + {surplus entry (2,1,1,2) in index x1all} + {surplus entry (2,2,2,3) in index x1all} + {surplus entry (2,2,1,4) in index x1all} + {surplus entry (1,1,2,5) in index x1all} + {surplus entry (1,1,1,6) in index x1all} + {surplus entry (1,2,2,7) in index x1all} + {surplus entry (1,2,1,8) in index x1all} +} + +do_execsql_test 3.3 { + DELETE FROM x1; + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); +} + +do_intck_test 3.4 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; + INSERT INTO imp(rowid) VALUES(-123); + INSERT INTO imp(rowid) VALUES(456); +} + +db close +sqlite3 db test.db + +do_intck_test 3.5 { + {entry (NULL,NULL,NULL,-123) missing from index x1all} + {entry (NULL,NULL,NULL,456) missing from index x1all} + {surplus entry (NULL,NULL,NULL,1) in index x1all} + {surplus entry (NULL,NULL,NULL,2) in index x1all} + {surplus entry (NULL,NULL,NULL,3) in index x1all} + {surplus entry (NULL,NULL,NULL,4) in index x1all} +} + +reset_db + +do_execsql_test 3.6 { + CREATE TABLE w1(a PRIMARY KEY, b, c); + INSERT INTO w1 VALUES(1.0, NULL, NULL); + INSERT INTO w1 VALUES(33.0, NULL, NULL); + INSERT INTO w1 VALUES(100.0, NULL, NULL); + CREATE INDEX w1bc ON w1(b, c); +} + +do_intck_test 3.7 { +} + +imposter_edit w1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE a=33; + INSERT INTO imp(a) VALUES(1234.5); + INSERT INTO imp(a) VALUES(-1234.5); +} + +do_intck_test 3.8 { + {surplus entry (33.0,2) in index sqlite_autoindex_w1_1} + {entry (1234.5,4) missing from index sqlite_autoindex_w1_1} + {entry (NULL,NULL,4) missing from index w1bc} + {entry (-1234.5,5) missing from index sqlite_autoindex_w1_1} + {entry (NULL,NULL,5) missing from index w1bc} + {surplus entry (NULL,NULL,2) in index w1bc} +} + +finish_test diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl new file mode 100644 index 000000000..1e216b59f --- /dev/null +++ b/ext/intck/intck_common.tcl @@ -0,0 +1,66 @@ +# 2024 Feb 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +ifcapable !vtab||!pragma { + proc return_if_no_intck {} { + finish_test + return -code return + } + return +} else { + proc return_if_no_intck {} {} +} + +proc do_intck {db {bSuspend 0}} { + set ic [sqlite3_intck $db main] + + set ret [list] + while {"SQLITE_OK"==[$ic step]} { + set msg [$ic message] + if {$msg!=""} { + lappend ret $msg + } + if {$bSuspend} { + $ic unlock + #puts "SQL: [$ic test_sql {}]" + #execsql_pp "EXPLAIN query plan [$ic test_sql {}]" + #explain_i [$ic test_sql {}] + } + } + + set err [$ic error] + if {[lindex $err 0]!="SQLITE_OK"} { + error $err + } + $ic close + + return $ret +} + +proc intck_sql {db tbl} { + set ic [sqlite3_intck $db main] + set sql [$ic test_sql $tbl] + $ic close + return $sql +} + +proc do_intck_test {tn expect} { + uplevel [list do_test $tn.a [list do_intck db] [list {*}$expect]] + uplevel [list do_test $tn.b [list do_intck db 1] [list {*}$expect]] +} + + diff --git a/ext/intck/intckbusy.test b/ext/intck/intckbusy.test new file mode 100644 index 000000000..7c65b686b --- /dev/null +++ b/ext/intck/intckbusy.test @@ -0,0 +1,49 @@ +# 2024 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckbusy +return_if_no_intck + + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 'two', 'three'); + INSERT INTO t1 VALUES(3, NULL, NULL); + CREATE INDEX i1 ON t1(b, c); +} + +sqlite3 db2 test.db + +do_execsql_test -db db2 1.1 { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_test 1.2 { + set ic [sqlite3_intck db main] + $ic step +} {SQLITE_BUSY} +do_test 1.3 { + $ic unlock +} {SQLITE_BUSY} +do_test 1.4 { + $ic error +} {SQLITE_BUSY {database is locked}} +do_test 1.4 { + $ic close +} {} + +finish_test + diff --git a/ext/intck/intckcorrupt.test b/ext/intck/intckcorrupt.test new file mode 100644 index 000000000..eee63b32f --- /dev/null +++ b/ext/intck/intckcorrupt.test @@ -0,0 +1,236 @@ +# 2024 Feb 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the intck extensions response +# to corruption at the b-tree level. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckcorrupt +return_if_no_intck + +#------------------------------------------------------------------------- +reset_db +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 356352 pagesize 4096 filename crash-acaae0347204ae.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 d0 00 00 00 .....@ ........ +| 32: 40 00 ea 00 00 00 00 00 00 40 00 00 00 40 00 00 @........@...@.. +| 96: 00 00 00 00 0d 00 00 00 04 0e 9c 00 0f ad 0f 4f ...............O +| 112: 0e fc 0e 9c 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3728: 00 00 00 00 00 00 00 00 00 00 00 00 5e 04 07 17 ............^... +| 3744: 1f 1f 01 81 0b 74 61 62 6c 65 74 31 5f 70 61 72 .....tablet1_par +| 3760: 65 6e 74 74 31 5f 70 61 72 65 6e 74 04 43 52 45 entt1_parent.CRE +| 3776: 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 70 61 ATE TABLE .t1_pa +| 3792: 72 65 6e 74 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 rent.(nodeno INT +| 3808: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3824: 2c 70 61 72 65 6e 74 6e 6f 64 65 29 51 03 06 17 ,parentnode)Q... +| 3840: 1b 1b 01 7b 74 61 62 6c 65 74 31 5f 6e 6f 64 65 ....tablet1_node +| 3856: 74 31 5f 6e 6f 64 65 03 43 52 45 41 54 45 20 54 t1_node.CREATE T +| 3872: 41 42 4c 45 20 22 74 31 5f 6e 6f 64 65 22 28 6e ABLE .t1_node.(n +| 3888: 6f 64 65 6e 6f 20 49 4e 54 45 47 45 52 20 50 52 odeno INTEGER PR +| 3904: 49 4d 41 52 59 20 4b 45 59 2c 64 61 74 61 29 5c IMARY KEY,data). +| 3920: 02 07 17 1d 1d 01 81 0b 74 61 62 6c 65 74 31 5f ........tablet1_ +| 3936: 72 6f 77 69 64 74 31 5f 72 6f 77 69 64 02 43 52 rowidt1_rowid.CR +| 3952: 45 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 72 EATE TABLE .t1_r +| 3968: 6f 77 69 64 22 28 72 6f 77 69 64 20 49 4e 54 45 owid.(rowid INTE +| 3984: 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c GER PRIMARY KEY, +| 4000: 6e 6f 64 65 6e 6f 2c 61 30 2c 61 31 29 51 01 07 nodeno,a0,a1)Q.. +| 4016: 17 11 11 08 81 0f 74 61 62 6c 65 74 31 74 31 43 ......tablet1t1C +| 4032: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 4048: 42 4c 45 20 74 31 20 55 53 49 4e 47 20 72 74 72 BLE t1 USING rtr +| 4064: 65 65 28 69 64 2c 78 30 20 50 52 49 4d 41 52 59 ee(id,x0 PRIMARY +| 4080: 20 4b 45 59 2c 70 61 72 65 6e 74 6e 6f 64 65 29 KEY,parentnode) +| page 2 offset 4096 +| 0: 51 03 06 17 1b 1b 01 7b 74 61 62 6c 65 74 31 5f Q.......tablet1_ +| 16: 6e 6f 64 65 74 31 5f 6e 6f 64 65 03 43 52 45 41 nodet1_node.CREA +| 32: 54 45 20 54 41 42 4c 45 20 22 74 31 5f 6e 6f 64 TE TABLE .t1_nod +| 48: 65 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 e.(nodeno INTEGE +| 64: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da +| 80: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl +| 96: 65 74 31 5f 72 6f 77 69 64 74 31 5f 72 6f 77 69 et1_rowidt1_rowi +| 112: 64 02 43 52 45 41 54 45 20 54 41 42 4c 45 00 00 d.CREATE TABLE.. +| 128: 01 0a 02 00 00 00 01 0e 0d 00 00 00 00 24 0e 0d .............$.. +| 144: 0c 1a 06 85 50 46 60 27 70 08 00 00 00 00 00 00 ....PF`'p....... +| 3824: 00 00 00 00 00 00 00 0d 0e 05 00 09 1d 00 74 6f ..............to +| 3840: 79 20 68 61 6c 66 10 0d 05 00 09 23 00 62 6f 74 y half.....#.bot +| 3856: 74 6f 6d 20 68 61 6c 66 0f 0c 05 00 09 21 00 72 tom half.....!.r +| 3872: 69 67 68 74 20 68 61 6c 66 0e 0b 05 00 09 1f 00 ight half....... +| 3888: 6c 65 66 74 20 43 15 f6 e6 f6 46 50 34 35 24 54 left C....FP45$T +| 3904: 15 44 52 05 44 14 24 c4 52 02 27 43 15 f6 e6 f6 .DR.D.$.R.'C.... +| 3920: 46 52 22 8e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 FR..odeno INTEGE +| 3936: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da +| 3952: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl +| 3968: 65 74 31 5f 72 6f 74 74 6f 6d 20 65 64 67 65 0f et1_rottom edge. +| 3984: 07 05 00 09 21 00 72 69 67 68 74 20 65 64 67 65 ....!.right edge +| 4000: 0e 06 05 00 09 1f 00 6c 65 66 74 20 65 64 67 65 .......left edge +| 4016: 0b 05 05 00 09 19 00 63 65 6e 74 65 72 17 04 05 .......center... +| 4032: 00 09 31 00 75 70 70 65 72 2d 72 69 67 68 74 20 ..1.upper-right +| 4048: 63 6f 72 6e 65 72 17 03 05 00 09 31 00 6c 6f 77 corner.....1.low +| 4064: 65 72 2d 72 69 67 68 74 20 63 6f 72 6e 65 72 16 er-right corner. +| 4080: 02 05 00 09 2f 00 75 70 70 65 72 2d 6c 65 66 74 ..../.upper-left +| page 3 offset 8192 +| 0: 20 63 6f 72 6e 65 72 16 01 05 00 09 2f 01 8c 6f corner...../..o +| 16: 77 65 72 2d 6c 53 51 4c 69 74 65 20 66 6f 72 6d wer-lSQLite form +| 32: 61 74 20 33 00 10 00 01 01 00 40 20 20 00 00 00 at 3......@ ... +| 48: 00 00 00 00 2f 00 00 0d eb 13 00 00 00 03 00 00 ..../........... +| 64: 00 04 00 00 00 00 00 00 00 06 00 00 00 01 00 00 ................ +| 80: 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................ +| page 6 offset 20480 +| 128: 00 00 00 00 00 00 00 00 97 3d 04 ae 7c 01 00 00 .........=..|... +| 624: 00 00 00 00 00 00 21 97 3d 04 ae 7c 01 00 00 00 ......!.=..|.... +| 1120: 00 00 00 00 00 20 97 3d 04 ae 7c 01 00 00 00 00 ..... .=..|..... +| 1616: 00 00 00 00 1f 97 3d 04 ae 7c 01 00 00 00 00 00 ......=..|...... +| 2112: 00 00 00 1e 97 3d 04 ae 7c 01 00 00 00 00 00 00 .....=..|....... +| 2608: 00 00 1d 97 d3 d0 4a e7 c0 00 00 00 00 00 00 00 ......J......... +| 3088: 00 00 00 00 00 00 00 00 00 00 00 00 01 f3 00 00 ................ +| 3600: 23 97 3d 04 ae 7c 01 00 00 00 00 00 00 00 00 00 #.=..|.......... +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 26 ...............& +| page 8 offset 28672 +| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0...... +| 1072: 97 4d 1e 14 00 ae 7c 00 00 00 00 00 00 00 00 00 .M....|......... +| 1088: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| page 10 offset 36864 +| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0...... +| 1072: 9a ee c1 80 fd 78 1f ce 1b ae eb b4 00 00 00 00 .....x.......... +| 1088: 13 20 ff 20 00 70 00 00 00 60 50 00 00 00 11 e0 . . .p...`P..... +| 1104: 00 00 00 70 00 00 00 60 50 05 35 14 c6 97 46 52 ...p...`P.5...FR +| 1120: 06 66 f7 26 d6 17 42 03 30 01 00 00 10 10 04 02 .f.&..B.0....... +| 1136: 02 00 00 00 00 00 00 00 00 40 00 00 00 00 00 00 .........@...... +| 1152: 00 00 00 00 00 40 00 00 00 40 00 00 00 00 00 00 .....@...@...... +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 ................ +| page 12 offset 45056 +| 0: 0d 00 00 00 01 04 30 00 04 30 e1 b4 30 97 4d 46 ......0..0..0.MF +| 16: 14 00 ae 7c 00 00 00 00 00 00 00 03 00 00 43 00 ...|..........C. +| page 47 offset 188416 +| 2512: 00 00 00 00 00 00 00 00 be 00 00 00 00 00 00 00 ................ +| page 87 offset 352256 +| 2512: 00 00 00 00 00 00 00 00 aa 00 00 00 00 00 00 00 ................ +| end crash-acaae0347204ae.db +}]} {} + +do_intck_test 1.1 { + {corruption found while reading database schema} +} + +#------------------------------------------------------------------------- +reset_db +do_test 2.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-3afa1ca9e9c1bd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 88 00 0f b8 0f 6d ...............m +| 112: 0f 3a 0f 0b 0e d5 0e 88 01 00 00 00 00 00 00 00 .:.............. +| 3712: 00 00 00 00 00 00 00 00 4b 06 06 17 25 25 01 5b ........K...%%.[ +| 3728: 74 61 62 6c 65 73 71 6c 69 74 65 5f 73 74 61 74 tablesqlite_stat +| 3744: 31 73 71 6c 69 74 65 5f 73 74 61 74 31 07 43 52 1sqlite_stat1.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 73 71 6c 69 74 EATE TABLE sqlit +| 3776: 65 5f 73 74 61 74 31 28 74 62 6c 2c 69 64 78 2c e_stat1(tbl,idx, +| 3792: 73 74 61 74 29 34 05 06 17 13 11 01 53 69 6e 64 stat)4......Sind +| 3808: 65 78 63 31 63 63 31 06 43 52 45 41 54 45 20 55 exc1cc1.CREATE U +| 3824: 4e 49 51 55 45 20 49 4e 44 45 58 20 63 31 63 20 NIQUE INDEX c1c +| 3840: 4f 4e 20 63 31 28 63 2c 20 62 29 2d 04 06 17 13 ON c1(c, b)-.... +| 3856: 11 01 45 69 6e 64 65 78 63 31 64 63 31 05 43 52 ..Eindexc1dc1.CR +| 3872: 45 41 54 45 20 49 4e 44 45 58 20 63 31 64 20 4f EATE INDEX c1d O +| 3888: 4e 20 63 31 28 64 2c 20 62 29 31 03 06 17 13 11 N c1(d, b)1..... +| 3904: 01 4d 69 6e 64 65 78 62 31 63 62 31 05 43 52 45 .Mindexb1cb1.CRE +| 3920: 41 54 45 20 55 4e 49 51 55 45 20 49 4e 44 45 58 ATE UNIQUE INDEX +| 3936: 20 62 31 63 20 4f 4e 20 62 31 28 63 29 49 02 06 b1c ON b1(c)I.. +| 3952: 17 11 11 0f 7f 74 61 62 6c 65 63 31 63 31 03 43 .....tablec1c1.C +| 3968: 52 45 41 54 45 20 54 41 42 4c 45 20 63 31 28 61 REATE TABLE c1(a +| 3984: 20 49 4e 54 20 50 52 49 4d 41 52 59 20 4b 45 59 INT PRIMARY KEY +| 4000: 2c 20 62 2c 20 63 2c 20 64 29 20 57 49 54 48 4f , b, c, d) WITHO +| 4016: 55 54 20 52 4f 57 49 44 46 01 06 17 11 11 01 79 UT ROWIDF......y +| 4032: 74 61 62 6c 65 62 31 62 31 02 43 52 45 41 54 45 tableb1b1.CREATE +| 4048: 20 54 41 42 4c 45 20 62 31 28 61 20 49 4e 54 20 TABLE b1(a INT +| 4064: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 2c 20 PRIMARY KEY, b, +| 4080: 63 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 c) WITHOUT ROWID +| page 2 offset 4096 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f e2 ................ +| 16: 0f da 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 06 ................ +| 4048: 67 07 07 04 01 0f 01 06 66 06 07 04 01 0f 01 05 g.......f....... +| 4064: 65 05 07 04 01 0f 01 04 64 04 07 04 01 0f 01 03 e.......d....... +| 4080: 63 03 07 04 01 0f 01 02 62 0f 05 04 09 0f 09 61 c.......b......a +| page 3 offset 8192 +| 0: 0a 00 00 00 07 0f bd 00 0f f9 0f ef 0f e5 0f db ................ +| 16: 0f d1 0f c7 0f bd 00 00 00 00 01 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 09 05 01 ................ +| 4032: 0f 01 01 07 61 07 07 09 05 01 0f 01 01 06 61 06 ....a.........a. +| 4048: 06 09 05 01 0f 01 01 05 61 05 05 09 05 01 0f 01 ........a....... +| 4064: 01 04 61 04 04 09 05 01 0f 01 01 03 61 03 03 09 ..a.........a... +| 4080: 05 01 0f 01 01 02 61 0f 02 06 05 09 0f 09 09 61 ......a........a +| page 4 offset 12288 +| 0: 0a 00 00 00 07 0f d8 00 0f fc 0f f0 0f ea 0f e4 ................ +| 16: 0f de 0f d8 0f f6 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 05 03 01 01 07 07 05 03 ................ +| 4064: 01 01 06 06 05 03 01 01 05 05 05 03 01 01 04 04 ................ +| 4080: 05 03 01 01 03 03 05 03 01 01 0f 02 03 03 09 09 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................ +| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a....... +| 4064: 61 05 07 04 01 1f 01 04 61 04 07 04 01 0f 01 03 a.......a....... +| 4080: 61 03 07 04 01 0f 01 02 61 02 05 04 09 0f 09 61 a.......a......a +| page 6 offset 20480 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f ea 0f e2 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................ +| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a....... +| 4064: 61 05 07 04 01 0f 01 04 61 04 07 04 01 0f 01 03 a.......a....... +| 4080: 61 03 07 04 01 0f 01 0f 61 02 05 04 09 0f 09 61 a.......a......a +| page 7 offset 24576 +| 0: 0d 00 00 00 05 0f 1c 00 0f f0 0f e0 0f d3 0f c5 ................ +| 16: 0f b8 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 0b 05 04 11 11 13 62 31 ..............b1 +| 4032: 62 31 37 20 31 0c 04 04 11 13 13 62 31 62 31 63 b17 1......b1b1c +| 4048: 37 20 31 0b 03 04 11 11 13 63 31 63 31 37 20 31 7 1......c1c17 1 +| 4064: 0e 02 04 11 13 07 63 31 63 31 64 37 20 31 20 31 ......c1c1d7 1 1 +| 4080: 0e 01 04 11 13 17 63 31 63 31 63 37 20 31 00 00 ......c1c1c7 1.. +| end crash-3afa1ca9e9c1bd.db +}]} {} + +do_intck_test 2.1 { + {corruption found while reading database schema} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3); +} + +do_test 3.1 { + set pgno [db one {SELECT rootpage FROM sqlite_schema WHERE name='t1'}] + db close + hexio_write test.db [expr ($pgno-1)*1024] 0000 +} {2} + +sqlite3 db test.db +do_intck_test 3.2 { + {corruption found while scanning database object i1} + {corruption found while scanning database object t1} +} + +finish_test + + diff --git a/ext/intck/intckfault.test b/ext/intck/intckfault.test new file mode 100644 index 000000000..0bc06e584 --- /dev/null +++ b/ext/intck/intckfault.test @@ -0,0 +1,42 @@ +# 2024 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckfault +return_if_no_intck + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 'two', 'three'); + INSERT INTO t1 VALUES(3, NULL, NULL); + CREATE INDEX i1 ON t1(b, c); +} + +do_faultsim_test 1 -faults oom-t* -prep { +} -body { + set ::ic [sqlite3_intck db main] + set nStep 0 + while {"SQLITE_OK"==[$::ic step]} { + incr nStep + if {$nStep==3} { $::ic unlock } + } + set res [$::ic error] + $::ic close + set res +} -test { + catch { $::ic close } + faultsim_test_result {0 {SQLITE_OK {}}} {0 {SQLITE_NOMEM {}}} {0 {SQLITE_NOMEM {out of memory}}} +} + +finish_test + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c new file mode 100644 index 000000000..ed169a266 --- /dev/null +++ b/ext/intck/sqlite3intck.c @@ -0,0 +1,940 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqlite3intck.h" +#include <string.h> +#include <assert.h> + +#include <stdio.h> +#include <stdlib.h> + +/* +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. +*/ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + return zRet; +} + +/* +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. +*/ +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; ii<p->nKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jj<ii-1; jj++){ + const char *zAlias = (const char*)sqlite3_column_name(p->pCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); + } + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; ii<p->nKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +/* +** Return true if argument c is an ascii whitespace character. +*/ +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(<sql>, <icol>); +*/ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY <index>", and then, if the index + ** is a partial index " WHERE <condition>". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" + + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } + } + intckFinalize(p, pStmt); + + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); + return zRet; +} + +/* +** Open a new integrity-check object. +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } + } + + *ppOut = pNew; + return rc; +} + +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } +} + +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; + } + } + + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } + } + } + } + + return p->rc; +} + +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +/* +** Return the error code and message. +*/ +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; +} + +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; +} diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h new file mode 100644 index 000000000..e08a86f28 --- /dev/null +++ b/ext/intck/sqlite3intck.h @@ -0,0 +1,171 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ +typedef struct sqlite3_intck sqlite3_intck; + +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); + +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); + +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_step(sqlite3_intck *pCk); + +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); + +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c new file mode 100644 index 000000000..d3a619b50 --- /dev/null +++ b/ext/intck/test_intck.c @@ -0,0 +1,233 @@ +/* +** 2010 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Code for testing all sorts of SQLite interfaces. This code +** is not included in the SQLite library. +*/ + +#include "sqlite3.h" +#include "sqlite3intck.h" +#include "tclsqlite.h" +#include <string.h> +#include <assert.h> + +/* In test1.c */ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +const char *sqlite3ErrName(int); + +typedef struct TestIntck TestIntck; +struct TestIntck { + sqlite3_intck *intck; +}; + +static int testIntckCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct Subcmd { + const char *zName; + int nArg; + const char *zExpect; + } aCmd[] = { + {"close", 0, ""}, /* 0 */ + {"step", 0, ""}, /* 1 */ + {"message", 0, ""}, /* 2 */ + {"error", 0, ""}, /* 3 */ + {"unlock", 0, ""}, /* 4 */ + {"test_sql", 1, ""}, /* 5 */ + {0 , 0} + }; + int rc = TCL_OK; + int iIdx = -1; + TestIntck *p = (TestIntck*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); + return TCL_ERROR; + } + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aCmd, sizeof(aCmd[0]), "SUB-COMMAND", 0, &iIdx + ); + if( rc ) return rc; + + if( objc!=2+aCmd[iIdx].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aCmd[iIdx].zExpect); + return TCL_ERROR; + } + + switch( iIdx ){ + case 0: assert( 0==strcmp("close", aCmd[iIdx].zName) ); { + Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0)); + break; + } + + case 1: assert( 0==strcmp("step", aCmd[iIdx].zName) ); { + rc = sqlite3_intck_step(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 2: assert( 0==strcmp("message", aCmd[iIdx].zName) ); { + const char *z = sqlite3_intck_message(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(z ? z : "", -1)); + break; + } + + case 3: assert( 0==strcmp("error", aCmd[iIdx].zName) ); { + const char *zErr = 0; + Tcl_Obj *pRes; + rc = sqlite3_intck_error(p->intck, 0); + pRes = Tcl_NewObj(); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(sqlite3ErrName(rc), -1) + ); + sqlite3_intck_error(p->intck, &zErr); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(zErr ? zErr : 0, -1) + ); + Tcl_SetObjResult(interp, pRes); + break; + } + + case 4: assert( 0==strcmp("unlock", aCmd[iIdx].zName) ); { + rc = sqlite3_intck_unlock(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 5: assert( 0==strcmp("test_sql", aCmd[iIdx].zName) ); { + const char *zObj = Tcl_GetString(objv[2]); + const char *zSql = sqlite3_intck_test_sql(p->intck, zObj[0] ? zObj : 0); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zSql, -1)); + break; + } + } + + return TCL_OK; +} + +/* +** Destructor for commands created by test_sqlite3_intck(). +*/ +static void testIntckFree(void *clientData){ + TestIntck *p = (TestIntck*)clientData; + sqlite3_intck_close(p->intck); + ckfree(p); +} + +/* +** tclcmd: sqlite3_intck DB DBNAME +*/ +static int test_sqlite3_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char zName[64]; + int iName = 0; + Tcl_CmdInfo info; + TestIntck *p = 0; + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + + p = (TestIntck*)ckalloc(sizeof(TestIntck)); + memset(p, 0, sizeof(TestIntck)); + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = 0; + + rc = sqlite3_intck_open(db, zDb, &p->intck); + if( rc!=SQLITE_OK ){ + ckfree(p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errstr(rc), -1)); + return TCL_ERROR; + } + + do { + sprintf(zName, "intck%d", iName++); + }while( Tcl_GetCommandInfo(interp, zName, &info)!=0 ); + Tcl_CreateObjCommand(interp, zName, testIntckCmd, (void*)p, testIntckFree); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zName, -1)); + + return TCL_OK; +} + +/* +** tclcmd: test_do_intck DB DBNAME +*/ +static int test_do_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + sqlite3_intck *pCk = 0; + Tcl_Obj *pRet = 0; + const char *zErr = 0; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + rc = sqlite3_intck_open(db, zDb, &pCk); + if( rc==SQLITE_OK ){ + while( sqlite3_intck_step(pCk)==SQLITE_OK ){ + const char *zMsg = sqlite3_intck_message(pCk); + if( zMsg ){ + Tcl_ListObjAppendElement(interp, pRet, Tcl_NewStringObj(zMsg, -1)); + } + } + rc = sqlite3_intck_error(pCk, &zErr); + } + if( rc!=SQLITE_OK ){ + if( zErr ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + }else{ + Tcl_SetObjResult(interp, pRet); + } + Tcl_DecrRefCount(pRet); + sqlite3_intck_close(pCk); + sqlite3_intck_close(0); + return rc ? TCL_ERROR : TCL_OK; +} + +int Sqlitetestintck_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_intck", test_sqlite3_intck, 0, 0); + Tcl_CreateObjCommand(interp, "test_do_intck", test_do_intck, 0, 0); + return TCL_OK; +} diff --git a/ext/jni/src/org/sqlite/jni/capi/SQLTester.java b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java index 81d6106be..c68785e2c 100644 --- a/ext/jni/src/org/sqlite/jni/capi/SQLTester.java +++ b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java @@ -851,11 +851,26 @@ class JsonBlockCommand extends TableResultCommand { //! --new command class NewDbCommand extends OpenDbCommand { public NewDbCommand(){ super(true); } + public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + if(argv.length>1){ + Util.unlink(argv[1]); + } + super.process(t, ts, argv); + } + } -//! Placeholder dummy/no-op commands +//! Placeholder dummy/no-op/unimplemented commands class NoopCommand extends Command { + private boolean verbose = false; + public NoopCommand(boolean verbose){ + this.verbose = verbose; + } + public NoopCommand(){} public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{ + if( this.verbose ){ + t.outln("Skipping unhandled command: "+argv[0]); + } } } @@ -1021,6 +1036,7 @@ class CommandDispatcher { case "db": rv = new DbCommand(); break; case "glob": rv = new GlobCommand(); break; case "json": rv = new JsonCommand(); break; + case "jsonglob": rv = new NoopCommand(true); break; case "json-block": rv = new JsonBlockCommand(); break; case "new": rv = new NewDbCommand(); break; case "notglob": rv = new NotGlobCommand(); break; @@ -1030,6 +1046,7 @@ class CommandDispatcher { case "print": rv = new PrintCommand(); break; case "result": rv = new ResultCommand(); break; case "run": rv = new RunCommand(); break; + case "stmt-cache": rv = new NoopCommand(); break; case "tableresult": rv = new TableResultCommand(); break; case "testcase": rv = new TestCaseCommand(); break; case "verbosity": rv = new VerbosityCommand(); break; diff --git a/ext/jni/src/org/sqlite/jni/test-script-interpreter.md b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md index a51d64d10..939f77e1b 100644 --- a/ext/jni/src/org/sqlite/jni/test-script-interpreter.md +++ b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md @@ -4,7 +4,7 @@ The purpose of the Test Script Interpreter is to read and interpret script files that contain SQL commands and desired results. The -interpreter will check results and report an discrepencies found. +interpreter will check results and report any discrepencies found. The test script files are ASCII text files. The filename always ends with ".test". Each script is evaluated independently; context does not carry @@ -87,6 +87,7 @@ Each command looks like an SQL comment. The command begins at the left margin (no leading space) and starts with exactly 2 minus signs ("-"). The command name consists of lowercase letters and maybe a "-" or two. Some commands have arguments. + The arguments are separated from the command name by one or more spaces. Commands have access to the input buffer and might reset the input buffer. @@ -159,9 +160,9 @@ the result buffer. This distinction does not matter for the --result command itself, but it is important for related commands like --glob and --notglob. Sometimes test cases will contains a bunch of SQL followed by multiple --glob and/or --notglob statements. All of the -globs should be evaluted agains the result buffer correct, but the SQL -should only be run once. This is accomplished by resetting the input -buffer but not the result buffer. +globs should be evaluated agains the result buffer, but the SQL should +only be run once. This is accomplished by resetting the input buffer +but not the result buffer. ### The --glob command @@ -187,7 +188,7 @@ The TEST-GLOB pattern is slightly different for a standard GLOB: ### The --notglob command The --notglob command works just like --glob except that it reports an -error if the GLOB does match, rather than if the GLOB does not matches. +error if the GLOB does match, rather than if the GLOB does not match. ### The --oom command diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index e7c2c9d5c..2d7f6584e 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -446,9 +446,9 @@ static int cksmRead( ** (2) checksum verification is enabled ** (3) we are not in the middle of checkpoint */ - if( iAmt>=512 /* (1) */ - && p->verifyCksm /* (2) */ - && !p->inCkpt /* (3) */ + if( iAmt>=512 && (iAmt & (iAmt-1))==0 /* (1) */ + && p->verifyCksm /* (2) */ + && !p->inCkpt /* (3) */ ){ u8 cksum[8]; cksmCompute((u8*)zBuf, iAmt-8, cksum); diff --git a/ext/misc/completion.c b/ext/misc/completion.c index 987595a3d..54abc0ae1 100644 --- a/ext/misc/completion.c +++ b/ext/misc/completion.c @@ -251,7 +251,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ zSql = sqlite3_mprintf( "%z%s" "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" - " JOIN pragma_table_info(sm.name,%Q) AS pti" + " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" " WHERE sm.type='table'", zSql, zSep, zDb, zDb ); diff --git a/ext/misc/compress.c b/ext/misc/compress.c index 2e7a31636..6b034eb45 100644 --- a/ext/misc/compress.c +++ b/ext/misc/compress.c @@ -89,7 +89,7 @@ static void uncompressFunc( unsigned int nIn; unsigned long int nOut; int rc; - int i; + unsigned int i; pIn = sqlite3_value_blob(argv[0]); nIn = sqlite3_value_bytes(argv[0]); diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 70546adfc..ca8090ed2 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -372,7 +372,9 @@ static int writeFile( #if !defined(_WIN32) && !defined(WIN32) if( S_ISLNK(mode) ){ const char *zTo = (const char*)sqlite3_value_text(pData); - if( zTo==0 || symlink(zTo, zFile)<0 ) return 1; + if( zTo==0 ) return 1; + unlink(zFile); + if( symlink(zTo, zFile)<0 ) return 1; }else #endif { @@ -458,13 +460,19 @@ static int writeFile( return 1; } #else - /* Legacy unix */ - struct timeval times[2]; - times[0].tv_usec = times[1].tv_usec = 0; - times[0].tv_sec = time(0); - times[1].tv_sec = mtime; - if( utimes(zFile, times) ){ - return 1; + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } } #endif } diff --git a/ext/misc/noop.c b/ext/misc/noop.c index d3a58670c..18c25e10f 100644 --- a/ext/misc/noop.c +++ b/ext/misc/noop.c @@ -38,6 +38,24 @@ static void noopfunc( sqlite3_result_value(context, argv[0]); } +/* +** Implementation of the multitype_text() function. +** +** The function returns its argument. The result will always have a +** TEXT value. But if the original input is numeric, it will also +** have that numeric value. +*/ +static void multitypeTextFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + (void)argc; + (void)sqlite3_value_text(argv[0]); + sqlite3_result_value(context, argv[0]); +} + #ifdef _WIN32 __declspec(dllexport) #endif @@ -64,5 +82,9 @@ int sqlite3_noop_init( rc = sqlite3_create_function(db, "noop_nd", 1, SQLITE_UTF8, 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "multitype_text", 1, + SQLITE_UTF8, + 0, multitypeTextFunc, 0, 0); return rc; } diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index d83bc5b83..cccbaf025 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -57,6 +57,11 @@ ** (12) The percentile(Y,P) is implemented as a single C99 source-code ** file that compiles into a shared-library or DLL that can be loaded ** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cond(Y,X) function is the equivalent of +** percentile(Y,X*100.0). */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -103,16 +108,32 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ double rPct; int eType; double y; - assert( argc==2 ); - - /* Requirement 3: P must be a number between 0 and 100 */ - eType = sqlite3_value_numeric_type(argv[1]); - rPct = sqlite3_value_double(argv[1]); - if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) - || rPct<0.0 || rPct>100.0 ){ - sqlite3_result_error(pCtx, "2nd argument to percentile() is not " - "a number between 0.0 and 100.0", -1); - return; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 50.0; + }else if( sqlite3_user_data(pCtx)==0 ){ + /* Requirement 3: P must be a number between 0 and 100 */ + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1]); + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>100.0 ){ + sqlite3_result_error(pCtx, "2nd argument to percentile() is not " + "a number between 0.0 and 100.0", -1); + return; + } + }else{ + /* Requirement 3: P must be a number between 0 and 1 */ + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1]); + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 ){ + sqlite3_result_error(pCtx, "2nd argument to percentile_cont() is not " + "a number between 0.0 and 1.0", -1); + return; + } + rPct *= 100.0; } /* Allocate the session context. */ @@ -165,14 +186,52 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ } /* -** Compare to doubles for sorting using qsort() +** Sort an array of doubles. */ -static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){ - double a = *(double*)pA; - double b = *(double*)pB; - if( a==b ) return 0; - if( a<b ) return -1; - return +1; +static void sortDoubles(double *a, int n){ + int iLt; /* Entries with index less than iLt are less than rPivot */ + int iGt; /* Entries with index iGt or more are greater than rPivot */ + int i; /* Loop counter */ + double rPivot; /* The pivot value */ + double rTmp; /* Temporary used to swap two values */ + + if( n<2 ) return; + if( n>5 ){ + rPivot = (a[0] + a[n/2] + a[n-1])/3.0; + }else{ + rPivot = a[n/2]; + } + iLt = i = 0; + iGt = n; + while( i<iGt ){ + if( a[i]<rPivot ){ + if( i>iLt ){ + rTmp = a[i]; + a[i] = a[iLt]; + a[iLt] = rTmp; + } + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + rTmp = a[i]; + a[i] = a[iGt]; + a[iGt] = rTmp; + }else{ + i++; + } + } + if( iLt>=2 ) sortDoubles(a, iLt); + if( n-iGt>=2 ) sortDoubles(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; i<n-1; i++){ + assert( a[i]<=a[i+1] ); + } +#endif } /* @@ -188,7 +247,7 @@ static void percentFinal(sqlite3_context *pCtx){ if( p==0 ) return; if( p->a==0 ) return; if( p->nUsed ){ - qsort(p->a, p->nUsed, sizeof(double), doubleCmp); + sortDoubles(p->a, p->nUsed); ix = (p->rPct-1.0)*(p->nUsed-1)*0.01; i1 = (unsigned)ix; i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; @@ -216,5 +275,15 @@ int sqlite3_percentile_init( rc = sqlite3_create_function(db, "percentile", 2, SQLITE_UTF8|SQLITE_INNOCUOUS, 0, 0, percentStep, percentFinal); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "median", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + 0, percentStep, percentFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "percentile_cont", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS, &percentStep, + 0, percentStep, percentFinal); + } return rc; } diff --git a/ext/misc/series.c b/ext/misc/series.c index abd6af7ad..f2ca75c25 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -90,6 +90,26 @@ ** and a very large cost if either start or stop are unavailable. This ** encourages the query planner to order joins such that the bounds of the ** series are well-defined. +** +** Update on 2024-08-22: +** xBestIndex now also looks for equality and inequality constraints against +** the value column and uses those constraints as additional bounds against +** the sequence range. Thus, a query like this: +** +** SELECT value FROM generate_series($SA,$EA) +** WHERE value BETWEEN $SB AND $EB; +** +** Is logically the same as: +** +** SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB)); +** +** Constraints on the value column can server as substitutes for constraints +** on the hidden start and stop columns. So, the following two queries +** are equivalent: +** +** SELECT value FROM generate_series($S,$E); +** SELECT value FROM generate_series WHERE value BETWEEN $S and $E; +** */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -103,16 +123,20 @@ SQLITE_EXTENSION_INIT1 ** index is ix. The 0th member is given by smBase. The sequence members ** progress per ix increment by smStep. */ -static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix){ - if( ix>=(sqlite3_uint64)LLONG_MAX ){ +static sqlite3_int64 genSeqMember( + sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix +){ + static const sqlite3_uint64 mxI64 = + ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; + if( ix>=mxI64 ){ /* Get ix into signed i64 range. */ - ix -= (sqlite3_uint64)LLONG_MAX; + ix -= mxI64; /* With 2's complement ALU, this next can be 1 step, but is split into * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (LLONG_MAX/2) * smStep; - smBase += (LLONG_MAX - LLONG_MAX/2) * smStep; + smBase += (mxI64/2) * smStep; + smBase += (mxI64 - mxI64/2) * smStep; } /* Under UBSAN (or on 1's complement machines), must do this last term * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ @@ -127,8 +151,10 @@ static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, typedef unsigned char u8; typedef struct SequenceSpec { - sqlite3_int64 iBase; /* Starting value ("start") */ - sqlite3_int64 iTerm; /* Given terminal value ("stop") */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ sqlite3_int64 iStep; /* Increment ("step") */ sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ @@ -321,9 +347,9 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; default: x = pCur->ss.iValueNow; break; } sqlite3_result_int64(ctx, x); @@ -331,7 +357,9 @@ static int seriesColumn( } #ifndef LARGEST_UINT64 +#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) #define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) +#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) #endif /* @@ -372,13 +400,18 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** parameter. (idxStr is not used in this implementation.) idxNum ** is a bitmask showing which constraints are available: ** -** 1: start=VALUE -** 2: stop=VALUE -** 4: step=VALUE -** -** Also, if bit 8 is set, that means that the series should be output -** in descending order rather than in ascending order. If bit 16 is -** set, then output must appear in ascending order. +** 0x0001: start=VALUE +** 0x0002: stop=VALUE +** 0x0004: step=VALUE +** 0x0008: descending order +** 0x0010: ascending order +** 0x0020: LIMIT VALUE +** 0x0040: OFFSET VALUE +** 0x0080: value=VALUE +** 0x0100: value>=VALUE +** 0x0200: value>VALUE +** 0x1000: value<=VALUE +** 0x2000: value<VALUE ** ** This routine should initialize the cursor and position it so that it ** is pointing at the first row, or pointing off the end of the table @@ -391,38 +424,149 @@ static int seriesFilter( ){ series_cursor *pCur = (series_cursor *)pVtabCursor; int i = 0; + int returnNoRows = 0; + sqlite3_int64 iMin = SMALLEST_INT64; + sqlite3_int64 iMax = LARGEST_INT64; + sqlite3_int64 iLimit = 0; + sqlite3_int64 iOffset = 0; + (void)idxStrUnused; - if( idxNum & 1 ){ + if( idxNum & 0x01 ){ pCur->ss.iBase = sqlite3_value_int64(argv[i++]); }else{ pCur->ss.iBase = 0; } - if( idxNum & 2 ){ + if( idxNum & 0x02 ){ pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); }else{ pCur->ss.iTerm = 0xffffffff; } - if( idxNum & 4 ){ + if( idxNum & 0x04 ){ pCur->ss.iStep = sqlite3_value_int64(argv[i++]); if( pCur->ss.iStep==0 ){ pCur->ss.iStep = 1; }else if( pCur->ss.iStep<0 ){ - if( (idxNum & 16)==0 ) idxNum |= 8; + if( (idxNum & 0x10)==0 ) idxNum |= 0x08; } }else{ pCur->ss.iStep = 1; } + + /* If there are constraints on the value column but there are + ** no constraints on the start, stop, and step columns, then + ** initialize the default range to be the entire range of 64-bit signed + ** integers. This range will contracted by the value column constraints + ** further below. + */ + if( (idxNum & 0x05)==0 && (idxNum & 0x0380)!=0 ){ + pCur->ss.iBase = SMALLEST_INT64; + } + if( (idxNum & 0x06)==0 && (idxNum & 0x3080)!=0 ){ + pCur->ss.iTerm = LARGEST_INT64; + } + pCur->ss.iOBase = pCur->ss.iBase; + pCur->ss.iOTerm = pCur->ss.iTerm; + + /* Extract the LIMIT and OFFSET values, but do not apply them yet. + ** The range must first be constrained by the limits on value. + */ + if( idxNum & 0x20 ){ + iLimit = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x40 ){ + iOffset = sqlite3_value_int64(argv[i++]); + } + } + + if( idxNum & 0x3380 ){ + /* Extract the maximum range of output values determined by + ** constraints on the "value" column. + */ + if( idxNum & 0x0080 ){ + iMin = iMax = sqlite3_value_int64(argv[i++]); + }else{ + if( idxNum & 0x0300 ){ + iMin = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x0200 ){ + if( iMin==LARGEST_INT64 ){ + returnNoRows = 1; + }else{ + iMin++; + } + } + } + if( idxNum & 0x3000 ){ + iMax = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x2000 ){ + if( iMax==SMALLEST_INT64 ){ + returnNoRows = 1; + }else{ + iMax--; + } + } + } + if( iMin>iMax ){ + returnNoRows = 1; + } + } + + /* Try to reduce the range of values to be generated based on + ** constraints on the "value" column. + */ + if( pCur->ss.iStep>0 ){ + sqlite3_int64 szStep = pCur->ss.iStep; + if( pCur->ss.iBase<iMin ){ + sqlite3_uint64 d = iMin - pCur->ss.iBase; + pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep; + } + if( pCur->ss.iTerm>iMax ){ + sqlite3_uint64 d = pCur->ss.iTerm - iMax; + pCur->ss.iTerm -= ((d+szStep-1)/szStep)*szStep; + } + }else{ + sqlite3_int64 szStep = -pCur->ss.iStep; + assert( szStep>0 ); + if( pCur->ss.iBase>iMax ){ + sqlite3_uint64 d = pCur->ss.iBase - iMax; + pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep; + } + if( pCur->ss.iTerm<iMin ){ + sqlite3_uint64 d = iMin - pCur->ss.iTerm; + pCur->ss.iTerm += ((d+szStep-1)/szStep)*szStep; + } + } + } + + /* Apply LIMIT and OFFSET constraints, if any */ + if( idxNum & 0x20 ){ + if( iOffset>0 ){ + pCur->ss.iBase += pCur->ss.iStep*iOffset; + } + if( iLimit>=0 ){ + sqlite3_int64 iTerm; + iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep; + if( pCur->ss.iStep<0 ){ + if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + }else{ + if( iTerm<pCur->ss.iTerm ) pCur->ss.iTerm = iTerm; + } + } + } + + for(i=0; i<argc; i++){ if( sqlite3_value_type(argv[i])==SQLITE_NULL ){ /* If any of the constraints have a NULL value, then return no rows. ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */ - pCur->ss.iBase = 1; - pCur->ss.iTerm = 0; - pCur->ss.iStep = 1; + returnNoRows = 1; break; } } - if( idxNum & 8 ){ + if( returnNoRows ){ + pCur->ss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; + } + if( idxNum & 0x08 ){ pCur->ss.isReversing = pCur->ss.iStep > 0; }else{ pCur->ss.isReversing = pCur->ss.iStep < 0; @@ -442,10 +586,35 @@ static int seriesFilter( ** ** The query plan is represented by bits in idxNum: ** -** (1) start = $value -- constraint exists -** (2) stop = $value -- constraint exists -** (4) step = $value -- constraint exists -** (8) output in descending order +** 0x0001 start = $num +** 0x0002 stop = $num +** 0x0004 step = $num +** 0x0008 output is in descending order +** 0x0010 output is in ascending order +** 0x0020 LIMIT $num +** 0x0040 OFFSET $num +** 0x0080 value = $num +** 0x0100 value >= $num +** 0x0200 value > $num +** 0x1000 value <= $num +** 0x2000 value < $num +** +** Only one of 0x0100 or 0x0200 will be returned. Similarly, only +** one of 0x1000 or 0x2000 will be returned. If the 0x0080 is set, then +** none of the 0xff00 bits will be set. +** +** The order of parameters passed to xFilter is as follows: +** +** * The argument to start= if bit 0x0001 is in the idxNum mask +** * The argument to stop= if bit 0x0002 is in the idxNum mask +** * The argument to step= if bit 0x0004 is in the idxNum mask +** * The argument to LIMIT if bit 0x0020 is in the idxNum mask +** * The argument to OFFSET if bit 0x0040 is in the idxNum mask +** * The argument to value=, or value>= or value> if any of +** bits 0x0380 are in the idxNum mask +** * The argument to value<= or value< if either of bits 0x3000 +** are in the mask +** */ static int seriesBestIndex( sqlite3_vtab *pVTab, @@ -453,10 +622,14 @@ static int seriesBestIndex( ){ int i, j; /* Loop over constraints */ int idxNum = 0; /* The query plan bitmask */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES int bStartSeen = 0; /* EQ constraint seen on the START column */ +#endif int unusableMask = 0; /* Mask of unusable constraints */ int nArg = 0; /* Number of arguments that seriesFilter() expects */ - int aIdx[3]; /* Constraints on start, stop, and step */ + int aIdx[7]; /* Constraints on start, stop, step, LIMIT, OFFSET, + ** and value. aIdx[5] covers value=, value>=, and + ** value>, aIdx[6] covers value<= and value< */ const struct sqlite3_index_constraint *pConstraint; /* This implementation assumes that the start, stop, and step columns @@ -464,28 +637,97 @@ static int seriesBestIndex( assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); - aIdx[0] = aIdx[1] = aIdx[2] = -1; + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = aIdx[5] = aIdx[6] = -1; pConstraint = pIdxInfo->aConstraint; for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ int iCol; /* 0 for start, 1 for stop, 2 for step */ int iMask; /* bitmask for those column */ - if( pConstraint->iColumn<SERIES_COLUMN_START ) continue; + int op = pConstraint->op; + if( op>=SQLITE_INDEX_CONSTRAINT_LIMIT + && op<=SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + if( pConstraint->usable==0 ){ + /* do nothing */ + }else if( op==SQLITE_INDEX_CONSTRAINT_LIMIT ){ + aIdx[3] = i; + idxNum |= 0x20; + }else{ + assert( op==SQLITE_INDEX_CONSTRAINT_OFFSET ); + aIdx[4] = i; + idxNum |= 0x40; + } + continue; + } + if( pConstraint->iColumn==SERIES_COLUMN_VALUE ){ + switch( op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: + case SQLITE_INDEX_CONSTRAINT_IS: { + idxNum |= 0x0080; + idxNum &= ~0x3300; + aIdx[5] = i; + aIdx[6] = -1; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0100; + idxNum &= ~0x0200; + aIdx[5] = i; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0200; + idxNum &= ~0x0100; + aIdx[5] = i; + bStartSeen = 1; + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x1000; + idxNum &= ~0x2000; + aIdx[6] = i; + break; + } + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x2000; + idxNum &= ~0x1000; + aIdx[6] = i; + break; + } + } + continue; + } iCol = pConstraint->iColumn - SERIES_COLUMN_START; assert( iCol>=0 && iCol<=2 ); iMask = 1 << iCol; - if( iCol==0 ) bStartSeen = 1; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( iCol==0 && op==SQLITE_INDEX_CONSTRAINT_EQ ){ + bStartSeen = 1; + } +#endif if( pConstraint->usable==0 ){ unusableMask |= iMask; continue; - }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + }else if( op==SQLITE_INDEX_CONSTRAINT_EQ ){ idxNum |= iMask; aIdx[iCol] = i; } } - for(i=0; i<3; i++){ + if( aIdx[3]==0 ){ + /* Ignore OFFSET if LIMIT is omitted */ + idxNum &= ~0x60; + aIdx[4] = 0; + } + for(i=0; i<7; i++){ if( (j = aIdx[i])>=0 ){ pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; + pIdxInfo->aConstraintUsage[j].omit = + !SQLITE_SERIES_CONSTRAINT_VERIFY || i>=3; } } /* The current generate_column() implementation requires at least one @@ -506,19 +748,22 @@ static int seriesBestIndex( ** this plan is unusable */ return SQLITE_CONSTRAINT; } - if( (idxNum & 3)==3 ){ + if( (idxNum & 0x03)==0x03 ){ /* Both start= and stop= boundaries are available. This is the ** the preferred case */ pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); pIdxInfo->estimatedRows = 1000; if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn==0 ){ if( pIdxInfo->aOrderBy[0].desc ){ - idxNum |= 8; + idxNum |= 0x08; }else{ - idxNum |= 16; + idxNum |= 0x10; } pIdxInfo->orderByConsumed = 1; } + }else if( (idxNum & 0x21)==0x21 ){ + /* We have start= and LIMIT */ + pIdxInfo->estimatedRows = 2500; }else{ /* If either boundary is missing, we have to generate a huge span ** of numbers. Make this case very expensive so that the query @@ -526,6 +771,9 @@ static int seriesBestIndex( pIdxInfo->estimatedRows = 2147483647; } pIdxInfo->idxNum = idxNum; +#ifdef SQLITE_INDEX_SCAN_HEX + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_HEX; +#endif return SQLITE_OK; } diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index ba3ea581f..72ca7278c 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -15,10 +15,20 @@ ** Two SQL functions are implemented: ** ** sha3(X,SIZE) -** sha3_query(Y,SIZE) +** sha3_agg(Y,SIZE) +** sha3_query(Z,SIZE) ** ** The sha3(X) function computes the SHA3 hash of the input X, or NULL if -** X is NULL. +** X is NULL. If inputs X is text, the UTF-8 rendering of that text is +** used to compute the hash. If X is a BLOB, then the binary data of the +** blob is used to compute the hash. If X is an integer or real number, +** then that number if converted into UTF-8 text and the hash is computed +** over the text. +** +** The sha3_agg(Y) function computes the SHA3 hash of all Y inputs. Since +** order is important for the hash, it is recommended that the Y expression +** by followed by an ORDER BY clause to guarantee that the inputs occur +** in the desired order. ** ** The sha3_query(Y) function evaluates all queries in the SQL statements of Y ** and returns a hash of their results. @@ -26,6 +36,68 @@ ** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm ** is used. If SIZE is included it must be one of the integers 224, 256, ** 384, or 512, to determine SHA3 hash variant that is computed. +** +** Because the sha3_agg() and sha3_query() functions compute a hash over +** multiple values, the values are encode to use include type information. +** +** In sha3_agg(), the sequence of bytes that gets hashed for each input +** Y depends on the datatype of Y: +** +** typeof(Y)='null' A single "N" is hashed. (One byte) +** +** typeof(Y)='integer' The data hash is the character "I" followed +** by an 8-byte big-endian binary of the +** 64-bit signed integer. (Nine bytes total.) +** +** typeof(Y)='real' The character "F" followed by an 8-byte +** big-ending binary of the double. (Nine +** bytes total.) +** +** typeof(Y)='text' The hash is over prefix "Tnnn:" followed +** by the UTF8 encoding of the text. The "nnn" +** in the prefix is the minimum-length decimal +** representation of the octet_length of the text. +** Notice the ":" at the end of the prefix, which +** is needed to separate the prefix from the +** content in cases where the content starts +** with a digit. +** +** typeof(Y)='blob' The hash is taken over prefix "Bnnn:" followed +** by the binary content of the blob. The "nnn" +** in the prefix is the mimimum-length decimal +** representation of the byte-length of the blob. +** +** According to the rules above, all of the following SELECT statements +** should return TRUE: +** +** SELECT sha3(1) = sha3('1'); +** +** SELECT sha3('hello') = sha3(x'68656c6c6f'); +** +** WITH a(x) AS (VALUES('xyzzy')) +** SELECT sha3_agg(x) = sha3('T5:xyzzy') FROM a; +** +** WITH a(x) AS (VALUES(x'010203')) +** SELECT sha3_agg(x) = sha3(x'42333a010203') FROM a; +** +** WITH a(x) AS (VALUES(0x123456)) +** SELECT sha3_agg(x) = sha3(x'490000000000123456') FROM a; +** +** WITH a(x) AS (VALUES(100.015625)) +** SELECT sha3_agg(x) = sha3(x'464059010000000000') FROM a; +** +** WITH a(x) AS (VALUES(NULL)) +** SELECT sha3_agg(x) = sha3('N') FROM a; +** +** +** In sha3_query(), individual column values are encoded as with +** sha3_agg(), but with the addition that a single "R" character is +** inserted at the start of each row. +** +** Note that sha3_agg() hashes rows for which Y is NULL. Add a FILTER +** clause if NULL rows should be excluded: +** +** SELECT sha3_agg(x ORDER BY rowid) FILTER(WHERE x NOT NULL) FROM t1; */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -75,6 +147,7 @@ struct SHA3Context { unsigned nRate; /* Bytes of input accepted per Keccak iteration */ unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + unsigned iSize; /* 224, 256, 358, or 512 */ }; /* @@ -404,6 +477,7 @@ static void KeccakF1600Step(SHA3Context *p){ */ static void SHA3Init(SHA3Context *p, int iSize){ memset(p, 0, sizeof(*p)); + p->iSize = iSize; if( iSize>=128 && iSize<=512 ){ p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; }else{ @@ -548,6 +622,60 @@ static void sha3_step_vformat( } /* +** Update a SHA3Context using a single sqlite3_value. +*/ +static void sha3UpdateFromValue(SHA3Context *p, sqlite3_value *pVal){ + switch( sqlite3_value_type(pVal) ){ + case SQLITE_NULL: { + SHA3Update(p, (const unsigned char*)"N",1); + break; + } + case SQLITE_INTEGER: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + sqlite3_int64 v = sqlite3_value_int64(pVal); + memcpy(&u, &v, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(p, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_value_double(pVal); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(p,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_text(pVal); + sha3_step_vformat(p,"T%d:",n2); + SHA3Update(p, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_value_bytes(pVal); + const unsigned char *z2 = sqlite3_value_blob(pVal); + sha3_step_vformat(p,"B%d:",n2); + SHA3Update(p, z2, n2); + break; + } + } +} + +/* ** Implementation of the sha3_query(SQL,SIZE) function. ** ** This function compiles and runs the SQL statement(s) given in the @@ -636,54 +764,7 @@ static void sha3QueryFunc( while( SQLITE_ROW==sqlite3_step(pStmt) ){ SHA3Update(&cx,(const unsigned char*)"R",1); for(i=0; i<nCol; i++){ - switch( sqlite3_column_type(pStmt,i) ){ - case SQLITE_NULL: { - SHA3Update(&cx, (const unsigned char*)"N",1); - break; - } - case SQLITE_INTEGER: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - sqlite3_int64 v = sqlite3_column_int64(pStmt,i); - memcpy(&u, &v, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'I'; - SHA3Update(&cx, x, 9); - break; - } - case SQLITE_FLOAT: { - sqlite3_uint64 u; - int j; - unsigned char x[9]; - double r = sqlite3_column_double(pStmt,i); - memcpy(&u, &r, 8); - for(j=8; j>=1; j--){ - x[j] = u & 0xff; - u >>= 8; - } - x[0] = 'F'; - SHA3Update(&cx,x,9); - break; - } - case SQLITE_TEXT: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_text(pStmt, i); - sha3_step_vformat(&cx,"T%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - case SQLITE_BLOB: { - int n2 = sqlite3_column_bytes(pStmt, i); - const unsigned char *z2 = sqlite3_column_blob(pStmt, i); - sha3_step_vformat(&cx,"B%d:",n2); - SHA3Update(&cx, z2, n2); - break; - } - } + sha3UpdateFromValue(&cx, sqlite3_column_value(pStmt,i)); } } sqlite3_finalize(pStmt); @@ -691,6 +772,44 @@ static void sha3QueryFunc( sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); } +/* +** xStep function for sha3_agg(). +*/ +static void sha3AggStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context *p; + p = (SHA3Context*)sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( p->nRate==0 ){ + int sz = 256; + if( argc==2 ){ + sz = sqlite3_value_int(argv[1]); + if( sz!=224 && sz!=384 && sz!=512 ){ + sz = 256; + } + } + SHA3Init(p, sz); + } + sha3UpdateFromValue(p, argv[0]); +} + + +/* +** xFinal function for sha3_agg(). +*/ +static void sha3AggFinal(sqlite3_context *context){ + SHA3Context *p; + p = (SHA3Context*)sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( p->iSize ){ + sqlite3_result_blob(context, SHA3Final(p), p->iSize/8, SQLITE_TRANSIENT); + } +} + + #ifdef _WIN32 __declspec(dllexport) @@ -712,6 +831,16 @@ int sqlite3_shathree_init( 0, sha3Func, 0, 0); } if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_agg", 2, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, sha3AggStep, sha3AggFinal); + } + if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8 | SQLITE_DIRECTONLY, 0, sha3QueryFunc, 0, 0); diff --git a/ext/misc/sqlar.c b/ext/misc/sqlar.c index be079a546..9f726f0b8 100644 --- a/ext/misc/sqlar.c +++ b/ext/misc/sqlar.c @@ -81,7 +81,7 @@ static void sqlarUncompressFunc( sqlite3_value **argv ){ uLong nData; - uLongf sz; + sqlite3_int64 sz; assert( argc==2 ); sz = sqlite3_value_int(argv[1]); @@ -89,14 +89,15 @@ static void sqlarUncompressFunc( if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ + uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); Bytef *pOut = sqlite3_malloc(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); - }else if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){ + }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ sqlite3_result_error(context, "error in uncompress()", -1); }else{ - sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT); + sqlite3_result_blob(context, pOut, szf, SQLITE_TRANSIENT); } sqlite3_free(pOut); } diff --git a/ext/misc/stmtrand.c b/ext/misc/stmtrand.c new file mode 100644 index 000000000..b5e3b89a3 --- /dev/null +++ b/ext/misc/stmtrand.c @@ -0,0 +1,97 @@ +/* +** 2024-05-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** An SQL function that return pseudo-random non-negative integers. +** +** SELECT stmtrand(123); +** +** A special feature of this function is that the same sequence of random +** integers is returned for each invocation of the statement. This makes +** the results repeatable, and hence useful for testing. The argument is +** an integer which is the seed for the random number sequence. The seed +** is used by the first invocation of this function only and is ignored +** for all subsequent calls within the same statement. +** +** Resetting a statement (sqlite3_reset()) also resets the random number +** sequence. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include <assert.h> +#include <string.h> + +/* State of the pseudo-random number generator */ +typedef struct Stmtrand { + unsigned int x, y; +} Stmtrand; + +/* auxdata key */ +#define STMTRAND_KEY (-4418371) + +/* +** Function: stmtrand(SEED) +** +** Return a pseudo-random number. +*/ +static void stmtrandFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stmtrand *p; + + p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); + if( p==0 ){ + unsigned int seed; + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + if( argc>=1 ){ + seed = (unsigned int)sqlite3_value_int(argv[0]); + }else{ + seed = 0; + } + p->x = seed | 1; + p->y = seed; + sqlite3_set_auxdata(context, STMTRAND_KEY, p, sqlite3_free); + p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + } + p->x = (p->x>>1) ^ ((1+~(p->x&1)) & 0xd0000001); + p->y = p->y*1103515245 + 12345; + sqlite3_result_int(context, (int)((p->x ^ p->y)&0x7fffffff)); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_stmtrand_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "stmtrand", 1, SQLITE_UTF8, 0, + stmtrandFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "stmtrand", 0, SQLITE_UTF8, 0, + stmtrandFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/vtablog.c b/ext/misc/vtablog.c index e414e7afa..2cc29c285 100644 --- a/ext/misc/vtablog.c +++ b/ext/misc/vtablog.c @@ -38,8 +38,9 @@ SQLITE_EXTENSION_INIT1 typedef struct vtablog_vtab vtablog_vtab; struct vtablog_vtab { sqlite3_vtab base; /* Base class - must be first */ + char *zDb; /* Schema name. argv[1] of xConnect/xCreate */ + char *zName; /* Table name. argv[2] of xConnect/xCreate */ int nRow; /* Number of rows in the table */ - int iInst; /* Instance number for this vtablog table */ int nCursor; /* Number of cursors created */ }; @@ -167,15 +168,14 @@ static int vtablogConnectCreate( char **pzErr, int isCreate ){ - static int nInst = 0; vtablog_vtab *pNew; int i; int rc; - int iInst = ++nInst; char *zSchema = 0; char *zNRow = 0; - printf("vtablog%s(tab=%d):\n", isCreate ? "Create" : "Connect", iInst); + printf("%s.%s.%s():\n", argv[1], argv[2], + isCreate ? "xCreate" : "xConnect"); printf(" argc=%d\n", argc); for(i=0; i<argc; i++){ printf(" argv[%d] = ", i); @@ -189,17 +189,18 @@ static int vtablogConnectCreate( for(i=3; i<argc; i++){ const char *z = argv[i]; if( vtablog_string_parameter(pzErr, "schema", z, &zSchema) ){ - return SQLITE_ERROR; + rc = SQLITE_ERROR; + goto vtablog_end_connect; } if( vtablog_string_parameter(pzErr, "rows", z, &zNRow) ){ - return SQLITE_ERROR; + rc = SQLITE_ERROR; + goto vtablog_end_connect; } } - if( zSchema==0 ){ - *pzErr = sqlite3_mprintf("no schema defined"); - return SQLITE_ERROR; + zSchema = sqlite3_mprintf("%s","CREATE TABLE x(a,b);"); } + printf(" schema = '%s'\n", zSchema); rc = sqlite3_declare_vtab(db, zSchema); if( rc==SQLITE_OK ){ pNew = sqlite3_malloc( sizeof(*pNew) ); @@ -208,8 +209,14 @@ static int vtablogConnectCreate( memset(pNew, 0, sizeof(*pNew)); pNew->nRow = 10; if( zNRow ) pNew->nRow = atoi(zNRow); - pNew->iInst = iInst; + printf(" nrow = %d\n", pNew->nRow); + pNew->zDb = sqlite3_mprintf("%s", argv[1]); + pNew->zName = sqlite3_mprintf("%s", argv[2]); } + +vtablog_end_connect: + sqlite3_free(zSchema); + sqlite3_free(zNRow); return rc; } static int vtablogCreate( @@ -237,7 +244,9 @@ static int vtablogConnect( */ static int vtablogDisconnect(sqlite3_vtab *pVtab){ vtablog_vtab *pTab = (vtablog_vtab*)pVtab; - printf("vtablogDisconnect(%d)\n", pTab->iInst); + printf("%s.%s.xDisconnect()\n", pTab->zDb, pTab->zName); + sqlite3_free(pTab->zDb); + sqlite3_free(pTab->zName); sqlite3_free(pVtab); return SQLITE_OK; } @@ -247,7 +256,9 @@ static int vtablogDisconnect(sqlite3_vtab *pVtab){ */ static int vtablogDestroy(sqlite3_vtab *pVtab){ vtablog_vtab *pTab = (vtablog_vtab*)pVtab; - printf("vtablogDestroy(%d)\n", pTab->iInst); + printf("%s.%s.xDestroy()\n", pTab->zDb, pTab->zName); + sqlite3_free(pTab->zDb); + sqlite3_free(pTab->zName); sqlite3_free(pVtab); return SQLITE_OK; } @@ -258,7 +269,8 @@ static int vtablogDestroy(sqlite3_vtab *pVtab){ static int vtablogOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ vtablog_vtab *pTab = (vtablog_vtab*)p; vtablog_cursor *pCur; - printf("vtablogOpen(tab=%d, cursor=%d)\n", pTab->iInst, ++pTab->nCursor); + printf("%s.%s.xOpen(cursor=%d)\n", pTab->zDb, pTab->zName, + ++pTab->nCursor); pCur = sqlite3_malloc( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); @@ -273,7 +285,7 @@ static int vtablogOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ static int vtablogClose(sqlite3_vtab_cursor *cur){ vtablog_cursor *pCur = (vtablog_cursor*)cur; vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; - printf("vtablogClose(tab=%d, cursor=%d)\n", pTab->iInst, pCur->iCursor); + printf("%s.%s.xClose(cursor=%d)\n", pTab->zDb, pTab->zName, pCur->iCursor); sqlite3_free(cur); return SQLITE_OK; } @@ -285,8 +297,9 @@ static int vtablogClose(sqlite3_vtab_cursor *cur){ static int vtablogNext(sqlite3_vtab_cursor *cur){ vtablog_cursor *pCur = (vtablog_cursor*)cur; vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; - printf("vtablogNext(tab=%d, cursor=%d) rowid %d -> %d\n", - pTab->iInst, pCur->iCursor, (int)pCur->iRowid, (int)pCur->iRowid+1); + printf("%s.%s.xNext(cursor=%d) rowid %d -> %d\n", + pTab->zDb, pTab->zName, pCur->iCursor, + (int)pCur->iRowid, (int)pCur->iRowid+1); pCur->iRowid++; return SQLITE_OK; } @@ -310,8 +323,8 @@ static int vtablogColumn( }else{ sqlite3_snprintf(sizeof(zVal),zVal,"{%d}%d", i, pCur->iRowid); } - printf("vtablogColumn(tab=%d, cursor=%d, i=%d): [%s]\n", - pTab->iInst, pCur->iCursor, i, zVal); + printf("%s.%s.xColumn(cursor=%d, i=%d): [%s]\n", + pTab->zDb, pTab->zName, pCur->iCursor, i, zVal); sqlite3_result_text(ctx, zVal, -1, SQLITE_TRANSIENT); return SQLITE_OK; } @@ -323,8 +336,8 @@ static int vtablogColumn( static int vtablogRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ vtablog_cursor *pCur = (vtablog_cursor*)cur; vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; - printf("vtablogRowid(tab=%d, cursor=%d): %d\n", - pTab->iInst, pCur->iCursor, (int)pCur->iRowid); + printf("%s.%s.xRowid(cursor=%d): %d\n", + pTab->zDb, pTab->zName, pCur->iCursor, (int)pCur->iRowid); *pRowid = pCur->iRowid; return SQLITE_OK; } @@ -337,8 +350,8 @@ static int vtablogEof(sqlite3_vtab_cursor *cur){ vtablog_cursor *pCur = (vtablog_cursor*)cur; vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; int rc = pCur->iRowid >= pTab->nRow; - printf("vtablogEof(tab=%d, cursor=%d): %d\n", - pTab->iInst, pCur->iCursor, rc); + printf("%s.%s.xEof(cursor=%d): %d\n", + pTab->zDb, pTab->zName, pCur->iCursor, rc); return rc; } @@ -417,7 +430,7 @@ static int vtablogFilter( ){ vtablog_cursor *pCur = (vtablog_cursor *)cur; vtablog_vtab *pTab = (vtablog_vtab*)cur->pVtab; - printf("vtablogFilter(tab=%d, cursor=%d):\n", pTab->iInst, pCur->iCursor); + printf("%s.%s.xFilter(cursor=%d):\n", pTab->zDb, pTab->zName, pCur->iCursor); pCur->iRowid = 0; return SQLITE_OK; } @@ -430,12 +443,37 @@ static int vtablogFilter( */ static int vtablogBestIndex( sqlite3_vtab *tab, - sqlite3_index_info *pIdxInfo + sqlite3_index_info *p ){ vtablog_vtab *pTab = (vtablog_vtab*)tab; - printf("vtablogBestIndex(tab=%d):\n", pTab->iInst); - pIdxInfo->estimatedCost = (double)500; - pIdxInfo->estimatedRows = 500; + int i; + printf("%s.%s.xBestIndex():\n", pTab->zDb, pTab->zName); + printf(" colUsed: 0x%016llx\n", p->colUsed); + printf(" nConstraint: %d\n", p->nConstraint); + for(i=0; i<p->nConstraint; i++){ + printf( + " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable, + sqlite3_vtab_collation(p,i)); + } + printf(" nOrderBy: %d\n", p->nOrderBy); + for(i=0; i<p->nOrderBy; i++){ + printf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } + p->estimatedCost = (double)500; + p->estimatedRows = 500; + printf(" idxNum=%d\n", p->idxNum); + printf(" idxStr=NULL\n"); + printf(" orderByConsumed=%d\n", p->orderByConsumed); + printf(" estimatedCost=%g\n", p->estimatedCost); + printf(" estimatedRows=%lld\n", p->estimatedRows); return SQLITE_OK; } @@ -454,7 +492,7 @@ static int vtablogUpdate( ){ vtablog_vtab *pTab = (vtablog_vtab*)tab; int i; - printf("vtablogUpdate(tab=%d):\n", pTab->iInst); + printf("%s.%s.xUpdate():\n", pTab->zDb, pTab->zName); printf(" argc=%d\n", argc); for(i=0; i<argc; i++){ printf(" argv[%d]=", i); @@ -464,12 +502,88 @@ static int vtablogUpdate( return SQLITE_OK; } +static int vtablogBegin(sqlite3_vtab *tab){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xBegin()\n", pTab->zDb, pTab->zName); + return SQLITE_OK; +} +static int vtablogSync(sqlite3_vtab *tab){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xSync()\n", pTab->zDb, pTab->zName); + return SQLITE_OK; +} +static int vtablogCommit(sqlite3_vtab *tab){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xCommit()\n", pTab->zDb, pTab->zName); + return SQLITE_OK; +} +static int vtablogRollback(sqlite3_vtab *tab){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xRollback()\n", pTab->zDb, pTab->zName); + return SQLITE_OK; +} +static int vtablogSavepoint(sqlite3_vtab *tab, int N){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xSavepoint(%d)\n", pTab->zDb, pTab->zName, N); + return SQLITE_OK; +} +static int vtablogRelease(sqlite3_vtab *tab, int N){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xRelease(%d)\n", pTab->zDb, pTab->zName, N); + return SQLITE_OK; +} +static int vtablogRollbackTo(sqlite3_vtab *tab, int N){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xRollbackTo(%d)\n", pTab->zDb, pTab->zName, N); + return SQLITE_OK; +} + +static int vtablogFindMethod( + sqlite3_vtab *tab, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xFindMethod(nArg=%d, zName=%s)\n", + pTab->zDb, pTab->zName, nArg, zName); + return SQLITE_OK; +} +static int vtablogRename(sqlite3_vtab *tab, const char *zNew){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xRename('%s')\n", pTab->zDb, pTab->zName, zNew); + sqlite3_free(pTab->zName); + pTab->zName = sqlite3_mprintf("%s", zNew); + return SQLITE_OK; +} + +/* Any table name that contains the text "shadow" is seen as a +** shadow table. Nothing else is. +*/ +static int vtablogShadowName(const char *zName){ + printf("vtablog.xShadowName('%s')\n", zName); + return sqlite3_strglob("*shadow*", zName)==0; +} + +static int vtablogIntegrity( + sqlite3_vtab *tab, + const char *zSchema, + const char *zTabName, + int mFlags, + char **pzErr +){ + vtablog_vtab *pTab = (vtablog_vtab*)tab; + printf("%s.%s.xIntegrity(mFlags=0x%x)\n", pTab->zDb, pTab->zName, mFlags); + return 0; +} + /* ** This following structure defines all the methods for the ** vtablog virtual table. */ static sqlite3_module vtablogModule = { - 0, /* iVersion */ + 4, /* iVersion */ vtablogCreate, /* xCreate */ vtablogConnect, /* xConnect */ vtablogBestIndex, /* xBestIndex */ @@ -483,17 +597,17 @@ static sqlite3_module vtablogModule = { vtablogColumn, /* xColumn - read data */ vtablogRowid, /* xRowid - read data */ vtablogUpdate, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ + vtablogBegin, /* xBegin */ + vtablogSync, /* xSync */ + vtablogCommit, /* xCommit */ + vtablogRollback, /* xRollback */ + vtablogFindMethod, /* xFindMethod */ + vtablogRename, /* xRename */ + vtablogSavepoint, /* xSavepoint */ + vtablogRelease, /* xRelease */ + vtablogRollbackTo, /* xRollbackTo */ + vtablogShadowName, /* xShadowName */ + vtablogIntegrity /* xIntegrity */ }; #ifdef _WIN32 diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index db327809f..fd6d13bc3 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -29,7 +29,9 @@ SQLITE_EXTENSION_INIT1 #include <stdio.h> #include <string.h> #include <assert.h> -#include <stdint.h> +#ifndef SQLITE_NO_STDINT +# include <stdint.h> +#endif #include <zlib.h> diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 15b05cede..feb7695d6 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -199,6 +199,7 @@ typedef unsigned int u32; typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; #endif /* @@ -885,6 +886,7 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ if( rc!=SQLITE_ROW ){ rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); pIter->zTbl = 0; + pIter->zDataTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); @@ -2979,7 +2981,7 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ u32 volatile *ptr; p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; + iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]); } } return iRet; diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index af794d80f..969d6208d 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -17,14 +17,7 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) #include "sqlite3rbu.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #include <assert.h> #include <string.h> @@ -432,11 +425,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){ } #else -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" int SqliteRbu_Init(Tcl_Interp *interp){ return TCL_OK; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */ #endif /* defined(SQLITE_TEST) */ diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index b6cb26ecd..001840b17 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -88,6 +88,15 @@ typedef unsigned int u32; typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; +typedef struct DbdataBuffer DbdataBuffer; + +/* +** Buffer type. +*/ +struct DbdataBuffer { + u8 *aBuf; + sqlite3_int64 nBuf; +}; /* Cursor object */ struct DbdataCursor { @@ -104,7 +113,7 @@ struct DbdataCursor { sqlite3_int64 iRowid; /* Only for the sqlite_dbdata table */ - u8 *pRec; /* Buffer containing current record */ + DbdataBuffer rec; sqlite3_int64 nRec; /* Size of pRec[] in bytes */ sqlite3_int64 nHdr; /* Size of header in bytes */ int iField; /* Current field number */ @@ -150,6 +159,31 @@ struct DbdataTable { ")" /* +** Ensure the buffer passed as the first argument is at least nMin bytes +** in size. If an error occurs while attempting to resize the buffer, +** SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int dbdataBufferSize(DbdataBuffer *pBuf, sqlite3_int64 nMin){ + if( nMin>pBuf->nBuf ){ + sqlite3_int64 nNew = nMin+16384; + u8 *aNew = (u8*)sqlite3_realloc64(pBuf->aBuf, nNew); + + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->aBuf = aNew; + pBuf->nBuf = nNew; + } + return SQLITE_OK; +} + +/* +** Release the allocation managed by buffer pBuf. +*/ +static void dbdataBufferFree(DbdataBuffer *pBuf){ + sqlite3_free(pBuf->aBuf); + memset(pBuf, 0, sizeof(*pBuf)); +} + +/* ** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual ** table. */ @@ -289,9 +323,9 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ pCsr->iField = 0; pCsr->bOnePage = 0; sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; + dbdataBufferFree(&pCsr->rec); pCsr->aPage = 0; + pCsr->nRec = 0; } /* @@ -433,67 +467,88 @@ static void dbdataValue( u8 *pData, sqlite3_int64 nData ){ - if( eType>=0 && dbdataValueBytes(eType)<=nData ){ - switch( eType ){ - case 0: - case 10: - case 11: - sqlite3_result_null(pCtx); - break; - - case 8: - sqlite3_result_int(pCtx, 0); - break; - case 9: - sqlite3_result_int(pCtx, 1); - break; - - case 1: case 2: case 3: case 4: case 5: case 6: case 7: { - sqlite3_uint64 v = (signed char)pData[0]; - pData++; - switch( eType ){ - case 7: - case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 4: v = (v<<8) + pData[0]; pData++; - case 3: v = (v<<8) + pData[0]; pData++; - case 2: v = (v<<8) + pData[0]; pData++; - } - - if( eType==7 ){ - double r; - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(pCtx, r); - }else{ - sqlite3_result_int64(pCtx, (sqlite3_int64)v); + if( eType>=0 ){ + if( dbdataValueBytes(eType)<=nData ){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; } - break; - } - - default: { - int n = ((eType-12) / 2); - if( eType % 2 ){ - switch( enc ){ -#ifndef SQLITE_OMIT_UTF16 - case SQLITE_UTF16BE: - sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; - case SQLITE_UTF16LE: - sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); - break; -#endif - default: - sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); - break; + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + switch( enc ){ + #ifndef SQLITE_OMIT_UTF16 + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + #endif + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } - }else{ - sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } } + }else{ + if( eType==7 ){ + sqlite3_result_double(pCtx, 0.0); + }else if( eType<7 ){ + sqlite3_result_int(pCtx, 0); + }else if( eType%2 ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + }else{ + sqlite3_result_blob(pCtx, "", 0, SQLITE_STATIC); + } } } } +/* This macro is a copy of the MX_CELL() macro in the SQLite core. Given +** a page-size, it returns the maximum number of cells that may be present +** on the page. */ +#define DBDATA_MX_CELL(pgsz) ((pgsz-8)/6) + +/* Maximum number of fields that may appear in a single record. This is +** the "hard-limit", according to comments in sqliteLimit.h. */ +#define DBDATA_MX_FIELD 32676 + /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ @@ -522,6 +577,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ + pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); + } } if( pTab->bPtr ){ @@ -539,7 +597,8 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } }else{ /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ + assert( pCsr->rec.aBuf!=0 || pCsr->nRec==0 ); + if( pCsr->nRec==0 ){ int bHasRowid = 0; int nPointer = 0; sqlite3_int64 nPayload = 0; @@ -566,23 +625,24 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ + int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; - iOff += 8 + nPointer + pCsr->iCell*2; - if( iOff>pCsr->nPage ){ + if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ - iOff = get_uint16(&pCsr->aPage[iOff]); + iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ - if( bNextPage || iOff>pCsr->nPage ){ + if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; + if( nPayload==0 ) nPayload = 1; } /* If this is a leaf intkey cell, load the rowid */ @@ -617,13 +677,13 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Allocate space for payload. And a bit more to catch small buffer ** overruns caused by attempting to read a varint or similar from ** near the end of a corrupt record. */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES); - pCsr->nRec = nPayload; + rc = dbdataBufferSize(&pCsr->rec, nPayload+DBDATA_PADDING_BYTES); + if( rc!=SQLITE_OK ) return rc; + assert( pCsr->rec.aBuf!=0 ); + assert( nPayload!=0 ); /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + memcpy(pCsr->rec.aBuf, &pCsr->aPage[iOff], nLocal); iOff += nLocal; /* Load content from overflow pages */ @@ -641,19 +701,22 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ nCopy = U-4; if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + memcpy(&pCsr->rec.aBuf[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } + nPayload -= nRem; } + memset(&pCsr->rec.aBuf[nPayload], 0, DBDATA_PADDING_BYTES); + pCsr->nRec = nPayload; - iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr); + iHdr = dbdataGetVarintU32(pCsr->rec.aBuf, &nHdr); if( nHdr>nPayload ) nHdr = 0; pCsr->nHdr = nHdr; - pCsr->pHdrPtr = &pCsr->pRec[iHdr]; - pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; + pCsr->pHdrPtr = &pCsr->rec.aBuf[iHdr]; + pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nHdr]; pCsr->iField = (bHasRowid ? -1 : 0); } } @@ -661,14 +724,16 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; - if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ + if( pCsr->pHdrPtr>=&pCsr->rec.aBuf[pCsr->nRec] + || pCsr->iField>=DBDATA_MX_FIELD + ){ bNextPage = 1; }else{ int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); szField = dbdataValueBytes(iType); - if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))<szField ){ - pCsr->pPtr = &pCsr->pRec[pCsr->nRec]; + if( (pCsr->nRec - (pCsr->pPtr - pCsr->rec.aBuf))<szField ){ + pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nRec]; }else{ pCsr->pPtr += szField; } @@ -678,20 +743,18 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( bNextPage ){ sqlite3_free(pCsr->aPage); - sqlite3_free(pCsr->pRec); pCsr->aPage = 0; - pCsr->pRec = 0; + pCsr->nRec = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; }else{ - if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->rec.aBuf[pCsr->nHdr] ){ return SQLITE_OK; } /* Advance to the next cell. The next iteration of the loop will load ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; + pCsr->nRec = 0; pCsr->iCell++; } } @@ -881,12 +944,12 @@ static int dbdataColumn( case DBDATA_COLUMN_VALUE: { if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); - }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){ + }else if( &pCsr->rec.aBuf[pCsr->nRec] >= pCsr->pPtr ){ sqlite3_int64 iType; dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( ctx, pCsr->enc, iType, pCsr->pPtr, - &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + &pCsr->rec.aBuf[pCsr->nRec] - pCsr->pPtr ); } break; diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 070dd03d6..11a43780d 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -342,7 +342,7 @@ foreach enc {utf8 utf16 utf16le utf16be} { DELETE FROM sqlite_schema WHERE name='t1'; } - proc my_sql_hook {sql} { + proc my_sql_hook2 {sql} { if {[string match "INSERT INTO lostandfound*" $sql]} { lappend ::script $sql } @@ -350,7 +350,7 @@ foreach enc {utf8 utf16 utf16le utf16be} { } do_test 18.$enc.2 { set ::script [list] - set R [sqlite3_recover_init_sql db main my_sql_hook] + set R [sqlite3_recover_init_sql db main my_sql_hook2] $R config lostandfound lostandfound $R run $R finish @@ -358,7 +358,18 @@ foreach enc {utf8 utf16 utf16le utf16be} { } {{INSERT INTO lostandfound VALUES(2, 2, 2, 1, 'abc', 'def')}} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 19.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + ALTER TABLE t1 ADD COLUMN c NOT NULL DEFAULT 13; + INSERT INTO t1 VALUES(3, 'three', 'hello world'); +} +do_recover_test 19.1 diff --git a/ext/recover/recovercorrupt2.test b/ext/recover/recovercorrupt2.test index 29acc27a3..6c216308f 100644 --- a/ext/recover/recovercorrupt2.test +++ b/ext/recover/recovercorrupt2.test @@ -524,5 +524,33 @@ do_test 7.1 { list [catch { $R finish } msg] $msg } {1 {file is not a database}} +reset_db +breakpoint +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 8192 pagesize 4096 filename db.sqlite +| page 1 offset 0 +| 0: ac ae b3 76 74 65 20 66 6f 72 6d 61 74 20 33 00 ...vte format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 76 8a 0d ff ff ff 1e 0f cb 00 0f cb 00 00 ..v............. +| 4032: 00 00 00 00 00 00 00 00 00 00 00 33 01 06 17 19 ...........3.... +| 4048: 19 01 43 74 61 62 6c 65 54 61 62 6c 65 30 54 61 ..CtableTable0Ta +| 4064: 62 6c 65 30 02 43 52 45 41 54 45 20 54 41 42 4c ble0.CREATE TABL +| 4080: 45 20 54 61 62 6c 65 30 20 28 43 6f 6c 30 20 29 E Table0 (Col0 ) +| page 2 offset 4096 +| 0: 0d 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 ................ +| end db.sqlite +}]} {} + +do_test 8.1 { + set R [sqlite3_recover_init db main test.db2] + catch { $R run } + list [catch { $R finish } msg] $msg +} {0 {}} + finish_test diff --git a/ext/recover/recovercorrupt3.test b/ext/recover/recovercorrupt3.test new file mode 100644 index 000000000..9a7c2d094 --- /dev/null +++ b/ext/recover/recovercorrupt3.test @@ -0,0 +1,549 @@ +# 2024 May 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recovercorrupt3 + +#| 0: d5 d5 9b d5 d5 d5 d5 d5 d5 d5 d5 d5 d5 d5 d5 d5 ................ +#| 16: 04 00 00 00 1d 00 00 00 00 00 00 00 5f 5f 5f 5f ............____ +#| 32: 5f 5f 5f 5f 5f 5f 5f 5f 71 5f 5f 5f 02 02 02 02 ________q___.... +#| 48: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +#| 64: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +#| 80: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +#| 96: 02 02 02 02 + +#------------------------------------------------------------------------- +reset_db +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 3821 pagesize 1024 filename clusterfuzz-testcase-sql_recovery_fuzzer-5803962339885056 +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 01 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 7a 70 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 112: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 128: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 144: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 160: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 176: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 192: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 208: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 224: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 240: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 256: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 272: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 288: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 a0 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 29 29 ..............)) +| 464: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 480: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 496: 29 29 29 dd dd dd dd dd dd dd dd dd dd dd dd dd )))............. +| 512: dd dd dd dd dd dd dd dd dd 6e 69 d2 e9 e9 e9 d2 .........ni..... +| 528: d2 d2 d2 d2 dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 544: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 560: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 576: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 592: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 608: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 624: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 640: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 656: dd dd dd dd dd dd dd da dd dd dd dd dd dd dd dd ................ +| 672: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 688: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 704: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 720: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 736: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 752: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 768: dd dd dd dd dd dd dd dd dd dd dd dd dd 29 29 29 .............))) +| 784: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 800: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 816: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 832: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 848: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 864: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 880: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 896: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 912: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 928: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 944: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 960: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 976: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 992: 29 29 29 29 29 29 29 29 29 29 dd dd dd dd dd dd ))))))))))...... +| 1008: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| page 2 offset 1024 +| 0: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 16: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 32: dd dd 6e 69 d2 e9 e9 e9 d2 d2 d2 d2 d2 dd dd dd ..ni............ +| 48: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 64: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 80: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 96: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 112: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 128: dd dd dd dd dd dd 29 29 29 29 29 29 29 29 29 29 ......)))))))))) +| 144: 29 29 29 29 29 29 29 29 29 ad a5 29 29 29 29 00 )))))))))..)))). +| 160: 75 9c 11 00 5b e5 64 28 7c ca 09 69 28 2d 69 00 u...[.d(|..i(-i. +| 176: 85 88 6c 81 48 83 a0 93 c0 c0 82 8b 81 84 85 f9 ..l.H........... +| 192: 88 7a 00 7f 00 96 40 7b 12 4b 84 75 a0 00 99 a0 .z....@..K.u.... +| 208: df a0 7e 81 c6 90 8f 7f 84 85 cc 84 82 90 88 60 ..~............` +| 224: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 240: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 256: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 272: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 288: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 464: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 480: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 496: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 512: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 528: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 544: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 560: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 576: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 592: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 608: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 624: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 640: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 656: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 672: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 688: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 704: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 720: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 736: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 752: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 768: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 784: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 800: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 816: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 832: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 848: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 864: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 880: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 896: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 912: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 928: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 944: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 960: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 976: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 992: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 1008: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| page 3 offset 2048 +| 0: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 16: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 32: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 48: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 64: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 80: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 96: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 112: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 128: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 144: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 160: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 176: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 192: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 208: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 224: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 240: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 256: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 272: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 288: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 464: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 480: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 496: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 512: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 528: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 544: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 560: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 576: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 592: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 608: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 624: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 640: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 656: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 672: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 688: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 704: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 720: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 736: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 752: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 768: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 784: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 800: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 816: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 832: 02 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 848: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 864: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 880: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 896: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 912: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 928: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 944: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 960: 80 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 976: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 992: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 1008: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| page 4 offset 3072 +| 0: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 16: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 32: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 48: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 64: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 80: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 96: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 112: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 128: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 144: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 160: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 176: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 192: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 208: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 224: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 240: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 256: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 272: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 288: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 464: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 480: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 496: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 512: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 528: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 544: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 560: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 576: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 592: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 608: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 624: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 640: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 656: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 672: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 688: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 704: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 720: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 736: 5f 5f 5f 5f 5f 5f 5f 00 d5 fe fe fe 08 00 00 00 _______......... +| end clusterfuzz-testcase-sql_recovery_fuzzer-5803962339885056 +}]} {} + +sqlite3_dbdata_init db +do_execsql_test 1.1 { + PRAGMA writable_schema = 1; +} + +do_test 1.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +#------------------------------------------------------------------------- +reset_db +do_test 2.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 3821 pagesize 1024 filename clusterfuzz-testcase-sql_recovery_fuzzer-5803962339885056 +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 04 00 01 01 00 40 20 20 00 00 00 01 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ................ +| 96: 00 2e 7a 70 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 112: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 128: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 144: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 160: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 176: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 192: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 208: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 224: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 240: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 256: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 272: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 288: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 a0 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 29 29 ..............)) +| 464: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 480: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 496: 29 29 29 dd dd dd dd dd dd dd dd dd dd dd dd dd )))............. +| 512: dd dd dd dd dd dd dd dd dd 6e 69 d2 e9 e9 e9 d2 .........ni..... +| 528: d2 d2 d2 d2 dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 544: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 560: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 576: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 592: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 608: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 624: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 640: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 656: dd dd dd dd dd dd dd da dd dd dd dd dd dd dd dd ................ +| 672: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 688: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 704: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 720: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 736: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 752: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 768: dd dd dd dd dd dd dd dd dd dd dd dd dd 29 29 29 .............))) +| 784: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 800: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 816: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 832: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 848: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 864: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 880: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 896: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 912: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 928: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 944: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 960: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 976: 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 29 )))))))))))))))) +| 992: 29 29 29 29 29 29 29 29 29 29 dd dd dd dd dd dd ))))))))))...... +| 1008: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| page 2 offset 1024 +| 0: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 16: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 32: dd dd 6e 69 d2 e9 e9 e9 d2 d2 d2 d2 d2 dd dd dd ..ni............ +| 48: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 64: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 80: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 96: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 112: dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd ................ +| 128: dd dd dd dd dd dd 29 29 29 29 29 29 29 29 29 29 ......)))))))))) +| 144: 29 29 29 29 29 29 29 29 29 ad a5 29 29 29 29 00 )))))))))..)))). +| 160: 75 9c 11 00 5b e5 64 28 7c ca 09 69 28 2d 69 00 u...[.d(|..i(-i. +| 176: 85 88 6c 81 48 83 a0 93 c0 c0 82 8b 81 84 85 f9 ..l.H........... +| 192: 88 7a 00 7f 00 96 40 7b 12 4b 84 75 a0 00 99 a0 .z....@..K.u.... +| 208: df a0 7e 81 c6 90 8f 7f 84 85 cc 84 82 90 88 60 ..~............` +| 224: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 240: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 256: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 272: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 288: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 464: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 480: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 496: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 512: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 528: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 544: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 560: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 576: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 592: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 608: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 624: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 640: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 656: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 672: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 688: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 704: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 720: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 736: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 752: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 768: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 784: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 800: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 816: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 832: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 848: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 864: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 880: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 896: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 912: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 928: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 944: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 960: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 976: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 992: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 1008: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| page 3 offset 2048 +| 0: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 16: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 32: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 48: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 64: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 80: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 96: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 112: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 128: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 144: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 160: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 176: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 192: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 208: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 224: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 240: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 256: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 272: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 288: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 464: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 480: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 496: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 512: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 528: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 544: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 560: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 576: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 592: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 608: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 624: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 640: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 656: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 672: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 688: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 704: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 720: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 736: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 752: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 768: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 784: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 800: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 816: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 832: 02 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 848: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 864: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 880: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 896: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 912: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 928: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 944: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 ................ +| 960: 80 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 976: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 992: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 1008: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| page 4 offset 3072 +| 0: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 16: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 32: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 48: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 64: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 80: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 96: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 112: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 128: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 144: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 160: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 176: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 192: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 208: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 224: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 240: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 256: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 272: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 288: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 304: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 320: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 336: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 352: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 368: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 384: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 400: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 416: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 432: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 448: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 464: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 480: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 496: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 512: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 528: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 544: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 560: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 576: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 592: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 608: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 624: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 640: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 656: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 672: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 688: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 704: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 720: 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 02 ................ +| 736: 5f 5f 5f 5f 5f 5f 5f 00 d5 fe fe fe 08 00 00 00 _______......... +| end clusterfuzz-testcase-sql_recovery_fuzzer-5803962339885056 +}]} {} + +sqlite3_dbdata_init db +do_execsql_test 2.1 { + PRAGMA writable_schema = 1; +} + +do_test 2.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +finish_test + diff --git a/ext/recover/recovercorrupt4.test b/ext/recover/recovercorrupt4.test new file mode 100644 index 000000000..25cfb88fa --- /dev/null +++ b/ext/recover/recovercorrupt4.test @@ -0,0 +1,63 @@ +# 2024 May 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recovercorrupt4 + +database_may_be_corrupt + +if {[permutation]!="inmemory_journal"} { + # This test cannot be run with the inmemory_journal permutation, as it + # must open a truncated, corrupt, database file. With the inmemory_journal + # permutation, this fails (SQLITE_CORRUPT error) when the [sqlite3] wrapper + # executes "PRAGMA journal_mode = memory". + do_execsql_test 1.0 { + CREATE TABLE rows(indexed INTEGER NOT NULL, unindexed INTEGER NOT NULL, filler BLOB NOT NULL DEFAULT 13); + -- CREATE UNIQUE INDEX rows_index ON rows(indexed); + INSERT INTO rows(indexed, unindexed, filler) VALUES(1, 1, x'31'); + INSERT INTO rows(indexed, unindexed, filler) VALUES(2, 2, x'32'); + INSERT INTO rows(indexed, unindexed, filler) VALUES(4, 4, x'34'); + INSERT INTO rows(indexed, unindexed, filler) VALUES(8, 8, randomblob(2048)); + } + + db close + + do_test 1.1 { + set sz [expr [file size test.db] - 1024] + set fd [open test.db] + fconfigure $fd -translation binary + + set data [read $fd $sz] + set fd2 [open test.db2 w] + fconfigure $fd2 -translation binary + puts -nonewline $fd2 $data + close $fd2 + set {} {} + } {} + + do_test 1.2 { + forcedelete test.db3 + sqlite3 db test.db2 + set R [sqlite3_recover_init db main test.db3] + $R run + $R finish + } {} + + do_test 1.3 { + sqlite3 db test.db3 + execsql { + SELECT indexed, unindexed FROM rows + } + } {1 1 2 2 4 4 8 8} +} + +finish_test diff --git a/ext/recover/recoverpgsz.test b/ext/recover/recoverpgsz.test index 1a91f0845..f921a7869 100644 --- a/ext/recover/recoverpgsz.test +++ b/ext/recover/recoverpgsz.test @@ -39,14 +39,14 @@ foreach {pgsz bOverflow} { set fd [open test.db] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary seek $fd $pgsz set pg1 [read $fd $pgsz] set pg2 [read $fd $pgsz] close $fd set fd2 [open test.db2 w] - fconfigure $fd2 -encoding binary -translation binary + fconfigure $fd2 -translation binary seek $fd2 $pgsz puts -nonewline $fd2 $pg1 close $fd2 @@ -71,7 +71,7 @@ foreach {pgsz bOverflow} { forcedelete test.db2 set fd2 [open test.db2 w] - fconfigure $fd2 -encoding binary -translation binary + fconfigure $fd2 -translation binary seek $fd2 $pgsz puts -nonewline $fd2 $pg2 close $fd2 @@ -95,6 +95,3 @@ foreach {pgsz bOverflow} { finish_test - - - diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index c445c5179..afa1ae861 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -363,8 +363,8 @@ static int recoverError( va_start(ap, zFmt); if( zFmt ){ z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); } + va_end(ap); sqlite3_free(p->zErrMsg); p->zErrMsg = z; p->errCode = errCode; @@ -1189,7 +1189,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zTbl, iRoot); + if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 1c333df8e..c1c0d8838 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -14,8 +14,7 @@ #include "sqlite3recover.h" #include "sqliteInt.h" - -#include <tcl.h> +#include "tclsqlite.h" #include <assert.h> #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -236,7 +235,7 @@ static int test_sqlite3_recover_init( zDb = Tcl_GetString(objv[2]); if( zDb[0]=='\0' ) zDb = 0; - pNew = ckalloc(sizeof(TestRecover)); + pNew = (TestRecover*)ckalloc(sizeof(TestRecover)); if( bSql==0 ){ zUri = Tcl_GetString(objv[3]); pNew->p = sqlite3_recover_init(db, zDb, zUri); @@ -308,4 +307,3 @@ int TestRecover_Init(Tcl_Interp *interp){ #endif return TCL_OK; } - diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 013bb0b5b..299b5b54b 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -694,11 +694,9 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ ** Clear the Rtree.pNodeBlob object */ static void nodeBlobReset(Rtree *pRtree){ - if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ - sqlite3_blob *pBlob = pRtree->pNodeBlob; - pRtree->pNodeBlob = 0; - sqlite3_blob_close(pBlob); - } + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); } /* @@ -742,7 +740,6 @@ static int nodeAcquire( &pRtree->pNodeBlob); } if( rc ){ - nodeBlobReset(pRtree); *ppNode = 0; /* If unable to open an sqlite3_blob on the desired row, that can only ** be because the shadow tables hold erroneous data. */ @@ -802,6 +799,7 @@ static int nodeAcquire( } *ppNode = pNode; }else{ + nodeBlobReset(pRtree); if( pNode ){ pRtree->nNodeRef--; sqlite3_free(pNode); @@ -946,6 +944,7 @@ static void nodeGetCoord( int iCoord, /* Which coordinate to extract */ RtreeCoord *pCoord /* OUT: Space to write result to */ ){ + assert( iCell<NCELL(pNode) ); readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -1135,7 +1134,9 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ sqlite3_finalize(pCsr->pReadAux); sqlite3_free(pCsr); pRtree->nCursor--; - nodeBlobReset(pRtree); + if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){ + nodeBlobReset(pRtree); + } return SQLITE_OK; } @@ -1720,7 +1721,11 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc==SQLITE_OK && ALWAYS(p) ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + if( p->iCell>=NCELL(pNode) ){ + rc = SQLITE_ABORT; + }else{ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } } return rc; } @@ -1738,6 +1743,7 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( rc ) return rc; if( NEVER(p==0) ) return SQLITE_OK; + if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else if( i<=pRtree->nDim2 ){ @@ -1835,6 +1841,8 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_OK; } +int sqlite3IntFloatCompare(i64,double); + /* ** Rtree virtual table module xFilter method. */ @@ -1864,7 +1872,8 @@ static int rtreeFilter( i64 iNode = 0; int eType = sqlite3_value_numeric_type(argv[0]); if( eType==SQLITE_INTEGER - || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) ){ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); }else{ @@ -3220,7 +3229,7 @@ constraint: static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; assert( pRtree->inWrTrans==0 ); - pRtree->inWrTrans++; + pRtree->inWrTrans = 1; return SQLITE_OK; } @@ -3234,6 +3243,9 @@ static int rtreeEndTransaction(sqlite3_vtab *pVtab){ nodeBlobReset(pRtree); return SQLITE_OK; } +static int rtreeRollback(sqlite3_vtab *pVtab){ + return rtreeEndTransaction(pVtab); +} /* ** The xRename method for rtree module virtual tables. @@ -3352,7 +3364,7 @@ static sqlite3_module rtreeModule = { rtreeBeginTransaction, /* xBegin - begin transaction */ rtreeEndTransaction, /* xSync - sync transaction */ rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeRollback, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 61664e152..e596df71d 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -797,4 +797,22 @@ do_test 23.0 { db eval {PRAGMA integrity_check;} } {ok} +reset_db +do_execsql_test 24.0 { + CREATE VIRTUAL TABLE rt1 USING rtree_i32(rid, c1, c2); + INSERT INTO rt1(rid, c1, c2) VALUES (9223372036854775807, 10, 18); +} + +do_execsql_test 24.1 { + SELECT (rid = (CAST (9223372036854775807 AS REAL))) + FROM rt1 WHERE + (rid = (CAST (9223372036854775807 AS REAL))); +} + +do_execsql_test 24.2 { + DELETE FROM rt1; + INSERT INTO rt1(rid, c1, c2) VALUES(1,2,3); + SELECT * FROM rt1 WHERE rid=1.005; +} {} + finish_test diff --git a/ext/rtree/rtreeJ.test b/ext/rtree/rtreeJ.test new file mode 100644 index 000000000..b091d2c68 --- /dev/null +++ b/ext/rtree/rtreeJ.test @@ -0,0 +1,273 @@ +# 2024-02-03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# ROLLBACK in the middle of an RTREE query +# +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set testprefix rtreeJ +ifcapable !rtree { finish_test ; return } + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1, 1, 1), (2, 2, 2); +} {} + +do_execsql_test 1.1 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + +# If a ROLLBACK occurs that backs out changes to the RTREE, then +# all pending queries to the RTREE are aborted. +# +do_test 1.2 { + db eval { + BEGIN; + INSERT INTO t1 VALUES(3, 3, 3); + INSERT INTO t1 VALUES(4, 4, 4); + } + set rc [catch { + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { ROLLBACK } + } + lappend res $id $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} + +do_execsql_test 1.3 { + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0} + +# A COMMIT of changes to the RTREE does not affect pending queries +# +do_test 1.4 { + set res {} + db eval { + BEGIN; + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(6, 6, 6); + } + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { COMMIT } + } + lappend res $id $x1 $x2 + } + set res +} {1 1.0 1.0 2 2.0 2.0 5 5.0 5.0 6 6.0 6.0} + +do_execsql_test 1.5 { + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 5 5.0 5.0 6 6.0 6.0} + +do_execsql_test 1.6 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,1,1),(2,2,2),(3,3,3),(4,4,4); + CREATE TABLE t2(x); + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0 4 4.0 4.0} + +# A rollback that does not affect the rtree table because +# the rtree table has not been written to does not cause +# a query abort. +# +do_test 1.7 { + set res {} + db eval { + BEGIN; + INSERT INTO t2(x) VALUES(12345); + } + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { ROLLBACK } + } + lappend res $id $x1 $x2 + } + set res +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0 4 4.0 4.0} + +# ROLLBACK TO that affects the RTREE does cause a query abort. +# +do_test 1.8 { + db eval { + DELETE FROM t1 WHERE rowid>1; + BEGIN; + DELETE FROM t2; + INSERT INTO t2(x) VALUES(23456); + SAVEPOINT 'one'; + INSERT INTO t1 VALUES(2,2,2),(3,3,3); + } + set rc [catch { + db eval { SELECT * FROM t1 } { + if {$id==1} { + db eval { ROLLBACK TO 'one'; } + } + lappend res $id $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} + +do_execsql_test 1.9 { + COMMIT; + SELECT * FROM t1; +} {1 1.0 1.0} + +# ROLLBACK TO that does not affect the RTREE does not cause a query abort. +# +do_execsql_test 1.10 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,1,1),(2,2,2),(3,3,3); + BEGIN; + DELETE FROM t2; + INSERT INTO t2(x) VALUES(34567); + SAVEPOINT 'one'; + INSERT INTO t2(x) VALUES('a string'); + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0} +do_test 1.11 { + set rc [catch { + set res {} + db eval { SELECT * FROM t1 } { + if {$id==2} { + # db eval { ROLLBACK TO 'one'; } + } + lappend res $id $x1 $x2 + } + set res + } msg] + list $rc $msg +} {0 {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0}} + +do_execsql_test 1.12 { + COMMIT; + SELECT * FROM t1; +} {1 1.0 1.0 2 2.0 2.0 3 3.0 3.0} + +#---------------------------------------------------------------------- + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1, 1, 1), (2, 2, 2); + CREATE TABLE t2(x); +} {} + +do_test 2.1 { + db eval { + BEGIN; + INSERT INTO t1 VALUES(3, 3, 3); + PRAGMA writable_schema = RESET; + } + + set rc [catch { + db eval { SELECT x1, x2 FROM t1 } { + if {$x1==1} { + db eval { ROLLBACK } + } + lappend res $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} + +do_execsql_test 2.1 { + CREATE TABLE bak_node(nodeno, data); + CREATE TABLE bak_parent(nodeno, parentnode); + CREATE TABLE bak_rowid(rowid, nodeno); +} +proc save_t1 {} { + db eval { + DELETE FROM bak_node; + DELETE FROM bak_parent; + DELETE FROM bak_rowid; + INSERT INTO bak_node SELECT * FROM t1_node; + INSERT INTO bak_parent SELECT * FROM t1_parent; + INSERT INTO bak_rowid SELECT * FROM t1_rowid; + } +} +proc restore_t1 {} { + db eval { + DELETE FROM t1_node; + DELETE FROM t1_parent; + DELETE FROM t1_rowid; + INSERT INTO t1_node SELECT * FROM bak_node; + INSERT INTO t1_parent SELECT * FROM bak_parent; + INSERT INTO t1_rowid SELECT * FROM bak_rowid; + } +} + +do_test 2.3 { + save_t1 + db eval { + INSERT INTO t1 VALUES(3, 3, 3); + } + set rc [catch { + db eval { SELECT rowid, x1, x2 FROM t1 } { + if {$x1==1} { + restore_t1 + } + lappend res $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} +do_execsql_test 2.4 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + +do_test 2.5 { + save_t1 + db eval { + INSERT INTO t1 VALUES(3, 3, 3); + } + set rc [catch { + db eval { SELECT x1 FROM t1 } { + if {$x1==1} { + restore_t1 + } + lappend res $x1 $x2 + } + } msg] + list $rc $msg +} {1 {query aborted}} +do_execsql_test 2.6 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + +do_test 2.7 { + save_t1 + db eval { + INSERT INTO t1 VALUES(3, 3, 3); + } + set ::res [list] + set rc [catch { + db eval { SELECT 'abc' FROM t1 } { + if {$::res==[list]} { + restore_t1 + set ::bDone 1 + } + lappend res abc + } + } msg] + set res +} {abc abc abc} +do_execsql_test 2.6 { + SELECT * FROM t1 +} {1 1.0 1.0 2 2.0 2.0} + + +finish_test diff --git a/ext/rtree/test_rtreedoc.c b/ext/rtree/test_rtreedoc.c index cdbcb2e8d..0c042d05a 100644 --- a/ext/rtree/test_rtreedoc.c +++ b/ext/rtree/test_rtreedoc.c @@ -14,11 +14,7 @@ */ #include "sqlite3.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" /* Solely for the UNUSED_PARAMETER() macro. */ #include "sqliteInt.h" @@ -90,7 +86,7 @@ static int invokeTclGeomCb( if( rc!=TCL_OK ){ rc = SQLITE_ERROR; }else{ - int nObj = 0; + Tcl_Size nObj = 0; Tcl_Obj **aObj = 0; pRes = Tcl_GetObjResult(interp); @@ -279,7 +275,7 @@ static int box_query(sqlite3_rtree_query_info *pInfo){ if( rc==SQLITE_OK ){ double rScore = 0.0; - int nObj = 0; + Tcl_Size nObj = 0; int eP = 0; Tcl_Obj **aObj = 0; Tcl_Obj *pRes = Tcl_GetObjResult(interp); diff --git a/ext/session/changesetfuzz1.test b/ext/session/changesetfuzz1.test index 20f5ac6de..d77fd53b9 100644 --- a/ext/session/changesetfuzz1.test +++ b/ext/session/changesetfuzz1.test @@ -27,7 +27,7 @@ if {$CF==""} { proc writefile {zFile data} { set fd [open $zFile w] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary puts -nonewline $fd $data close $fd } @@ -81,4 +81,3 @@ do_test 1.2 { finish_test - diff --git a/ext/session/session4.test b/ext/session/session4.test index de183a69a..55cb76f15 100644 --- a/ext/session/session4.test +++ b/ext/session/session4.test @@ -11,7 +11,10 @@ # This file implements regression tests for the session module. # -package require Tcl 8.6 +if {$tcl_version<8.6} { + puts "This module requires Tcl 8.6 or later" + return +} if {![info exists testdir]} { set testdir [file join [file dirname [info script]] .. .. test] @@ -136,7 +139,7 @@ foreach {tn blob} { do_test 2.$tn { set changeset [binary decode hex $blob] #set fd [open x.change w+] -#fconfigure $fd -encoding binary -translation binary +#fconfigure $fd -translation binary #puts -nonewline $fd $changeset #close $fd list [catch { sqlite3changeset_apply db $changeset xConflict } msg] $msg diff --git a/ext/session/sessionchange.test b/ext/session/sessionchange.test new file mode 100644 index 000000000..c1c28622e --- /dev/null +++ b/ext/session/sessionchange.test @@ -0,0 +1,101 @@ +# 2011 March 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionchange + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + set C [changeset_from_sql { + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(10, 20, 30); + }] + + set res [list] + set iter [sqlite3changeset_start $C] + while {[$iter next]=="SQLITE_ROW"} { + lappend res [$iter data] + } + $iter finalize + + set res +} [list \ + {INSERT t1 0 X.. {} {i 10 i 20 i 30}} \ + {INSERT t1 0 X.. {} {i 1 i 2 i 3}} \ +] + + +do_test 1.2 { + sqlite3changegroup grp + set iter [sqlite3changeset_start $C] + while {[$iter next]=="SQLITE_ROW"} { + grp add_change $iter + } + $iter finalize + + set res [list] + grp output + sqlite3session_foreach c [grp output] { lappend res $c } + grp delete + set res +} [list \ + {INSERT t1 0 X.. {} {i 10 i 20 i 30}} \ + {INSERT t1 0 X.. {} {i 1 i 2 i 3}} \ +] + +do_test 1.3.1 { + set iter [sqlite3changeset_start $C] + sqlite3changegroup grp + list [catch { grp add_change $iter } msg] $msg +} {1 SQLITE_ERROR} +do_test 1.3.2 { + while {[$iter next]=="SQLITE_ROW"} { } + list [catch { grp add_change $iter } msg] $msg +} {1 SQLITE_ERROR} +grp delete +$iter finalize + +do_test 1.4 { + set res [list] + set iter [sqlite3changeset_start -invert $C] + while {[$iter next]=="SQLITE_ROW"} { + lappend res [$iter data] + } + $iter finalize + set res +} [list \ + {DELETE t1 0 X.. {i 10 i 20 i 30} {}} \ + {DELETE t1 0 X.. {i 1 i 2 i 3} {}} \ +] + +do_test 1.5 { + sqlite3changegroup grp + set iter [sqlite3changeset_start -invert $C] + $iter next + list [catch { grp add_change $iter } msg] $msg +} {1 SQLITE_ERROR} +$iter finalize +grp delete + + + +finish_test diff --git a/ext/session/sessionconflict.test b/ext/session/sessionconflict.test new file mode 100644 index 000000000..25b260299 --- /dev/null +++ b/ext/session/sessionconflict.test @@ -0,0 +1,76 @@ +# 2011 March 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionconflict + +db close +sqlite3_shutdown +test_sqlite3_log log +proc log {code msg} { puts "LOG $code $msg" } +sqlite3 db test.db + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 1.0 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } +} {} + +do_execsql_test -db db2 1.1 { + INSERT INTO t1 VALUES(6, 6, 6); +} + +proc xConflict {args} { + return "ABORT" +} + + +do_test 1.2 { + set chng [changeset_from_sql { + UPDATE t1 SET b=10, c=10 WHERE a=1; + UPDATE t1 SET b=444 WHERE a=2; + INSERT INTO t1 VALUES(4, 4, 4); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(6, 6, 6); + }] + + execsql BEGIN db2 + set res [list [catch { sqlite3changeset_apply db2 $chng xConflict } msg] $msg] + execsql ROLLBACK db2 + set res +} {1 SQLITE_ABORT} + +do_execsql_test -db db2 1.3 { + SELECT * FROM t1; +} { + 1 1 1 + 2 2 2 + 3 3 3 + 6 6 6 +} + + + +finish_test diff --git a/ext/session/sessiondiff.test b/ext/session/sessiondiff.test index b00af0eb3..461ce17c9 100644 --- a/ext/session/sessiondiff.test +++ b/ext/session/sessiondiff.test @@ -47,7 +47,7 @@ proc database_cksum {db1} { proc readfile {filename} { set fd [open $filename] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set data [read $fd] close $fd set data diff --git a/ext/session/sessionfault.test b/ext/session/sessionfault.test index 96e966b41..9c1b2b999 100644 --- a/ext/session/sessionfault.test +++ b/ext/session/sessionfault.test @@ -367,6 +367,7 @@ do_execsql_test 7.prep1 { } faultsim_save_and_close +unset -nocomplain res set res [list] for {set ::i 0} {$::i < 480} {incr ::i 4} { lappend res "INSERT t1 0 X. {} {i $::i i $::i}" diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test index 2757d6044..16dc4e272 100644 --- a/ext/session/sessionstat1.test +++ b/ext/session/sessionstat1.test @@ -92,7 +92,7 @@ do_test 2.1 { } {} do_execsql_test -db db2 2.2 { - SELECT * FROM sqlite_stat1 + SELECT * FROM sqlite_stat1 ORDER BY tbl, idx } { t1 sqlite_autoindex_t1_1 {32 1} t1 t1b {32 4} @@ -104,7 +104,7 @@ do_test 2.3 { } {} do_execsql_test -db db2 2.4 { - SELECT * FROM sqlite_stat1 + SELECT * FROM sqlite_stat1 ORDER BY tbl, idx; } { t1 sqlite_autoindex_t1_1 {32 1} t1 t1b {32 4} diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 4f682a13d..7a8132bfa 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -2348,9 +2348,7 @@ void sqlite3session_delete(sqlite3_session *pSession){ ** associated hash-tables. */ sessionDeleteTable(pSession, pSession->pTable); - /* Assert that all allocations have been freed and then free the - ** session object itself. */ - // assert( pSession->nMalloc==0 ); + /* Free the session object. */ sqlite3_free(pSession); } @@ -3687,14 +3685,14 @@ static int sessionChangesetNextOne( p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + /* If the iterator is already at the end of the changeset, return DONE. */ if( p->in.iNext>=p->in.nData ){ return SQLITE_DONE; } - sessionDiscardData(&p->in); - p->in.iCurrent = p->in.iNext; - op = p->in.aData[p->in.iNext++]; while( op=='T' || op=='P' ){ if( pbNew ) *pbNew = 1; @@ -5429,6 +5427,7 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + SessionBuffer rec; sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ @@ -5727,108 +5726,128 @@ static int sessionChangesetExtendRecord( } /* -** Add all changes in the changeset traversed by the iterator passed as -** the first argument to the changegroup hash tables. +** Locate or create a SessionTable object that may be used to add the +** change currently pointed to by iterator pIter to changegroup pGrp. +** If successful, set output variable (*ppTab) to point to the table +** object and return SQLITE_OK. Otherwise, if some error occurs, return +** an SQLite error code and leave (*ppTab) set to NULL. */ -static int sessionChangesetToHash( - sqlite3_changeset_iter *pIter, /* Iterator to read from */ - sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ - int bRebase /* True if hash table is for rebasing */ +static int sessionChangesetFindTable( + sqlite3_changegroup *pGrp, + const char *zTab, + sqlite3_changeset_iter *pIter, + SessionTable **ppTab ){ - u8 *aRec; - int nRec; int rc = SQLITE_OK; SessionTable *pTab = 0; - SessionBuffer rec = {0, 0, 0}; + int nTab = (int)strlen(zTab); + u8 *abPK = 0; + int nCol = 0; - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ - const char *zNew; - int nCol; - int op; - int iHash; - int bIndirect; - SessionChange *pChange; - SessionChange *pExist = 0; - SessionChange **pp; - - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - break; - } + *ppTab = 0; + sqlite3changeset_pk(pIter, &abPK, &nCol); - sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); - if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ - /* Search the list for a matching table */ - int nNew = (int)strlen(zNew); - u8 *abPK; + /* Search the list for an existing table */ + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; + } - sqlite3changeset_pk(pIter, &abPK, 0); - for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; + /* If one was not found above, create a new table now */ + if( !pTab ){ + SessionTable **ppNew; + + pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nTab+1); + if( !pTab ){ + return SQLITE_NOMEM; + } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zTab, nTab+1); + + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + return rc; } - if( !pTab ){ - SessionTable **ppTab; + } - pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nNew+1); - if( !pTab ){ - rc = SQLITE_NOMEM; - break; - } - memset(pTab, 0, sizeof(SessionTable)); - pTab->nCol = nCol; - pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); - pTab->zName = (char*)&pTab->abPK[nCol]; - memcpy(pTab->zName, zNew, nNew+1); - - if( pGrp->db ){ - pTab->nCol = 0; - rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); - sqlite3_free(pTab); - break; - } - } + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppNew=&pGrp->pList; *ppNew; ppNew=&(*ppNew)->pNext); + *ppNew = pTab; + } - /* The new object must be linked on to the end of the list, not - ** simply added to the start of it. This is to ensure that the - ** tables within the output of sqlite3changegroup_output() are in - ** the right order. */ - for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); - *ppTab = pTab; - } + /* Check that the table is compatible. */ + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + rc = SQLITE_SCHEMA; + } - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ - rc = SQLITE_SCHEMA; - break; - } - } + *ppTab = pTab; + return rc; +} - if( nCol<pTab->nCol ){ - assert( pGrp->db ); - rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, &rec); - if( rc ) break; - aRec = rec.aBuf; - nRec = rec.nBuf; - } +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + int rc = SQLITE_OK; + int nCol = 0; + int op = 0; + int iHash = 0; + int bIndirect = 0; + SessionChange *pChange = 0; + SessionChange *pExist = 0; + SessionChange **pp = 0; + SessionTable *pTab = 0; + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; - if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ - rc = SQLITE_NOMEM; - break; - } + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + const char *zTab = 0; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK && nCol<pTab->nCol ){ + SessionBuffer *pBuf = &pGrp->rec; + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); + aRec = pBuf->aBuf; + nRec = pBuf->nBuf; + assert( pGrp->db ); + } + + if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. */ iHash = sessionChangeHash( pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); - - /* Search for existing entry. If found, remove it from the hash table. - ** Code below may link it back in. - */ for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; @@ -5843,19 +5862,41 @@ static int sessionChangesetToHash( break; } } + } + if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); - if( rc ) break; - if( pChange ){ - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; - pTab->nEntry++; - } + } + if( rc==SQLITE_OK && pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ + int bRebase /* True if hash table is for rebasing */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + + while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ + rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + if( rc!=SQLITE_OK ) break; } - sqlite3_free(rec.aBuf); if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -5984,6 +6025,23 @@ int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void *pData){ } /* +** Add a single change to a changeset-group. +*/ +int sqlite3changegroup_add_change( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter +){ + if( pIter->in.iCurrent==pIter->in.iNext + || pIter->rc!=SQLITE_OK + || pIter->bInvert + ){ + /* Iterator does not point to any valid entry or is an INVERT iterator. */ + return SQLITE_ERROR; + } + return sessionOneChangeToHash(pGrp, pIter, 0); +} + +/* ** Obtain a buffer containing a changeset representing the concatenation ** of all changesets added to the group so far. */ @@ -6032,6 +6090,7 @@ void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); + sqlite3_free(pGrp->rec.aBuf); sqlite3_free(pGrp); } } @@ -6433,6 +6492,7 @@ int sqlite3rebaser_rebase_strm( void sqlite3rebaser_delete(sqlite3_rebaser *p){ if( p ){ sessionDeleteTable(0, p->grp.pList); + sqlite3_free(p->grp.rec.aBuf); sqlite3_free(p); } } diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 160ea8786..f950d419b 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -1058,6 +1058,30 @@ int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); /* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + +/* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup ** diff --git a/ext/session/test_session.c b/ext/session/test_session.c index af42351ba..79ad000e1 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -5,14 +5,7 @@ #include "sqlite3session.h" #include <assert.h> #include <string.h> -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; @@ -517,11 +510,11 @@ struct TestConflictHandler { }; static int test_obj_eq_string(Tcl_Obj *p, const char *z){ - int n; - int nObj; + Tcl_Size n; + Tcl_Size nObj; char *zObj; - n = (int)strlen(z); + n = (Tcl_Size)strlen(z); zObj = Tcl_GetStringFromObj(p, &nObj); return (nObj==n && (n==0 || 0==memcmp(zObj, z, n))); @@ -796,7 +789,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( Tcl_CmdInfo info; /* Database Tcl command (objv[1]) info */ int rc; /* Return code from changeset_invert() */ void *pChangeset; /* Buffer containing changeset */ - int nChangeset; /* Size of buffer aChangeset in bytes */ + Tcl_Size nChangeset; /* Size of buffer aChangeset in bytes */ TestConflictHandler ctx; TestStreamInput sStr; void *pRebase = 0; @@ -853,18 +846,18 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( if( sStr.nStream==0 ){ if( bV2==0 ){ - rc = sqlite3changeset_apply(db, nChangeset, pChangeset, + rc = sqlite3changeset_apply(db, (int)nChangeset, pChangeset, (objc==5)?test_filter_handler:0, test_conflict_handler, (void *)&ctx ); }else{ - rc = sqlite3changeset_apply_v2(db, nChangeset, pChangeset, + rc = sqlite3changeset_apply_v2(db, (int)nChangeset, pChangeset, (objc==5)?test_filter_handler:0, test_conflict_handler, (void *)&ctx, &pRebase, &nRebase, flags ); } }else{ sStr.aData = (unsigned char*)pChangeset; - sStr.nData = nChangeset; + sStr.nData = (int)nChangeset; if( bV2==0 ){ rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr, (objc==5) ? test_filter_handler : 0, @@ -927,7 +920,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_apply_replace_all( Tcl_CmdInfo info; /* Database Tcl command (objv[1]) info */ int rc; /* Return code from changeset_invert() */ void *pChangeset; /* Buffer containing changeset */ - int nChangeset; /* Size of buffer aChangeset in bytes */ + Tcl_Size nChangeset; /* Size of buffer aChangeset in bytes */ if( objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB CHANGESET"); @@ -940,7 +933,8 @@ static int SQLITE_TCLAPI test_sqlite3changeset_apply_replace_all( db = *(sqlite3 **)info.objClientData; pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); - rc = sqlite3changeset_apply(db, nChangeset, pChangeset, 0, replace_handler,0); + rc = sqlite3changeset_apply(db, (int)nChangeset, pChangeset, + 0, replace_handler,0); if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); } @@ -959,6 +953,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_invert( Tcl_Obj *CONST objv[] ){ int rc; /* Return code from changeset_invert() */ + Tcl_Size nn; TestStreamInput sIn; /* Input stream */ TestSessionsBlob sOut; /* Output blob */ @@ -970,7 +965,8 @@ static int SQLITE_TCLAPI test_sqlite3changeset_invert( memset(&sIn, 0, sizeof(sIn)); memset(&sOut, 0, sizeof(sOut)); sIn.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); - sIn.aData = Tcl_GetByteArrayFromObj(objv[1], &sIn.nData); + sIn.aData = Tcl_GetByteArrayFromObj(objv[1], &nn); + sIn.nData = (int)nn; if( sIn.nStream ){ rc = sqlite3changeset_invert_strm( @@ -999,6 +995,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat( Tcl_Obj *CONST objv[] ){ int rc; /* Return code from changeset_invert() */ + Tcl_Size nn; TestStreamInput sLeft; /* Input stream */ TestStreamInput sRight; /* Input stream */ @@ -1011,8 +1008,10 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat( memset(&sLeft, 0, sizeof(sLeft)); memset(&sRight, 0, sizeof(sRight)); - sLeft.aData = Tcl_GetByteArrayFromObj(objv[1], &sLeft.nData); - sRight.aData = Tcl_GetByteArrayFromObj(objv[2], &sRight.nData); + sLeft.aData = Tcl_GetByteArrayFromObj(objv[1], &nn); + sLeft.nData = (int)nn; + sRight.aData = Tcl_GetByteArrayFromObj(objv[2], &nn); + sRight.nData = (int)nn; sLeft.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); sRight.nStream = sLeft.nStream; @@ -1038,6 +1037,64 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat( return rc; } +static Tcl_Obj *testIterData(sqlite3_changeset_iter *pIter){ + Tcl_Obj *pVar = 0; + int nCol; /* Number of columns in table */ + int nCol2; /* Number of columns in table */ + int op; /* SQLITE_INSERT, UPDATE or DELETE */ + const char *zTab; /* Name of table change applies to */ + Tcl_Obj *pOld; /* Vector of old.* values */ + Tcl_Obj *pNew; /* Vector of new.* values */ + int bIndirect; + + char *zPK; + unsigned char *abPK; + int i; + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + pVar = Tcl_NewObj(); + + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj( + op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : + "DELETE", -1 + )); + + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zTab, -1)); + Tcl_ListObjAppendElement(0, pVar, Tcl_NewBooleanObj(bIndirect)); + + zPK = ckalloc(nCol+1); + memset(zPK, 0, nCol+1); + sqlite3changeset_pk(pIter, &abPK, &nCol2); + assert( nCol==nCol2 ); + for(i=0; i<nCol; i++){ + zPK[i] = (abPK[i] ? 'X' : '.'); + } + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zPK, -1)); + ckfree(zPK); + + pOld = Tcl_NewObj(); + if( op!=SQLITE_INSERT ){ + for(i=0; i<nCol; i++){ + sqlite3_value *pVal; + sqlite3changeset_old(pIter, i, &pVal); + test_append_value(pOld, pVal); + } + } + pNew = Tcl_NewObj(); + if( op!=SQLITE_DELETE ){ + for(i=0; i<nCol; i++){ + sqlite3_value *pVal; + sqlite3changeset_new(pIter, i, &pVal); + test_append_value(pNew, pVal); + } + } + Tcl_ListObjAppendElement(0, pVar, pOld); + Tcl_ListObjAppendElement(0, pVar, pNew); + + return pVar; +} + /* ** sqlite3session_foreach VARNAME CHANGESET SCRIPT */ @@ -1048,7 +1105,7 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( Tcl_Obj *CONST objv[] ){ void *pChangeset; - int nChangeset; + Tcl_Size nChangeset; sqlite3_changeset_iter *pIter; int rc; Tcl_Obj *pVarname; @@ -1090,19 +1147,19 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( if( isInvert ){ int f = SQLITE_CHANGESETSTART_INVERT; if( sStr.nStream==0 ){ - rc = sqlite3changeset_start_v2(&pIter, nChangeset, pChangeset, f); + rc = sqlite3changeset_start_v2(&pIter, (int)nChangeset, pChangeset, f); }else{ void *pCtx = (void*)&sStr; sStr.aData = (unsigned char*)pChangeset; - sStr.nData = nChangeset; + sStr.nData = (int)nChangeset; rc = sqlite3changeset_start_v2_strm(&pIter, testStreamInput, pCtx, f); } }else{ if( sStr.nStream==0 ){ - rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); + rc = sqlite3changeset_start(&pIter, (int)nChangeset, pChangeset); }else{ sStr.aData = (unsigned char*)pChangeset; - sStr.nData = nChangeset; + sStr.nData = (int)nChangeset; rc = sqlite3changeset_start_strm(&pIter, testStreamInput, (void*)&sStr); } } @@ -1111,67 +1168,8 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( } while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ - int nCol; /* Number of columns in table */ - int nCol2; /* Number of columns in table */ - int op; /* SQLITE_INSERT, UPDATE or DELETE */ - const char *zTab; /* Name of table change applies to */ - Tcl_Obj *pVar; /* Tcl value to set $VARNAME to */ - Tcl_Obj *pOld; /* Vector of old.* values */ - Tcl_Obj *pNew; /* Vector of new.* values */ - int bIndirect; - - char *zPK; - unsigned char *abPK; - int i; - - /* Test that _fk_conflicts() returns SQLITE_MISUSE if called on this - ** iterator. */ - int nDummy; - if( SQLITE_MISUSE!=sqlite3changeset_fk_conflicts(pIter, &nDummy) ){ - sqlite3changeset_finalize(pIter); - return TCL_ERROR; - } - - sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); - pVar = Tcl_NewObj(); - Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj( - op==SQLITE_INSERT ? "INSERT" : - op==SQLITE_UPDATE ? "UPDATE" : - "DELETE", -1 - )); - - Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zTab, -1)); - Tcl_ListObjAppendElement(0, pVar, Tcl_NewBooleanObj(bIndirect)); - - zPK = ckalloc(nCol+1); - memset(zPK, 0, nCol+1); - sqlite3changeset_pk(pIter, &abPK, &nCol2); - assert( nCol==nCol2 ); - for(i=0; i<nCol; i++){ - zPK[i] = (abPK[i] ? 'X' : '.'); - } - Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zPK, -1)); - ckfree(zPK); - - pOld = Tcl_NewObj(); - if( op!=SQLITE_INSERT ){ - for(i=0; i<nCol; i++){ - sqlite3_value *pVal; - sqlite3changeset_old(pIter, i, &pVal); - test_append_value(pOld, pVal); - } - } - pNew = Tcl_NewObj(); - if( op!=SQLITE_DELETE ){ - for(i=0; i<nCol; i++){ - sqlite3_value *pVal; - sqlite3changeset_new(pIter, i, &pVal); - test_append_value(pNew, pVal); - } - } - Tcl_ListObjAppendElement(0, pVar, pOld); - Tcl_ListObjAppendElement(0, pVar, pNew); - + Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ + pVar = testIterData(pIter); Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ @@ -1238,9 +1236,9 @@ static int SQLITE_TCLAPI test_rebaser_cmd( assert( rc==SQLITE_OK ); switch( iSub ){ case 0: { /* configure */ - int nRebase = 0; + Tcl_Size nRebase = 0; unsigned char *pRebase = Tcl_GetByteArrayFromObj(objv[2], &nRebase); - rc = sqlite3rebaser_configure(p, nRebase, pRebase); + rc = sqlite3rebaser_configure(p, (int)nRebase, pRebase); break; } @@ -1251,10 +1249,12 @@ static int SQLITE_TCLAPI test_rebaser_cmd( default: { /* rebase */ TestStreamInput sStr; /* Input stream */ TestSessionsBlob sOut; /* Output blob */ + Tcl_Size nn; memset(&sStr, 0, sizeof(sStr)); memset(&sOut, 0, sizeof(sOut)); - sStr.aData = Tcl_GetByteArrayFromObj(objv[2], &sStr.nData); + sStr.aData = Tcl_GetByteArrayFromObj(objv[2], &nn); + sStr.nData = nn; sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); if( sStr.nStream ){ @@ -1393,7 +1393,7 @@ static int SQLITE_TCLAPI test_changeset( Tcl_Obj *CONST objv[] ){ void *pChangeset = 0; /* Buffer containing changeset */ - int nChangeset = 0; /* Size of buffer aChangeset in bytes */ + Tcl_Size nChangeset = 0; /* Size of buffer aChangeset in bytes */ int rc = SQLITE_OK; char *z = 0; @@ -1404,7 +1404,7 @@ static int SQLITE_TCLAPI test_changeset( pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[1], &nChangeset); Tcl_ResetResult(interp); - rc = sqlite3_test_changeset(nChangeset, pChangeset, &z); + rc = sqlite3_test_changeset((int)nChangeset, pChangeset, &z); if( rc!=SQLITE_OK ){ char *zErr = sqlite3_mprintf("(%d) - \"%s\"", rc, z); Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); @@ -1459,6 +1459,12 @@ struct TestChangegroup { sqlite3_changegroup *pGrp; }; +typedef struct TestChangeIter TestChangeIter; +struct TestChangeIter { + sqlite3_changeset_iter *pIter; +}; + + /* ** Destructor for Tcl changegroup command object. */ @@ -1491,6 +1497,7 @@ static int SQLITE_TCLAPI test_changegroup_cmd( { "add", 1, "CHANGESET", }, /* 1 */ { "output", 0, "", }, /* 2 */ { "delete", 0, "", }, /* 3 */ + { "add_change", 1, "ITERATOR", }, /* 4 */ { 0 } }; int rc = TCL_OK; @@ -1522,9 +1529,9 @@ static int SQLITE_TCLAPI test_changegroup_cmd( }; case 1: { /* add */ - int nByte = 0; + Tcl_Size nByte = 0; const u8 *aByte = Tcl_GetByteArrayFromObj(objv[2], &nByte); - rc = sqlite3changegroup_add(p->pGrp, nByte, (void*)aByte); + rc = sqlite3changegroup_add(p->pGrp, (int)nByte, (void*)aByte); if( rc!=SQLITE_OK ) rc = test_session_error(interp, rc, 0); break; }; @@ -1542,6 +1549,24 @@ static int SQLITE_TCLAPI test_changegroup_cmd( break; }; + case 4: { /* add_change */ + Tcl_CmdInfo cmdInfo; /* Database Tcl command (objv[2]) info */ + TestChangeIter *pIter = 0; + const char *zIter = Tcl_GetString(objv[2]); + if( 0==Tcl_GetCommandInfo(interp, zIter, &cmdInfo) ){ + Tcl_AppendResult(interp, "no such iter: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + + pIter = (struct TestChangeIter*)cmdInfo.objClientData; + + rc = sqlite3changegroup_add_change(p->pGrp, pIter->pIter); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + break; + }; + default: { /* delete */ assert( iSub==3 ); Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); @@ -1585,6 +1610,115 @@ static int SQLITE_TCLAPI test_sqlite3changegroup( return TCL_OK; } +extern const char *sqlite3ErrName(int); + +/* +** Destructor for Tcl iterator command object. +*/ +static void test_iter_del(void *clientData){ + TestChangeIter *p = (TestChangeIter*)clientData; + sqlite3changeset_finalize(p->pIter); + ckfree(p); +} + +static int SQLITE_TCLAPI test_iter_cmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static const char *aSub[] = { + "next", /* 0 */ + "data", /* 1 */ + "finalize", /* 2 */ + 0 + }; + int iSub = 0; + + TestChangeIter *p = (TestChangeIter*)clientData; + int rc = SQLITE_OK; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CMD"); + return TCL_ERROR; + } + + if( Tcl_GetIndexFromObj(interp, objv[1], aSub, "sub-command", 0, &iSub) ){ + return TCL_ERROR; + } + switch( iSub ){ + case 0: + rc = sqlite3changeset_next(p->pIter); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + case 1: + Tcl_SetObjResult(interp, testIterData(p->pIter)); + break; + case 2: + rc = sqlite3changeset_finalize(p->pIter); + p->pIter = 0; + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + default: + assert( 0 ); + break; + } + + return TCL_OK; +} + +/* +** Tclcmd: sqlite3changeset_start ?-invert? CHANGESET +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_start( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int isInvert = 0; + void *pChangeset = 0; /* Buffer containing changeset */ + Tcl_Size nChangeset = 0; /* Size of buffer aChangeset in bytes */ + TestChangeIter *pNew = 0; + sqlite3_changeset_iter *pIter = 0; + int flags = 0; + int rc = SQLITE_OK; + + static int iCmd = 1; + char zCmd[64]; + + if( objc==3 ){ + Tcl_Size n = 0; + const char *z = Tcl_GetStringFromObj(objv[1], &n); + isInvert = (n>=2 && sqlite3_strnicmp(z, "-invert", (int)n)==0); + } + + if( objc!=2 && (objc!=3 || !isInvert) ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-invert? CHANGESET"); + return TCL_ERROR; + } + + flags = isInvert ? SQLITE_CHANGESETSTART_INVERT : 0; + pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[objc-1], &nChangeset); + rc = sqlite3changeset_start_v2(&pIter, (int)nChangeset, pChangeset, flags); + if( rc!=SQLITE_OK ){ + char *zErr = sqlite3_mprintf( + "error in sqlite3changeset_start_v2() - %d", rc + ); + Tcl_AppendResult(interp, zErr, (char*)0); + return TCL_ERROR; + } + + pNew = (TestChangeIter*)ckalloc(sizeof(TestChangeIter)); + pNew->pIter = pIter; + + sprintf(zCmd, "csiter%d", iCmd++); + Tcl_CreateObjCommand(interp, zCmd, test_iter_cmd, (void*)pNew, test_iter_del); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1)); + return TCL_OK; +} + int TestSession_Init(Tcl_Interp *interp){ struct Cmd { const char *zCmd; @@ -1592,6 +1726,7 @@ int TestSession_Init(Tcl_Interp *interp){ } aCmd[] = { { "sqlite3session", test_sqlite3session }, { "sqlite3changegroup", test_sqlite3changegroup }, + { "sqlite3changeset_start", test_sqlite3changeset_start }, { "sqlite3session_foreach", test_sqlite3session_foreach }, { "sqlite3changeset_invert", test_sqlite3changeset_invert }, { "sqlite3changeset_concat", test_sqlite3changeset_concat }, diff --git a/ext/userauth/user-auth.txt b/ext/userauth/user-auth.txt index ba4eabc13..9d6ba2333 100644 --- a/ext/userauth/user-auth.txt +++ b/ext/userauth/user-auth.txt @@ -1,3 +1,15 @@ +*********************************** NOTICE ************************************ +* This extension is deprecated. The SQLite developers do not maintain this * +* extension. At some point in the future, it might disappear from the source * +* tree. * +* * +* If you are using this extension and think it should be supported moving * +* forward, visit the SQLite Forum (https://sqlite.org/forum) and argue your * +* case there. * +* * +* This deprecation notice was added on 2024-01-22. * +******************************************************************************* + Activate the user authentication logic by including the ext/userauth/userauth.c source code file in the build and adding the -DSQLITE_USER_AUTHENTICATION compile-time option. diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index f0cff463a..51531627b 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -43,8 +43,9 @@ # which generates the makefile code, rather than using $(call) and # $(eval), or at least centralize the setup of the numerous vars # related to each build variant $(JS_BUILD_MODES). (Update: an -# external script was attempted but it's even less legible than the -# $(eval) indirection going on in this file. +# external script was attempted but generating properly-escaped +# makefile code from within a shell script is even less legible +# than the $(eval) indirection going on in this file.) # default: all #default: quick @@ -138,8 +139,10 @@ dir.api := api dir.jacc := jaccwabyt dir.common := common dir.fiddle := fiddle +dir.fiddle-debug := fiddle-debug dir.tool := $(dir.top)/tool -CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ +CLEAN_FILES += *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ $(dir.fiddle)/*~ \ + $(dir.fiddle-debug)/* ######################################################################## # dir.dout = output dir for deliverables. @@ -230,6 +233,14 @@ SQLITE_OPT = \ # can be used to find errant uses of sqlite3_js_vfs_create_file() # in client code. +######################################################################## +# minimal=1 disables all "extraneous" stuff from sqlite3-wasm.c, the +# goal being to create a WASM file with only the core APIs. +minimal ?= 0 +ifeq (1,$(minimal)) + SQLITE_OPT += -DSQLITE_WASM_MINIMAL +endif + ########################################################################@ # It's important that sqlite3.h be built to completion before any # other parts of the build run, thus we use .NOTPARALLEL to disable @@ -288,6 +299,12 @@ endif # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the # cleanup process for the dist build to work properly. +# +# Slight caveat: this uses the version info from the in-tree +# sqlite3.c/h, which may diff from a user-provided $(sqlite3.c). The +# end result is that the generated JS files may have static version +# info from $(bin.version-info) which differ from their runtime-emited +# version info (e.g. from sqlite3_libversion()). bin.version-info := $(dir.top)/version-info .NOTPARALLEL: $(bin.version-info) $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile @@ -306,8 +323,9 @@ DISTCLEAN_FILES += $(bin.stripccomments) ######################################################################## -# C-PP.FILTER: a $(call)able to transform $(1) to $(2) via ./c-pp -f -# $(1) ... +# C-PP.FILTER: a $(call)able to transform $(1) to $(2) via: +# +# ./c-pp -f $(1) -o $(2) $(3) # # Historical notes: # @@ -333,19 +351,26 @@ DISTCLEAN_FILES += $(bin.stripccomments) # # Note that the SQLITE_... build flags used here have NO EFFECT on the # JS/WASM build. They are solely for use with $(bin.c-pp) itself. +# +# -D... flags which should be included in all invocations should be +# appended to $(C-PP.FILTER.global). bin.c-pp := ./c-pp $(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=3 +C-PP.FILTER.global ?= +ifeq (1,$(SQLITE_C_IS_SEE)) + C-PP.FILTER.global += -Denable-see +endif define C-PP.FILTER # Create $2 from $1 using $(bin.c-pp) # $1 = Input file: c-pp -f $(1).js # $2 = Output file: c-pp -o $(2).js # $3 = optional c-pp -D... flags $(2): $(1) $$(MAKEFILE) $$(bin.c-pp) - $$(bin.c-pp) -f $(1) -o $$@ $(3) + $$(bin.c-pp) -f $(1) -o $$@ $(3) $(C-PP.FILTER.global) CLEAN_FILES += $(2) endef # /end C-PP.FILTER @@ -371,8 +396,8 @@ emcc_opt_full := $(emcc_opt) -g3 # ^^^ -flto improves runtime speed at -O0 considerably but doubles # build time. # -# ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no -# way around that except to use -g3, but -g3 causes the binary file +# ^^^^ (-O3, -Oz, -Os) all minify symbol names and there appears to be +# no way around that except to use -g3, but -g3 causes the binary file # size to absolutely explode (approx. 5x larger). This minification # utterly breaks the resulting module, making it unsable except as # self-contained/self-referential-only code, as ALL of the exported @@ -396,11 +421,22 @@ emcc_opt_full := $(emcc_opt) -g3 ######################################################################## # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.main := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) -EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.main) +EXPORTED_FUNCTIONS.api.core := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core) +EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.core) ifeq (1,$(SQLITE_C_IS_SEE)) EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see) endif +ifeq (0,$(minimal)) + EXPORTED_FUNCTIONS.api.in += \ + $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-auth) \ + $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-preupdate) \ + $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-session) \ + $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-vtab) +else + $(info ========================================) + $(info This is a minimal-mode build) + $(info ========================================) +endif EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api $(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ @@ -424,19 +460,21 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js # Emscripten glue: sqlite3-api.jses += $(dir.common)/whwasmutil.js sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js -# sqlite3-api-glue.js Glues the previous part together: -sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js +# sqlite3-api-glue Glues the previous part together with sqlite: +sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js # $(sqlite3-api-build-version.js) = library version info sqlite3-api.jses += $(sqlite3-api-build-version.js) -# sqlite3-api-oo1.js = the oo1 API: -sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js -# sqlite3-api-worker.js = the Worker1 API: -sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js -# sqlite3-v-helper = helper APIs for VFSes and VTABLEs: -sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js -# sqlite3-vfs-opfs.c-pp.js = the first OPFS VFS: +# sqlite3-api-oo1 = the oo1 API: +sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js +# sqlite3-api-worker = the Worker1 API: +sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js +# sqlite3-vfs-helper = helper APIs for VFSes: +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js +# sqlite3-vtab-helper = helper APIs for VTABLEs: +sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js +# sqlite3-vfs-opfs = the first OPFS VFS: sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js -# sqlite3-vfs-opfs-sahpool.c-pp.js = the second OPFS VFS: +# sqlite3-vfs-opfs-sahpool = the second OPFS VFS: sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js # sqlite3-api-cleanup.js = "finalizes" the build and cleans up # any extraneous global symbols which are needed temporarily @@ -449,13 +487,14 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # the first OPFS VFS and necessarily an external file. SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js SOAP.js.bld := $(dir.dout)/$(notdir $(SOAP.js)) -sqlite3-api.ext.jses += $(SOAP.js.bld) +# +# $(sqlite3-api.ext.jses) = API-related files which are standalone files, +# not part of the amalgamation. +# +sqlite3-api.ext.jses := $(SOAP.js.bld) $(SOAP.js.bld): $(SOAP.js) cp $< $@ -all quick: $(sqlite3-api.ext.jses) -q: quick - ######################################################################## # $(sqlite3-api*.*js) contain the core library code but not the # Emscripten-related glue which deals with loading sqlite3.wasm. In @@ -528,6 +567,10 @@ emcc.jsflags += -sSTRICT_JS=0 # 3.1.31. The fix for that in newer emcc's is to throw a built-time # error if STRICT_JS is used together with those options. +# emcc.jsflags += -sSTRICT=1 +# -sSTRICT=1 Causes failures about unknown symbols which the build +# tools should be installing, e.g. __syscall_geteuid32 + # -sENVIRONMENT values for the various build modes: emcc.environment.vanilla := web,worker emcc.environment.bundler-friendly := $(emcc.environment.vanilla) @@ -548,6 +591,11 @@ emcc.environment.node := node # time with 16mb+ memory and 3X time when starting with 8MB. However, # such test results are inconsistent due to browser internals which # are opaque to us. +# +# 2024-03-04: emsdk 3.1.55 replaces INITIAL_MEMORY with INITIAL_HEAP, +# but also says (in its changelog): "Note that it is currently not +# supported in all configurations (#21071)." +# https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md emcc.jsflags += -sALLOW_MEMORY_GROWTH emcc.INITIAL_MEMORY.128 := 134217728 emcc.INITIAL_MEMORY.96 := 100663296 @@ -652,6 +700,9 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) ######################################################################## # $(sqlite3-license-version.js) contains the license header and # in-comment build version info. +# +# Maintenance reminder: there are awk binaries out there which do not +# support -e SCRIPT. $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) \ $(MAKEFILE) @echo "Making $@..."; { \ @@ -659,8 +710,8 @@ $(sqlite3-license-version.js): $(sqlite3.h) $(sqlite3-license-version-header.js) echo '/*'; \ echo '** This code was built from sqlite3 version...'; \ echo "**"; \ - awk -e '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' \ - -e '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ + awk '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' $(sqlite3.h); \ + awk '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo "**"; \ echo "** Using the Emscripten SDK version $(emcc.version)."; \ echo '*/'; \ @@ -810,13 +861,13 @@ pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) # $4 = resulting sqlite-api JS/MJS file # $5 = resulting JS/MJS file # $6 = -D... flags for $(bin.c-pp) -# $7 = emcc -sXYZ flags (CURRENTLY UNUSED - was factored out) +# $7 = optional extra flags for emcc # # Maintenance reminder: be careful not to introduce spaces around args # ($1, $2), otherwise string concatenation will malfunction. # -# emcc.environment.$(2) must be set to a value for emcc's -# -sENVIRONMENT flag. +# Before calling this, emcc.environment.$(2) must be set to a value +# for emcc's -sENVIRONMENT flag. # # $(cflags.$(1)) and $(cflags.$(1).$(2)) may be defined to append # CFLAGS to a given build mode. @@ -923,18 +974,39 @@ sqlite3-worker1.js.in := $(dir.api)/sqlite3-worker1.c-pp.js sqlite3-worker1-promiser.js.in := $(dir.api)/sqlite3-worker1-promiser.c-pp.js sqlite3-worker1.js := $(dir.dout)/sqlite3-worker1.js sqlite3-worker1-promiser.js := $(dir.dout)/sqlite3-worker1-promiser.js -sqlite3-worker1-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs +sqlite3-worker1-promiser.mjs := $(dir.dout)/sqlite3-worker1-promiser.mjs +sqlite3-worker1-bundler-friendly.mjs := $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs sqlite3-worker1-promiser-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js $(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1.js))) -$(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1-bundler-friendly.js),\ +$(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1-bundler-friendly.mjs),\ $(c-pp.D.sqlite3-bundler-friendly))) $(eval $(call C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),$(sqlite3-worker1-promiser.js))) $(eval $(call C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),\ $(sqlite3-worker1-promiser-bundler-friendly.js),\ $(c-pp.D.sqlite3-bundler-friendly))) -$(sqlite3-bundler-friendly.mjs): $(sqlite3-worker1-bundler-friendly.js) \ +$(eval $(call C-PP.FILTER,$(sqlite3-worker1-promiser.js.in),$(sqlite3-worker1-promiser.mjs),\ + -Dtarget=es6-module -Dtarget=es6-bundler-friendly)) +$(sqlite3-bundler-friendly.mjs): $(sqlite3-worker1-bundler-friendly.mjs) \ $(sqlite3-worker1-promiser-bundler-friendly.js) -$(sqlite3.js) $(sqlite3.mjs): $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js) +$(eval $(call C-PP.FILTER,demo-worker1-promiser.c-pp.js,demo-worker1-promiser.js)) +$(eval $(call C-PP.FILTER,demo-worker1-promiser.c-pp.js,demo-worker1-promiser.mjs,\ + -Dtarget=es6-module)) +$(eval $(call C-PP.FILTER,demo-worker1-promiser.c-pp.html,demo-worker1-promiser.html)) +$(eval $(call C-PP.FILTER,demo-worker1-promiser.c-pp.html,demo-worker1-promiser-esm.html,\ + -Dtarget=es6-module)) +all: $(sqlite3-worker1.js) \ + $(sqlite3-worker1-promiser.js) $(sqlite3-worker1-promiser.mjs) + +demo-worker1-promiser.html: $(sqlite3-worker1-promiser.js) demo-worker1-promiser.js +demo-worker1-promiser-esm.html: $(sqlite3-worker1-promiser.mjs) demo-worker1-promiser.mjs +all: demo-worker1-promiser.html demo-worker1-promiser-esm.html + +sqlite3-api.ext.jses += \ + $(sqlite3-worker1-promiser.mjs) \ + $(sqlite3-worker1-bundler-friendly.mjs) \ + $(sqlite3-worker1.js) +all quick: $(sqlite3-api.ext.jses) +q: quick ######################################################################## # batch-runner.js is part of one of the test apps which reads in SQL @@ -975,6 +1047,9 @@ emcc.speedtest1.common += -sABORTING_MALLOC emcc.speedtest1.common += -sSTRICT_JS=0 emcc.speedtest1.common += -sMODULARIZE emcc.speedtest1.common += -Wno-limited-postlink-optimizations +emcc.speedtest1.common += -Wno-unused-main +# ^^^^ -Wno-unused-main is for emcc 3.1.52+. speedtest1 has a wasm_main() which is +# exported and called by the JS code. EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) emcc.speedtest1.common += -sSTACK_SIZE=512KB emcc.speedtest1.common += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.speedtest1) @@ -1003,9 +1078,9 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.main) +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.core) @echo "Making $@ ..." - @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.main); } > $@ + @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.core); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm emcc.flags.speedtest1-vanilla := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM @@ -1021,6 +1096,7 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cfiles) \ $(emcc.speedtest1.common) \ $(emcc.flags.speedtest1-vanilla) $(pre-post-speedtest1-vanilla.flags) \ $(SQLITE_OPT) \ + -USQLITE_WASM_MINIMAL \ -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cfiles) -lm @@ -1128,7 +1204,7 @@ endif # Push files to public wasm-testing.sqlite.org server wasm-testing.include = *.js *.mjs *.html \ ./tests \ - $(dir.dout) $(dir.common) $(dir.fiddle) $(dir.jacc) + $(dir.dout) $(dir.common) $(dir.fiddle) $(dir.fiddle-debug) $(dir.jacc) wasm-testing.exclude = sql/speedtest1.sql wasm-testing.dir = /jail/sites/wasm-testing wasm-testing.dest ?= wasm-testing:$(wasm-testing.dir) @@ -1177,3 +1253,4 @@ endif # Run local web server for the test/demo pages. httpd: althttpd -max-age 1 -enable-sab 1 -page index.html + diff --git a/ext/wasm/SQLTester/SQLTester.mjs b/ext/wasm/SQLTester/SQLTester.mjs index a72399aef..71c5d4c53 100644 --- a/ext/wasm/SQLTester/SQLTester.mjs +++ b/ext/wasm/SQLTester/SQLTester.mjs @@ -174,11 +174,17 @@ const Rx = newObj({ squiggly: /[{}]/ }); + + const Util = newObj({ toss, - unlink: function(fn){ - return 0==sqlite3.wasm.sqlite3_wasm_vfs_unlink(0,fn); + unlink: function f(fn){ + if(!f.unlink){ + f.unlink = sqlite3.wasm.xWrap('sqlite3__wasm_vfs_unlink','int', + ['*','string']); + } + return 0==f.unlink(0,fn); }, argvToString: (list)=>{ @@ -197,7 +203,7 @@ const Util = newObj({ utf8Encode: (str)=>__utf8Encoder.encode(str), - strglob: sqlite3.wasm.xWrap('sqlite3_wasm_SQLTester_strglob','int', + strglob: sqlite3.wasm.xWrap('sqlite3__wasm_SQLTester_strglob','int', ['string','string']) })/*Util*/; diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth new file mode 100644 index 000000000..085090821 --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth @@ -0,0 +1 @@ +_sqlite3_set_authorizer diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core index 57cd61eb9..db47ee7db 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core @@ -10,6 +10,7 @@ _sqlite3_bind_int64 _sqlite3_bind_null _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index +_sqlite3_bind_parameter_name _sqlite3_bind_pointer _sqlite3_bind_text _sqlite3_busy_handler @@ -23,7 +24,7 @@ _sqlite3_collation_needed _sqlite3_column_blob _sqlite3_column_bytes _sqlite3_column_count -_sqlite3_column_count +_sqlite3_column_decltype _sqlite3_column_double _sqlite3_column_int _sqlite3_column_int64 @@ -40,15 +41,13 @@ _sqlite3_create_collation _sqlite3_create_collation_v2 _sqlite3_create_function _sqlite3_create_function_v2 -_sqlite3_create_module -_sqlite3_create_module_v2 _sqlite3_create_window_function _sqlite3_data_count _sqlite3_db_filename _sqlite3_db_handle _sqlite3_db_name +_sqlite3_db_readonly _sqlite3_db_status -_sqlite3_declare_vtab _sqlite3_deserialize _sqlite3_drop_modules _sqlite3_errcode @@ -65,6 +64,8 @@ _sqlite3_free _sqlite3_get_auxdata _sqlite3_get_autocommit _sqlite3_initialize +_sqlite3_interrupt +_sqlite3_is_interrupted _sqlite3_keyword_count _sqlite3_keyword_name _sqlite3_keyword_check @@ -80,12 +81,6 @@ _sqlite3_open_v2 _sqlite3_overload_function _sqlite3_prepare_v2 _sqlite3_prepare_v3 -_sqlite3_preupdate_blobwrite -_sqlite3_preupdate_count -_sqlite3_preupdate_depth -_sqlite3_preupdate_hook -_sqlite3_preupdate_new -_sqlite3_preupdate_old _sqlite3_progress_handler _sqlite3_randomness _sqlite3_realloc @@ -108,7 +103,6 @@ _sqlite3_result_zeroblob _sqlite3_result_zeroblob64 _sqlite3_rollback_hook _sqlite3_serialize -_sqlite3_set_authorizer _sqlite3_set_auxdata _sqlite3_set_last_insert_rowid _sqlite3_shutdown @@ -117,6 +111,8 @@ _sqlite3_sql _sqlite3_status _sqlite3_status64 _sqlite3_step +_sqlite3_stmt_busy +_sqlite3_stmt_explain _sqlite3_stmt_isexplain _sqlite3_stmt_readonly _sqlite3_stmt_status @@ -160,45 +156,3 @@ _sqlite3_vtab_in_next _sqlite3_vtab_nochange _sqlite3_vtab_on_conflict _sqlite3_vtab_rhs_value -_sqlite3changegroup_add -_sqlite3changegroup_add_strm -_sqlite3changegroup_delete -_sqlite3changegroup_new -_sqlite3changegroup_output -_sqlite3changegroup_output_strm -_sqlite3changeset_apply -_sqlite3changeset_apply_strm -_sqlite3changeset_apply_v2 -_sqlite3changeset_apply_v2_strm -_sqlite3changeset_concat -_sqlite3changeset_concat_strm -_sqlite3changeset_conflict -_sqlite3changeset_finalize -_sqlite3changeset_fk_conflicts -_sqlite3changeset_invert -_sqlite3changeset_invert_strm -_sqlite3changeset_new -_sqlite3changeset_next -_sqlite3changeset_old -_sqlite3changeset_op -_sqlite3changeset_pk -_sqlite3changeset_start -_sqlite3changeset_start_strm -_sqlite3changeset_start_v2 -_sqlite3changeset_start_v2_strm -_sqlite3session_attach -_sqlite3session_changeset -_sqlite3session_changeset_size -_sqlite3session_changeset_strm -_sqlite3session_config -_sqlite3session_create -_sqlite3session_delete -_sqlite3session_diff -_sqlite3session_enable -_sqlite3session_indirect -_sqlite3session_isempty -_sqlite3session_memory_used -_sqlite3session_object_config -_sqlite3session_patchset -_sqlite3session_patchset_strm -_sqlite3session_table_filter diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate new file mode 100644 index 000000000..5c57a76b6 --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate @@ -0,0 +1,6 @@ +_sqlite3_preupdate_blobwrite +_sqlite3_preupdate_count +_sqlite3_preupdate_depth +_sqlite3_preupdate_hook +_sqlite3_preupdate_new +_sqlite3_preupdate_old diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session new file mode 100644 index 000000000..5b7b53f95 --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session @@ -0,0 +1,42 @@ +_sqlite3changegroup_add +_sqlite3changegroup_add_strm +_sqlite3changegroup_delete +_sqlite3changegroup_new +_sqlite3changegroup_output +_sqlite3changegroup_output_strm +_sqlite3changeset_apply +_sqlite3changeset_apply_strm +_sqlite3changeset_apply_v2 +_sqlite3changeset_apply_v2_strm +_sqlite3changeset_concat +_sqlite3changeset_concat_strm +_sqlite3changeset_conflict +_sqlite3changeset_finalize +_sqlite3changeset_fk_conflicts +_sqlite3changeset_invert +_sqlite3changeset_invert_strm +_sqlite3changeset_new +_sqlite3changeset_next +_sqlite3changeset_old +_sqlite3changeset_op +_sqlite3changeset_pk +_sqlite3changeset_start +_sqlite3changeset_start_strm +_sqlite3changeset_start_v2 +_sqlite3changeset_start_v2_strm +_sqlite3session_attach +_sqlite3session_changeset +_sqlite3session_changeset_size +_sqlite3session_changeset_strm +_sqlite3session_config +_sqlite3session_create +_sqlite3session_delete +_sqlite3session_diff +_sqlite3session_enable +_sqlite3session_indirect +_sqlite3session_isempty +_sqlite3session_memory_used +_sqlite3session_object_config +_sqlite3session_patchset +_sqlite3session_patchset_strm +_sqlite3session_table_filter diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab new file mode 100644 index 000000000..1f6de9682 --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab @@ -0,0 +1,11 @@ +_sqlite3_create_module +_sqlite3_create_module_v2 +_sqlite3_declare_vtab +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index eb0f073cf..ebd4aaacb 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -78,10 +78,12 @@ browser client: a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-v-helper.js`**\ - Installs `sqlite3.vfs` and `sqlite3.vtab`, namespaces which contain - helpers for use by downstream code which creates `sqlite3_vfs` - and `sqlite3_module` implementations. +- **`sqlite3-vfs-helper.js`**\ + Installs the `sqlite3.vfs` namespace, which contain helpers for use + by downstream code which creates `sqlite3_vfs` implementations. +- **`sqlite3-vtab-helper.js`**\ + Installs the `sqlite3.vtab` namespace, which contain helpers for use + by downstream code which creates `sqlite3_module` implementations. - **`sqlite3-vfs-opfs.c-pp.js`**\ is an sqlite3 VFS implementation which supports the Origin-Private FileSystem (OPFS) as a storage layer to provide persistent storage diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 0e27e1fd9..7fd82a7d6 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -19,8 +19,10 @@ Module.postRun.push(function(Module/*the Emscripten-style module object*/){ - sqlite3-api-glue.js => glues previous parts together - sqlite3-api-oo.js => SQLite3 OO API #1 - sqlite3-api-worker1.js => Worker-based API - - sqlite3-vfs-helper.js => Internal-use utilities for... - - sqlite3-vfs-opfs.js => OPFS VFS + - sqlite3-vfs-helper.c-pp.js => Utilities for VFS impls + - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls + - sqlite3-vfs-opfs.c-pp.js => OPFS VFS + - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS - sqlite3-api-cleanup.js => final API cleanup - post-js-footer.js => closes this postRun() function */ diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index 29efb3e07..e5eb0cfeb 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -14,7 +14,8 @@ previous steps of the sqlite3-api.js bootstrapping process: sqlite3-api-prologue.js, whwasmutil.js, and jaccwabyt.js. It initializes the main API pieces so that the downstream components - (e.g. sqlite3-api-oo1.js) have all that they need. + (e.g. sqlite3-api-oo1.js) have all of the infrastructure that they + need. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; @@ -94,6 +95,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], + ["sqlite3_bind_parameter_name", "string", "sqlite3_stmt*", "int"], ["sqlite3_bind_pointer", "int", "sqlite3_stmt*", "int", "*", "string:static", "*"], ["sqlite3_busy_handler","int", [ @@ -114,6 +116,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], ["sqlite3_column_count", "int", "sqlite3_stmt*"], + ["sqlite3_column_decltype", "string", "sqlite3_stmt*", "int"], ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], @@ -151,6 +154,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_db_filename", "string", "sqlite3*", "string"], ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], ["sqlite3_db_name", "string", "sqlite3*", "int"], + ["sqlite3_db_readonly", "int", "sqlite3*", "string"], ["sqlite3_db_status", "int", "sqlite3*", "int", "*", "*", "int"], ["sqlite3_errcode", "int", "sqlite3*"], ["sqlite3_errmsg", "string", "sqlite3*"], @@ -191,10 +195,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_get_autocommit", "int", "sqlite3*"], ["sqlite3_get_auxdata", "*", "sqlite3_context*", "int"], ["sqlite3_initialize", undefined], - /*["sqlite3_interrupt", undefined, "sqlite3*" - ^^^ we cannot actually currently support this because JS is - single-threaded and we don't have a portable way to access a DB - from 2 SharedWorkers concurrently. ],*/ + ["sqlite3_interrupt", undefined, "sqlite3*"], + ["sqlite3_is_interrupted", "int", "sqlite3*"], ["sqlite3_keyword_count", "int"], ["sqlite3_keyword_name", "int", ["int", "**", "*"]], ["sqlite3_keyword_check", "int", ["string", "int"]], @@ -242,26 +244,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }), '*' ]], - ["sqlite3_set_authorizer", "int", [ - "sqlite3*", - new wasm.xWrap.FuncPtrAdapter({ - name: "sqlite3_set_authorizer::xAuth", - signature: "i(pi"+"ssss)", - contextKey: (argv, argIndex)=>argv[0/*(sqlite3*)*/], - callProxy: (callback)=>{ - return (pV, iCode, s0, s1, s2, s3)=>{ - try{ - s0 = s0 && wasm.cstrToJs(s0); s1 = s1 && wasm.cstrToJs(s1); - s2 = s2 && wasm.cstrToJs(s2); s3 = s3 && wasm.cstrToJs(s3); - return callback(pV, iCode, s0, s1, s2, s3) || 0; - }catch(e){ - return e.resultCode || capi.SQLITE_ERROR; - } - } - } - }), - "*"/*pUserData*/ - ]], ["sqlite3_set_auxdata", undefined, [ "sqlite3_context*", "int", "*", new wasm.xWrap.FuncPtrAdapter({ @@ -275,8 +257,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_sql", "string", "sqlite3_stmt*"], ["sqlite3_status", "int", "int", "*", "*", "int"], ["sqlite3_step", "int", "sqlite3_stmt*"], - ["sqlite3_stmt_isexplain", "int", ["sqlite3_stmt*"]], - ["sqlite3_stmt_readonly", "int", ["sqlite3_stmt*"]], + ["sqlite3_stmt_busy", "int", "sqlite3_stmt*"], + ["sqlite3_stmt_readonly", "int", "sqlite3_stmt*"], ["sqlite3_stmt_status", "int", "sqlite3_stmt*", "int", "int"], ["sqlite3_strglob", "int", "string","string"], ["sqlite3_stricmp", "int", "string", "string"], @@ -321,6 +303,38 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_vfs_unregister", "int", "sqlite3_vfs*"] ]/*wasm.bindingSignatures*/; + if( !!wasm.exports.sqlite3_stmt_explain ){ + wasm.bindingSignatures.push( + ["sqlite3_stmt_explain", "int", "sqlite3_stmt*", "int"], + ["sqlite3_stmt_isexplain", "int", "sqlite3_stmt*"] + ); + } + + if( !!wasm.exports.sqlite3_set_authorizer ){ + wasm.bindingSignatures.push( + ["sqlite3_set_authorizer", "int", [ + "sqlite3*", + new wasm.xWrap.FuncPtrAdapter({ + name: "sqlite3_set_authorizer::xAuth", + signature: "i(pi"+"ssss)", + contextKey: (argv, argIndex)=>argv[0/*(sqlite3*)*/], + callProxy: (callback)=>{ + return (pV, iCode, s0, s1, s2, s3)=>{ + try{ + s0 = s0 && wasm.cstrToJs(s0); s1 = s1 && wasm.cstrToJs(s1); + s2 = s2 && wasm.cstrToJs(s2); s3 = s3 && wasm.cstrToJs(s3); + return callback(pV, iCode, s0, s1, s2, s3) || 0; + }catch(e){ + return e.resultCode || capi.SQLITE_ERROR; + } + } + } + }), + "*"/*pUserData*/ + ]] + ); + }/* sqlite3_set_authorizer() */ + if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ /* ^^^ "the problem" is that this is an option feature and the build-time function-export list does not currently take @@ -328,7 +342,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); } - if(wasm.exports.sqlite3_activate_see instanceof Function){ +//#if enable-see + if(wasm.exports.sqlite3_key_v2 instanceof Function){ + /** + This code is capable of using an SEE build but note that an SEE + WASM build is generally incompatible with SEE's license + conditions. It is permitted for use internally in organizations + which have licensed SEE, but not for public sites because + exposing an SEE build of sqlite3.wasm effectively provides all + clients with a working copy of the commercial SEE code. + */ wasm.bindingSignatures.push( ["sqlite3_key", "int", "sqlite3*", "string", "int"], ["sqlite3_key_v2","int","sqlite3*","string","*","int"], @@ -337,10 +360,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_activate_see", undefined, "string"] ); } +//#endif enable-see + /** Functions which require BigInt (int64) support are separated from the others because we need to conditionally bind them or apply dummy impls, depending on the capabilities of the environment. + (That said: we never actually build without BigInt support, + and such builds are untested.) Note that not all of these functions directly require int64 but are only for use with APIs which require int64. For example, @@ -350,41 +377,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], ["sqlite3_changes64","i64", ["sqlite3*"]], ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], - ["sqlite3_create_module", "int", - ["sqlite3*","string","sqlite3_module*","*"]], - ["sqlite3_create_module_v2", "int", - ["sqlite3*","string","sqlite3_module*","*","*"]], - ["sqlite3_declare_vtab", "int", ["sqlite3*", "string:flexible"]], ["sqlite3_deserialize", "int", "sqlite3*", "string", "*", "i64", "i64", "int"] /* Careful! Short version: de/serialize() are problematic because they might use a different allocator than the user for managing the deserialized block. de/serialize() are ONLY safe to use with - sqlite3_malloc(), sqlite3_free(), and its 64-bit variants. */, + sqlite3_malloc(), sqlite3_free(), and its 64-bit variants. Because + of this, the canonical builds of sqlite3.wasm/js guarantee that + sqlite3.wasm.alloc() and friends use those allocators. Custom builds + may not guarantee that, however. */, ["sqlite3_drop_modules", "int", ["sqlite3*", "**"]], ["sqlite3_last_insert_rowid", "i64", ["sqlite3*"]], ["sqlite3_malloc64", "*","i64"], ["sqlite3_msize", "i64", "*"], ["sqlite3_overload_function", "int", ["sqlite3*","string","int"]], - ["sqlite3_preupdate_blobwrite", "int", "sqlite3*"], - ["sqlite3_preupdate_count", "int", "sqlite3*"], - ["sqlite3_preupdate_depth", "int", "sqlite3*"], - ["sqlite3_preupdate_hook", "*", [ - "sqlite3*", - new wasm.xWrap.FuncPtrAdapter({ - name: 'sqlite3_preupdate_hook', - signature: "v(ppippjj)", - contextKey: (argv)=>argv[0/* sqlite3* */], - callProxy: (callback)=>{ - return (p,db,op,zDb,zTbl,iKey1,iKey2)=>{ - callback(p, db, op, wasm.cstrToJs(zDb), wasm.cstrToJs(zTbl), - iKey1, iKey2); - }; - } - }), - "*" - ]], - ["sqlite3_preupdate_new", "int", ["sqlite3*", "int", "**"]], - ["sqlite3_preupdate_old", "int", ["sqlite3*", "int", "**"]], ["sqlite3_realloc64", "*","*", "i64"], ["sqlite3_result_int64", undefined, "*", "i64"], ["sqlite3_result_zeroblob64", "int", "*", "i64"], @@ -407,23 +412,59 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ "*" ]], ["sqlite3_uri_int64", "i64", ["sqlite3_filename", "string", "i64"]], - ["sqlite3_value_int64","i64", "sqlite3_value*"], - ["sqlite3_vtab_collation","string","sqlite3_index_info*","int"], - ["sqlite3_vtab_distinct","int", "sqlite3_index_info*"], - ["sqlite3_vtab_in","int", "sqlite3_index_info*", "int", "int"], - ["sqlite3_vtab_in_first", "int", "sqlite3_value*", "**"], - ["sqlite3_vtab_in_next", "int", "sqlite3_value*", "**"], - /*["sqlite3_vtab_config" is variadic and requires a hand-written - proxy.] */ - ["sqlite3_vtab_nochange","int", "sqlite3_context*"], - ["sqlite3_vtab_on_conflict","int", "sqlite3*"], - ["sqlite3_vtab_rhs_value","int", "sqlite3_index_info*", "int", "**"] + ["sqlite3_value_int64","i64", "sqlite3_value*"] ]; + if( wasm.bigIntEnabled && !!wasm.exports.sqlite3_declare_vtab ){ + wasm.bindingSignatures.int64.push( + ["sqlite3_create_module", "int", + ["sqlite3*","string","sqlite3_module*","*"]], + ["sqlite3_create_module_v2", "int", + ["sqlite3*","string","sqlite3_module*","*","*"]], + ["sqlite3_declare_vtab", "int", ["sqlite3*", "string:flexible"]], + ["sqlite3_vtab_collation","string","sqlite3_index_info*","int"], + ["sqlite3_vtab_distinct","int", "sqlite3_index_info*"], + ["sqlite3_vtab_in","int", "sqlite3_index_info*", "int", "int"], + ["sqlite3_vtab_in_first", "int", "sqlite3_value*", "**"], + ["sqlite3_vtab_in_next", "int", "sqlite3_value*", "**"], + /*["sqlite3_vtab_config" is variadic and requires a hand-written + proxy.] */ + ["sqlite3_vtab_nochange","int", "sqlite3_context*"], + ["sqlite3_vtab_on_conflict","int", "sqlite3*"], + ["sqlite3_vtab_rhs_value","int", "sqlite3_index_info*", "int", "**"] + ); + }/* virtual table APIs */ + + if(wasm.bigIntEnabled && !!wasm.exports.sqlite3_preupdate_hook){ + wasm.bindingSignatures.int64.push( + ["sqlite3_preupdate_blobwrite", "int", "sqlite3*"], + ["sqlite3_preupdate_count", "int", "sqlite3*"], + ["sqlite3_preupdate_depth", "int", "sqlite3*"], + ["sqlite3_preupdate_hook", "*", [ + "sqlite3*", + new wasm.xWrap.FuncPtrAdapter({ + name: 'sqlite3_preupdate_hook', + signature: "v(ppippjj)", + contextKey: (argv)=>argv[0/* sqlite3* */], + callProxy: (callback)=>{ + return (p,db,op,zDb,zTbl,iKey1,iKey2)=>{ + callback(p, db, op, wasm.cstrToJs(zDb), wasm.cstrToJs(zTbl), + iKey1, iKey2); + }; + } + }), + "*" + ]], + ["sqlite3_preupdate_new", "int", ["sqlite3*", "int", "**"]], + ["sqlite3_preupdate_old", "int", ["sqlite3*", "int", "**"]] + ); + } /* preupdate API */ + // Add session/changeset APIs... - if(wasm.bigIntEnabled && !!wasm.exports.sqlite3changegroup_add){ - /* ACHTUNG: 2022-12-23: the session/changeset API bindings are - COMPLETELY UNTESTED. */ + if(wasm.bigIntEnabled + && !!wasm.exports.sqlite3changegroup_add + && !!wasm.exports.sqlite3session_create + && !!wasm.exports.sqlite3_preupdate_hook /* required by the session API */){ /** FuncPtrAdapter options for session-related callbacks with the native signature "i(ps)". This proxy converts the 2nd argument @@ -601,16 +642,25 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Functions which are intended solely for API-internal use by the WASM components, not client code. These get installed into - sqlite3.wasm. Some of them get exposed to clients via variants - named sqlite3_js_...(). + sqlite3.util. Some of them get exposed to clients via variants + in sqlite3_js_...(). + + 2024-01-11: these were renamed, with two underscores in the + prefix, to ensure that clients do not accidentally depend on + them. They have always been documented as internal-use-only, so + no clients "should" be depending on the old names. */ - wasm.bindingSignatures.wasm = [ - ["sqlite3_wasm_db_reset", "int", "sqlite3*"], - ["sqlite3_wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"], - ["sqlite3_wasm_vfs_create_file", "int", - "sqlite3_vfs*","string","*", "int"], - ["sqlite3_wasm_posix_create_file", "int", "string","*", "int"], - ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] + wasm.bindingSignatures.wasmInternal = [ + ["sqlite3__wasm_db_reset", "int", "sqlite3*"], + ["sqlite3__wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"], + [/* DO NOT USE. This is deprecated since 2023-08-11 because it can + trigger assert() in debug builds when used with file sizes + which are not sizes to a multiple of a valid db page size. */ + "sqlite3__wasm_vfs_create_file", "int", "sqlite3_vfs*","string","*", "int" + ], + ["sqlite3__wasm_posix_create_file", "int", "string","*", "int"], + ["sqlite3__wasm_vfs_unlink", "int", "sqlite3_vfs*","string"], + ["sqlite3__wasm_qfmt_token","string:dealloc", "string","int"] ]; /** @@ -652,7 +702,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Use case: sqlite3_bind_pointer() and sqlite3_result_pointer() call for "a static string and preferably a string - literal". This converter is used to ensure that the string + literal." This converter is used to ensure that the string value seen by those functions is long-lived and behaves as they need it to. */ @@ -674,14 +724,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `sqlite3_vfs*` via capi.sqlite3_vfs.pointer. */ const __xArgPtr = wasm.xWrap.argAdapter('*'); - const nilType = function(){}/*a class no value can ever be an instance of*/; + const nilType = function(){ + /*a class which no value can ever be an instance of*/ + }; wasm.xWrap.argAdapter('sqlite3_filename', __xArgPtr) ('sqlite3_context*', __xArgPtr) ('sqlite3_value*', __xArgPtr) ('void*', __xArgPtr) ('sqlite3_changegroup*', __xArgPtr) ('sqlite3_changeset_iter*', __xArgPtr) - //('sqlite3_rebaser*', __xArgPtr) ('sqlite3_session*', __xArgPtr) ('sqlite3_stmt*', (v)=> __xArgPtr((v instanceof (sqlite3?.oo1?.Stmt || nilType)) @@ -689,12 +740,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ('sqlite3*', (v)=> __xArgPtr((v instanceof (sqlite3?.oo1?.DB || nilType)) ? v.pointer : v)) - ('sqlite3_index_info*', (v)=> - __xArgPtr((v instanceof (capi.sqlite3_index_info || nilType)) - ? v.pointer : v)) - ('sqlite3_module*', (v)=> - __xArgPtr((v instanceof (capi.sqlite3_module || nilType)) - ? v.pointer : v)) /** `sqlite3_vfs*`: @@ -717,6 +762,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return __xArgPtr((v instanceof (capi.sqlite3_vfs || nilType)) ? v.pointer : v); }); + if( wasm.exports.sqlite3_declare_vtab ){ + wasm.xWrap.argAdapter('sqlite3_index_info*', (v)=> + __xArgPtr((v instanceof (capi.sqlite3_index_info || nilType)) + ? v.pointer : v)) + ('sqlite3_module*', (v)=> + __xArgPtr((v instanceof (capi.sqlite3_module || nilType)) + ? v.pointer : v) + ); + } const __xRcPtr = wasm.xWrap.resultAdapter('*'); wasm.xWrap.resultAdapter('sqlite3*', __xRcPtr) @@ -742,8 +796,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ for(const e of wasm.bindingSignatures){ capi[e[0]] = wasm.xWrap.apply(null, e); } - for(const e of wasm.bindingSignatures.wasm){ - wasm[e[0]] = wasm.xWrap.apply(null, e); + for(const e of wasm.bindingSignatures.wasmInternal){ + util[e[0]] = wasm.xWrap.apply(null, e); } /* For C API functions which cannot work properly unless @@ -765,9 +819,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ implicitly making it part of the public interface. */ delete wasm.bindingSignatures; - if(wasm.exports.sqlite3_wasm_db_error){ + if(wasm.exports.sqlite3__wasm_db_error){ const __db_err = wasm.xWrap( - 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' + 'sqlite3__wasm_db_error', 'int', 'sqlite3*', 'int', 'string' ); /** Sets the given db's error state. Accepts: @@ -785,7 +839,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Returns the resulting code. Pass (pDb,0,0) to clear the error state. */ - util.sqlite3_wasm_db_error = function(pDb, resultCode, message){ + util.sqlite3__wasm_db_error = function(pDb, resultCode, message){ if(resultCode instanceof sqlite3.WasmAllocError){ resultCode = capi.SQLITE_NOMEM; message = 0 /*avoid allocating message string*/; @@ -796,17 +850,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return pDb ? __db_err(pDb, resultCode, message) : resultCode; }; }else{ - util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ - console.warn("sqlite3_wasm_db_error() is not exported.",arguments); + util.sqlite3__wasm_db_error = function(pDb,errCode,msg){ + console.warn("sqlite3__wasm_db_error() is not exported.",arguments); return errCode; }; } }/*xWrap() bindings*/ {/* Import C-level constants and structs... */ - const cJson = wasm.xCall('sqlite3_wasm_enum_json'); + const cJson = wasm.xCall('sqlite3__wasm_enum_json'); if(!cJson){ - toss("Maintenance required: increase sqlite3_wasm_enum_json()'s", + toss("Maintenance required: increase sqlite3__wasm_enum_json()'s", "static buffer size!"); } //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); @@ -877,7 +931,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ delete capi[k]; } capi.sqlite3_vtab_config = wasm.xWrap( - 'sqlite3_wasm_vtab_config','int',[ + 'sqlite3__wasm_vtab_config','int',[ 'sqlite3*', 'int', 'int'] ); }/* end vtab-related setup */ @@ -889,7 +943,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ consistency with non-special-case wrappings. */ const __dbArgcMismatch = (pDb,f,n)=>{ - return util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE, + return util.sqlite3__wasm_db_error(pDb, capi.SQLITE_MISUSE, f+"() requires "+n+" argument"+ (1===n?"":'s')+"."); }; @@ -898,7 +952,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ argument and require SQLITE_UTF8. Sets the db error code to SQLITE_FORMAT and returns that code. */ const __errEncoding = (pDb)=>{ - return util.sqlite3_wasm_db_error( + return util.sqlite3__wasm_db_error( pDb, capi.SQLITE_FORMAT, "SQLITE_UTF8 is the only supported encoding." ); }; @@ -995,12 +1049,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'sqlite3_update_hook' ]) { const x = wasm.exports[name]; + if( !x ){ + /* assume it was built without this API */ + continue; + } closeArgs.length = x.length/*==argument count*/ /* recall that undefined entries translate to 0 when passed to WASM. */; try{ capi[name](...closeArgs) } catch(e){ - console.warn("close-time call of",name+"(",closeArgs,") threw:",e); + sqlite3.config.warn("close-time call of",name+"(",closeArgs,") threw:",e); } } const m = __dbCleanupMap(pDb, 0); @@ -1051,7 +1109,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; }/*sqlite3_close_v2()*/ - if(capi.sqlite3session_table_filter){ + if(capi.sqlite3session_create){ const __sqlite3SessionDelete = wasm.xWrap( 'sqlite3session_delete', undefined, ['sqlite3_session*'] ); @@ -1128,7 +1186,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } return rc; }catch(e){ - return util.sqlite3_wasm_db_error(pDb, e); + return util.sqlite3__wasm_db_error(pDb, e); } }; @@ -1254,7 +1312,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return rc; }catch(e){ console.error("sqlite3_create_function_v2() setup threw:",e); - return util.sqlite3_wasm_db_error(pDb, e, "Creation of UDF threw: "+e); + return util.sqlite3__wasm_db_error(pDb, e, "Creation of UDF threw: "+e); } }; @@ -1299,7 +1357,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return rc; }catch(e){ console.error("sqlite3_create_window_function() setup threw:",e); - return util.sqlite3_wasm_db_error(pDb, e, "Creation of UDF threw: "+e); + return util.sqlite3__wasm_db_error(pDb, e, "Creation of UDF threw: "+e); } }; /** @@ -1394,7 +1452,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ case 'string': return __prepare.basic(pDb, xSql, xSqlLen, prepFlags, ppStmt, null); case 'number': return __prepare.full(pDb, xSql, xSqlLen, prepFlags, ppStmt, pzTail); default: - return util.sqlite3_wasm_db_error( + return util.sqlite3__wasm_db_error( pDb, capi.SQLITE_MISUSE, "Invalid SQL argument type for sqlite3_prepare_v2/v3()." ); @@ -1438,7 +1496,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }else if('string'===typeof text){ [p, n] = wasm.allocCString(text); }else{ - return util.sqlite3_wasm_db_error( + return util.sqlite3__wasm_db_error( capi.sqlite3_db_handle(pStmt), capi.SQLITE_MISUSE, "Invalid 3rd argument type for sqlite3_bind_text()." ); @@ -1446,7 +1504,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return __bindText(pStmt, iCol, p, n, capi.SQLITE_WASM_DEALLOC); }catch(e){ wasm.dealloc(p); - return util.sqlite3_wasm_db_error( + return util.sqlite3__wasm_db_error( capi.sqlite3_db_handle(pStmt), e ); } @@ -1472,7 +1530,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }else if('string'===typeof pMem){ [p, n] = wasm.allocCString(pMem); }else{ - return util.sqlite3_wasm_db_error( + return util.sqlite3__wasm_db_error( capi.sqlite3_db_handle(pStmt), capi.SQLITE_MISUSE, "Invalid 3rd argument type for sqlite3_bind_blob()." ); @@ -1480,7 +1538,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return __bindBlob(pStmt, iCol, p, n, capi.SQLITE_WASM_DEALLOC); }catch(e){ wasm.dealloc(p); - return util.sqlite3_wasm_db_error( + return util.sqlite3__wasm_db_error( capi.sqlite3_db_handle(pStmt), e ); } @@ -1504,11 +1562,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ case capi.SQLITE_CONFIG_SORTERREF_SIZE: // 28 /* int nByte */ case capi.SQLITE_CONFIG_STMTJRNL_SPILL: // 26 /* int nByte */ case capi.SQLITE_CONFIG_URI:// 17 /* int */ - return wasm.exports.sqlite3_wasm_config_i(op, args[0]); + return wasm.exports.sqlite3__wasm_config_i(op, args[0]); case capi.SQLITE_CONFIG_LOOKASIDE: // 13 /* int int */ - return wasm.exports.sqlite3_wasm_config_ii(op, args[0], args[1]); + return wasm.exports.sqlite3__wasm_config_ii(op, args[0], args[1]); case capi.SQLITE_CONFIG_MEMDB_MAXSIZE: // 29 /* sqlite3_int64 */ - return wasm.exports.sqlite3_wasm_config_j(op, args[0]); + return wasm.exports.sqlite3__wasm_config_j(op, args[0]); case capi.SQLITE_CONFIG_GETMALLOC: // 5 /* sqlite3_mem_methods* */ case capi.SQLITE_CONFIG_GETMUTEX: // 11 /* sqlite3_mutex_methods* */ case capi.SQLITE_CONFIG_GETPCACHE2: // 19 /* sqlite3_pcache_methods2* */ @@ -1529,6 +1587,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ case capi.SQLITE_CONFIG_SQLLOG: // 21 /* xSqllog, void* */ case capi.SQLITE_CONFIG_WIN32_HEAPSIZE: // 23 /* int nByte */ default: + /* maintenance note: we specifically do not include + SQLITE_CONFIG_ROWID_IN_VIEW here, on the grounds that + it's only for legacy support and no apps written with + this API require that. */ return capi.SQLITE_NOTFOUND; } }; @@ -1574,11 +1636,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if( pKvvfs ){/* kvvfs-specific glue */ if(util.isUIThread()){ const kvvfsMethods = new capi.sqlite3_kvvfs_methods( - wasm.exports.sqlite3_wasm_kvvfs_methods() + wasm.exports.sqlite3__wasm_kvvfs_methods() ); delete capi.sqlite3_kvvfs_methods; - const kvvfsMakeKey = wasm.exports.sqlite3_wasm_kvvfsMakeKeyOnPstack, + const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKeyOnPstack, pstack = wasm.pstack; const kvvfsStorage = (zClass)=> @@ -1587,7 +1649,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Implementations for members of the object referred to by - sqlite3_wasm_kvvfs_methods(). We swap out the native + sqlite3__wasm_kvvfs_methods(). We swap out the native implementations with these, which use localStorage or sessionStorage for their backing store. */ @@ -1667,5 +1729,181 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }/*pKvvfs*/ + /* Warn if client-level code makes use of FuncPtrAdapter. */ wasm.xWrap.FuncPtrAdapter.warnOnUse = true; + + const StructBinder = sqlite3.StructBinder + /* we require a local alias b/c StructBinder is removed from the sqlite3 + object during the final steps of the API cleanup. */; + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructBinder.StructType + target object. + + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error, e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc. + + As a special case, if the given function is a pointer, then + `wasm.functionEntry()` is used to validate that it is a known + function. If so, it is used as-is with no extra level of proxying + or cleanup, else an exception is thrown. It is legal to pass a + value of 0, indicating a NULL pointer, with the caveat that 0 + _is_ a legal function pointer in WASM but it will not be accepted + as such _here_. (Justification: the function at address zero must + be one which initially came from the WASM module, not a method we + want to bind to a virtual table or VFS.) + + This function returns a proxy for itself which is bound to tgt + and takes 2 args (name,func). That function returns the same + thing as this one, permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + + ACHTUNG: because we cannot generically know how to transform JS + exceptions into result codes, the installed functions do no + automatic catching of exceptions. It is critical, to avoid + undefined behavior in the C layer, that methods mapped via + this function do not throw. The exception, as it were, to that + rule is... + + If applyArgcCheck is true then each JS function (as opposed to + function pointers) gets wrapped in a proxy which asserts that it + is passed the expected number of arguments, throwing if the + argument count does not match expectations. That is only intended + for dev-time usage for sanity checking, and may leave the C + environment in an undefined state. + */ + const installMethod = function callee( + tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck + ){ + if(!(tgt instanceof StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + }else if(!(func instanceof Function) && !wasm.isPtr(func)){ + toss("Usage errror: expecting a Function or WASM pointer to one."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + } + if(!callee.argcProxy){ + callee.argcProxy = function(tgt, funcName, func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch for", + tgt.structInfo.name+"::"+funcName + +": Native signature is:",sig); + } + return func.apply(this, args); + } + }; + /* An ondispose() callback for use with + StructBinder-created types. */ + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name,"does not have a function pointer signature:",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = (applyArgcCheck && !wasm.isPtr(func)) + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(tgt, memKey, func, sigN) + : func; + if(wasm.isPtr(fProxy)){ + if(fProxy && !wasm.functionEntry(fProxy)){ + toss("Pointer",fProxy,"is not a WASM function table entry."); + } + tgt[memKey] = fProxy; + }else{ + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ + tgt.addOnDispose('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + } + return (n,f)=>callee(tgt, n, f, applyArgcCheck); + }/*installMethod*/; + installMethod.installMethodArgcCheck = false; + + /** + Installs methods into the given StructBinder.StructType-type + instance. Each entry in the given methods object must map to a + known member of the given StructType, else an exception will be + triggered. See installMethod() for more details, including the + semantics of the 3rd argument. + + As an exception to the above, if any two or more methods in the + 2nd argument are the exact same function, installMethod() is + _not_ called for the 2nd and subsequent instances, and instead + those instances get assigned the same method pointer which is + created for the first instance. This optimization is primarily to + accommodate special handling of sqlite3_module::xConnect and + xCreate methods. + + On success, returns its first argument. Throws on error. + */ + const installMethods = function( + structInstance, methods, applyArgcCheck = installMethod.installMethodArgcCheck + ){ + const seen = new Map /* map of <Function, memberName> */; + for(const k of Object.keys(methods)){ + const m = methods[k]; + const prior = seen.get(m); + if(prior){ + const mkey = structInstance.memberKey(k); + structInstance[mkey] = structInstance[structInstance.memberKey(prior)]; + }else{ + installMethod(structInstance, k, m, applyArgcCheck); + seen.set(m, k); + } + } + return structInstance; + }; + + /** + Equivalent to calling installMethod(this,...arguments) with a + first argument of this object. If called with 1 or 2 arguments + and the first is an object, it's instead equivalent to calling + installMethods(this,...arguments). + */ + StructBinder.StructType.prototype.installMethod = function callee( + name, func, applyArgcCheck = installMethod.installMethodArgcCheck + ){ + return (arguments.length < 3 && name && 'object'===typeof name) + ? installMethods(this, ...arguments) + : installMethod(this, ...arguments); + }; + + /** + Equivalent to calling installMethods() with a first argument + of this object. + */ + StructBinder.StructType.prototype.installMethods = function( + methods, applyArgcCheck = installMethod.installMethodArgcCheck + ){ + return installMethods(this, methods, applyArgcCheck); + }; + }); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index 160d59db5..3d6a24c77 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -82,10 +82,111 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** A map of sqlite3_vfs pointers to SQL code or a callback function to run when the DB constructor opens a database with the given - VFS. In the latter case, the call signature is (theDbObject,sqlite3Namespace) - and the callback is expected to throw on error. + VFS. In the latter case, the call signature is + (theDbObject,sqlite3Namespace) and the callback is expected to + throw on error. */ - const __vfsPostOpenSql = Object.create(null); + const __vfsPostOpenCallback = Object.create(null); + +//#if enable-see + /** + Converts ArrayBuffer or Uint8Array ba into a string of hex + digits. + */ + const byteArrayToHex = function(ba){ + if( ba instanceof ArrayBuffer ){ + ba = new Uint8Array(ba); + } + const li = []; + const digits = "0123456789abcdef"; + for( const d of ba ){ + li.push( digits[(d & 0xf0) >> 4], digits[d & 0x0f] ); + } + return li.join(''); + }; + + /** + Internal helper to apply an SEE key to a just-opened + database. Requires that db be-a DB object which has just been + opened, opt be the options object processed by its ctor, and opt + must have either the key, hexkey, or textkey properties, either + as a string, an ArrayBuffer, or a Uint8Array. + + This is a no-op in non-SEE builds. It throws on error and returns + without side effects if none of the key/textkey/hexkey options + are set. It throws if more than one is set or if any are set to + values of an invalid type. + + Returns true if it applies the key, else an unspecified falsy + value. Note that applying the key does not imply that the key is + correct, only that it was passed on to the db. + */ + const dbCtorApplySEEKey = function(db,opt){ + if( !capi.sqlite3_key_v2 ) return; + let keytype; + let key; + const check = (opt.key ? 1 : 0) + (opt.hexkey ? 1 : 0) + (opt.textkey ? 1 : 0); + if( !check ) return; + else if( check>1 ){ + toss3(capi.SQLITE_MISUSE, + "Only ONE of (key, hexkey, textkey) may be provided."); + } + if( opt.key ){ + /* It is not legal to bind an argument to PRAGMA key=?, so we + convert it to a hexkey... */ + keytype = 'key'; + key = opt.key; + if('string'===typeof key){ + key = new TextEncoder('utf-8').encode(key); + } + if((key instanceof ArrayBuffer) || (key instanceof Uint8Array)){ + key = byteArrayToHex(key); + keytype = 'hexkey'; + }else{ + toss3(capi.SQLITE_MISUSE, + "Invalid value for the 'key' option. Expecting a string,", + "ArrayBuffer, or Uint8Array."); + return; + } + }else if( opt.textkey ){ + /* For textkey we need it to be in string form, so convert it to + a string if it's a byte array... */ + keytype = 'textkey'; + key = opt.textkey; + if(key instanceof ArrayBuffer){ + key = new Uint8Array(key); + } + if(key instanceof Uint8Array){ + key = new TextDecoder('utf-8').decode(key); + }else if('string'!==typeof key){ + toss3(capi.SQLITE_MISUSE, + "Invalid value for the 'textkey' option. Expecting a string,", + "ArrayBuffer, or Uint8Array."); + } + }else if( opt.hexkey ){ + keytype = 'hexkey'; + key = opt.hexkey; + if((key instanceof ArrayBuffer) || (key instanceof Uint8Array)){ + key = byteArrayToHex(key); + }else if('string'!==typeof key){ + toss3(capi.SQLITE_MISUSE, + "Invalid value for the 'hexkey' option. Expecting a string,", + "ArrayBuffer, or Uint8Array."); + } + /* else assume it's valid hex codes */ + }else{ + return; + } + let stmt; + try{ + stmt = db.prepare("PRAGMA "+keytype+"="+util.sqlite3__wasm_qfmt_token(key, 1)); + stmt.step(); + return true; + }finally{ + if(stmt) stmt.finalize(); + } + }; +//#endif enable-see /** A proxy for DB class constructors. It must be called with the @@ -175,16 +276,28 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); try{ +//#if enable-see + dbCtorApplySEEKey(this,opt); +//#endif // Check for per-VFS post-open SQL/callback... - const pVfs = capi.sqlite3_js_db_vfs(pDb); - if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); - const postInitSql = __vfsPostOpenSql[pVfs]; - if(postInitSql instanceof Function){ - postInitSql(this, sqlite3); - }else if(postInitSql){ - checkSqlite3Rc( - pDb, capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0) - ); + const pVfs = capi.sqlite3_js_db_vfs(pDb) + || toss3("Internal error: cannot get VFS for new db handle."); + const postInitSql = __vfsPostOpenCallback[pVfs]; + if(postInitSql){ + /** + Reminder: if this db is encrypted and the client did _not_ pass + in the key, any init code will fail, causing the ctor to throw. + We don't actually know whether the db is encrypted, so we cannot + sensibly apply any heuristics which skip the init code only for + encrypted databases for which no key has yet been supplied. + */ + if(postInitSql instanceof Function){ + postInitSql(this, sqlite3); + }else{ + checkSqlite3Rc( + pDb, capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0) + ); + } } }catch(e){ this.close(); @@ -193,18 +306,28 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; /** - Sets SQL which should be exec()'d on a DB instance after it is - opened with the given VFS pointer. The SQL may be any type - supported by the "string:flexible" function argument conversion. - Alternately, the 2nd argument may be a function, in which case it - is called with (theOo1DbObject,sqlite3Namespace) at the end of - the DB() constructor. The function must throw on error, in which - case the db is closed and the exception is propagated. This - function is intended only for use by DB subclasses or sqlite3_vfs + Sets a callback which should be called after a db is opened with + the given sqlite3_vfs pointer. The 2nd argument must be a + function, which gets called with + (theOo1DbObject,sqlite3Namespace) at the end of the DB() + constructor. The function must throw on error, in which case the + db is closed and the exception is propagated. This function is + intended only for use by DB subclasses or sqlite3_vfs implementations. + + Prior to 2024-07-22, it was legal to pass SQL code as the second + argument, but that can interfere with a client's ability to run + pragmas which must be run before anything else, namely (pragma + locking_mode=exclusive) for use with WAL mode. That capability + had only ever been used as an internal detail of the two OPFS + VFSes, and they no longer use it that way. */ - dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ - __vfsPostOpenSql[pVfs] = sql; + dbCtorHelper.setVfsPostOpenCallback = function(pVfs, callback){ + if( !(callback instanceof Function)){ + toss3("dbCtorHelper.setVfsPostOpenCallback() should not be used with "+ + "a non-function argument.",arguments); + } + __vfsPostOpenCallback[pVfs] = callback; }; /** @@ -280,6 +403,36 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `flags`: open-mode flags - `vfs`: the VFS fname +//#if enable-see + + SEE-capable builds optionally support ONE of the following + additional options: + + - `key`, `hexkey`, or `textkey`: encryption key as a string, + ArrayBuffer, or Uint8Array. These flags function as documented + for the SEE pragmas of the same names. Using a byte array for + `hexkey` is equivalent to the same series of hex codes in + string form, so `'666f6f'` is equivalent to + `Uint8Array([0x66,0x6f,0x6f])`. A `textkey` byte array is + assumed to be UTF-8. A `key` string is transformed into a UTF-8 + byte array, and a `key` byte array is transformed into a + `hexkey` with the same bytes. + + In non-SEE builds, these options are ignored. In SEE builds, + `PRAGMA key/textkey/hexkey=X` is executed immediately after + opening the db. If more than one of the options is provided, + or any option has an invalid argument type, an exception is + thrown. + + Note that some DB subclasses may run post-initialization SQL + code, e.g. to set a busy-handler timeout or tweak the page cache + size. Such code is run _after_ the SEE key is applied. If no key + is supplied and the database is encrypted, execution of the + post-initialization SQL will fail, causing the constructor to + throw. + +//#endif enable-see + The `filename` and `vfs` arguments may be either JS strings or C-strings allocated via WASM. `flags` is required to be a JS string (because it's specific to this API, which is specific @@ -288,7 +441,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ For purposes of passing a DB instance to C-style sqlite3 functions, the DB object's read-only `pointer` property holds its `sqlite3*` pointer value. That property can also be used to check - whether this DB instance is still open. + whether this DB instance is still open: it will evaluate to + `undefined` after the DB object's close() method is called. In the main window thread, the filenames `":localStorage:"` and `":sessionStorage:"` are special: they cause the db to use either @@ -341,6 +495,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ statement's preparation and when it is stepped may invalidate it. - `parameterCount`: the number of bindable parameters in the query. + + As a general rule, most methods of this class will throw if + called on an instance which has been finalized. For brevity's + sake, the method docs do not all repeat this warning. */ const Stmt = function(){ if(BindTypes!==arguments[2]){ @@ -433,40 +591,56 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ out.returnVal = ()=>opt.resultRows; } if(opt.callback || opt.resultRows){ - switch((undefined===opt.rowMode) - ? 'array' : opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; - case 'array': out.cbArg = (stmt)=>stmt.get([]); break; - case 'stmt': - if(Array.isArray(opt.resultRows)){ - toss3("exec(): invalid rowMode for a resultRows array: must", - "be one of 'array', 'object',", - "a result column number, or column name reference."); - } - out.cbArg = (stmt)=>stmt; + switch((undefined===opt.rowMode) ? 'array' : opt.rowMode) { + case 'object': + out.cbArg = (stmt,cache)=>{ + if( !cache.columnNames ) cache.columnNames = stmt.getColumnNames([]); + /* https://sqlite.org/forum/forumpost/3632183d2470617d: + conversion of rows to objects (key/val pairs) is + somewhat expensive for large data sets because of the + native-to-JS conversion of the column names. If we + instead cache the names and build objects from that + list of strings, it can run twice as fast. The + difference is not noticeable for small data sets but + becomes human-perceivable when enough rows are + involved. */ + const row = stmt.get([]); + const rv = Object.create(null); + for( const i in cache.columnNames ) rv[cache.columnNames[i]] = row[i]; + return rv; + }; + break; + case 'array': out.cbArg = (stmt)=>stmt.get([]); break; + case 'stmt': + if(Array.isArray(opt.resultRows)){ + toss3("exec(): invalid rowMode for a resultRows array: must", + "be one of 'array', 'object',", + "a result column number, or column name reference."); + } + out.cbArg = (stmt)=>stmt; + break; + default: + if(util.isInt32(opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(opt.rowMode); break; - default: - if(util.isInt32(opt.rowMode)){ - out.cbArg = (stmt)=>stmt.get(opt.rowMode); - break; - }else if('string'===typeof opt.rowMode - && opt.rowMode.length>1 - && '$'===opt.rowMode[0]){ - /* "$X": fetch column named "X" (case-sensitive!). Prior - to 2022-12-14 ":X" and "@X" were also permitted, but - having so many options is unnecessary and likely to - cause confusion. */ - const $colName = opt.rowMode.substr(1); - out.cbArg = (stmt)=>{ - const rc = stmt.get(Object.create(null))[$colName]; - return (undefined===rc) - ? toss3(capi.SQLITE_NOTFOUND, - "exec(): unknown result column:",$colName) - : rc; - }; - break; - } - toss3("Invalid rowMode:",opt.rowMode); + }else if('string'===typeof opt.rowMode + && opt.rowMode.length>1 + && '$'===opt.rowMode[0]){ + /* "$X": fetch column named "X" (case-sensitive!). Prior + to 2022-12-14 ":X" and "@X" were also permitted, but + having so many options is unnecessary and likely to + cause confusion. */ + const $colName = opt.rowMode.substr(1); + out.cbArg = (stmt)=>{ + const rc = stmt.get(Object.create(null))[$colName]; + return (undefined===rc) + ? toss3(capi.SQLITE_NOTFOUND, + "exec(): unknown result column:",$colName) + : rc; + }; + break; + } + toss3("Invalid rowMode:",opt.rowMode); } } return out; @@ -884,10 +1058,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ and names. */) ? 0 : 1; evalFirstResult = false; if(arg.cbArg || resultRows){ + const cbArgCache = Object.create(null) + /* 2nd arg for arg.cbArg, used by (at least) row-to-object + converter */; for(; stmt.step(); stmt._lockedByExec = false){ - if(0===gotColNames++) stmt.getColumnNames(opt.columnNames); + if(0===gotColNames++){ + stmt.getColumnNames(cbArgCache.columnNames = (opt.columnNames || [])); + } stmt._lockedByExec = true; - const row = arg.cbArg(stmt); + const row = arg.cbArg(stmt,cbArgCache); if(resultRows) resultRows.push(row); if(callback && false === callback.call(opt, row, stmt)){ break; @@ -1522,14 +1701,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ they are larger than 32 bits, else double or int32, depending on whether they have a fractional part. Booleans are bound as integer 0 or 1. It is not expected the distinction of binding - doubles which have no fractional parts is integers is + doubles which have no fractional parts and integers is significant for the majority of clients due to sqlite3's data - typing model. If [BigInt] support is enabled then this - routine will bind BigInt values as 64-bit integers if they'll - fit in 64 bits. If that support disabled, it will store the - BigInt as an int32 or a double if it can do so without loss - of precision. If the BigInt is _too BigInt_ then it will - throw. + typing model. If BigInt support is enabled then this routine + will bind BigInt values as 64-bit integers if they'll fit in + 64 bits. If that support disabled, it will store the BigInt + as an int32 or a double if it can do so without loss of + precision. If the BigInt is _too BigInt_ then it will throw. - Strings are bound as strings (use bindAsBlob() to force blob binding). @@ -1635,9 +1813,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, /** Steps the statement one time. If the result indicates that a - row of data is available, a truthy value is returned. - If no row of data is available, a falsy - value is returned. Throws on error. + row of data is available, a truthy value is returned. If no + row of data is available, a falsy value is returned. Throws on + error. */ step: function(){ affirmNotLockedByExec(this, 'step()'); @@ -1657,6 +1835,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Functions exactly like step() except that... 1) On success, it calls this.reset() and returns this object. + 2) On error, it throws and does not call reset(). This is intended to simplify constructs like: @@ -1680,7 +1859,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ throws. On success, it returns true if the step indicated that a row of - data was available, else it returns false. + data was available, else it returns a falsy value. This is intended to simplify use cases such as: @@ -1698,6 +1877,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ catch(e){/*ignored*/} } }, + /** Fetches the value from the given 0-based column index of the current data row, throwing if index is out of range. @@ -1706,7 +1886,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ an exception is thrown. By default it will determine the data type of the result - automatically. If passed a second arugment, it must be one + automatically. If passed a second argument, it must be one of the enumeration values for sqlite3 types, which are defined as members of the sqlite3 module: SQLITE_INTEGER, SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB. Any other value, @@ -1722,7 +1902,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ If ndx is a plain object, this function behaves even differentlier: it assigns the properties of the object to - the values of their corresponding result columns. + the values of their corresponding result columns and returns + that object. Blobs are returned as Uint8Array instances. @@ -1868,6 +2049,39 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return (affirmStmtOpen(this).parameterCount ? capi.sqlite3_bind_parameter_index(this.pointer, name) : undefined); + }, + /** + If this statement has named bindable parameters and the given + index refers to one, its name is returned, else null is + returned. If this statement has no bound parameters, undefined + is returned. + + Added in 3.47. + */ + getParamName: function(ndx){ + return (affirmStmtOpen(this).parameterCount + ? capi.sqlite3_bind_parameter_name(this.pointer, ndx) + : undefined); + }, + + /** + Behaves like sqlite3_stmt_busy() but throws if this statement + is closed and returns a value of type boolean instead of integer. + + Added in 3.47. + */ + isBusy: function(){ + return 0!==capi.sqlite3_stmt_busy(affirmStmtOpen(this)); + }, + + /** + Behaves like sqlite3_stmt_readonly() but throws if this statement + is closed and returns a value of type boolean instead of integer. + + Added in 3.47. + */ + isReadOnly: function(){ + return 0!==capi.sqlite3_stmt_readonly(affirmStmtOpen(this)); } }/*Stmt.prototype*/; @@ -1906,16 +2120,26 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Functionally equivalent to DB(storageName,'c','kvvfs') except that it throws if the given storage name is not one of 'local' or 'session'. + + As of version 3.46, the argument may optionally be an options + object in the form: + + { + filename: 'session'|'local', + ... etc. (all options supported by the DB ctor) + } + + noting that the 'vfs' option supported by main DB + constructor is ignored here: the vfs is always 'kvvfs'. */ sqlite3.oo1.JsStorageDb = function(storageName='session'){ + const opt = dbCtorHelper.normalizeArgs(...arguments); + storageName = opt.filename; if('session'!==storageName && 'local'!==storageName){ toss3("JsStorageDb db name must be one of 'session' or 'local'."); } - dbCtorHelper.call(this, { - filename: storageName, - flags: 'c', - vfs: "kvvfs" - }); + opt.vfs = 'kvvfs'; + dbCtorHelper.call(this, opt); }; const jdb = sqlite3.oo1.JsStorageDb; jdb.prototype = Object.create(DB.prototype); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index ef1154f6b..c8db3698c 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -37,7 +37,7 @@ This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so - that it can be used without any _direct_ dependency on + that it can be used without any direct dependency on Emscripten. (Note the default values for the config object!) The config object is only honored the first time this is called. Subsequent calls ignore the argument and return the same @@ -98,14 +98,37 @@ The returned object is the top-level sqlite3 namespace object. + + Client code may optionally assign sqlite3ApiBootstrap.defaultConfig + an object-type value before calling sqlite3ApiBootstrap() (without + arguments) in order to tell that call to use this object as its + default config value. The intention of this is to provide + downstream clients with a reasonably flexible approach for plugging + in an environment-suitable configuration without having to define a + new global-scope symbol. + + However, because clients who access this library via an + Emscripten-hosted module will not have an opportunity to call + sqlite3ApiBootstrap() themselves, nor to access it before it is + called, an alternative option for setting the configuration is to + define globalThis.sqlite3ApiConfig to an object. If it is set, it + is used instead of sqlite3ApiBootstrap.defaultConfig if + sqlite3ApiBootstrap() is called without arguments. + + Both sqlite3ApiBootstrap.defaultConfig and + globalThis.sqlite3ApiConfig get deleted by sqlite3ApiBootstrap() + because any changes to them made after that point would have no + useful effect. */ 'use strict'; globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( apiConfig = (globalThis.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) ){ if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ - console.warn("sqlite3ApiBootstrap() called multiple times.", - "Config and external initializers are ignored on calls after the first."); + (sqlite3ApiBootstrap.sqlite3.config || console).warn( + "sqlite3ApiBootstrap() called multiple times.", + "Config and external initializers are ignored on calls after the first." + ); return sqlite3ApiBootstrap.sqlite3; } const config = Object.assign(Object.create(null),{ @@ -114,8 +137,16 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( bigIntEnabled: (()=>{ if('undefined'!==typeof Module){ /* Emscripten module will contain HEAPU64 when built with - -sWASM_BIGINT=1, else it will not. */ - return !!Module.HEAPU64; + -sWASM_BIGINT=1, else it will not. + + As of emsdk 3.1.55, when building in strict mode, HEAPxyz + are only available if _explicitly_ included in the exports, + else they are not. We do not (as of 2024-03-04) use -sSTRICT + for the canonical builds. + */ + if( !!Module.HEAPU64 ) return true; + /* Else fall through and hope for the best. Nobody _really_ + builds this without BigInt support, do they? */ } return !!globalThis.BigInt64Array; })(), @@ -149,6 +180,15 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( config[k] = config[k](); } }); + + /** + Eliminate any confusion about whether these config objects may + be used after library initialization by eliminating the outward-facing + objects... + */ + delete globalThis.sqlite3ApiConfig; + delete sqlite3ApiBootstrap.defaultConfig; + /** The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members @@ -205,7 +245,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( The exception's message is created by concatenating its arguments with a space between each, except for the - two-args-with-an-objec form and that the first argument will + two-args-with-an-object form and that the first argument will get coerced to a string, as described above, if it's an integer. @@ -1061,7 +1101,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( are undefined if the passed-in value did not come from this.pointer. */ - restore: wasm.exports.sqlite3_wasm_pstack_restore, + restore: wasm.exports.sqlite3__wasm_pstack_restore, /** Attempts to allocate the given number of bytes from the pstack. On success, it zeroes out a block of memory of the @@ -1083,7 +1123,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( if('string'===typeof n && !(n = wasm.sizeofIR(n))){ WasmAllocError.toss("Invalid value for pstack.alloc(",arguments[0],")"); } - return wasm.exports.sqlite3_wasm_pstack_alloc(n) + return wasm.exports.sqlite3__wasm_pstack_alloc(n) || WasmAllocError.toss("Could not allocate",n, "bytes from the pstack."); }, @@ -1163,10 +1203,10 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( */ pointer: { configurable: false, iterable: true, writeable: false, - get: wasm.exports.sqlite3_wasm_pstack_ptr + get: wasm.exports.sqlite3__wasm_pstack_ptr //Whether or not a setter as an alternative to restore() is //clearer or would just lead to confusion is unclear. - //set: wasm.exports.sqlite3_wasm_pstack_restore + //set: wasm.exports.sqlite3__wasm_pstack_restore }, /** sqlite3.wasm.pstack.quota to the total number of bytes @@ -1175,7 +1215,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( */ quota: { configurable: false, iterable: true, writeable: false, - get: wasm.exports.sqlite3_wasm_pstack_quota + get: wasm.exports.sqlite3__wasm_pstack_quota }, /** sqlite3.wasm.pstack.remaining resolves to the amount of space @@ -1183,7 +1223,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( */ remaining: { configurable: false, iterable: true, writeable: false, - get: wasm.exports.sqlite3_wasm_pstack_remaining + get: wasm.exports.sqlite3__wasm_pstack_remaining } })/*wasm.pstack properties*/; @@ -1238,9 +1278,10 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( option is not available on the client. Though the mount point name returned by this function is intended - to remain stable, clients should not hard-coded it anywhere. Always call this function to get the path. + to remain stable, clients should not hard-coded it + anywhere. Always call this function to get the path. - Note that this function is a no-op in must builds of this + Note that this function is a no-op in most builds of this library, as the WASMFS capability requires a custom build. */ @@ -1256,14 +1297,14 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( } try{ if(pdir && 0===wasm.xCallWrapped( - 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + 'sqlite3__wasm_init_wasmfs', 'i32', ['string'], pdir )){ return __wasmfsOpfsDir = pdir; }else{ return __wasmfsOpfsDir = ""; } }catch(e){ - // sqlite3_wasm_init_wasmfs() is not available + // sqlite3__wasm_init_wasmfs() is not available return __wasmfsOpfsDir = ""; } }; @@ -1365,7 +1406,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( const zSchema = schema ? (wasm.isPtr(schema) ? schema : wasm.scopedAllocCString(''+schema)) : 0; - let rc = wasm.exports.sqlite3_wasm_db_serialize( + let rc = wasm.exports.sqlite3__wasm_db_serialize( pDb, zSchema, ppOut, pSize, 0 ); if(rc){ @@ -1391,7 +1432,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( or not provided, then "main" is assumed. */ capi.sqlite3_js_db_vfs = - (dbPointer, dbName=0)=>wasm.sqlite3_wasm_db_vfs(dbPointer, dbName); + (dbPointer, dbName=0)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); /** A thin wrapper around capi.sqlite3_aggregate_context() which @@ -1449,7 +1490,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( if(!util.isInt32(dataLen) || dataLen<0){ SQLite3Error.toss("Invalid 3rd argument for sqlite3_js_posix_create_file()."); } - const rc = wasm.sqlite3_wasm_posix_create_file(filename, pData, dataLen); + const rc = util.sqlite3__wasm_posix_create_file(filename, pData, dataLen); if(rc) SQLite3Error.toss("Creation of file failed with sqlite3 result code", capi.sqlite3_js_rc_str(rc)); }finally{ @@ -1551,7 +1592,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( SQLite3Error.toss("Invalid 4th argument for sqlite3_js_vfs_create_file()."); } try{ - const rc = wasm.sqlite3_wasm_vfs_create_file(vfs, filename, pData, dataLen); + const rc = util.sqlite3__wasm_vfs_create_file(vfs, filename, pData, dataLen); if(rc) SQLite3Error.toss("Creation of file failed with sqlite3 result code", capi.sqlite3_js_rc_str(rc)); }finally{ @@ -1672,12 +1713,12 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( */ capi.sqlite3_db_config = function(pDb, op, ...args){ if(!this.s){ - this.s = wasm.xWrap('sqlite3_wasm_db_config_s','int', + this.s = wasm.xWrap('sqlite3__wasm_db_config_s','int', ['sqlite3*', 'int', 'string:static'] /* MAINDBNAME requires a static string */); - this.pii = wasm.xWrap('sqlite3_wasm_db_config_pii', 'int', + this.pii = wasm.xWrap('sqlite3__wasm_db_config_pii', 'int', ['sqlite3*', 'int', '*','int', 'int']); - this.ip = wasm.xWrap('sqlite3_wasm_db_config_ip','int', + this.ip = wasm.xWrap('sqlite3__wasm_db_config_ip','int', ['sqlite3*', 'int', 'int','*']); } switch(op){ @@ -1798,7 +1839,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** Calls either sqlite3_result_error_nomem(), if e is-a WasmAllocError, or sqlite3_result_error(). In the latter case, - the second arugment is coerced to a string to create the error + the second argument is coerced to a string to create the error message. The first argument is a (sqlite3_context*). Returns void. diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.c-pp.js index 3099c19ec..991862545 100644 --- a/ext/wasm/api/sqlite3-api-worker1.js +++ b/ext/wasm/api/sqlite3-api-worker1.c-pp.js @@ -359,6 +359,7 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +const util = sqlite3.util; sqlite3.initWorker1API = function(){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; @@ -409,12 +410,12 @@ sqlite3.initWorker1API = function(){ if(db){ delete this.dbs[getDbId(db)]; const filename = db.filename; - const pVfs = sqlite3.wasm.sqlite3_wasm_db_vfs(db.pointer, 0); + const pVfs = util.sqlite3__wasm_db_vfs(db.pointer, 0); db.close(); const ddNdx = this.dbList.indexOf(db); if(ddNdx>=0) this.dbList.splice(ddNdx, 1); if(alsoUnlink && filename && pVfs){ - sqlite3.wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); + util.sqlite3__wasm_vfs_unlink(pVfs, filename); } } }, @@ -458,11 +459,6 @@ sqlite3.initWorker1API = function(){ return wState.dbList[0] && getDbId(wState.dbList[0]); }; - const guessVfs = function(filename){ - const m = /^file:.+(vfs=(\w+))/.exec(filename); - return sqlite3.capi.sqlite3_vfs_find(m ? m[2] : 0); - }; - const isSpecialDbFilename = (n)=>{ return ""===n || ':'===n[0]; }; @@ -484,36 +480,8 @@ sqlite3.initWorker1API = function(){ toss("Throwing because of simulateError flag."); } const rc = Object.create(null); - let byteArray, pVfs; oargs.vfs = args.vfs; - if(isSpecialDbFilename(args.filename)){ - oargs.filename = args.filename || ""; - }else{ - oargs.filename = args.filename; - byteArray = args.byteArray; - if(byteArray) pVfs = guessVfs(args.filename); - } - if(pVfs){ - /* 2022-11-02: this feature is as-yet untested except that - sqlite3_wasm_vfs_create_file() has been tested from the - browser dev console. */ - let pMem; - try{ - pMem = sqlite3.wasm.allocFromTypedArray(byteArray); - const rc = sqlite3.wasm.sqlite3_wasm_vfs_create_file( - pVfs, oargs.filename, pMem, byteArray.byteLength - ); - if(rc) sqlite3.SQLite3Error.toss(rc); - }catch(e){ - throw new sqlite3.SQLite3Error( - e.name+' creating '+args.filename+": "+e.message, { - cause: e - } - ); - }finally{ - if(pMem) sqlite3.wasm.dealloc(pMem); - } - } + oargs.filename = args.filename || ""; const db = wState.open(oargs); rc.filename = db.filename; rc.persistent = !!sqlite3.capi.sqlite3_js_db_uses_vfs(db.pointer, "opfs"); @@ -603,7 +571,6 @@ sqlite3.initWorker1API = function(){ }); rc.version = sqlite3.version; rc.vfsList = sqlite3.capi.sqlite3_js_vfs_list(); - rc.opfsEnabled = !!sqlite3.opfs; return rc; }, @@ -630,12 +597,6 @@ sqlite3.initWorker1API = function(){ toss: function(ev){ toss("Testing worker exception"); - }, - - 'opfs-tree': async function(ev){ - if(!sqlite3.opfs) toss("OPFS support is unavailable."); - const response = await sqlite3.opfs.treeList(); - return response; } }/*wMsgHandler*/; diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index cafd296c6..ca5583971 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -51,7 +51,7 @@ */ "use strict"; const wPost = (type,...args)=>postMessage({type, payload:args}); -const installAsyncProxy = function(self){ +const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; if(globalThis.window === globalThis){ toss("This code cannot run from the main thread.", @@ -87,35 +87,6 @@ const installAsyncProxy = function(self){ const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); const error = (...args)=>logImpl(0, ...args); - const metrics = Object.create(null); - metrics.reset = ()=>{ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - }; - metrics.dump = ()=>{ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - } - console.log(globalThis?.location?.href, - "metrics for",globalThis?.location?.href,":\n", - metrics, - "\nTotal of",n,"op(s) for",t,"ms", - "approx",w,"ms spent waiting on OPFS APIs."); - console.log("Serialization metrics:",metrics.s11n); - }; /** __openFiles is a map of sqlite3_file pointers (integers) to @@ -173,10 +144,10 @@ const installAsyncProxy = function(self){ /** If the given file-holding object has a sync handle attached to it, - that handle is remove and asynchronously closed. Though it may + that handle is removed and asynchronously closed. Though it may sound sensible to continue work as soon as the close() returns (noting that it's asynchronous), doing so can cause operations - performed soon afterwards, e.g. a call to getSyncHandle() to fail + performed soon afterwards, e.g. a call to getSyncHandle(), to fail because they may happen out of order from the close(). OPFS does not guaranty that the actual order of operations is retained in such cases. i.e. always "await" on the result of this function. @@ -265,23 +236,34 @@ const installAsyncProxy = function(self){ this.name = 'GetSyncHandleError'; } }; + + /** + Attempts to find a suitable SQLITE_xyz result code for Error + object e. Returns either such a translation or rc if if it does + not know how to translate the exception. + */ GetSyncHandleError.convertRc = (e,rc)=>{ - if(1){ - return ( - e instanceof GetSyncHandleError - && ((e.cause.name==='NoModificationAllowedError') - /* Inconsistent exception.name from Chrome/ium with the - same exception.message text: */ - || (e.cause.name==='DOMException' - && 0===e.cause.message.indexOf('Access Handles cannot'))) - ) ? ( - /*console.warn("SQLITE_BUSY",e),*/ - state.sq3Codes.SQLITE_BUSY - ) : rc; - }else{ - return rc; + if( e instanceof GetSyncHandleError ){ + if( e.cause.name==='NoModificationAllowedError' + /* Inconsistent exception.name from Chrome/ium with the + same exception.message text: */ + || (e.cause.name==='DOMException' + && 0===e.cause.message.indexOf('Access Handles cannot')) ){ + return state.sq3Codes.SQLITE_BUSY; + }else if( 'NotFoundError'===e.cause.name ){ + /** + Maintenance reminder: SQLITE_NOTFOUND, though it looks like + a good match, has different semantics than NotFoundError + and is not suitable here. + */ + return state.sq3Codes.SQLITE_CANTOPEN; + } + }else if( 'NotFoundError'===e?.name ){ + return state.sq3Codes.SQLITE_CANTOPEN; } - } + return rc; + }; + /** Returns the sync access handle associated with the given file handle object (which must be a valid handle object, as created by @@ -293,6 +275,20 @@ const installAsyncProxy = function(self){ times. If acquisition still fails at that point it will give up and propagate the exception. Client-level code will see that as an I/O error. + + 2024-06-12: there is a rare race condition here which has been + reported a single time: + + https://sqlite.org/forum/forumpost/9ee7f5340802d600 + + What appears to be happening is that file we're waiting for a + lock on is deleted while we wait. What currently happens here is + that a locking exception is thrown but the exception type is + NotFoundError. In such cases, we very probably should attempt to + re-open/re-create the file an obtain the lock on it (noting that + there's another race condition there). That's easy to say but + creating a viable test for that condition has proven challenging + so far. */ const getSyncHandle = async (fh,opName)=>{ if(!fh.syncHandle){ @@ -348,37 +344,6 @@ const installAsyncProxy = function(self){ }; /** - We track 2 different timers: the "metrics" timer records how much - time we spend performing work. The "wait" timer records how much - time we spend waiting on the underlying OPFS timer. See the calls - to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() - throughout this file to see how they're used. - */ - const __mTimer = Object.create(null); - __mTimer.op = undefined; - __mTimer.start = undefined; - const mTimeStart = (op)=>{ - __mTimer.start = performance.now(); - __mTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[__mTimer.op].time += performance.now() - __mTimer.start - ); - const __wTimer = Object.create(null); - __wTimer.op = undefined; - __wTimer.start = undefined; - const wTimeStart = (op)=>{ - __wTimer.start = performance.now(); - __wTimer.op = op; - //metrics[op] || toss("Maintenance required: missing metrics for",op); - }; - const wTimeEnd = ()=>( - metrics[__wTimer.op].wait += performance.now() - __wTimer.start - ); - - /** Gets set to true by the 'opfs-async-shutdown' command to quit the wait loop. This is only intended for debugging purposes: we cannot inspect this file's state while the tight waitLoop() is running and @@ -388,37 +353,24 @@ const installAsyncProxy = function(self){ /** Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods - methods, as well as helpers like mkdir(). Maintenance reminder: - members are in alphabetical order to simplify finding them. + methods, as well as helpers like mkdir(). */ const vfsAsyncImpls = { - 'opfs-async-metrics': async ()=>{ - mTimeStart('opfs-async-metrics'); - metrics.dump(); - storeAndNotify('opfs-async-metrics', 0); - mTimeEnd(); - }, 'opfs-async-shutdown': async ()=>{ flagAsyncShutdown = true; storeAndNotify('opfs-async-shutdown', 0); }, mkdir: async (dirname)=>{ - mTimeStart('mkdir'); let rc = 0; - wTimeStart('mkdir'); try { await getDirForFilename(dirname+"/filepart", true); }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); } storeAndNotify('mkdir', rc); - mTimeEnd(); }, xAccess: async (filename)=>{ - mTimeStart('xAccess'); /* OPFS cannot support the full range of xAccess() queries sqlite3 calls for. We can essentially just tell if the file is accessible, but if it is then it's automatically writable @@ -431,26 +383,20 @@ const installAsyncProxy = function(self){ accessible, non-0 means not accessible. */ let rc = 0; - wTimeStart('xAccess'); try{ const [dh, fn] = await getDirForFilename(filename); await dh.getFileHandle(fn); }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR; - }finally{ - wTimeEnd(); } storeAndNotify('xAccess', rc); - mTimeEnd(); }, xClose: async function(fid/*sqlite3_file pointer*/){ const opName = 'xClose'; - mTimeStart(opName); __implicitLocks.delete(fid); const fh = __openFiles[fid]; let rc = 0; - wTimeStart(opName); if(fh){ delete __openFiles[fid]; await closeSyncHandle(fh); @@ -462,15 +408,11 @@ const installAsyncProxy = function(self){ state.s11n.serialize(); rc = state.sq3Codes.SQLITE_NOTFOUND; } - wTimeEnd(); storeAndNotify(opName, rc); - mTimeEnd(); }, xDelete: async function(...args){ - mTimeStart('xDelete'); const rc = await vfsAsyncImpls.xDeleteNoWait(...args); storeAndNotify('xDelete', rc); - mTimeEnd(); }, xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ /* The syncDir flag is, for purposes of the VFS API's semantics, @@ -486,7 +428,6 @@ const installAsyncProxy = function(self){ is false. */ let rc = 0; - wTimeStart('xDelete'); try { while(filename){ const [hDir, filenamePart] = await getDirForFilename(filename, false); @@ -502,14 +443,11 @@ const installAsyncProxy = function(self){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR_DELETE; } - wTimeEnd(); return rc; }, xFileSize: async function(fid/*sqlite3_file pointer*/){ - mTimeStart('xFileSize'); const fh = __openFiles[fid]; let rc = 0; - wTimeStart('xFileSize'); try{ const sz = await (await getSyncHandle(fh,'xFileSize')).getSize(); state.s11n.serialize(Number(sz)); @@ -518,19 +456,15 @@ const installAsyncProxy = function(self){ rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR); } await releaseImplicitLock(fh); - wTimeEnd(); storeAndNotify('xFileSize', rc); - mTimeEnd(); }, xLock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xLock'); const fh = __openFiles[fid]; let rc = 0; const oldLockType = fh.xLock; fh.xLock = lockType; if( !fh.syncHandle ){ - wTimeStart('xLock'); try { await getSyncHandle(fh,'xLock'); __implicitLocks.delete(fid); @@ -539,18 +473,14 @@ const installAsyncProxy = function(self){ rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); fh.xLock = oldLockType; } - wTimeEnd(); } storeAndNotify('xLock',rc); - mTimeEnd(); }, xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags/*SQLITE_OPEN_...*/, opfsFlags/*OPFS_...*/){ const opName = 'xOpen'; - mTimeStart(opName); const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); - wTimeStart('xOpen'); try{ let hDir, filenamePart; try { @@ -558,12 +488,17 @@ const installAsyncProxy = function(self){ }catch(e){ state.s11n.storeException(1,e); storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); - mTimeEnd(); - wTimeEnd(); return; } + if( state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN & opfsFlags ){ + try{ + await hDir.removeEntry(filenamePart); + }catch(e){ + /* ignoring */ + //warn("Ignoring failed Unlink of",filename,":",e); + } + } const hFile = await hDir.getFileHandle(filenamePart, {create}); - wTimeEnd(); const fh = Object.assign(Object.create(null),{ fid: fid, filenameAbs: filename, @@ -578,76 +513,50 @@ const installAsyncProxy = function(self){ fh.releaseImplicitLocks = (opfsFlags & state.opfsFlags.OPFS_UNLOCK_ASAP) || state.opfsFlags.defaultUnlockAsap; - if(0 /* this block is modelled after something wa-sqlite - does but it leads to immediate contention on journal files. - Update: this approach reportedly only works for DELETE journal - mode. */ - && (0===(flags & state.sq3Codes.SQLITE_OPEN_MAIN_DB))){ - /* sqlite does not lock these files, so go ahead and grab an OPFS - lock. */ - fh.xLock = "xOpen"/* Truthy value to keep entry from getting - flagged as auto-locked. String value so - that we can easily distinguish is later - if needed. */; - await getSyncHandle(fh,'xOpen'); - } __openFiles[fid] = fh; storeAndNotify(opName, 0); }catch(e){ - wTimeEnd(); error(opName,e); state.s11n.storeException(1,e); storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } - mTimeEnd(); }, xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xRead'); let rc = 0, nRead; const fh = __openFiles[fid]; try{ - wTimeStart('xRead'); nRead = (await getSyncHandle(fh,'xRead')).read( fh.sabView.subarray(0, n), {at: Number(offset64)} ); - wTimeEnd(); if(nRead < n){/* Zero-fill remaining bytes */ fh.sabView.fill(0, nRead, n); rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - if(undefined===nRead) wTimeEnd(); error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } await releaseImplicitLock(fh); storeAndNotify('xRead',rc); - mTimeEnd(); }, xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ - mTimeStart('xSync'); const fh = __openFiles[fid]; let rc = 0; if(!fh.readOnly && fh.syncHandle){ try { - wTimeStart('xSync'); await fh.syncHandle.flush(); }catch(e){ state.s11n.storeException(2,e); rc = state.sq3Codes.SQLITE_IOERR_FSYNC; } - wTimeEnd(); } storeAndNotify('xSync',rc); - mTimeEnd(); }, xTruncate: async function(fid/*sqlite3_file pointer*/,size){ - mTimeStart('xTruncate'); let rc = 0; const fh = __openFiles[fid]; - wTimeStart('xTruncate'); try{ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); @@ -657,33 +566,27 @@ const installAsyncProxy = function(self){ rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } await releaseImplicitLock(fh); - wTimeEnd(); storeAndNotify('xTruncate',rc); - mTimeEnd(); }, xUnlock: async function(fid/*sqlite3_file pointer*/, lockType/*SQLITE_LOCK_...*/){ - mTimeStart('xUnlock'); let rc = 0; const fh = __openFiles[fid]; - if( state.sq3Codes.SQLITE_LOCK_NONE===lockType - && fh.syncHandle ){ - wTimeStart('xUnlock'); + if( fh.syncHandle + && state.sq3Codes.SQLITE_LOCK_NONE===lockType + /* Note that we do not differentiate between lock types in + this VFS. We're either locked or unlocked. */ ){ try { await closeSyncHandle(fh) } catch(e){ state.s11n.storeException(1,e); rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - wTimeEnd(); } storeAndNotify('xUnlock',rc); - mTimeEnd(); }, xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ - mTimeStart('xWrite'); let rc; const fh = __openFiles[fid]; - wTimeStart('xWrite'); try{ affirmNotRO('xWrite', fh); rc = ( @@ -697,9 +600,7 @@ const installAsyncProxy = function(self){ rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } await releaseImplicitLock(fh); - wTimeEnd(); storeAndNotify('xWrite',rc); - mTimeEnd(); } }/*vfsAsyncImpls*/; @@ -733,8 +634,6 @@ const installAsyncProxy = function(self){ } }; state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); const argc = viewU8[0]; const rc = argc ? [] : null; if(argc){ @@ -759,12 +658,9 @@ const installAsyncProxy = function(self){ } if(clear) viewU8[0] = 0; //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; return rc; }; state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; if(args.length){ //log("serialize():",args); const typeIds = []; @@ -795,7 +691,6 @@ const installAsyncProxy = function(self){ }else{ viewU8[0] = 0; } - metrics.s11n.serialize.time += performance.now() - t; }; state.s11n.storeException = state.asyncS11nExceptions @@ -877,7 +772,6 @@ const installAsyncProxy = function(self){ } }); initS11n(); - metrics.reset(); log("init state",state); wPost('opfs-async-inited'); waitLoop(); @@ -890,9 +784,6 @@ const installAsyncProxy = function(self){ waitLoop(); } break; - case 'opfs-async-metrics': - metrics.dump(); - break; } }; wPost('opfs-async-loaded'); @@ -911,5 +802,5 @@ if(!globalThis.SharedArrayBuffer){ !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ - installAsyncProxy(self); + installAsyncProxy(); } diff --git a/ext/wasm/api/sqlite3-vfs-helper.c-pp.js b/ext/wasm/api/sqlite3-vfs-helper.c-pp.js new file mode 100644 index 000000000..4d29c7b91 --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-helper.c-pp.js @@ -0,0 +1,103 @@ +/* +** 2022-11-30 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ + +/** + This file installs sqlite3.vfs, a namespace of helpers for use in + the creation of JavaScript implementations of sqlite3_vfs. +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; + const vfs = Object.create(null); + sqlite3.vfs = vfs; + + /** + Uses sqlite3_vfs_register() to register this + sqlite3.capi.sqlite3_vfs instance. This object must have already + been filled out properly. If the first argument is truthy, the + VFS is registered as the default VFS, else it is not. + + On success, returns this object. Throws on error. + */ + capi.sqlite3_vfs.prototype.registerVfs = function(asDefault=false){ + if(!(this instanceof sqlite3.capi.sqlite3_vfs)){ + toss("Expecting a sqlite3_vfs-type argument."); + } + const rc = capi.sqlite3_vfs_register(this, asDefault ? 1 : 0); + if(rc){ + toss("sqlite3_vfs_register(",this,") failed with rc",rc); + } + if(this.pointer !== capi.sqlite3_vfs_find(this.$zName)){ + toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", + this); + } + return this; + }; + + /** + A wrapper for + sqlite3.StructBinder.StructType.prototype.installMethods() or + registerVfs() to reduce installation of a VFS and/or its I/O + methods to a single call. + + Accepts an object which contains the properties "io" and/or + "vfs", each of which is itself an object with following properties: + + - `struct`: an sqlite3.StructBinder.StructType-type struct. This + must be a populated (except for the methods) object of type + sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the + "vfs" entry). + + - `methods`: an object mapping sqlite3_io_methods method names + (e.g. 'xClose') to JS implementations of those methods. The JS + implementations must be call-compatible with their native + counterparts. + + For each of those object, this function passes its (`struct`, + `methods`, (optional) `applyArgcCheck`) properties to + installMethods(). + + If the `vfs` entry is set then: + + - Its `struct` property's registerVfs() is called. The + `vfs` entry may optionally have an `asDefault` property, which + gets passed as the argument to registerVfs(). + + - If `struct.$zName` is falsy and the entry has a string-type + `name` property, `struct.$zName` is set to the C-string form of + that `name` value before registerVfs() is called. That string + gets added to the on-dispose state of the struct. + + On success returns this object. Throws on error. + */ + vfs.installVfs = function(opt){ + let count = 0; + const propList = ['io','vfs']; + for(const key of propList){ + const o = opt[key]; + if(o){ + ++count; + o.struct.installMethods(o.methods, !!o.applyArgcCheck); + if('vfs'===key){ + if(!o.struct.$zName && 'string'===typeof o.name){ + o.struct.addOnDispose( + o.struct.$zName = wasm.allocCString(o.name) + ); + } + o.struct.registerVfs(!!o.asDefault); + } + } + } + if(!count) toss("Misuse: installVfs() options object requires at least", + "one of:", propList); + return this; + }; +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 870073cc0..d423bb0bb 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -78,8 +78,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.SQLITE_OPEN_MAIN_DB | capi.SQLITE_OPEN_MAIN_JOURNAL | capi.SQLITE_OPEN_SUPER_JOURNAL | - capi.SQLITE_OPEN_WAL /* noting that WAL support is - unavailable in the WASM build.*/; + capi.SQLITE_OPEN_WAL; /** Subdirectory of the VFS's space where "opaque" (randomly-named) files are stored. Changing this effectively invalidates the data @@ -102,7 +101,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ clearOnInit: false, /* Logging verbosity 3+ == everything, 2 == warnings+errors, 1 == errors only. */ - verbosity: 2 + verbosity: 2, + forceReinitIfPreviouslyFailed: false }); /** Logging routines, from most to least serious. */ @@ -1004,9 +1004,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return true; }; - /** Only for testing a rejection case. */ - let instanceCounter = 0; - /** installOpfsSAHPoolVfs() asynchronously initializes the OPFS SyncAccessHandle (a.k.a. SAH) Pool VFS. It returns a Promise which @@ -1081,12 +1078,26 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ the default directory. If no directory is explicitly provided then a directory name is synthesized from the `name` option. - Peculiarities of this VFS: + + - `forceReinitIfPreviouslyFailed`: (default=`false`) Is a fallback option + to assist in working around certain flaky environments which may + mysteriously fail to permit access to OPFS sync access handles on + an initial attempt but permit it on a second attemp. This option + should never be used but is provided for those who choose to + throw caution to the wind and trust such environments. If this + option is truthy _and_ the previous attempt to initialize this + VFS with the same `name` failed, the VFS will attempt to + initialize a second time instead of returning the cached + failure. See discussion at: + <https://github.com/sqlite/sqlite-wasm/issues/79> + + + Peculiarities of this VFS vis a vis other SQLite VFSes: - Paths given to it _must_ be absolute. Relative paths will not be properly recognized. This is arguably a bug but correcting it requires some hoop-jumping in routines which have no business - doing tricks. + doing such tricks. - It is possible to install multiple instances under different names, each sandboxed from one another inside their own private @@ -1137,8 +1148,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ existing content. Throws if the pool has no available file slots, on I/O error, or if the input does not appear to be a database. In the latter case, only a cursory examination is made. - Note that this routine is _only_ for importing database files, - not arbitrary files, the reason being that this VFS will + Results are undefined if the given db name refers to an opened + db. Note that this routine is _only_ for importing database + files, not arbitrary files, the reason being that this VFS will automatically clean up any non-database files so importing them is pointless. @@ -1206,13 +1218,25 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ handles are currently in use, e.g. by an sqlite3 db. */ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ - const vfsName = options.name || optionDefaults.name; - if(0 && 2===++instanceCounter){ - throw new Error("Just testing rejection."); + options = Object.assign(Object.create(null), optionDefaults, (options||{})); + const vfsName = options.name; + if(options.$testThrowPhase1){ + throw options.$testThrowPhase1; } if(initPromises[vfsName]){ - //console.warn("Returning same OpfsSAHPool result",options,vfsName,initPromises[vfsName]); - return initPromises[vfsName]; + try { + const p = await initPromises[vfsName]; + //log("installOpfsSAHPoolVfs() returning cached result",options,vfsName,p); + return p; + }catch(e){ + //log("installOpfsSAHPoolVfs() got cached failure",options,vfsName,e); + if( options.forceReinitIfPreviouslyFailed ){ + delete initPromises[vfsName]; + /* Fall through and try again. */ + }else{ + throw e; + } + } } if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || @@ -1237,8 +1261,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ensues. */ return initPromises[vfsName] = apiVersionCheck().then(async function(){ - if(options.$testThrowInInit){ - throw options.$testThrowInInit; + if(options.$testThrowPhase2){ + throw options.$testThrowPhase2; } const thePool = new OpfsSAHPool(options); return thePool.isReady.then(async()=>{ @@ -1254,24 +1278,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ oo1.DB.dbCtorHelper.call(this, opt); }; OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); - // yes or no? OpfsSAHPoolDb.PoolUtil = poolUtil; poolUtil.OpfsSAHPoolDb = OpfsSAHPoolDb; - oo1.DB.dbCtorHelper.setVfsPostOpenSql( - theVfs.pointer, - function(oo1Db, sqlite3){ - sqlite3.capi.sqlite3_exec(oo1Db, [ - /* See notes in sqlite3-vfs-opfs.js */ - "pragma journal_mode=DELETE;", - "pragma cache_size=-16384;" - ], 0, 0, 0); - } - ); }/*extend sqlite3.oo1*/ thePool.log("VFS initialized."); return poolUtil; }).catch(async (e)=>{ await thePool.removeVfs().catch(()=>{}); - return e; + throw e; }); }).catch((err)=>{ //error("rejecting promise:",err); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index af89f216f..2d11b3583 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -38,10 +38,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Significant notes and limitations: - - As of this writing, OPFS is still very much in flux and only - available in bleeding-edge versions of Chrome (v102+, noting that - that number will increase as the OPFS API matures). - - The OPFS features used here are only available in dedicated Worker threads. This file tries to detect that case, resulting in a rejected Promise if those features do not seem to be available. @@ -62,15 +58,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The argument may optionally be a plain object with the following configuration options: - - proxyUri: as described above + - proxyUri: name of the async proxy JS file. - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables logging of errors. 2 enables logging of warnings and errors. 3 - additionally enables debugging info. + additionally enables debugging info. Logging is performed + via the sqlite3.config.{log|warn|error}() functions. - - sanityChecks (=false): if true, some basic sanity tests are - run on the OPFS VFS API after it's initialized, before the - returned Promise resolves. + - sanityChecks (=false): if true, some basic sanity tests are run on + the OPFS VFS API after it's initialized, before the returned + Promise resolves. This is only intended for testing and + development of the VFS, not client-side use. On success, the Promise resolves to the top-most sqlite3 namespace object and that object gets a new object installed in its @@ -245,7 +243,8 @@ const installOpfsVfs = function callee(options){ opfsIoMethods.$iVersion = 1; opfsVfs.$iVersion = 2/*yes, two*/; opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/*sure, why not?*/; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; opfsVfs.$zName = wasm.allocCString("opfs"); // All C-side memory of opfsVfs is zeroed out, but just to be explicit: opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; @@ -391,6 +390,7 @@ const installOpfsVfs = function callee(options){ 'SQLITE_ACCESS_EXISTS', 'SQLITE_ACCESS_READWRITE', 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', 'SQLITE_ERROR', 'SQLITE_IOERR', 'SQLITE_IOERR_ACCESS', @@ -422,13 +422,28 @@ const installOpfsVfs = function callee(options){ }); state.opfsFlags = Object.assign(Object.create(null),{ /** - Flag for use with xOpen(). "opfs-unlock-asap=1" enables - this. See defaultUnlockAsap, below. + Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" + enables this. See defaultUnlockAsap, below. */ OPFS_UNLOCK_ASAP: 0x01, /** + Flag for use with xOpen(). URI flag "delete-before-open=1" + tells the VFS to delete the db file before attempting to open + it. This can be used, e.g., to replace a db which has been + corrupted (without forcing us to expose a delete/unlink() + function in the public API). + + Failure to unlink the file is ignored but may lead to + downstream errors. An unlink can fail if, e.g., another tab + has the handle open. + + It goes without saying that deleting a file out from under another + instance results in Undefined Behavior. + */ + OPFS_UNLINK_BEFORE_OPEN: 0x02, + /** If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that locks at + access handle (i.e. an OPFS lock) will release that lock at the end of the call which acquires it. If false, such "autolocks" are not released until the VFS is idle for some brief amount of time. @@ -455,9 +470,22 @@ const installOpfsVfs = function callee(options){ Atomics.notify(state.sabOPView, state.opIds.whichOp) /* async thread will take over here */; const t = performance.now(); - Atomics.wait(state.sabOPView, state.opIds.rc, -1) - /* When this wait() call returns, the async half will have - completed the operation and reported its results. */; + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync. Calling + wait() here until it returns 'not-equal' gets them back in + sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its results + in the state.opIds.rc slot of the SAB. */ const rc = Atomics.load(state.sabOPView, state.opIds.rc); metrics[op].wait += performance.now() - t; if(rc && state.asyncS11nExceptions){ @@ -704,9 +732,13 @@ const installOpfsVfs = function callee(options){ involve an inherent race condition. For the time being, pending a better solution, we simply report whether the given pFile is open. + + Update 2024-06-12: based on forum discussions, this + function now always sets pOut to 0 (false): + + https://sqlite.org/forum/forumpost/a2f573b00cda1372 */ - const f = __openFiles[pFile]; - wasm.poke(pOut, f.lockType ? 1 : 0, 'i32'); + wasm.poke(pOut, 0, 'i32'); return 0; }, xClose: function(pFile){ @@ -722,7 +754,6 @@ const installOpfsVfs = function callee(options){ return rc; }, xDeviceCharacteristics: function(pFile){ - //debug("xDeviceCharacteristics(",pFile,")"); return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; }, xFileControl: function(pFile, opId, pArg){ @@ -874,13 +905,17 @@ const installOpfsVfs = function callee(options){ let opfsFlags = 0; if(0===zName){ zName = randomFilename(); - }else if('number'===typeof zName){ + }else if(wasm.isPtr(zName)){ if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ /* -----------------------^^^^^ MUST pass the untranslated C-string here. */ opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; } + if(capi.sqlite3_uri_boolean(zName, "delete-before-open", 0)){ + opfsFlags |= state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN; + } zName = wasm.cstrToJs(zName); + //warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); } const fh = Object.create(null); fh.fid = pFile; @@ -974,7 +1009,9 @@ const installOpfsVfs = function callee(options){ }; /** Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't. + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. */ opfsUtil.entryExists = async function(fsEntryName){ try { @@ -994,27 +1031,6 @@ const installOpfsVfs = function callee(options){ opfsUtil.randomFilename = randomFilename; /** - Re-registers the OPFS VFS. This is intended only for odd use - cases which have to call sqlite3_shutdown() as part of their - initialization process, which will unregister the VFS - registered by installOpfsVfs(). If passed a truthy value, the - OPFS VFS is registered as the default VFS, else it is not made - the default. Returns the result of the the - sqlite3_vfs_register() call. - - Design note: the problem of having to re-register things after - a shutdown/initialize pair is more general. How to best plug - that in to the library is unclear. In particular, we cannot - hook in to any C-side calls to sqlite3_initialize(), so we - cannot add an after-initialize callback mechanism. - */ - opfsUtil.registerVfs = (asDefault=false)=>{ - return wasm.exports.sqlite3_vfs_register( - opfsVfs.pointer, asDefault ? 1 : 0 - ); - }; - - /** Returns a promise which resolves to an object which represents all files and directories in the OPFS tree. The top-most object has two properties: `dirs` is an array of directory entries @@ -1111,9 +1127,9 @@ const installOpfsVfs = function callee(options){ }; /** - Traverses the OPFS filesystem, calling a callback for each one. - The argument may be either a callback function or an options object - with any of the following properties: + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: - `callback`: function which gets called for each filesystem entry. It gets passed 3 arguments: 1) the @@ -1143,11 +1159,6 @@ const installOpfsVfs = function callee(options){ but that promise has no specific meaning: the traversal it performs is synchronous. The promise must be used to catch any exceptions propagated by the callback, however. - - TODO: add an option which specifies whether to traverse - depth-first or breadth-first. We currently do depth-first but - an incremental file browsing widget would benefit more from - breadth-first. */ opfsUtil.traverse = async function(opt){ const defaultOpt = { @@ -1213,16 +1224,18 @@ const installOpfsVfs = function callee(options){ Asynchronously imports the given bytes (a byte array or ArrayBuffer) into the given database file. + Results are undefined if the given db name refers to an opened + db. + If passed a function for its second argument, its behaviour - changes to async and it imports its data in chunks fed to it by - the given callback function. It calls the callback (which may - be async) repeatedly, expecting either a Uint8Array or - ArrayBuffer (to denote new input) or undefined (to denote - EOF). For so long as the callback continues to return - non-undefined, it will append incoming data to the given - VFS-hosted database file. When called this way, the resolved - value of the returned Promise is the number of bytes written to - the target file. + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. It very specifically requires the input to be an SQLite3 database and throws if that's not the case. It does so in @@ -1275,40 +1288,13 @@ const installOpfsVfs = function callee(options){ OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( opfsVfs.pointer, function(oo1Db, sqlite3){ /* Set a relatively high default busy-timeout handler to help OPFS dbs deal with multi-tab/multi-worker contention. */ sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - sqlite3.capi.sqlite3_exec(oo1Db, [ - /* As of July 2023, the PERSIST journal mode on OPFS is - somewhat slower than DELETE or TRUNCATE (it was faster - before Chrome version 108 or 109). TRUNCATE and DELETE - have very similar performance on OPFS. - - Roy Hashimoto notes that TRUNCATE and PERSIST modes may - decrease OPFS concurrency because multiple connections - can open the journal file in those modes: - - https://github.com/rhashimoto/wa-sqlite/issues/68 - - Given that, and the fact that testing has not revealed - any appreciable difference between performance of - TRUNCATE and DELETE modes on OPFS, we currently (as of - 2023-07-13) default to DELETE mode. - */ - "pragma journal_mode=DELETE;", - /* - This vfs benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We - currently rely on setting a non-default cache size when - building sqlite3.wasm. If that policy changes, the cache - can be set here. - */ - "pragma cache_size=-16384;" - ], 0, 0, 0); } ); }/*extend sqlite3.oo1*/ diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js index e63da8afc..d78fbdbf3 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js @@ -10,19 +10,17 @@ */ /** - This file installs sqlite3.vfs, and object which exists to assist - in the creation of JavaScript implementations of sqlite3_vfs, along - with its virtual table counterpart, sqlite3.vtab. + This file installs sqlite3.vtab, a namespace of helpers for use in + the creation of JavaScript implementations virtual tables. If built + without virtual table support then this function does nothing. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( !sqlite3.wasm.exports.sqlite3_declare_vtab ){ + return; + } const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; - const vfs = Object.create(null), vtab = Object.create(null); - - const StructBinder = sqlite3.StructBinder - /* we require a local alias b/c StructBinder is removed from the sqlite3 - object during the final steps of the API cleanup. */; - sqlite3.vfs = vfs; + const vtab = Object.create(null); sqlite3.vtab = vtab; const sii = capi.sqlite3_index_info; @@ -73,257 +71,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error, e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc. - - As a special case, if the given function is a pointer, then - `wasm.functionEntry()` is used to validate that it is a known - function. If so, it is used as-is with no extra level of proxying - or cleanup, else an exception is thrown. It is legal to pass a - value of 0, indicating a NULL pointer, with the caveat that 0 - _is_ a legal function pointer in WASM but it will not be accepted - as such _here_. (Justification: the function at address zero must - be one which initially came from the WASM module, not a method we - want to bind to a virtual table or VFS.) - - This function returns a proxy for itself which is bound to tgt - and takes 2 args (name,func). That function returns the same - thing as this one, permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - - ACHTUNG: because we cannot generically know how to transform JS - exceptions into result codes, the installed functions do no - automatic catching of exceptions. It is critical, to avoid - undefined behavior in the C layer, that methods mapped via - this function do not throw. The exception, as it were, to that - rule is... - - If applyArgcCheck is true then each JS function (as opposed to - function pointers) gets wrapped in a proxy which asserts that it - is passed the expected number of arguments, throwing if the - argument count does not match expectations. That is only intended - for dev-time usage for sanity checking, and will leave the C - environment in an undefined state. - */ - const installMethod = function callee( - tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck - ){ - if(!(tgt instanceof StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - }else if(!(func instanceof Function) && !wasm.isPtr(func)){ - toss("Usage errror: expecting a Function or WASM pointer to one."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt, n, f, applyArgcCheck); - } - if(!callee.argcProxy){ - callee.argcProxy = function(tgt, funcName, func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch for", - tgt.structInfo.name+"::"+funcName - +": Native signature is:",sig); - } - return func.apply(this, args); - } - }; - /* An ondispose() callback for use with - StructBinder-created types. */ - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name,"does not have a function pointer signature:",sigN); - } - const memKey = tgt.memberKey(name); - const fProxy = (applyArgcCheck && !wasm.isPtr(func)) - /** This middle-man proxy is only for use during development, to - confirm that we always pass the proper number of - arguments. We know that the C-level code will always use the - correct argument count. */ - ? callee.argcProxy(tgt, memKey, func, sigN) - : func; - if(wasm.isPtr(fProxy)){ - if(fProxy && !wasm.functionEntry(fProxy)){ - toss("Pointer",fProxy,"is not a WASM function table entry."); - } - tgt[memKey] = fProxy; - }else{ - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ - tgt.addOnDispose('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - } - return (n,f)=>callee(tgt, n, f, applyArgcCheck); - }/*installMethod*/; - installMethod.installMethodArgcCheck = false; - - /** - Installs methods into the given StructType-type instance. Each - entry in the given methods object must map to a known member of - the given StructType, else an exception will be triggered. See - installMethod() for more details, including the semantics of the - 3rd argument. - - As an exception to the above, if any two or more methods in the - 2nd argument are the exact same function, installMethod() is - _not_ called for the 2nd and subsequent instances, and instead - those instances get assigned the same method pointer which is - created for the first instance. This optimization is primarily to - accommodate special handling of sqlite3_module::xConnect and - xCreate methods. - - On success, returns its first argument. Throws on error. - */ - const installMethods = function( - structInstance, methods, applyArgcCheck = installMethod.installMethodArgcCheck - ){ - const seen = new Map /* map of <Function, memberName> */; - for(const k of Object.keys(methods)){ - const m = methods[k]; - const prior = seen.get(m); - if(prior){ - const mkey = structInstance.memberKey(k); - structInstance[mkey] = structInstance[structInstance.memberKey(prior)]; - }else{ - installMethod(structInstance, k, m, applyArgcCheck); - seen.set(m, k); - } - } - return structInstance; - }; - - /** - Equivalent to calling installMethod(this,...arguments) with a - first argument of this object. If called with 1 or 2 arguments - and the first is an object, it's instead equivalent to calling - installMethods(this,...arguments). - */ - StructBinder.StructType.prototype.installMethod = function callee( - name, func, applyArgcCheck = installMethod.installMethodArgcCheck - ){ - return (arguments.length < 3 && name && 'object'===typeof name) - ? installMethods(this, ...arguments) - : installMethod(this, ...arguments); - }; - - /** - Equivalent to calling installMethods() with a first argument - of this object. - */ - StructBinder.StructType.prototype.installMethods = function( - methods, applyArgcCheck = installMethod.installMethodArgcCheck - ){ - return installMethods(this, methods, applyArgcCheck); - }; - - /** - Uses sqlite3_vfs_register() to register this - sqlite3.capi.sqlite3_vfs. This object must have already been - filled out properly. If the first argument is truthy, the VFS is - registered as the default VFS, else it is not. - - On success, returns this object. Throws on error. - */ - capi.sqlite3_vfs.prototype.registerVfs = function(asDefault=false){ - if(!(this instanceof sqlite3.capi.sqlite3_vfs)){ - toss("Expecting a sqlite3_vfs-type argument."); - } - const rc = capi.sqlite3_vfs_register(this, asDefault ? 1 : 0); - if(rc){ - toss("sqlite3_vfs_register(",this,") failed with rc",rc); - } - if(this.pointer !== capi.sqlite3_vfs_find(this.$zName)){ - toss("BUG: sqlite3_vfs_find(vfs.$zName) failed for just-installed VFS", - this); - } - return this; - }; - - /** - A wrapper for installMethods() or registerVfs() to reduce - installation of a VFS and/or its I/O methods to a single - call. - - Accepts an object which contains the properties "io" and/or - "vfs", each of which is itself an object with following properties: - - - `struct`: an sqlite3.StructType-type struct. This must be a - populated (except for the methods) object of type - sqlite3_io_methods (for the "io" entry) or sqlite3_vfs (for the - "vfs" entry). - - - `methods`: an object mapping sqlite3_io_methods method names - (e.g. 'xClose') to JS implementations of those methods. The JS - implementations must be call-compatible with their native - counterparts. - - For each of those object, this function passes its (`struct`, - `methods`, (optional) `applyArgcCheck`) properties to - installMethods(). - - If the `vfs` entry is set then: - - - Its `struct` property's registerVfs() is called. The - `vfs` entry may optionally have an `asDefault` property, which - gets passed as the argument to registerVfs(). - - - If `struct.$zName` is falsy and the entry has a string-type - `name` property, `struct.$zName` is set to the C-string form of - that `name` value before registerVfs() is called. That string - gets added to the on-dispose state of the struct. - - On success returns this object. Throws on error. - */ - vfs.installVfs = function(opt){ - let count = 0; - const propList = ['io','vfs']; - for(const key of propList){ - const o = opt[key]; - if(o){ - ++count; - installMethods(o.struct, o.methods, !!o.applyArgcCheck); - if('vfs'===key){ - if(!o.struct.$zName && 'string'===typeof o.name){ - o.struct.addOnDispose( - o.struct.$zName = wasm.allocCString(o.name) - ); - } - o.struct.registerVfs(!!o.asDefault); - } - } - } - if(!count) toss("Misuse: installVfs() options object requires at least", - "one of:", propList); - return this; - }; - - /** Internal factory function for xVtab and xCursor impls. */ const __xWrapFactory = function(methodName,StructType){ @@ -457,30 +204,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vtab.xIndexInfo = (pIdxInfo)=>new capi.sqlite3_index_info(pIdxInfo); /** - Given an error object, this function returns - sqlite3.capi.SQLITE_NOMEM if (e instanceof - sqlite3.WasmAllocError), else it returns its - second argument. Its intended usage is in the methods - of a sqlite3_vfs or sqlite3_module: - - ``` - try{ - let rc = ... - return rc; - }catch(e){ - return sqlite3.vtab.exceptionToRc(e, sqlite3.capi.SQLITE_XYZ); - // where SQLITE_XYZ is some call-appropriate result code. - } - ``` - */ - /**vfs.exceptionToRc = vtab.exceptionToRc = - (e, defaultRc=capi.SQLITE_ERROR)=>( - (e instanceof sqlite3.WasmAllocError) - ? capi.SQLITE_NOMEM - : defaultRc - );*/ - - /** Given an sqlite3_module method name and error object, this function returns sqlite3.capi.SQLITE_NOMEM if (e instanceof sqlite3.WasmAllocError), else it returns its second argument. Its @@ -526,20 +249,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vtab.xError.errorReporter = 1 ? console.error.bind(console) : false; /** - "The problem" with this is that it introduces an outer function with - a different arity than the passed-in method callback. That means we - cannot do argc validation on these. Additionally, some methods (namely - xConnect) may have call-specific error handling. It would be a shame to - hard-coded that per-method support in this function. - */ - /** vtab.methodCatcher = function(methodName, method, defaultErrRc=capi.SQLITE_ERROR){ - return function(...args){ - try { method(...args); } - }catch(e){ return vtab.xError(methodName, e, defaultRc) } - }; - */ - - /** A helper for sqlite3_vtab::xRowid() and xUpdate() implementations. It must be passed the final argument to one of those methods (an output pointer to an int64 row ID) and the @@ -685,12 +394,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ remethods[k] = fwrap(k, m); } } - installMethods(mod, remethods, false); + mod.installMethods(remethods, false); }else{ // No automatic exception handling. Trust the client // to not throw. - installMethods( - mod, methods, !!opt.applyArgcCheck/*undocumented option*/ + mod.installMethods( + methods, !!opt.applyArgcCheck/*undocumented option*/ ); } if(0===mod.$iVersion){ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 618d0f085..7f7e69689 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -104,8 +104,8 @@ #ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS # define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 #endif -#ifndef SQLITE_ENABLE_FTS4 -# define SQLITE_ENABLE_FTS4 1 +#ifndef SQLITE_ENABLE_FTS5 +# define SQLITE_ENABLE_FTS5 1 #endif #ifndef SQLITE_ENABLE_MATH_FUNCTIONS # define SQLITE_ENABLE_MATH_FUNCTIONS 1 @@ -172,6 +172,48 @@ # define SQLITE_EXTRA_INIT sqlite3_wasm_extra_init #endif +/* +** If SQLITE_WASM_MINIMAL is defined, undefine most of the ENABLE +** macros. +*/ +#ifdef SQLITE_WASM_MINIMAL +# undef SQLITE_ENABLE_DBPAGE_VTAB +# undef SQLITE_ENABLE_DBSTAT_VTAB +# undef SQLITE_ENABLE_EXPLAIN_COMMENTS +# undef SQLITE_ENABLE_FTS5 +# undef SQLITE_ENABLE_OFFSET_SQL_FUNC +# undef SQLITE_ENABLE_PREUPDATE_HOOK +# undef SQLITE_ENABLE_RTREE +# undef SQLITE_ENABLE_SESSION +# undef SQLITE_ENABLE_STMTVTAB +# undef SQLITE_OMIT_AUTHORIZATION +# define SQLITE_OMIT_AUTHORIZATION +/*Reminder re. custom sqlite3.c: + + fossil clean -x + ./configure + OPTS='-DSQLITE_OMIT_VIRTUALTABLE -DSQLITE_OMIT_EXPLAIN -DSQLITE_OMIT_TRIGGER' make -e sqlite3 +*/ +/*Requires a custom sqlite3.c +# undef SQLITE_OMIT_TRIGGER +# define SQLITE_OMIT_TRIGGER +*/ +/*TODO (requires build tweaks) +# undef SQLITE_OMIT_WINDOWFUNC +# define SQLITE_OMIT_WINDOWFUNC +*/ +/*Requires a custom sqlite3.c +# undef SQLITE_OMIT_EXPLAIN +# define SQLITE_OMIT_EXPLAIN +*/ +/*Requires a custom sqlite3.c +# undef SQLITE_OMIT_VIRTUALTABLE +# define SQLITE_OMIT_VIRTUALTABLE +*/ +# undef SQLITE_OMIT_JSON +# define SQLITE_OMIT_JSON +#endif + #include <assert.h> /* @@ -238,28 +280,28 @@ ** Another option is to malloc() a chunk of our own and call that our ** "stack". */ -SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_end(void){ +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_end(void){ extern void __heap_base /* see https://stackoverflow.com/questions/10038964 */; return &__heap_base; } -SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_begin(void){ +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_begin(void){ extern void __data_end; return &__data_end; } static void * pWasmStackPtr = 0; -SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_ptr(void){ - if(!pWasmStackPtr) pWasmStackPtr = sqlite3_wasm_stack_end(); +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_ptr(void){ + if(!pWasmStackPtr) pWasmStackPtr = sqlite3__wasm_stack_end(); return pWasmStackPtr; } -SQLITE_WASM_EXPORT void sqlite3_wasm_stack_restore(void * p){ +SQLITE_WASM_EXPORT void sqlite3__wasm_stack_restore(void * p){ pWasmStackPtr = p; } -SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_alloc(int n){ +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_alloc(int n){ if(n<=0) return 0; n = (n + 7) & ~7 /* align to 8-byte boundary */; - unsigned char * const p = (unsigned char *)sqlite3_wasm_stack_ptr(); - unsigned const char * const b = (unsigned const char *)sqlite3_wasm_stack_begin(); + unsigned char * const p = (unsigned char *)sqlite3__wasm_stack_ptr(); + unsigned const char * const b = (unsigned const char *)sqlite3__wasm_stack_begin(); if(b + n >= p || b + n < b/*overflow*/) return 0; return pWasmStackPtr = p - n; } @@ -267,7 +309,7 @@ SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_alloc(int n){ /* ** State for the "pseudo-stack" allocator implemented in -** sqlite3_wasm_pstack_xyz(). In order to avoid colliding with +** sqlite3__wasm_pstack_xyz(). In order to avoid colliding with ** Emscripten-controled stack space, it carves out a bit of stack ** memory to use for that purpose. This memory ends up in the ** WASM-managed memory, such that routines which manipulate the wasm @@ -291,14 +333,14 @@ static struct { /* ** Returns the current pstack position. */ -SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_ptr(void){ +SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_ptr(void){ return PStack.pPos; } /* ** Sets the pstack position poitner to p. Results are undefined if the -** given value did not come from sqlite3_wasm_pstack_ptr(). +** given value did not come from sqlite3__wasm_pstack_ptr(). */ -SQLITE_WASM_EXPORT void sqlite3_wasm_pstack_restore(unsigned char * p){ +SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); assert(0==((unsigned long long)p & 0x7)); if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ @@ -313,7 +355,7 @@ SQLITE_WASM_EXPORT void sqlite3_wasm_pstack_restore(unsigned char * p){ ** JS code from having to do so, and most uses of the pstack will ** call for doing so). */ -SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_alloc(int n){ +SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_alloc(int n){ if( n<=0 ) return 0; //if( n & 0x7 ) n += 8 - (n & 0x7) /* align to 8-byte boundary */; n = (n + 7) & ~7 /* align to 8-byte boundary */; @@ -324,9 +366,9 @@ SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_alloc(int n){ } /* ** Return the number of bytes left which can be -** sqlite3_wasm_pstack_alloc()'d. +** sqlite3__wasm_pstack_alloc()'d. */ -SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_remaining(void){ +SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_remaining(void){ assert(PStack.pPos >= PStack.pBegin); assert(PStack.pPos <= PStack.pEnd); return (int)(PStack.pPos - PStack.pBegin); @@ -337,7 +379,7 @@ SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_remaining(void){ ** any space which is currently allocated. This value is a ** compile-time constant. */ -SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_quota(void){ +SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_quota(void){ return (int)(PStack.pEnd - PStack.pBegin); } @@ -356,7 +398,7 @@ SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_quota(void){ ** Returns err_code. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ +int sqlite3__wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ if( db!=0 ){ if( 0!=zMsg ){ const int nMsg = sqlite3Strlen30(zMsg); @@ -380,7 +422,7 @@ struct WasmTestStruct { }; typedef struct WasmTestStruct WasmTestStruct; SQLITE_WASM_EXPORT -void sqlite3_wasm_test_struct(WasmTestStruct * s){ +void sqlite3__wasm_test_struct(WasmTestStruct * s){ if(s){ s->v4 *= 2; s->v8 = s->v4 * 2; @@ -408,7 +450,7 @@ void sqlite3_wasm_test_struct(WasmTestStruct * s){ ** increased. In debug builds that will trigger an assert(). */ SQLITE_WASM_EXPORT -const char * sqlite3_wasm_enum_json(void){ +const char * sqlite3__wasm_enum_json(void){ static char aBuffer[1024 * 20] = {0} /* where the JSON goes */; int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; @@ -425,7 +467,7 @@ const char * sqlite3_wasm_enum_json(void){ /* Core output macros... */ #define lenCheck assert(zPos < zEnd - 128 \ - && "sqlite3_wasm_enum_json() buffer is too small."); \ + && "sqlite3__wasm_enum_json() buffer is too small."); \ if( zPos >= zEnd - 128 ) return 0 #define outf(format,...) \ zPos += snprintf(zPos, ((size_t)(zEnd - zPos)), format, __VA_ARGS__); \ @@ -499,6 +541,7 @@ const char * sqlite3_wasm_enum_json(void){ } _DefGroup; DefGroup(changeset){ +#ifdef SQLITE_CHANGESETSTART_INVERT DefInt(SQLITE_CHANGESETSTART_INVERT); DefInt(SQLITE_CHANGESETAPPLY_NOSAVEPOINT); DefInt(SQLITE_CHANGESETAPPLY_INVERT); @@ -513,6 +556,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_CHANGESET_OMIT); DefInt(SQLITE_CHANGESET_REPLACE); DefInt(SQLITE_CHANGESET_ABORT); +#endif } _DefGroup; DefGroup(config){ @@ -545,6 +589,10 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_CONFIG_SMALL_MALLOC); DefInt(SQLITE_CONFIG_SORTERREF_SIZE); DefInt(SQLITE_CONFIG_MEMDB_MAXSIZE); + /* maintenance note: we specifically do not include + SQLITE_CONFIG_ROWID_IN_VIEW here, on the grounds that + it's only for legacy support and no apps written with + this API require that. */ } _DefGroup; DefGroup(dataTypes) { @@ -560,7 +608,6 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_DBCONFIG_LOOKASIDE); DefInt(SQLITE_DBCONFIG_ENABLE_FKEY); DefInt(SQLITE_DBCONFIG_ENABLE_TRIGGER); - DefInt(SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER); DefInt(SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION); DefInt(SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE); DefInt(SQLITE_DBCONFIG_ENABLE_QPSG); @@ -855,8 +902,10 @@ const char * sqlite3_wasm_enum_json(void){ } _DefGroup; DefGroup(session){ +#ifdef SQLITE_SESSION_CONFIG_STRMSIZE DefInt(SQLITE_SESSION_CONFIG_STRMSIZE); DefInt(SQLITE_SESSION_OBJCONFIG_SIZE); +#endif } _DefGroup; DefGroup(sqlite3Status){ @@ -918,6 +967,7 @@ const char * sqlite3_wasm_enum_json(void){ } _DefGroup; DefGroup(vtab) { +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_MINIMAL) DefInt(SQLITE_INDEX_SCAN_UNIQUE); DefInt(SQLITE_INDEX_CONSTRAINT_EQ); DefInt(SQLITE_INDEX_CONSTRAINT_GT); @@ -945,6 +995,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_FAIL); //DefInt(SQLITE_ABORT); // Also an error code DefInt(SQLITE_REPLACE); +#endif /*!SQLITE_OMIT_VIRTUALTABLE*/ } _DefGroup; #undef DefGroup @@ -1104,6 +1155,7 @@ const char * sqlite3_wasm_enum_json(void){ } _StructBinder; #undef CurrentStruct +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_MINIMAL) /** ** Workaround: in order to map the various inner structs from ** sqlite3_index_info, we have to uplift those into constructs we @@ -1180,6 +1232,8 @@ const char * sqlite3_wasm_enum_json(void){ } _StructBinder; #undef CurrentStruct +#endif /*!SQLITE_OMIT_VIRTUALTABLE*/ + #if SQLITE_WASM_TESTS #define CurrentStruct WasmTestStruct StructBinder { @@ -1220,7 +1274,7 @@ const char * sqlite3_wasm_enum_json(void){ ** call is returned. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ +int sqlite3__wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); if( zName && pVfs && pVfs->xDelete ){ @@ -1238,7 +1292,7 @@ int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ ** given name is open. */ SQLITE_WASM_EXPORT -sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ +sqlite3_vfs * sqlite3__wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ sqlite3_vfs * pVfs = 0; sqlite3_file_control(pDb, zDbName ? zDbName : "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); @@ -1261,7 +1315,7 @@ sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ ** SQLITE_MISUSE if pDb is NULL. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_reset(sqlite3 *pDb){ +int sqlite3__wasm_db_reset(sqlite3 *pDb){ int rc = SQLITE_MISUSE; if( pDb ){ sqlite3_table_column_metadata(pDb, "main", 0, 0, 0, 0, 0, 0, 0); @@ -1288,11 +1342,11 @@ int sqlite3_wasm_db_reset(sqlite3 *pDb){ ** takes no measures to ensure that is the case. ** ** This implementation appears to work fine, but -** sqlite3_wasm_db_serialize() is arguably the better way to achieve +** sqlite3__wasm_db_serialize() is arguably the better way to achieve ** this. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_export_chunked( sqlite3* pDb, +int sqlite3__wasm_db_export_chunked( sqlite3* pDb, int (*xCallback)(unsigned const char *zOut, int n) ){ sqlite3_int64 nSize = 0; sqlite3_int64 nPos = 0; @@ -1343,7 +1397,7 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb, ** sqlite3_free() to free it. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, +int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, unsigned char **pOut, sqlite3_int64 *nOut, unsigned int mFlags ){ unsigned char * z; @@ -1366,7 +1420,7 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** this function's out-of-scope use of the sqlite3_vfs/file/io_methods ** APIs leads to triggering of assertions in the core library. Its use ** is now deprecated and VFS-specific APIs for importing files need to -** be found to replace it. sqlite3_wasm_posix_create_file() is +** be found to replace it. sqlite3__wasm_posix_create_file() is ** suitable for the "unix" family of VFSes. ** ** Creates a new file using the I/O API of the given VFS, containing @@ -1407,7 +1461,7 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** support is disabled or unavailable. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, +int sqlite3__wasm_vfs_create_file( sqlite3_vfs *pVfs, const char *zFilename, const unsigned char * pData, int nData ){ @@ -1497,7 +1551,7 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, ** SQLITE_IOERR on error. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_posix_create_file( const char *zFilename, +int sqlite3__wasm_posix_create_file( const char *zFilename, const unsigned char * pData, int nData ){ int rc; @@ -1520,17 +1574,17 @@ int sqlite3_wasm_posix_create_file( const char *zFilename, ** for use by the sqlite project's own JS/WASM bindings. ** ** Allocates sqlite3KvvfsMethods.nKeySize bytes from -** sqlite3_wasm_pstack_alloc() and returns 0 if that allocation fails, +** sqlite3__wasm_pstack_alloc() and returns 0 if that allocation fails, ** else it passes that string to kvstorageMakeKey() and returns a ** NUL-terminated pointer to that string. It is up to the caller to -** use sqlite3_wasm_pstack_restore() to free the returned pointer. +** use sqlite3__wasm_pstack_restore() to free the returned pointer. */ SQLITE_WASM_EXPORT -char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, +char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, const char *zKeyIn){ assert(sqlite3KvvfsMethods.nKeySize>24); char *zKeyOut = - (char *)sqlite3_wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); + (char *)sqlite3__wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); if(zKeyOut){ kvstorageMakeKey(zClass, zKeyIn, zKeyOut); } @@ -1545,10 +1599,11 @@ char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, ** I/O methods and associated state. */ SQLITE_WASM_EXPORT -sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ +sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_WASM_MINIMAL) /* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. @@ -1560,7 +1615,7 @@ sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ ** valid value. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ +int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ switch(op){ case SQLITE_VTAB_DIRECTONLY: case SQLITE_VTAB_INNOCUOUS: @@ -1571,6 +1626,7 @@ int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ return SQLITE_MISUSE; } } +#endif /*!SQLITE_OMIT_VIRTUALTABLE*/ /* ** This function is NOT part of the sqlite3 public API. It is strictly @@ -1580,7 +1636,7 @@ int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ ** (int,int*) variadic args. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ +int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ switch(op){ case SQLITE_DBCONFIG_ENABLE_FKEY: case SQLITE_DBCONFIG_ENABLE_TRIGGER: @@ -1613,7 +1669,7 @@ int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ ** (void*,int,int) variadic args. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ +int sqlite3__wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ switch(op){ case SQLITE_DBCONFIG_LOOKASIDE: return sqlite3_db_config(pDb, op, pArg1, arg2, arg3); @@ -1629,7 +1685,7 @@ int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int ** (const char *) variadic args. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ +int sqlite3__wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ switch(op){ case SQLITE_DBCONFIG_MAINDBNAME: return sqlite3_db_config(pDb, op, zArg); @@ -1646,7 +1702,7 @@ int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ ** a single integer argument. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_config_i(int op, int arg){ +int sqlite3__wasm_config_i(int op, int arg){ return sqlite3_config(op, arg); } @@ -1658,7 +1714,7 @@ int sqlite3_wasm_config_i(int op, int arg){ ** two int arguments. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ +int sqlite3__wasm_config_ii(int op, int arg1, int arg2){ return sqlite3_config(op, arg1, arg2); } @@ -1670,39 +1726,28 @@ int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ ** a single i64 argument. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_config_j(int op, sqlite3_int64 arg){ +int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ return sqlite3_config(op, arg); } -#if 0 -// Pending removal after verification of a workaround discussed in the -// forum post linked to below. /* ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** -** Returns a pointer to sqlite3_free(). In compliant browsers the -** return value, when passed to sqlite3.wasm.exports.functionEntry(), -** must resolve to the same function as -** sqlite3.wasm.exports.sqlite3_free. i.e. from a dev console where -** sqlite3 is exported globally, the following must be true: -** -** ``` -** sqlite3.wasm.functionEntry( -** sqlite3.wasm.exports.sqlite3_wasm_ptr_to_sqlite3_free() -** ) === sqlite3.wasm.exports.sqlite3_free -** ``` -** -** Using a function to return this pointer, as opposed to exporting it -** via sqlite3_wasm_enum_json(), is an attempt to work around a -** Safari-specific quirk covered at -** https://sqlite.org/forum/info/e5b20e1feb37a19a. -**/ +** If z is not NULL, returns the result of passing z to +** sqlite3_mprintf()'s %Q modifier (if addQuotes is true) or %q (if +** addQuotes is 0). Returns NULL if z is NULL or on OOM. +*/ SQLITE_WASM_EXPORT -void * sqlite3_wasm_ptr_to_sqlite3_free(void){ - return (void*)sqlite3_free; +char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ + char * rc = 0; + if( z ){ + rc = addQuotes + ? sqlite3_mprintf("%Q", z) + : sqlite3_mprintf("%q", z); + } + return rc; } -#endif #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) #include <emscripten/wasmfs.h> @@ -1729,7 +1774,7 @@ void * sqlite3_wasm_ptr_to_sqlite3_free(void){ ** defined, SQLITE_NOTFOUND is returned without side effects. */ SQLITE_WASM_EXPORT -int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ +int sqlite3__wasm_init_wasmfs(const char *zMountPoint){ static backend_t pOpfs = 0; if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; if( !pOpfs ){ @@ -1749,7 +1794,7 @@ int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ } #else SQLITE_WASM_EXPORT -int sqlite3_wasm_init_wasmfs(const char *zUnused){ +int sqlite3__wasm_init_wasmfs(const char *zUnused){ //emscripten_console_warn("WASMFS OPFS is not compiled in."); if(zUnused){/*unused*/} return SQLITE_NOTFOUND; @@ -1759,51 +1804,51 @@ int sqlite3_wasm_init_wasmfs(const char *zUnused){ #if SQLITE_WASM_TESTS SQLITE_WASM_EXPORT -int sqlite3_wasm_test_intptr(int * p){ +int sqlite3__wasm_test_intptr(int * p){ return *p = *p * 2; } SQLITE_WASM_EXPORT -void * sqlite3_wasm_test_voidptr(void * p){ +void * sqlite3__wasm_test_voidptr(void * p){ return p; } SQLITE_WASM_EXPORT -int64_t sqlite3_wasm_test_int64_max(void){ +int64_t sqlite3__wasm_test_int64_max(void){ return (int64_t)0x7fffffffffffffff; } SQLITE_WASM_EXPORT -int64_t sqlite3_wasm_test_int64_min(void){ - return ~sqlite3_wasm_test_int64_max(); +int64_t sqlite3__wasm_test_int64_min(void){ + return ~sqlite3__wasm_test_int64_max(); } SQLITE_WASM_EXPORT -int64_t sqlite3_wasm_test_int64_times2(int64_t x){ +int64_t sqlite3__wasm_test_int64_times2(int64_t x){ return x * 2; } SQLITE_WASM_EXPORT -void sqlite3_wasm_test_int64_minmax(int64_t * min, int64_t *max){ - *max = sqlite3_wasm_test_int64_max(); - *min = sqlite3_wasm_test_int64_min(); +void sqlite3__wasm_test_int64_minmax(int64_t * min, int64_t *max){ + *max = sqlite3__wasm_test_int64_max(); + *min = sqlite3__wasm_test_int64_min(); /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ } SQLITE_WASM_EXPORT -int64_t sqlite3_wasm_test_int64ptr(int64_t * p){ - /*printf("sqlite3_wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ +int64_t sqlite3__wasm_test_int64ptr(int64_t * p){ + /*printf("sqlite3__wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ return *p = *p * 2; } SQLITE_WASM_EXPORT -void sqlite3_wasm_test_stack_overflow(int recurse){ - if(recurse) sqlite3_wasm_test_stack_overflow(recurse); +void sqlite3__wasm_test_stack_overflow(int recurse){ + if(recurse) sqlite3__wasm_test_stack_overflow(recurse); } /* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ SQLITE_WASM_EXPORT -char * sqlite3_wasm_test_str_hello(int fail){ +char * sqlite3__wasm_test_str_hello(int fail){ char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ memcpy(s, "hello", 5); @@ -1838,12 +1883,12 @@ char * sqlite3_wasm_test_str_hello(int fail){ ** optional + or - sign in front, or a hexadecimal ** literal of the form 0x... */ -static int sqlite3_wasm_SQLTester_strnotglob(const char *zGlob, const char *z){ +static int sqlite3__wasm_SQLTester_strnotglob(const char *zGlob, const char *z){ int c, c2; int invert; int seen; typedef int (*recurse_f)(const char *,const char *); - static const recurse_f recurse = sqlite3_wasm_SQLTester_strnotglob; + static const recurse_f recurse = sqlite3__wasm_SQLTester_strnotglob; while( (c = (*(zGlob++)))!=0 ){ if( c=='*' ){ @@ -1918,11 +1963,10 @@ static int sqlite3_wasm_SQLTester_strnotglob(const char *zGlob, const char *z){ } SQLITE_WASM_EXPORT -int sqlite3_wasm_SQLTester_strglob(const char *zGlob, const char *z){ - return !sqlite3_wasm_SQLTester_strnotglob(zGlob, z); +int sqlite3__wasm_SQLTester_strglob(const char *zGlob, const char *z){ + return !sqlite3__wasm_SQLTester_strnotglob(zGlob, z); } - #endif /* SQLITE_WASM_TESTS */ #undef SQLITE_WASM_EXPORT diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index cd78ed4bc..55e497ead 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -42,9 +42,13 @@ - `onready` (optional, but...): this callback is called with no arguments when the worker fires its initial 'sqlite3-api'/'worker1-ready' message, which it does when - sqlite3.initWorker1API() completes its initialization. This is - the simplest way to tell the worker to kick off work at the - earliest opportunity. + sqlite3.initWorker1API() completes its initialization. This is the + simplest way to tell the worker to kick off work at the earliest + opportunity, and the only way to know when the worker module has + completed loading. The irony of using a callback for this, instead + of returning a promise from sqlite3Worker1Promiser() is not lost on + the developers: see sqlite3Worker1Promiser.v2() which uses a + Promise instead. - `onunhandled` (optional): a callback which gets passed the message event object for any worker.onmessage() events which @@ -156,6 +160,7 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi if(!config.worker) config.worker = callee.defaultConfig.worker; if('function'===typeof config.worker) config.worker = config.worker(); let dbId; + let promiserFunc; config.worker.onmessage = function(ev){ ev = ev.data; debug('worker1.onmessage',ev); @@ -163,7 +168,7 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi if(!msgHandler){ if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) { /*fired one time when the Worker1 API initializes*/ - if(config.onready) config.onready(); + if(config.onready) config.onready(promiserFunc); return; } msgHandler = handlerMap[ev.type] /* check for exec per-row callback */; @@ -192,7 +197,7 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi try {msgHandler.resolve(ev)} catch(e){msgHandler.reject(e)} }/*worker.onmessage()*/; - return function(/*(msgType, msgArgs) || (msgEnvelope)*/){ + return promiserFunc = function(/*(msgType, msgArgs) || (msgEnvelope)*/){ let msg; if(1===arguments.length){ msg = arguments[0]; @@ -202,7 +207,7 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi msg.args = arguments[1]; msg.dbId = msg.args.dbId; }else{ - toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); + toss("Invalid arguments for sqlite3Worker1Promiser()-created factory."); } if(!msg.dbId && msg.type!=='open') msg.dbId = dbId; msg.messageId = genMsgId(msg); @@ -246,9 +251,10 @@ globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfi return p; }; }/*sqlite3Worker1Promiser()*/; + globalThis.sqlite3Worker1Promiser.defaultConfig = { worker: function(){ -//#if target=es6-bundler-friendly +//#if target=es6-module return new Worker(new URL("sqlite3-worker1-bundler-friendly.mjs", import.meta.url),{ type: 'module' }); @@ -269,14 +275,72 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { return new Worker(theJs + globalThis.location.search); //#endif } -//#ifnot target=es6-bundler-friendly +//#ifnot target=es6-module .bind({ currentScript: globalThis?.document?.currentScript }) //#endif , onerror: (...args)=>console.error('worker1 promiser error',...args) -}; +}/*defaultConfig*/; + +/** + sqlite3Worker1Promiser.v2(), added in 3.46, works identically to + sqlite3Worker1Promiser() except that it returns a Promise instead + of relying an an onready callback in the config object. The Promise + resolves to the same factory function which + sqlite3Worker1Promiser() returns. + + If config is-a function or is an object which contains an onready + function, that function is replaced by a proxy which will resolve + after calling the original function and will reject if that + function throws. +*/ +sqlite3Worker1Promiser.v2 = function(config){ + let oldFunc; + if( 'function' == typeof config ){ + oldFunc = config; + config = {}; + }else if('function'===typeof config?.onready){ + oldFunc = config.onready; + delete config.onready; + } + const promiseProxy = Object.create(null); + config = Object.assign((config || Object.create(null)),{ + onready: async function(func){ + try { + if( oldFunc ) await oldFunc(func); + promiseProxy.resolve(func); + } + catch(e){promiseProxy.reject(e)} + } + }); + const p = new Promise(function(resolve,reject){ + promiseProxy.resolve = resolve; + promiseProxy.reject = reject; + }); + try{ + this.original(config); + }catch(e){ + promiseProxy.reject(e); + } + return p; +}.bind({ + /* We do this because clients are + recommended to delete globalThis.sqlite3Worker1Promiser. */ + original: sqlite3Worker1Promiser +}); + +//#if target=es6-module +/** + When built as a module, we export sqlite3Worker1Promiser.v2() + instead of sqlite3Worker1Promise() because (A) its interface is more + conventional for ESM usage and (B) the ESM option export option for + this API did not exist until v2 was created, so there's no backwards + incompatibility. +*/ +export default sqlite3Worker1Promiser.v2; +//#endif /* target=es6-module */ //#else /* Built with the omit-oo1 flag. */ //#endif ifnot omit-oo1 diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 0437ef35d..25400d48e 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -156,8 +156,7 @@ - 'dealloc()` must behave like C's `free()`, accepting either a pointer returned from its allocation counterpart or the values - null/0 (for which it must be a no-op). allocating N bytes of - memory and returning its pointer. In Emscripten this is + null/0 (for which it must be a no-op). In Emscripten this is conventionally made available via `Module['_free']`. APIs which require allocation routines are explicitly documented as @@ -730,22 +729,23 @@ globalThis.WhWasmUtilInstaller = function(target){ }; /** - The counterpart of peek(), this sets a numeric value at - the given WASM heap address, using the type to define how many - bytes are written. Throws if given an invalid type. See - peek() for details about the type argument. If the 3rd - argument ends with `*` then it is treated as a pointer type and - this function behaves as if the 3rd argument were `i32`. + The counterpart of peek(), this sets a numeric value at the given + WASM heap address, using the 3rd argument to define how many + bytes are written. Throws if given an invalid type. See peek() + for details about the `type` argument. If the 3rd argument ends + with `*` then it is treated as a pointer type and this function + behaves as if the 3rd argument were `i32`. If the first argument is an array, it is treated like a list of pointers and the given value is written to each one. - Returns `this`. (Prior to 2022-12-09 it returns this function.) + Returns `this`. (Prior to 2022-12-09 it returned this function.) - ACHTUNG: calling this often, e.g. in a loop, can have a noticably - painful impact on performance. Rather than doing so, use - heapForSize() to fetch the heap object and assign directly to it - or use the heap's set() method. + ACHTUNG: calling this often, e.g. in a loop to populate a large + chunk of memory, can have a noticably painful impact on + performance. Rather than doing so, use heapForSize() to fetch the + heap object and assign directly to it or use the heap's set() + method. */ target.poke = function(ptr, value, type='i8'){ if (type.endsWith('*')) type = ptrIR; @@ -1382,15 +1382,19 @@ globalThis.WhWasmUtilInstaller = function(target){ conversion of argument or return types, but see xWrap() and xCallWrapped() for variants which do. + If the first argument is a function is is assumed to be + a WASM-bound function and is used as-is instead of looking up + the function via xGet(). + As a special case, if passed only 1 argument after the name and that argument in an Array, that array's entries become the function arguments. (This is not an ambiguous case because it's not legal to pass an Array object to a WASM function.) */ target.xCall = function(fname, ...args){ - const f = target.xGet(fname); + const f = (fname instanceof Function) ? fname : target.xGet(fname); if(!(f instanceof Function)) toss("Exported symbol",fname,"is not a function."); - if(f.length!==args.length) __argcMismatch(fname,f.length) + if(f.length!==args.length) __argcMismatch(((f===fname) ? f.name : fname),f.length) /* This is arguably over-pedantic but we want to help clients keep from shooting themselves in the foot when calling C APIs. */; return (2===arguments.length && Array.isArray(arguments[1])) @@ -1537,7 +1541,7 @@ globalThis.WhWasmUtilInstaller = function(target){ jsFuncToWasm(). - bindScope (string): one of ('transient', 'context', - 'singleton'). Bind scopes are: + 'singleton', 'permanent'). Bind scopes are: - 'transient': it will convert JS functions to WASM only for the duration of the xWrap()'d function call, using @@ -1787,11 +1791,29 @@ globalThis.WhWasmUtilInstaller = function(target){ const __xResultAdapterCheck = (t)=>xResult.get(t) || toss("Result adapter not found:",t); + /** + Fetches the xWrap() argument adapter mapped to t, calls it, + passing in all remaining arguments, and returns the result. + Throws if t is not mapped to an argument converter. + */ cache.xWrap.convertArg = (t,...args)=>__xArgAdapterCheck(t)(...args); + /** + Identical to convertArg() except that it does not perform + an is-defined check on the mapping to t before invoking it. + */ cache.xWrap.convertArgNoCheck = (t,...args)=>xArg.get(t)(...args); + /** + Fetches the xWrap() result adapter mapped to t, calls it, passing + it v, and returns the result. Throws if t is not mapped to an + argument converter. + */ cache.xWrap.convertResult = (t,v)=>(null===t ? v : (t ? __xResultAdapterCheck(t)(v) : undefined)); + /** + Identical to convertResult() except that it does not perform an + is-defined check on the mapping to t before invoking it. + */ cache.xWrap.convertResultNoCheck = (t,v)=>(null===t ? v : (t ? xResult.get(t)(v) : undefined)); @@ -1903,15 +1925,15 @@ globalThis.WhWasmUtilInstaller = function(target){ const C-string, encoded as UTF-8, copies it to a JS string, and returns that JS string. - - `string:dealloc` or `utf8:dealloc) (results): treats the result value - as a non-const UTF-8 C-string, ownership of which has just been - transfered to the caller. It copies the C-string to a JS - string, frees the C-string, and returns the JS string. If such - a result value is NULL, the JS result is `null`. Achtung: when - using an API which returns results from a specific allocator, - e.g. `my_malloc()`, this conversion _is not legal_. Instead, an - equivalent conversion which uses the appropriate deallocator is - required. For example: + - `string:dealloc` or `utf8:dealloc` (results): treats the result + value as a non-const UTF-8 C-string, ownership of which has + just been transfered to the caller. It copies the C-string to a + JS string, frees the C-string, and returns the JS string. If + such a result value is NULL, the JS result is `null`. Achtung: + when using an API which returns results from a specific + allocator, e.g. `my_malloc()`, this conversion _is not + legal_. Instead, an equivalent conversion which uses the + appropriate deallocator is required. For example: ```js target.xWrap.resultAdapter('string:my_free',(i)=>{ @@ -2012,8 +2034,12 @@ globalThis.WhWasmUtilInstaller = function(target){ arguments may be passed in after that one, and what those arguments are, is _not_ part of the public interface and is _not_ stable. + + Maintenance reminder: the Ember framework modifies the core + Array type, breaking for-in loops. */ - for(const i in args) args[i] = cxw.convertArgNoCheck( + let i = 0; + for(; i < args.length; ++i) args[i] = cxw.convertArgNoCheck( argTypes[i], args[i], args, i ); return cxw.convertResultNoCheck(resultType, xf.apply(null,args)); diff --git a/ext/wasm/demo-worker1-promiser.html b/ext/wasm/demo-worker1-promiser.c-pp.html index e99131e6c..e0b487bdf 100644 --- a/ext/wasm/demo-worker1-promiser.html +++ b/ext/wasm/demo-worker1-promiser.c-pp.html @@ -6,7 +6,11 @@ <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> <link rel="stylesheet" href="common/emscripten.css"/> <link rel="stylesheet" href="common/testing.css"/> +//#if target=es6-module + <title>worker-promise (via ESM) tests</title> +//#else <title>worker-promise tests</title> +//#endif </head> <body> <header id='titlebar'><span>worker-promise tests</span></header> @@ -22,13 +26,17 @@ </figure> <div class="emscripten" id="module-status">Downloading...</div> <div class="emscripten"> - <progress value="0" max="100" id="module-progress" hidden='1'></progress> + <progress value="0" max="100" id="module-progress" hidden='1'></progress> </div><!-- /emscripten bits --> <div>Most stuff on this page happens in the dev console.</div> <hr> <div id='test-output'></div> <script src="common/SqliteTestUtil.js"></script> +//#if target=es6-module + <script src="demo-worker1-promiser.mjs" type="module"></script> +//#else <script src="jswasm/sqlite3-worker1-promiser.js"></script> <script src="demo-worker1-promiser.js"></script> +//#endif </body> </html> diff --git a/ext/wasm/demo-worker1-promiser.js b/ext/wasm/demo-worker1-promiser.c-pp.js index c2d24623a..f6fc9568a 100644 --- a/ext/wasm/demo-worker1-promiser.js +++ b/ext/wasm/demo-worker1-promiser.c-pp.js @@ -13,9 +13,15 @@ Demonstration of the sqlite3 Worker API #1 Promiser: a Promise-based proxy for for the sqlite3 Worker #1 API. */ -'use strict'; -(function(){ - const T = self.SqliteTestUtil; +//#if target=es6-module +import {default as promiserFactory} from "./jswasm/sqlite3-worker1-promiser.mjs"; +//#else +"use strict"; +const promiserFactory = globalThis.sqlite3Worker1Promiser.v2; +delete globalThis.sqlite3Worker1Promiser; +//#endif +(async function(){ + const T = globalThis.SqliteTestUtil; const eOutput = document.querySelector('#test-output'); const warn = console.warn.bind(console); const error = console.error.bind(console); @@ -33,31 +39,35 @@ logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); }; - //why is this triggered even when we catch() a Promise? - //window.addEventListener('unhandledrejection', function(event) { - // warn('unhandledrejection',event); - //}); - const promiserConfig = { - worker: ()=>{ - const w = new Worker("jswasm/sqlite3-worker1.js"); - w.onerror = (event)=>error("worker.onerror",event); - return w; +//#ifnot target=es6-module + /** + The v1 interfaces uses an onready function. The v2 interface optionally + accepts one but does not require it. If provided, it is called _before_ + the promise is resolved, and the promise is rejected if onready() throws. + */ + onready: function(f){ + /* f === the function returned by promiserFactory(). + Ostensibly (f === workerPromise) but this function is + called before the promiserFactory() Promise resolves, so + before workerPromise is set. */ + console.warn("This is the v2 interface - you don't need an onready() function."); }, +//#endif debug: 1 ? undefined : (...args)=>console.debug('worker debug',...args), onunhandled: function(ev){ error("Unhandled worker message:",ev.data); }, - onready: function(){ - self.sqlite3TestModule.setStatus(null)/*hide the HTML-side is-loading spinner*/; - runTests(); - }, onerror: function(ev){ error("worker1 error:",ev); } }; - const workerPromise = self.sqlite3Worker1Promiser(promiserConfig); - delete self.sqlite3Worker1Promiser; + const workerPromise = await promiserFactory(promiserConfig) + .then((func)=>{ + console.log("Init complete. Starting tests momentarily."); + globalThis.sqlite3TestModule.setStatus(null)/*hide the HTML-side is-loading spinner*/; + return func; + }); const wtest = async function(msgType, msgArgs, callback){ if(2===arguments.length && 'function'===typeof msgArgs){ @@ -271,5 +281,5 @@ }).finally(()=>logHtml('',"That's all, folks!")); }/*runTests2()*/; - log("Init complete, but async init bits may still be running."); + runTests(); })(); diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 5d610e37b..e820e066d 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -19,10 +19,15 @@ MAKEFILE.dist := $(lastword $(MAKEFILE_LIST)) # built, and won't be built until we expand the dependencies. Thus we # have to use a temporary name for the archive until we can get # that binary built. +ifeq (1,$(SQLITE_C_IS_SEE)) +dist-name-extra := -see +else +dist-name-extra := +endif ifeq (,$(filter snapshot,$(MAKECMDGOALS))) -dist-name-prefix := sqlite-wasm +dist-name-prefix := sqlite-wasm$(dist-name-extra) else -dist-name-prefix := sqlite-wasm-snapshot-$(shell /usr/bin/date +%Y%m%d) +dist-name-prefix := sqlite-wasm$(dist-name-extra)-snapshot-$(shell /usr/bin/date +%Y%m%d) endif dist-name := $(dist-name-prefix)-TEMP @@ -49,12 +54,18 @@ dist.top.extras := \ tester1.js tester1.mjs \ demo-jsstorage.html demo-jsstorage.js \ demo-worker1.html demo-worker1.js \ - demo-worker1-promiser.html demo-worker1-promiser.js -dist.jswasm.extras := $(sqlite3-api.ext.jses) $(sqlite3.wasm) + demo-worker1-promiser.html demo-worker1-promiser.js \ + demo-worker1-promiser-esm.html demo-worker1-promiser.mjs +dist.jswasm.extras := $(sqlite3.wasm) \ + $(sqlite3-api.ext.jses) dist.common.extras := \ $(wildcard $(dir.common)/*.css) \ $(dir.common)/SqliteTestUtil.js +#$(info sqlite3-worker1-promiser.mjs = $(sqlite3-worker1-promiser.mjs)) +#$(info sqlite3-worker1.js = $(sqlite3-worker1.js)) +#$(info sqlite3-api.ext.jses = $(sqlite3-api.ext.jses)) +#$(info dist.jswasm.extras = $(dist.jswasm.extras)) .PHONY: dist snapshot # DIST_STRIP_COMMENTS $(call)able to be used in stripping C-style # from the dist copies of certain files. @@ -67,7 +78,8 @@ endef # STRIP_K1.js = list of JS files which need to be passed through # $(bin.stripcomments) with a single -k flag. STRIP_K1.js := $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js) \ - $(sqlite3-worker1-bundler-friendly.js) $(sqlite3-worker1-promiser-bundler-friendly.js) + $(sqlite3-worker1-bundler-friendly.js) \ + $(sqlite3-api.ext.jses) # STRIP_K2.js = list of JS files which need to be passed through # $(bin.stripcomments) with two -k flags. STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) \ @@ -88,6 +100,7 @@ STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) \ dist: \ $(bin.stripccomments) $(bin.version-info) \ $(dist.build) $(STRIP_K1.js) $(STRIP_K2.js) \ + $(dist.jswasm.extras) $(dist.common.extras) \ $(MAKEFILE) $(MAKEFILE.dist) @echo "Making end-user deliverables..." @rm -fr $(dist-dir.top) diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 57141d7e2..1f0d9e2ad 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -9,16 +9,18 @@ MAKEFILE.fiddle := $(lastword $(MAKEFILE_LIST)) # shell.c and its build flags... make-np-0 := make -C $(dir.top) -n -p make-np-1 := sed -e 's/(TOP)/(dir.top)/g' +# Extract SHELL_OPT and SHELL_DEP from the top-most makefile and import +# them as vars here... $(eval $(shell $(make-np-0) | grep -e '^SHELL_OPT ' | $(make-np-1))) -$(eval $(shell $(make-np-0) | grep -e '^SHELL_SRC ' | $(make-np-1))) +$(eval $(shell $(make-np-0) | grep -e '^SHELL_DEP ' | $(make-np-1))) # ^^^ can't do that in 1 invocation b/c newlines get stripped ifeq (,$(SHELL_OPT)) $(error Could not parse SHELL_OPT from $(dir.top)/Makefile.) endif -ifeq (,$(SHELL_SRC)) -$(error Could not parse SHELL_SRC from $(dir.top)/Makefile.) +ifeq (,$(SHELL_DEP)) +$(error Could not parse SHELL_DEP from $(dir.top)/Makefile.) endif -$(dir.top)/shell.c: $(SHELL_SRC) $(dir.top)/tool/mkshellc.tcl $(sqlite3.c) +$(dir.top)/shell.c: $(SHELL_DEP) $(dir.top)/tool/mkshellc.tcl $(sqlite3.c) $(MAKE) -C $(dir.top) shell.c # /shell.c ######################################################################## @@ -40,9 +42,17 @@ fiddle.emcc-flags = \ -sEXPORTED_FUNCTIONS=@$(abspath $(EXPORTED_FUNCTIONS.fiddle)) \ -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory \ $(SQLITE_OPT) $(SHELL_OPT) \ + -USQLITE_WASM_MINIMAL \ -DSQLITE_SHELL_FIDDLE # -D_POSIX_C_SOURCE is needed for strdup() with emcc +# Flags specifically for debug builds of fiddle. Performance suffers +# greatly in debug builds. +fiddle.emcc-flags.debug := $(fiddle.emcc-flags) \ + -DSQLITE_DEBUG \ + -DSQLITE_ENABLE_SELECTTRACE \ + -DSQLITE_ENABLE_WHERETRACE + fiddle.EXPORTED_FUNCTIONS.in := \ EXPORTED_FUNCTIONS.fiddle.in \ $(EXPORTED_FUNCTIONS.api) @@ -50,37 +60,52 @@ fiddle.EXPORTED_FUNCTIONS.in := \ $(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE.fiddle) sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ -fiddle-module.js := $(dir.fiddle)/fiddle-module.js -fiddle-module.wasm := $(subst .js,.wasm,$(fiddle-module.js)) fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.c) - -fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) -$(fiddle.SOAP.js): $(SOAP.js) - cp $< $@ - $(eval $(call call-make-pre-post,fiddle-module,vanilla)) -$(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \ - $(EXPORTED_FUNCTIONS.fiddle) \ - $(fiddle.cses) $(pre-post-fiddle-module-vanilla.deps) $(fiddle.SOAP.js) - $(emcc.bin) -o $@ $(fiddle.emcc-flags) \ - $(pre-post-fiddle-module-vanilla.flags) \ - $(fiddle.cses) - $(maybe-wasm-strip) $(fiddle-module.wasm) - gzip < $@ > $@.gz - gzip < $(fiddle-module.wasm) > $(fiddle-module.wasm).gz -$(dir.fiddle)/fiddle.js.gz: $(dir.fiddle)/fiddle.js - gzip < $< > $@ +######################################################################## +# emit rules for one of the two fiddle builds. $1 must be +# either $(dir.fiddle) or $(dir.fiddle-debug). $2 must be empty +# in the former case and .debug in the latter. +define make-fiddle-rules +fiddle-module.js$(2) := $(1)/fiddle-module.js +fiddle-module.wasm$(2) := $$(subst .js,.wasm,$$(fiddle-module.js$(2))) +$$(fiddle-module.js$(2)): $$(MAKEFILE) $$(MAKEFILE.fiddle) \ + $$(EXPORTED_FUNCTIONS.fiddle) \ + $$(fiddle.cses) $$(pre-post-fiddle-module-vanilla.deps) $$(SOAP.js) + @test -d "$$(dir $$@)" || mkdir -p "$$(dir $$@)" + $$(emcc.bin) -o $$@ $$(fiddle.emcc-flags$(2)) \ + $$(pre-post-fiddle-module-vanilla.flags) \ + $$(fiddle.cses) + $$(maybe-wasm-strip) $$(fiddle-module.wasm$(2)) + @cp -p $$(SOAP.js) $$(dir $$@) + @if [[ x.debug = x$(2) ]]; then \ + cp -p $$(dir.fiddle)/index.html \ + $$(dir.fiddle)/fiddle.js \ + $$(dir.fiddle)/fiddle-worker.js \ + $$(dir $$@)/.; \ + fi + @for i in $(1)/*.*js $(1)/*.html $(1)/*.wasm; do \ + test -f $$$${i} || continue; \ + gzip < $$$${i} > $$$${i}.gz; \ + done +fiddle$(2): $$(fiddle-module.js$(2)) $(1)/fiddle.js.gz +endef + +$(eval $(call make-fiddle-rules,$(dir.fiddle))) +$(eval $(call make-fiddle-rules,$(dir.fiddle-debug),.debug)) +fiddle: $(fiddle-module.js) $(fiddle-module.js.debug) +fiddle.debug: $(fiddle-module.js.debug) clean: clean-fiddle clean-fiddle: - rm -f $(fiddle-module.js) $(fiddle-module.js).gz \ - $(fiddle-module.wasm) $(fiddle-module.wasm).gz \ - $(dir.fiddle)/$(SOAP.js) \ - $(dir.fiddle)/fiddle-module.worker.js \ + rm -f $(fiddle-module.js) \ + $(fiddle-module.wasm) \ + $(dir.fiddle)/sqlite3-opfs-*.js \ + $(dir.fiddle)/*.gz \ EXPORTED_FUNCTIONS.fiddle -.PHONY: fiddle -fiddle: $(fiddle-module.js) $(dir.fiddle)/fiddle.js.gz + rm -fr $(dir.fiddle-debug) +.PHONY: fiddle fiddle.debug all: fiddle ######################################################################## diff --git a/ext/wasm/fiddle/emscripten.css b/ext/wasm/fiddle/emscripten.css deleted file mode 100644 index 7e3dc811d..000000000 --- a/ext/wasm/fiddle/emscripten.css +++ /dev/null @@ -1,24 +0,0 @@ -/* emcscript-related styling, used during the module load/intialization processes... */ -.emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } -div.emscripten { text-align: center; } -div.emscripten_border { border: 1px solid black; } -#module-spinner { overflow: visible; } -#module-spinner > * { - margin-top: 1em; -} -.spinner { - height: 50px; - width: 50px; - margin: 0px auto; - animation: rotation 0.8s linear infinite; - border-left: 10px solid rgb(0,150,240); - border-right: 10px solid rgb(0,150,240); - border-bottom: 10px solid rgb(0,150,240); - border-top: 10px solid rgb(100,0,200); - border-radius: 100%; - background-color: rgb(200,100,250); -} -@keyframes rotation { - from {transform: rotate(0deg);} - to {transform: rotate(360deg);} -} diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index 67f61008f..27d915eb2 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -166,11 +166,10 @@ stdout("SQLite version", capi.sqlite3_libversion(), capi.sqlite3_sourceid().substr(0,19)); stdout('Welcome to the "fiddle" shell.'); - if(sqlite3.opfs){ + if(capi.sqlite3_vfs_find("opfs")){ stdout("\nOPFS is available. To open a persistent db, use:\n\n", " .open file:name?vfs=opfs\n\nbut note that some", "features (e.g. upload) do not yet work with OPFS."); - sqlite3.opfs.registerVfs(); } stdout('\nEnter ".help" for usage hints.'); this.exec([ // initialization commands... @@ -317,7 +316,7 @@ }; console.warn("Unknown fiddle-worker message type:",ev); }; - + /** emscripten module for use with build mode -sMODULARIZE. */ @@ -374,9 +373,7 @@ "for use in the dev console.", sqlite3); globalThis.sqlite3 = sqlite3; const dbVfs = sqlite3.wasm.xWrap('fiddle_db_vfs', "*", ['string']); - fiddleModule.fsUnlink = (fn)=>{ - return sqlite3.wasm.sqlite3_wasm_vfs_unlink(dbVfs(0), fn); - }; + fiddleModule.fsUnlink = (fn)=>fiddleModule.FS.unlink(fn); wMsg('fiddle-ready'); }).catch(e=>{ console.error("Fiddle worker init failed:",e); diff --git a/ext/wasm/fiddle/fiddle.js b/ext/wasm/fiddle/fiddle.js index 2a3d1746f..d28589835 100644 --- a/ext/wasm/fiddle/fiddle.js +++ b/ext/wasm/fiddle/fiddle.js @@ -403,8 +403,10 @@ E('#btn-reset').addEventListener('click',()=>SF.resetDb()); const taInput = E('#input'); const btnClearIn = E('#btn-clear'); + const selectExamples = E('#select-examples'); btnClearIn.addEventListener('click',function(){ taInput.value = ''; + selectExamples.selectedIndex = 0; },false); // Ctrl-enter and shift-enter both run the current SQL. taInput.addEventListener('keydown',function(ev){ @@ -733,16 +735,15 @@ ]}, //{name: "Timer on", sql: ".timer on"}, // ^^^ re-enable if emscripten re-enables getrusage() + {name: "Box Mode", sql: ".mode box"}, {name: "Setup table T", sql:[ ".nullvalue NULL\n", "CREATE TABLE t(a,b);\n", "INSERT INTO t(a,b) VALUES('abc',123),('def',456),(NULL,789),('ghi',012);\n", "SELECT * FROM t;\n" ]}, - {name: "Table list", sql: ".tables"}, - {name: "Box Mode", sql: ".mode box"}, - {name: "JSON Mode", sql: ".mode json"}, - {name: "Mandlebrot", sql:[ + {name: "sqlite_schema", sql: "select * from sqlite_schema"}, + {name: "Mandelbrot", sql:[ "WITH RECURSIVE", " xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2),\n", " yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0),\n", @@ -760,7 +761,13 @@ " FROM m2 GROUP BY cy\n", " )\n", "SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;\n", - ]} + ]}, + {name: "JSON pretty-print", + sql: "select json_pretty(json_object('ex',json('[52,3.14159]')))" + }, + {name: "JSON pretty-print (with tabs)", + sql: "select json_pretty(json_object('ex',json('[52,3.14159]')),char(0x09))" + } ]; const newOpt = function(lbl,val){ const o = document.createElement('option'); diff --git a/ext/wasm/fiddle/index.html b/ext/wasm/fiddle/index.html index 272f1aca7..5f8647b03 100644 --- a/ext/wasm/fiddle/index.html +++ b/ext/wasm/fiddle/index.html @@ -171,6 +171,31 @@ display: flex; flex-direction: column-reverse; } + + /* emcscript-related styling, used during the module load/intialization processes... */ + .emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } + div.emscripten { text-align: center; } + div.emscripten_border { border: 1px solid black; } + #module-spinner { overflow: visible; } + #module-spinner > * { + margin-top: 1em; + } + .spinner { + height: 50px; + width: 50px; + margin: 0px auto; + animation: rotation 0.8s linear infinite; + border-left: 10px solid rgb(0,150,240); + border-right: 10px solid rgb(0,150,240); + border-bottom: 10px solid rgb(0,150,240); + border-top: 10px solid rgb(100,0,200); + border-radius: 100%; + background-color: rgb(200,100,250); + } + @keyframes rotation { + from {transform: rotate(0deg);} + to {transform: rotate(360deg);} + } </style> </head> <body> diff --git a/ext/wasm/index-dist.html b/ext/wasm/index-dist.html index f5bcdc1cb..7b778b020 100644 --- a/ext/wasm/index-dist.html +++ b/ext/wasm/index-dist.html @@ -97,6 +97,8 @@ wrapper is significantly easier to use, however.</li> <li><a href='demo-worker1-promiser.html'>demo-worker1-promiser</a>: a demo of the Promise-based wrapper of the Worker1 API.</li> + <li><a href='demo-worker1-promiser-esm.html'>demo-worker1-promiser-esm</a>: + same as the previous demo except loads the promiser from an ESM module.</li> </ul> </li> </ul> diff --git a/ext/wasm/index.html b/ext/wasm/index.html index ebbfd6763..d12a3aa03 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -84,6 +84,8 @@ wrapper is significantly easier to use, however.</li> <li><a href='demo-worker1-promiser.html'>demo-worker1-promiser</a>: a demo of the Promise-based wrapper of the Worker1 API.</li> + <li><a href='demo-worker1-promiser-esm.html'>demo-worker1-promiser-esm</a>: + same as the previous demo except loads the promiser from an ESM module.</li> </ul> </li> <li>speedtest1 ports (sqlite3's primary benchmarking tool)... diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 3abc589b5..5261c8393 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -111,10 +111,6 @@ self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ const S = globalThis.S = App.sqlite3 = sqlite3; log("Loaded speedtest1 module. Setting up..."); - App.vfsUnlink = function(pDb, fname){ - const pVfs = S.wasm.sqlite3_wasm_db_vfs(pDb, 0); - if(pVfs) S.wasm.sqlite3_wasm_vfs_unlink(pVfs, fname||0); - }; App.pDir = wasmfsDir(S.wasm); App.wasm = S.wasm; //if(App.pDir) log("Persistent storage:",pDir); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 36ca4c976..c49a82d36 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -63,7 +63,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; /* Predicate for tests/groups. */ const testIsTodo = ()=>false; const haveWasmCTests = ()=>{ - return !!wasm.exports.sqlite3_wasm_test_intptr; + return !!wasm.exports.sqlite3__wasm_test_intptr; }; const hasOpfs = ()=>{ return globalThis.FileSystemHandle @@ -461,7 +461,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; try{ sqlite3.SQLite3Error.toss("resultCode check") } catch(e){ T.assert(capi.SQLITE_ERROR === e.resultCode) - .assert('resultCode check' === e.message); + .assert('resultCode check' === e.message); } }) //////////////////////////////////////////////////////////////////// @@ -722,7 +722,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //log("xCall()..."); { - const pJson = w.xCall('sqlite3_wasm_enum_json'); + const pJson = w.xCall('sqlite3__wasm_enum_json'); T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); } @@ -736,9 +736,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.mustThrowMatching(()=>fw(1), /requires 0 arg/); let rc = fw(); T.assert('string'===typeof rc).assert(rc.length>5); - rc = w.xCallWrapped('sqlite3_wasm_enum_json','*'); + rc = w.xCallWrapped('sqlite3__wasm_enum_json','*'); T.assert(rc>0 && Number.isFinite(rc)); - rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); + rc = w.xCallWrapped('sqlite3__wasm_enum_json','utf8'); T.assert('string'===typeof rc).assert(rc.length>300); @@ -821,7 +821,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(haveWasmCTests()){ if(!sqlite3.config.useStdAlloc){ - fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:dealloc',['i32']); + fw = w.xWrap('sqlite3__wasm_test_str_hello', 'utf8:dealloc',['i32']); rc = fw(0); T.assert('hello'===rc); rc = fw(1); @@ -831,14 +831,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(w.bigIntEnabled){ w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); - fw = w.xWrap('sqlite3_wasm_test_int64_times2','thrice','twice'); + fw = w.xWrap('sqlite3__wasm_test_int64_times2','thrice','twice'); rc = fw(1); T.assert(12n===rc); w.scopedAllocCall(function(){ const pI1 = w.scopedAlloc(8), pI2 = pI1+4; w.pokePtr([pI1, pI2], 0); - const f = w.xWrap('sqlite3_wasm_test_int64_minmax',undefined,['i64*','i64*']); + const f = w.xWrap('sqlite3__wasm_test_int64_minmax',undefined,['i64*','i64*']); const [r1, r2] = w.peek64([pI1, pI2]); T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); }); @@ -942,7 +942,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). assert(0===wts.$ppV).assert(0===wts.$xFunc); const testFunc = - W.xGet('sqlite3_wasm_test_struct'/*name gets mangled in -O3 builds!*/); + W.xGet('sqlite3__wasm_test_struct'/*name gets mangled in -O3 builds!*/); let counter = 0; //log("wts.pointer =",wts.pointer); const wtsFunc = function(arg){ @@ -1128,7 +1128,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.g('sqlite3.oo1') .t('Create db', function(sqlite3){ const dbFile = '/tester1.db'; - wasm.sqlite3_wasm_vfs_unlink(0, dbFile); + sqlite3.util.sqlite3__wasm_vfs_unlink(0, dbFile); const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); db.onclose = { disposeAfter: [], @@ -1459,7 +1459,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; rv = db.exec("SELECT 1 WHERE 0",{rowMode: 0}); T.assert(Array.isArray(rv)).assert(0===rv.length); if(wasm.bigIntEnabled && haveWasmCTests()){ - const mI = wasm.xCall('sqlite3_wasm_test_int64_max'); + const mI = wasm.xCall('sqlite3__wasm_test_int64_max'); const b = BigInt(Number.MAX_SAFE_INTEGER * 2); T.assert(b === db.selectValue("SELECT "+b)). assert(b === db.selectValue("SELECT ?", b)). @@ -1475,14 +1475,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let st = db.prepare("update t set b=:b where a='blob'"); try { - T.assert(0===st.columnCount); + T.assert(0===st.columnCount) + .assert( false===st.isReadOnly() ); const ndx = st.getParamIndex(':b'); T.assert(1===ndx); st.bindAsBlob(ndx, "ima blob") /*step() skipped intentionally*/.reset(true); } finally { T.assert(0===st.finalize()) - .assert(undefined===st.finalize()); + .assert(undefined===st.finalize()); } try { @@ -1509,6 +1510,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// .t({ name: "sqlite3_set_authorizer()", + predicate: ()=>!!wasm.exports.sqlite3_set_authorizer || "Missing sqlite3_set_authorizer()", test:function(sqlite3){ T.assert(capi.SQLITE_IGNORE>0) .assert(capi.SQLITE_DENY>0); @@ -1685,7 +1687,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(n>0 && db2.selectValue(sql) === n); }finally{ db2.close(); - wasm.sqlite3_wasm_vfs_unlink(0, filename); + sqlite3.util.sqlite3__wasm_vfs_unlink(0, filename); } } }/*sqlite3_js_posix_create_file()*/) @@ -2064,6 +2066,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }) //////////////////////////////////////////////////////////////////// + .t("Read-only", function(sqlite3){ + T.assert( 0===capi.sqlite3_db_readonly(this.db, "main") ); + const db = new sqlite3.oo1.DB('file://'+this.db.filename+'?mode=ro'); + T.assert( 1===capi.sqlite3_db_readonly(db, "main") ); + T.assert( -1===capi.sqlite3_db_readonly(db, "nope") ); + db.close(); + }) + + //////////////////////////////////////////////////////////////////// .t({ name: 'C-side WASM tests', predicate: ()=>(haveWasmCTests() || "Not compiled in."), @@ -2075,7 +2086,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; try{ ptrInt = w.scopedAlloc(4); w.poke32(ptrInt,origValue); - const cf = w.xGet('sqlite3_wasm_test_intptr'); + const cf = w.xGet('sqlite3__wasm_test_intptr'); const oldPtrInt = ptrInt; T.assert(origValue === w.peek32(ptrInt)); const rc = cf(ptrInt); @@ -2090,13 +2101,13 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const v64 = ()=>w.peek64(pi64) T.assert(v64() == o64); //T.assert(o64 === w.peek64(pi64)); - const cf64w = w.xGet('sqlite3_wasm_test_int64ptr'); + const cf64w = w.xGet('sqlite3__wasm_test_int64ptr'); cf64w(pi64); T.assert(v64() == BigInt(2 * o64)); cf64w(pi64); T.assert(v64() == BigInt(4 * o64)); - const biTimes2 = w.xGet('sqlite3_wasm_test_int64_times2'); + const biTimes2 = w.xGet('sqlite3__wasm_test_int64_times2'); T.assert(BigInt(2 * o64) === biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError in the call :/ */)); @@ -2106,13 +2117,13 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const g64 = (p)=>w.peek64(p); w.poke64([pMin, pMax], 0); const minMaxI64 = [ - w.xCall('sqlite3_wasm_test_int64_min'), - w.xCall('sqlite3_wasm_test_int64_max') + w.xCall('sqlite3__wasm_test_int64_min'), + w.xCall('sqlite3__wasm_test_int64_max') ]; T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); - w.xCall('sqlite3_wasm_test_int64_minmax', pMin, pMax); + w.xCall('sqlite3__wasm_test_int64_minmax', pMin, pMax); T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); //log("pMin",g64(pMin), "pMax",g64(pMax)); @@ -2142,7 +2153,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t({ name: 'virtual table #1: eponymous w/ manual exception handling', - predicate: ()=>!!capi.sqlite3_index_info, + predicate: ()=>!!capi.sqlite3_create_module || "Missing vtab support", test: function(sqlite3){ const VT = sqlite3.vtab; const tmplCols = Object.assign(Object.create(null),{ @@ -2339,7 +2350,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t({ name: 'virtual table #2: non-eponymous w/ automated exception wrapping', - predicate: ()=>!!capi.sqlite3_index_info, + predicate: ()=>!!capi.sqlite3_create_module || "Missing vtab support", test: function(sqlite3){ const VT = sqlite3.vtab; const tmplCols = Object.assign(Object.create(null),{ @@ -2560,7 +2571,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t('Close db', function(){ T.assert(this.db).assert(wasm.isPtr(this.db.pointer)); - //wasm.sqlite3_wasm_db_reset(this.db); // will leak virtual tables! + //wasm.sqlite3__wasm_db_reset(this.db); // will leak virtual tables! this.db.close(); T.assert(!this.db.pointer); }) @@ -2587,7 +2598,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const pVfs = capi.sqlite3_vfs_find('kvvfs'); T.assert(pVfs); const JDb = this.JDb = sqlite3.oo1.JsStorageDb; - const unlink = this.kvvfsUnlink = ()=>{JDb.clearStorage(filename)}; + const unlink = this.kvvfsUnlink = ()=>JDb.clearStorage(this.kvvfsDbFile); unlink(); let db = new JDb(filename); try { @@ -2605,6 +2616,60 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } }/*kvvfs sanity checks*/) +//#if enable-see + .t({ + name: 'kvvfs with SEE encryption', + predicate: ()=>(isUIThread() + || "Only available in main thread."), + test: function(sqlite3){ + this.kvvfsUnlink(); + let db; + const encOpt1 = 1 + ? {textkey: 'foo'} + : {key: 'foo'}; + const encOpt2 = encOpt1.textkey + ? encOpt1 + : {hexkey: new Uint8Array([0x66,0x6f,0x6f]/*==>"foo"*/)} + try{ + db = new this.JDb({ + filename: this.kvvfsDbFile, + ...encOpt1 + }); + db.exec([ + "create table t(a,b);", + "insert into t(a,b) values(1,2),(3,4)" + ]); + db.close(); + let err; + try{ + db = new this.JDb({ + filename: this.kvvfsDbFile, + flags: 'ct' + }); + T.assert(db) /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("sessionStorage =",sessionStorage); + }catch(e){ + err = e; + }finally{ + db.close(); + } + T.assert(err,"Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB==err.resultCode, + "Expecting NOTADB"); + db = new sqlite3.oo1.DB({ + filename: this.kvvfsDbFile, + vfs: 'kvvfs', + ...encOpt2 + }); + T.assert( 4===db.selectValue('select sum(a) from t') ); + }finally{ + if( db ) db.close(); + this.kvvfsUnlink(); + } + } + })/*kvvfs with SEE*/ +//#endif enable-see ;/* end kvvfs tests */ //////////////////////////////////////////////////////////////////////// @@ -2697,7 +2762,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; })/* commit/rollback/update hooks */ .t({ name: "sqlite3_preupdate_hook()", - predicate: ()=>wasm.bigIntEnabled || "Pre-update hook requires int64", + predicate: ()=>capi.sqlite3_preupdate_hook || "Missing pre-update hook API", test: function(sqlite3){ const db = new sqlite3.oo1.DB(':memory:', 1 ? 'c' : 'ct'); const countHook = Object.create(null); @@ -2768,9 +2833,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.g('Session API') .t({ name: 'Session API sanity checks', - predicate: ()=>!!capi.sqlite3changegroup_add, + predicate: ()=>!!capi.sqlite3changegroup_add || "Missing session API", test: function(sqlite3){ - warn("The session API tests could use some expansion."); + //warn("The session API tests could use some expansion."); const db1 = new sqlite3.oo1.DB(), db2 = new sqlite3.oo1.DB(); const sqlInit = [ "create table t(rowid INTEGER PRIMARY KEY,a,b); ", @@ -2805,7 +2870,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert('b4' === db1.selectValue('select b from t where rowid=4')) .assert(3 === db1.selectValue('select count(*) from t')); - const testSessionEnable = false; + const testSessionEnable = + false /* it's not yet clear whether these test failures are + broken tests or broken bindings. */; if(testSessionEnable){ rc = capi.sqlite3session_enable(pSession, 0); T.assert( 0 === rc ) @@ -2816,7 +2883,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert( capi.sqlite3session_enable(pSession, -1) > 0 ) .assert(undefined === db1.selectValue('select a from t where rowid=2')); }else{ - warn("sqlite3session_enable() tests are currently disabled."); + //warn("sqlite3session_enable() tests are currently disabled."); } let db1Count = db1.selectValue("select count(*) from t"); T.assert( db1Count === (testSessionEnable ? 2 : 3) ); @@ -2888,18 +2955,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t({ name: 'OPFS db sanity checks', test: async function(sqlite3){ + T.assert(capi.sqlite3_vfs_find('opfs')); + const opfs = sqlite3.opfs; const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; - const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs'); - T.assert(pVfs); - const unlink = this.opfsUnlink = - (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)}; - unlink(); - let db = new sqlite3.oo1.OpfsDb(filename); + const fileUri = 'file://'+filename+'?delete-before-open=1'; + const initSql = [ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]; + let db = new sqlite3.oo1.OpfsDb(fileUri); try { - db.exec([ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]); + db.exec(initSql); T.assert(3 === db.selectValue('select count(*) from p')); db.close(); db = new sqlite3.oo1.OpfsDb(filename); @@ -2911,7 +2977,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule; && 0===this.opfsDbExport.byteLength % 512); }finally{ db.close(); - unlink(); + } + T.assert(await opfs.entryExists(filename)); + try { + db = new sqlite3.oo1.OpfsDb(fileUri); + db.exec(initSql) /* will throw if delete-before-open did not work */; + T.assert(3 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); } } }/*OPFS db sanity checks*/) @@ -2919,15 +2992,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule; name: 'OPFS import', test: async function(sqlite3){ let db; + const filename = this.opfsDbFile; try { const exp = this.opfsDbExport; - const filename = this.opfsDbFile; delete this.opfsDbExport; this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp); db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); T.assert(6 === db.selectValue('select count(*) from p')). assert( this.opfsImportSize == exp.byteLength ); db.close(); + const unlink = this.opfsUnlink = + (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn); this.opfsUnlink(filename); T.assert(!(await sqlite3.opfs.entryExists(filename))); // Try again with a function as an input source: @@ -2954,11 +3029,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; name: '(Internal-use) OPFS utility APIs', test: async function(sqlite3){ const filename = this.opfsDbFile; - const pVfs = this.opfsVfs; const unlink = this.opfsUnlink; - T.assert(filename && pVfs && !!unlink); + T.assert(filename && !!unlink); delete this.opfsDbFile; - delete this.opfsVfs; delete this.opfsUnlink; /************************************************************** ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended @@ -3041,11 +3114,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(db instanceof sqlite3.oo1.DB) .assert(1 === u1.getFileCount()); db.exec([ + 'pragma locking_mode=exclusive;', + 'pragma journal_mode=wal;' + /* WAL mode only works in this VFS if locking_mode=exclusive + is invoked prior to the first db access, as this build + does not have the shared-memory APIs needed for WAL without + exclusive-mode locking. See: + + https://sqlite.org/wal.html#use_of_wal_without_shared_memory + + Note that WAL mode here DOES NOT add any concurrency capabilities + to this VFS, but it MAY provide slightly improved performance + over the other journaling modes. + */, 'create table t(a);', 'insert into t(a) values(1),(2),(3)' ]); - T.assert(1 === u1.getFileCount()); - T.assert(3 === db.selectValue('select count(*) from t')); + T.assert(2 === u1.getFileCount() /* one is the journal file */) + .assert(3 === db.selectValue('select count(*) from t')) + .assert('wal'===db.selectValue('pragma journal_mode')); db.close(); T.assert(1 === u1.getFileCount()); db = new u2.OpfsSAHPoolDb(dbName); @@ -3065,6 +3152,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert( dbytes.byteLength == nWrote ); let db2 = new u1.OpfsSAHPoolDb(dbName2); T.assert(db2 instanceof sqlite3.oo1.DB) + .assert('wal' !== db2.selectValue("pragma journal_mode") + /* importDb() unsets the WAL-mode header for + historical reasons. Because clients must + explicitly enable pragma locking_mode=exclusive + before using WAL, that behavior is retained. */) .assert(3 === db2.selectValue('select count(*) from t')); db2.close(); T.assert(true === u1.unlink(dbName2)) @@ -3117,20 +3209,108 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); let cErr, u3; - conf2.$testThrowInInit = new Error("Testing throwing during init."); + conf2.$testThrowPhase2 = new Error("Testing throwing during init."); conf2.name = sahPoolConfig.name+'-err'; const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); - T.assert(P3 === conf2.$testThrowInInit) + T.assert(P3 === conf2.$testThrowPhase2) .assert(cErr === P3) .assert(undefined === u3) .assert(!sqlite3.capi.sqlite3_vfs_find(conf2.name)); + delete conf2.$testThrowPhase2; + T.assert(cErr === await inst(conf2).catch(e=>e), + "Init result is cached even if it failed"); + + /* Ensure that the forceReinitIfPreviouslyFailed fallback bypasses + the VFS init cache... */ + cErr = u3 = undefined; + conf2.forceReinitIfPreviouslyFailed = true; + conf2.verbosity = 3; + const P3b = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); + T.assert(undefined === cErr) + .assert(P3b === u3) + .assert(P3b === await inst(conf2)) + .assert(true === await u3.removeVfs()) + .assert(false === await P3b.removeVfs()); } }/*OPFS SAH Pool sanity checks*/) //////////////////////////////////////////////////////////////////////// + T.g('Misc. APIs') + .t('bind_parameter_...', function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec("create table t(a)"); + const stmt = db.prepare("insert into t(a) values($a)"); + T.assert( 1===capi.sqlite3_bind_parameter_count(stmt) ) + .assert( 1===capi.sqlite3_bind_parameter_index(stmt, "$a") ) + .assert( 0===capi.sqlite3_bind_parameter_index(stmt, ":a") ) + .assert( 1===stmt.getParamIndex("$a") ) + .assert( 0===stmt.getParamIndex(":a") ) + .assert( "$a"===capi.sqlite3_bind_parameter_name(stmt, 1) ) + .assert( null===capi.sqlite3_bind_parameter_name(stmt, 0) ) + .assert( "$a"===stmt.getParamName(1) ) + .assert( null===stmt.getParamName(0) ); + stmt.finalize(); + db.close(); + }) + + //////////////////////////////////////////////////////////////////// + .t("Misc. stmt_...", function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec("create table t(a doggiebiscuits); insert into t(a) values(123)"); + const stmt = db.prepare("select a, a+1 from t"); + T.assert( stmt.isReadOnly() ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); + let n = 0; + while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ + ++n; + T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), + "Because stmt is busy" ) + .assert( capi.sqlite3_stmt_busy(stmt) ) + .assert( stmt.isBusy() ) + .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) + .assert( true===stmt.isReadOnly() ); + const sv = capi.sqlite3_column_value(stmt, 0); + T.assert( 123===capi.sqlite3_value_int(sv) ) + .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) + .assert( null===capi.sqlite3_column_decltype(stmt,1) ); + } + T.assert( 1===n ) + .assert( 0===capi.sqlite3_stmt_busy(stmt) ) + .assert( !stmt.isBusy() ); + stmt.finalize(); + db.close(); + }) + + //////////////////////////////////////////////////////////////////// + .t("interrupt", function(sqlite3){ + const db = new sqlite3.oo1.DB(); + T.assert( 0===capi.sqlite3_is_interrupted(db) ); + capi.sqlite3_interrupt(db); + T.assert( 0!==capi.sqlite3_is_interrupted(db) ); + db.close(); + }) + + //////////////////////////////////////////////////////////////////////// T.g('Bug Reports') .t({ name: 'Delete via bound parameter in subquery', + predicate: function(sqlite3){ + const d = new sqlite3.oo1.DB(); + try{ + d.exec("create virtual table f using fts5(x)"); + return true; + }catch(e){ + return "FTS5 is not available"; + }finally{ + d.close(); + } + }, test: function(sqlite3){ // Testing https://sqlite.org/forum/forumpost/40ce55bdf5 // with the exception that that post uses "external content" @@ -3209,6 +3389,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; print: log, printErr: error }).then(async function(sqlite3){ + TestUtil.assert(!!sqlite3.util); log("Done initializing WASM/JS bits. Running tests..."); sqlite3.config.warn("Installing sqlite3 bits as global S for local dev/test purposes."); globalThis.S = sqlite3; @@ -3227,9 +3408,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; logClass('warning',"BigInt/int64 support is disabled."); } if(haveWasmCTests()){ - log("sqlite3_wasm_test_...() APIs are available."); + log("sqlite3__wasm_test_...() APIs are available."); }else{ - logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); + logClass('warning',"sqlite3__wasm_test_...() APIs unavailable."); } log("registered vfs list =",capi.sqlite3_js_vfs_list().join(', ')); TestUtil.runTests(sqlite3); diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 595ab2452..54ed04a4f 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -35,6 +35,9 @@ re-opening the tab usually resolves it, but sometimes restarting the browser is required. </p> + <p> + Links for various testing options: <ul id='testlinks'></ul> + </p> <div class='input-wrapper'> <input type='checkbox' id='cb-log-reverse'> <label for='cb-log-reverse'>Reverse log order?</label> diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 14cd6f514..1848901af 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -113,6 +113,31 @@ } }; + /* Set up links to launch this tool with various combinations of + flags... */ + const eTestLinks = document.querySelector('#testlinks'); + const optArgs = function(obj){ + const li = []; + for(const k of ['interval','iterations','workers','verbose','unlock-asap']){ + if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); + } + return li.join('&'); + }; + for(const opt of [ + {interval: 1000, workers: 5, iterations: 30}, + {interval: 500, workers: 5, iterations: 30}, + {interval: 250, workers: 3, iterations: 30}, + {interval: 600, workers: 5, iterations: 100} + ]){ + const li = document.createElement('li'); + eTestLinks.appendChild(li); + const a = document.createElement('a'); + li.appendChild(a); + const args = optArgs(opt); + a.setAttribute('href', '?'+args); + a.innerText = args; + } + stdout("Launching",options.workerCount,"workers. Options:",options); workers.uri = ( 'worker.js?' @@ -230,7 +230,6 @@ SRC += \ SRC += \ $(TOP)/ext/misc/stmt.c - # FTS5 things # FTS5_HDR = \ @@ -365,6 +364,7 @@ TESTSRC += \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ + $(TOP)/ext/misc/stmtrand.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c \ @@ -375,7 +375,9 @@ TESTSRC += \ $(TOP)/ext/rtree/test_rtreedoc.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/test_recover.c + $(TOP)/ext/recover/test_recover.c \ + $(TOP)/ext/intck/test_intck.c \ + $(TOP)/ext/intck/sqlite3intck.c #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c @@ -659,7 +661,7 @@ target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c touch target_source sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl src-verify - tclsh $(TOP)/tool/mksqlite3c.tcl + tclsh $(TOP)/tool/mksqlite3c.tcl $(EXTRA_SRC) cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c @@ -671,7 +673,7 @@ sqlite3ext.h: target_source cp tsrc/sqlite3ext.h . sqlite3.c-debug: target_source $(TOP)/tool/mksqlite3c.tcl src-verify - tclsh $(TOP)/tool/mksqlite3c.tcl --linemacros=1 + tclsh $(TOP)/tool/mksqlite3c.tcl --linemacros=1 $(EXTRA_SRC) echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c @@ -717,8 +719,7 @@ opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl tclsh $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl - cat parse.h $(TOP)/src/vdbe.c | \ - tclsh $(TOP)/tool/mkopcodeh.tcl >opcodes.h + cat parse.h $(TOP)/src/vdbe.c | tclsh $(TOP)/tool/mkopcodeh.tcl >opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # @@ -741,32 +742,37 @@ keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c ./mkkeywordhash >keywordhash.h -# Source files that go into making shell.c -SHELL_SRC = \ - $(TOP)/src/shell.c.in \ - $(TOP)/ext/misc/appendvfs.c \ - $(TOP)/ext/misc/completion.c \ - $(TOP)/ext/misc/base64.c \ - $(TOP)/ext/misc/base85.c \ - $(TOP)/ext/misc/decimal.c \ - $(TOP)/ext/misc/fileio.c \ - $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/regexp.c \ - $(TOP)/ext/misc/series.c \ - $(TOP)/ext/misc/shathree.c \ - $(TOP)/ext/misc/sqlar.c \ - $(TOP)/ext/misc/uint.c \ - $(TOP)/ext/expert/sqlite3expert.c \ - $(TOP)/ext/expert/sqlite3expert.h \ - $(TOP)/ext/misc/zipfile.c \ - $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/misc/pcachetrace.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/sqlite3recover.h \ - $(TOP)/src/test_windirent.c - -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl +# Source and header files that shell.c depends on +SHELL_DEP = \ + $(TOP)/src/shell.c.in \ + $(TOP)/ext/consio/console_io.c \ + $(TOP)/ext/consio/console_io.h \ + $(TOP)/ext/expert/sqlite3expert.c \ + $(TOP)/ext/expert/sqlite3expert.h \ + $(TOP)/ext/intck/sqlite3intck.c \ + $(TOP)/ext/intck/sqlite3intck.h \ + $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ + $(TOP)/ext/misc/completion.c \ + $(TOP)/ext/misc/decimal.c \ + $(TOP)/ext/misc/fileio.c \ + $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/pcachetrace.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/shathree.c \ + $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/zipfile.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ + $(TOP)/src/test_windirent.c \ + $(TOP)/src/test_windirent.h + +shell.c: $(SHELL_DEP) $(TOP)/tool/mkshellc.tcl tclsh $(TOP)/tool/mkshellc.tcl >shell.c @@ -1,48 +1,50 @@ -C Fix\sa\sfew\scompiler\sgripes. -D 2024-01-10T20:13:38.259 +C Merge\sall\sthe\slatests\strunk\senhancements\sinto\sthe\swin-dupe-crt-fio\sbranch. +D 2024-08-27T13:57:43.910 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 24be65ae641c5727bbc247d60286a15ecc24fb80f14f8fb2d32533bf0ec96e79 +F Makefile.in 101cb90f75c16ca3f8e7088364322441f9b01d6fdcef4a9c7f5fdcf78a3601a4 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc df56c06ef05add5ebcdcc9d4823fb2ec66ac16b06acb6971a7420859025886fa -F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 -F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 +F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065 +F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c +F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 +F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 +F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am adedc1324b6a87fdd1265ddd336d2fb7d4f36a0e77b86ea553ae7cc4ea239347 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.msc ac338c36a338f6b49475da71930f45145a181d6b551578d5a7a64f113ef27b2c +F autoconf/Makefile.msc 7ac6c331fc3b8aa57b6782db995b8c0e49230352decd4e2662fd07c06a9ed623 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 42cfd21d0b19dc7d5d85fb5c405c5f3c6a4c923021c39128f6ba685355d8fd56 F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d5893b277 F autoconf/tea/Makefile.in 106a96f2f745d41a0f6193f1de98d7355830b65d45032c18cd7c90295ec24196 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac 4c32b08691a5b296206b38422b53b92b65be3d3f6b3dd6552a50981a61f5acda +F autoconf/tea/configure.ac 0deb5d6c49c8119f75f436488219fc043127d72057af5dfba2c9ce096a5734bc F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 -F autoconf/tea/tclconfig/tcl.m4 debe13280bd5a9d76dc34e7919cd9ed3a1408c7320400900357128c2d1abb723 +F autoconf/tea/tclconfig/tcl.m4 c6e5f2fc7178f40d087403daa044ef3b86a8e30793f3b121bdcbdf152c6a776a F autoconf/tea/win/makefile.vc 2c478a9a962e48b2bf9062734e04d7c63c556e217095419173f9d7938d7d78f7 F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb -F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 +F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure bcb1042e92775424a1021d2f4c89c78a699a6225df01fa8c593df7df0be6ad10 x -F configure.ac f25bd7843120f2c2b8bc9db5a92b0502bbdd28e68907415c3d42fc8e57c657b9 +F configure 49523f0a070b583cea040d26eff53a65fb0893eca4663b1343a4d5a9a964da53 x +F configure.ac a100ebf7a07f5dedd319ef547dd467d1676ed059b85a7877aa9c44ac309f7000 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd -F doc/compile-for-windows.md 50b27d77be96195c66031a3181cb8684ed822327ea834e07f9c014213e5e3bcf +F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f F doc/jsonb.md 5fab4b8613aa9153fbeb6259297bd4697988af8b3d23900deba588fa7841456b -F doc/lemon.html 44a53a1d2b42d7751f7b2f478efb23c978e258d794bfd172442307a755b9fa44 +F doc/lemon.html 8b266ff711d2ec7f867c3dca37634963f48a630329908cc282beebfa8c708706 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 -F doc/testrunner.md 8d36ec692cf4994bb66d84a4645b9afa1ce9d47dc12cbf8d437c5a5fb6ddeedb +F doc/testrunner.md 15583cf8c7d8a1c3378fd5d4319ca769a14c4d950a5df9b015d01d5be290dc69 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a @@ -51,164 +53,169 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c 3f635b6dd2ddbc722aeaa738d415768c9a40dedff53894975816dcbd7711145f x +F ext/consio/console_io.c 7f3cb6490ea6e3bd51b606bd3a37cd1d26d0bf07ccec21dd75e20bf4ee1a2a59 x F ext/consio/console_io.h b5ebe34aa15b357621ebbea3d3f2e2b24750d4280b5802516409e23947fd9ee5 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test 0dd5cb096d66bed593e33053a3b364f6ef52ed72064bf5cf298364636dbf3cd6 -F ext/expert/sqlite3expert.c 90446bb1183429308c68ceb23e00cb8252f43b8886cf5efa3fc7e833b5fa24c8 +F ext/expert/expert1.test 661f873fd451127edf822ef0d520088faa319135f6a15bd10be6801ac284ac9b +F ext/expert/sqlite3expert.c 8b09aeb2b95a9fca8b6628b522bf4d69aa746ff64c38eb1e99a9b5fad8cf03b9 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b -F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 +F ext/expert/test_expert.c b767b2039a0df707eb3147e86bcf68b252d8455d9a41774b1a836cd052ceca70 F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c7cc3bf59ee F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c d01dfb641fc04efeeadcb94d7a8342eb07d71c1a3a3852ec8ab5e64c1fcdfff9 +F ext/fts3/fts3.c c922380b62bd15bce953dae3350337acbd0fff07c10cdc805819409791eb480a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h be688580701d41340de73384e3acc8c55be12a438583207444bd5e20f9ef426c +F ext/fts3/fts3Int.h 968f7d7cae541a6926146e9fd3fb2b2ccbd3845b7890a8ed03de0c06ac776682 F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 F ext/fts3/fts3_expr.c 903bfb9433109fffb10e910d7066c49cbf8eeae316adc93f0499c4da7dfc932a F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 F ext/fts3/fts3_porter.c e19807ce0ae31c1c6e9898e89ecc93183d7ec224ea101af039722a4f49e5f2b8 -F ext/fts3/fts3_snippet.c 4d6523e3eddeb7b46e7a82b3476a0a86a0c04821e0e2b8dd40f45ee28057cb13 -F ext/fts3/fts3_term.c 845f0e2456b1be42f7f1bec1da1dfc05bc347531eff90775ffc6698902c281de -F ext/fts3/fts3_test.c d8d7b2734f894e8a489987447658e374cdd3a3bc8575c401decf1911cb7c6454 +F ext/fts3/fts3_snippet.c c38117a2e4dcc9485a170a57a6134423955247b230fef7073c46fa9c51239540 +F ext/fts3/fts3_term.c 6a96027ad364001432545fe43322b6af04ed28bb5619ec51af1f59d0710d6d69 +F ext/fts3/fts3_test.c 7a9cb3d61774134211bf4bfdf1adcb581a1a0377f2d050a121ae7ab44baef0e3 F ext/fts3/fts3_tokenize_vtab.c 7fd9ef364f257b97218b9c331f2378e307375c592f70fd541f714e747d944962 -F ext/fts3/fts3_tokenizer.c 6d8fc150c48238955d5182bf661498db0dd473c8a2a80e00c16994a646fa96e7 +F ext/fts3/fts3_tokenizer.c defede96b5dd5d658edfae77355b9c31ea65236eedc7bbe1adbc50d645cca5bc F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 5bb4721330ca589f906e72bb824dd4080b313c6d4c4231fa541e9db32dc67982 +F ext/fts3/fts3_write.c 81cd8f7e8003e427a1801e04842776b731af26dd93af206e4e66ea5ae319cad1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 +F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb -F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h ecba24fed7b359b3a53016bb07e411b3b4c9cdf163aa141006536423a63b611e -F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 -F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 +F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 +F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0 +F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 +F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c e91156ebdcc08d837f4f324168f69f3c0d7fdef0e521fd561efb48ef3297b696 +F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea +F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c bb1965c3965f6fe5f64160bf1c0694a9684a790a783f293a76da1d38d319b258 -F ext/fts5/fts5_main.c 1c6c7df07fd0d8b37548887dcdc00a8cc31fd1dc13bf9091f3350a9f23e6353c -F ext/fts5/fts5_storage.c f9e31b0d155e9b2c92d5d3a09ad7a56b937fbf1c7f962e10f4ca6281349f3934 -F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 +F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe +F ext/fts5/fts5_main.c 3d8b778f65fe5be218f6a8e4019048f092c0c09621b035ce3e8804093036b578 +F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 +F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b -F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13 -F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00 +F ext/fts5/fts5_tokenize.c ae9c4fa93174ef06ffc138bd4280a1c37f7e13624d3d2706aad4b80573f23c41 +F ext/fts5/fts5_unicode2.c 6f9b0fb79a8facaed76628ffd4eb9c16d7f2b84b52872784f617cf3422a9b043 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c 209e0c151e108d5f3621fa24b91e9b02f3750ee6c3f9ccec312df39481b68a09 +F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 3378732aae2a7d9a4b9b5c40bde678d4259ca16bd490883325aecc4747bcb384 -F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95 -F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390 -F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d -F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de -F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7 -F ext/fts5/test/fts5af.test 2329b6c6e6e9243371022dd9439892ff1850b590ae9ce073e3a83eefaf993a81 -F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd -F ext/fts5/test/fts5ah.test 2f047dfe89dc8611fa53e3d8bfc453b79cff037aa423c8d171e91e645745aa2c -F ext/fts5/test/fts5ai.test bc97e4758cc93e06bf851d61c98fdf4e8b8f8315ee28a84fb15f916360856414 -F ext/fts5/test/fts5aj.test 745020852d85f5dd49d11cb7ad11d3cc6dafc4fe6d6d24bc0875ac8f43ee4149 -F ext/fts5/test/fts5ak.test f459a64c9d38698af72a7c657ab6349bca96150241dd69fcce752634b2742d41 -F ext/fts5/test/fts5al.test 00c4c1c6a1366b73aa48ce2068c634520867c3cf7f5d1676ebbb775ee1f35734 -F ext/fts5/test/fts5alter.test 5565f7e4605512b69171ac18ca84398603f9f6456dbe377beeca97e83cc242cd -F ext/fts5/test/fts5auto.test 78989e6527ce69c9eddbef7392fea5c10b0010cd2b2ae68eec7bc869c471e691 -F ext/fts5/test/fts5aux.test ed3596469f85a6cff5f6060e0cd9e3f9602051d8db2b497f5d12c85d39f20a62 -F ext/fts5/test/fts5auxdata.test eacc97ff04892f1a5f3d4df5a73f8bcbc3955ea1d12c9f24137eb1fc079e7611 +F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe +F ext/fts5/test/fts5aa.test 015c81b84d53bfcedd77d624202c8b02e9f0cbbb4b51688e3a9c9f90bccbb4ac +F ext/fts5/test/fts5ab.test 4bdb619fee409e11417e8827e320b857e42e926a01a0408fc9f143ec870a6ced +F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b +F ext/fts5/test/fts5ad.test 058e616612964e61d19f70295f0e6eaedceb4b29b1fbf4f859615ef7e779dc22 +F ext/fts5/test/fts5ae.test 3d49edbd50bb0684199a2e7568aeb30d1d29718f5c0f61751983740fa836d15f +F ext/fts5/test/fts5af.test ae81f08b8da4c5f9b3ec1ef538a4ab6b7c278e92fa9058d6dc5d842c5d9771b9 +F ext/fts5/test/fts5ag.test 6667807b5d3fbf460892e756763fbe3d87a2fffe345a06514ba010ca6f6641f7 +F ext/fts5/test/fts5ah.test e1f01314b35745a30e1b494b46045b82005d71cae74f1ebd9f1338566b77f9fc +F ext/fts5/test/fts5ai.test cbe26d78030998f535bc103f37915350b137a822c71a9db439a077d7666a3539 +F ext/fts5/test/fts5aj.test 53c8508dab4acca3e691a4c51eca4b3b018319ab8635e540103d5bbdc91543c9 +F ext/fts5/test/fts5ak.test 25e2f8afdcff30d98ca9dee8c5cacca2f26db17501c9401f16d99ee036f70e8d +F ext/fts5/test/fts5al.test f0e655606771b2b5dbaf70e7f0044d560257cf3531d5eea40df58d0d7add8c39 +F ext/fts5/test/fts5alter.test ebbee06419c2d3cee5ef7ebb5ba6a9996f1aa374035361c0acd37368cc5f64f3 +F ext/fts5/test/fts5auto.test 2278de323172ced485d2844cb1357d00036ac1665f27e70fa1a48ce57bf31c2c +F ext/fts5/test/fts5aux.test 27210687338133b1e9bc0dd669322fca59fd432439f40b126895e2d7c2f899d6 +F ext/fts5/test/fts5aux2.test 4f59ac5e7c06c430a9f4890877e10f7b4708e46897422ee6743d27b0a8d01497 +F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06cffec0b37ce4624161 F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd -F ext/fts5/test/fts5bigpl.test 6466c89b38439f0aba26ac09e232a6b963f29b1cbe1304f6a664fe1e7a8f5fd3 +F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a +F ext/fts5/test/fts5blob.test caa33369e93e99ff494cd1103506ae34c5afbc0bcc369ed5e58e135144e33689 +F ext/fts5/test/fts5cat.test bf67dd335f964482ee658287521b81e2b88697b45eb7f73933e15f198ed447cb F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 -F ext/fts5/test/fts5colset.test 7031ce84fb4d312df5a99fc4e7b324e660ccb513c97eccdef469bfd52d3d0f8f -F ext/fts5/test/fts5columnsize.test 45459ce4dd9fd853b6044cdc9674921bff89e3d840f348ca8c1630f9edbf5482 -F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825ce437d26afe0817f +F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 +F ext/fts5/test/fts5columnsize.test 0af91d63985afdf663455d4b572b935238380140d74079eac362760866d3297b +F ext/fts5/test/fts5config.test 017daf10d2642496e97402baa0134de8b5b46b9c37e53c229cd9ab711d21522c F ext/fts5/test/fts5conflict.test bf6030a77dbb1bedfcc42e589ed7980846c995765d77460551e448b56d741244 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4 -F ext/fts5/test/fts5content.test 282b373c58c8e798568ec2ced18b23f29bffa8d61317a0e51a035000ad6cd731 -F ext/fts5/test/fts5contentless.test 1cd1237894eeff11feb1ff8180044eac0b17dde22c181f7a722f2dcbfdb3377c -F ext/fts5/test/fts5contentless2.test 14c83bdacf8230f5f7ca74ecf2926b87d8a7cb788a69ce9937020428ac4fe192 -F ext/fts5/test/fts5contentless3.test 353d871c5ea08992aed3e2ebda0b1bdc35116cd24fe330fe7cf05be1e2b49fd7 -F ext/fts5/test/fts5contentless4.test dd33ead36b048c9447b81ec358bd4a27166c49ffaac65a54e95eabf59f338947 -F ext/fts5/test/fts5contentless5.test 96041cbf5ef781a68a5d0f0d18a88030c47a52b156b17876ed6ce36e80e27a7e -F ext/fts5/test/fts5corrupt.test b6d4034b682bb3387bc44c510c71b3c67d4349e4df139490fc0b69e6a972b99f -F ext/fts5/test/fts5corrupt2.test 99e7e23a58b4d89eb7167c6de1669cbc595cd3c79ab333e0eb56405473319e77 -F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78 -F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3 -F ext/fts5/test/fts5corrupt5.test f9dbed6785c38d891c0b947fd6e973e9c4eb44991e1322f8d7e5281a019276f2 -F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4 -F ext/fts5/test/fts5corrupt7.test 80ad7f683a8bda2404731bb77e8c3dbbb620c1f6cc583cca8239f6accd6338c0 +F ext/fts5/test/fts5content.test d5c0c2142e64cb305f0968de70c01f8e59dbc3ecc56520c22e739e5dd99ea3bb +F ext/fts5/test/fts5contentless.test 606f063b29ba0f46d4b79aa36cdd1ef4dab5de53eae8c881d731af75a4894aca +F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e2739b358439e9bdcf56ced35f +F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a +F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8 +F ext/fts5/test/fts5contentless5.test 40cdcb4fe751672450829c5a96bd32c25fc2f6076279dd2ce5c58ac9a390132a +F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd +F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4 +F ext/fts5/test/fts5corrupt3.test 621e9bca3e7299f487e1b29ff4179d9fc9560f5847dfc5b50a16010c9d2a0e5f +F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 +F ext/fts5/test/fts5corrupt5.test 11b47126f5772cc37b67e3e8b2ed05895c4d07c05338bc07e4eea225bfe32c76 +F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 +F ext/fts5/test/fts5corrupt7.test 4e830875c33b9ea3c4cf1ba71e692b63893cbb4faae8c69b1071889dc26e211c F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 -F ext/fts5/test/fts5dlidx.test b90852c55881b29dbac6380b274de27beae623ac4b6d567c6c8fb9cdc315a86e -F ext/fts5/test/fts5doclist.test faa9e9cc3c0645fa6203667cb5f007c359447c6ee66753f71a58175c2497cacd -F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 -F ext/fts5/test/fts5eb.test 5f0a86e9fe4715912e6bfa556368aae96d13c61a481373f24daa40429d5d5ca1 +F ext/fts5/test/fts5dlidx.test a7c42b0a74dc7c8aa1a46d586e0aadda4b6cc42c24450f8d3774b21166e93159 +F ext/fts5/test/fts5doclist.test b7cb84758504519746957802db9cd31187bb4e0028b89d9087ba06e26cc4155f +F ext/fts5/test/fts5ea.test cefdf66024550fa7920c03395c71ce5046235ed1a1a7a469d79b19e7aad5afb5 +F ext/fts5/test/fts5eb.test 401f756fdb77083aeba8b696c1e0ad4d834c39dbd6f17e492bb55a2ad64b4296 +F ext/fts5/test/fts5expr.test c7e208813df7a90badc856fde3796da79569b39382e0fdb43042127f3b8e06a7 F ext/fts5/test/fts5fault1.test d28a65caee75db6897c3cf1358c5230d3bb2a3bf7fb31062c19c7e5382b3d2bd F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421c20a0756251fe344 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522 F ext/fts5/test/fts5fault4.test 87a10d0caee57da587c7588b0c8d25d2930197399b4812ad1e4d574c75324cee F ext/fts5/test/fts5fault5.test a336e4e11847de24c9497f80cce18e00bb3fab7fb11f97d04eb9af898900a762 -F ext/fts5/test/fts5fault6.test a0fc0a8f99e4b16500c31dfc7e38e1defe0f1693ac47650517ac7b723b1956f8 +F ext/fts5/test/fts5fault6.test 40f49976c6ca8927bf7d65d0b8df46009d7ea172e1d4050b294610e7ea0a2979 F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd164090be139e9e97ed4c -F ext/fts5/test/fts5fault8.test 318238659d35f82ad215ecb57ca4c87486ea85d45dbeedaee42f148ff5105ee2 +F ext/fts5/test/fts5fault8.test 9353fe6a2a993c3231e09c49b0f4a12c8d306319555ff2ca6672b5b86fe9b0dd F ext/fts5/test/fts5fault9.test 098e6b894bbdf9b2192f994a30f4043673fb3f338b6b8ab1624c704422f39119 F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c53ca85af12290c8c F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05cf54f5b045b729ab5 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 -F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3 -F ext/fts5/test/fts5faultH.test 57f53c87ffd59be0265840f2b54a16811f9cb9012db86aad9b41d0d14d85dfe3 -F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079 -F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717 +F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 +F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d +F ext/fts5/test/fts5faultI.test e1e8a927122a8c3be8614a59717e838cad0505ba0a6f404983da8430e7aeb14d +F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 +F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e -F ext/fts5/test/fts5hash.test dc7bc7e0cdeb42cfce31294ad2f8fcf43192bfd0145bb7f3ecc5465d8c72696f -F ext/fts5/test/fts5integrity.test 0d249d351163e17e2227aa9850e68193f88a7813d16cc7d51287e554bf915b3d -F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fdde7ea00e78ea283d227 -F ext/fts5/test/fts5lastrowid.test be98fe3e03235296585b72daad7aed5717ba0062bae5e5c18dd6e04e194c6b28 +F ext/fts5/test/fts5hash.test fd3e0367fbf0b0944d6936fdb22696350f57b9871069c6766251578a103e8a14 +F ext/fts5/test/fts5integrity.test 646796671205dae46af5bb12a49b5696483cfe8e12d71d21454940b13ace95ab +F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba1044c168db0e106f9ac92c +F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a -F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6 +F ext/fts5/test/fts5locale.test 797cf6f5e017462ab11313ce884b9f1df8ff063811e74ef42190cd19ed6b600b +F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 +F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test 89dc46e37951b7f6653809f4abf6b1ca2f1fa62259efaf719339288f76fb6be9 +F ext/fts5/test/fts5misc.test 60bb2be4a2d83d7a45047c1812781e2e337a27efa539d86356ef7f4acaf08eab F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 -F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd +F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618834bf1fcc3e7b84da -F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 -F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 -F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39 -F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634 -F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0 -F ext/fts5/test/fts5origintext4.test 0c4e4514b68d9ddb15e5a538d9d234da85747a3fd62432265dbdba5c8708e457 -F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d -F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b -F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a -F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 -F ext/fts5/test/fts5porter2.test 0d251a673f02fa13ca7f011654873b3add20745f7402f108600a23e52d8c7457 -F ext/fts5/test/fts5prefix.test a0fa67b06650f2deaa7bf27745899d94e0fb547ad9ecbd08bfad98c04912c056 -F ext/fts5/test/fts5prefix2.test ad751d4a5b029726ee908a7664e27d27bde7584218b8d7944c2a323afd381432 -F ext/fts5/test/fts5query.test ac363b17a442620bb0780e93c24f16a5f963dfe2f23dc85647b869efcfada728 -F ext/fts5/test/fts5rank.test 30f29e278cd7fb8831ba4f082feb74d8eb90c463bf07113ae200afc2b467ef32 -F ext/fts5/test/fts5rebuild.test 55d6f17715cddbf825680dd6551efbc72ed916d8cf1cde40a46fc5d785b451e7 -F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415 -F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff171013003d8297203a20ad6 -F ext/fts5/test/fts5savepoint.test 050796b24929325cdbbb2fbfe2794816ae95d298e940ae15032200c2f4a73725 +F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47 +F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed +F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 +F ext/fts5/test/fts5origintext.test 2015f69bc8abd111152a8e66211fd2d45026378001e07c054159aa4f84e6691d +F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba +F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b +F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 +F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b +F ext/fts5/test/fts5phrase.test 6260b8be40d51ec287992cd983a5e58a9be92e5dc6e01d48cbce1ad0e95a99d8 +F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626 +F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0 +F ext/fts5/test/fts5porter2.test 94f0e4351e2c99b4e74f1fae05a4ddf1cb5b926620a8c14554160d075ddc7a59 +F ext/fts5/test/fts5prefix.test c0b7842f1a2d830c0b146cd438a95ea4c5a25635719ed0d973ffe41907338b83 +F ext/fts5/test/fts5prefix2.test a5bb43b8a2687efafa7ac4e5ccff6812015cf8cf18e3086bb0eb3126f30fbbf6 +F ext/fts5/test/fts5query.test 0320a7a4b58a6e3e50ec8910b301649da90ace675001f9e0bf6392750ad4591d +F ext/fts5/test/fts5rank.test 47c1e8e5d84754ff18e012fdd629776088b5a15de41bdd24957581cf084d8a00 +F ext/fts5/test/fts5rebuild.test 83e72d77636378833233fadc7cb7517a2fa446ea7d1f94dd526ba3e7e104b9f5 +F ext/fts5/test/fts5restart.test 9af2084b8e065130037b95f05f3f220bb7973903a7701e2c5fb916dff7cf80c5 +F ext/fts5/test/fts5rowid.test 8632829fec04996832a4cfb4f0bd89721ba65b7e398c1731741bdb63f070e1a3 +F ext/fts5/test/fts5savepoint.test 7f373184cf2d6c1c472d2bc732e1fce62211ffe023f13e381db0f5e4fd06e41d F ext/fts5/test/fts5secure.test a02f771742fb2b1b9bdcb4bf523bcf2d0aa1ff597831d40fe3e72aaa6d0ec40f F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test 6d066828d225b0dbe5db818d4d6165df7bb70210e68a577e858e8762400d5a23 @@ -216,38 +223,48 @@ F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b98 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a -F ext/fts5/test/fts5secure8.test eb3579e9d58b0acad97e8082dee1f99b2d393198f03500b453c2b25761c0c298 -F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b -F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b -F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd -F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f -F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb -F ext/fts5/test/fts5synonym2.test e2f6ff68c4fbe12a866a3a87510f553d9dac99bcb74c10b56487c4c0a562fcf5 +F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0 +F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 +F ext/fts5/test/fts5simple.test ed7c3815c9fa1c16166258cb98edb2e014c63c7589958d76c5487df0df913d61 +F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc +F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 +F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 +F ext/fts5/test/fts5synonym2.test 58f357b997cf2fedeeb9d0de4db9f880fa96fa2fe27a743bfe7d7b96895bdd87 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 -F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43 -F ext/fts5/test/fts5tokenizer2.test cb5428c7cfb3b6a74b7adfcde65506e329112003e8dffa7501d01c2d18d02569 -F ext/fts5/test/fts5trigram.test 6c4e37864f3e7d90673db5563d9736d7e40080ab94d10ebdffa94c1b77941da0 -F ext/fts5/test/fts5trigram2.test 9fe4207f8a4241747aff1005258b564958588d21bfd240d6cd4c2e955d31c156 -F ext/fts5/test/fts5ubsan.test 783d5a8d13ebfa169e634940228db54540780e3ba7a87ad1e4510e61440bf64b +F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 +F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6 +F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 +F ext/fts5/test/fts5trigram.test fb9ee982edd76280ce979905a2251081cd04ae4c470248bd5d391b2d096430ab +F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61 +F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 -F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9 -F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e -F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c75ae10f4afc6faa0744 -F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687 -F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d +F ext/fts5/test/fts5unicode.test 41898f7e476e6515cd4b737c02a442cda5a580a74509788aa9072a2074948e0e +F ext/fts5/test/fts5unicode2.test 3bbd30152f9f760bf13886e5b1e5ec23ff62f56758ddda5d9c775a6082fb4c7c +F ext/fts5/test/fts5unicode3.test f4891a3dac3b49c3d7c0fdb29566e9eb0ecff35263370c89f9661b1952b20818 +F ext/fts5/test/fts5unicode4.test 728c8f0caafb05567f524ad313d9f8b780fa45987b8a8df04eff87923c74b4d0 +F ext/fts5/test/fts5unindexed.test 168838d2c385e131120bbf5b516d2432a5fabc4caa2259c932e1d49ae209a4ae F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc -F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67 -F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760 -F ext/fts5/test/fts5vocab2.test 1b1f0059f762ffb404213d35dac013e010621f08128589b6ec7bec59d9a710f3 +F ext/fts5/test/fts5version.test c22d163c17e60a99f022cbc52de5a48bb7f84deaa00fe15e9bc4c3aa1996204e +F ext/fts5/test/fts5vocab.test 2a2bdb60d0998fa3124d541b6d30b019504918dc43a6584645b63a24be72f992 +F ext/fts5/test/fts5vocab2.test bbba149c254375d00055930c1a501c9a51e80b0d20bf7b98f3e9fa3b03786373 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl 3eba8e9bee4221ed165f3304b51b2a74a705f4ec5df3d044573a2be539534af8 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 -F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 +F ext/icu/icu.c 3add8197e0a86c1761771a39500ebae749438bcf1836160b407a56b4eaa8721c F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 +F ext/intck/intck1.test f3a3cba14b6aeff145ffa5515546dd22f7510dad91512e519f43b92b56514012 +F ext/intck/intck2.test d2457c7e5e5b688046d15ebe08a1e1427cc5e7a6dc8d6af215f42e8bcaf67304 +F ext/intck/intck_common.tcl a61fd2697ae55b0a3d89847ca0b590c6e0d8ff64bebb70920d93724799894159 +F ext/intck/intckbusy.test d5ed4ef85a4b1dc1dee2484bd14a4bb68529659cca743327df0c775f005fa387 +F ext/intck/intckcorrupt.test f6c302792326fb3db9dcfc70b554c55369bc4b52882eaaf039cfe0b74c821029 +F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028f051e985efab +F ext/intck/sqlite3intck.c 0d10df36e2b7b438aa80ecd3f5e584d41b747586b038258fe6b407f66b81e7c5 +F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 +F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -277,7 +294,7 @@ F ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java 01bc0c238eed2d5f9 F ext/jni/src/org/sqlite/jni/capi/ResultCode.java 8141171f1bcf9f46eef303b9d3c5dc2537a25ad1628f3638398d8a60cacefa7f F ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java e172210a2080e851ebb694c70e9f0bf89284237795e38710a7f5f1b61e3f6787 F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385155fa3b8011a5cca0bb3c28468c7131c1a5 -F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 09bee15aa0eedac68d767ae21d9a6a62a31ade59182a3ccbf036d6463d9e30b1 +F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 0b25cde8c5fa77f3e7ad92368acf195c5c64fb1c5273b8ee71b2d7ab812aab34 F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615 F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java addf120e0e76e5be1ff2260daa7ce305ff9b5fafd64153a7a28e9d8f000a815f F ext/jni/src/org/sqlite/jni/capi/Tester1.java e5fa17301b7266c1cbe4bcce67788e08e45871c7c72c153d515abb37e501de0a @@ -303,7 +320,7 @@ F ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java 1efd1220ea328a32f2d2a1b F ext/jni/src/org/sqlite/jni/fts5/fts5_api.java a8e88c3783d21cec51b0748568a96653fead88f8f4953376178d9c7385b197ea F ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java 9e2b954d210d572552b28aca523b272fae14bd41e318921b22f65b728d5bf978 F ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java 92bdaa3893bd684533004d64ade23d329843f809cd0d0f4f1a2856da6e6b4d90 -F ext/jni/src/org/sqlite/jni/test-script-interpreter.md f9f25126127045d051e918fe59004a1485311c50a13edbf18c79a6ff9160030e +F ext/jni/src/org/sqlite/jni/test-script-interpreter.md 9bf7e9cab1183287b048bb77baee4b266f0c15baf1b624feec12fbf00cfa7e94 F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483 F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 43c43adfb7866098aadaaca1620028a6ec82d5193149970019b1cce9eb59fb03 F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java 27b141f5914c7cb0e40e90a301d5e05b77f3bd42236834a68031b7086381fafd @@ -371,16 +388,16 @@ F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0e F ext/misc/btreeinfo.c cb952620eedf5c0b7625b678f0f08e54d2ec0011d4e50efda5ebdc97f3df7d04 F ext/misc/carray.c 34fac63770971611c5285de0a9f0ac67d504eaf66be891f637add9290f1c76a5 F ext/misc/carray.h 503209952ccf2431c7fd899ebb92bf46bf7635b38aace42ec8aa1b8d7b6e98a5 -F ext/misc/cksumvfs.c 9224e33cc0cb6aa61ff1d7d7b8fd6fe56beca9f9c47954fa4ae0a69bef608f69 +F ext/misc/cksumvfs.c 3a7931dd30667be6348af919f3f9e6188dfd7646b42af8e399a499b327f5bd63 F ext/misc/closure.c 0e04f52d93e678dd6f950f195f365992edf3c380df246f3d80425cba4c13891e -F ext/misc/completion.c ef78835483b43ac18c96be312b90b615d8368189909be03513ab7a9338131298 -F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 +F ext/misc/completion.c cb978c88d5577821323617a8ea775ce1b920e02dcdb593858f02044a4d008eea +F ext/misc/compress.c 2c79a74330e0e0ba6cb3f7397f8ba5af12d46377ef5d3ee075e12dd8a6ed57f0 F ext/misc/csv.c 575c2c05fba0a451586a4d42c2c81e711780c41e797126f198d8d9e0a308dcdb F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 172cf81a8634e6a0f0bedaf71a8372fee63348cf5a3c4e1b78bb233c35889fdc F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c d88e60f63557d76d4e38acffda5556b2ab42e98f5d830897f22aba65930d975c +F ext/misc/fileio.c 916638042f318701460485032e33981056747d0f92e6757aa9499f2363ea7047 F ext/misc/fossildelta.c 8c026e086e406e2b69947f1856fa3b848fff5379962276430d10085b8756b05a F ext/misc/fuzzer.c 8b28acf1a7e95d50e332bdd47e792ff27054ad99d3f9bc2e91273814d4b31a5a F ext/misc/ieee754.c 62a90978204d2c956d5036eb89e548e736ca5fac0e965912867ddd7bb833256d @@ -389,10 +406,10 @@ F ext/misc/memtrace.c 7c0d115d2ef716ad0ba632c91e05bd119cb16c1aedf3bec9f06196ead2 F ext/misc/memvfs.c 7dffa8cc89c7f2d73da4bd4ccea1bcbd2bd283e3bb4cea398df7c372a197291b F ext/misc/mmapwarm.c a81af4aaec00f24f308e2f4c19bf1d88f3ac3ce848c36daa7a4cd38145c4080d F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58f34edd -F ext/misc/noop.c 81efe4cad9ec740e64388b14281cb983e6e2c223fed43eb77ab3e34946e0c1ab +F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5a0 F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 -F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 +F ext/misc/percentile.c af1941dc87d45dd0c2698a3087fbfe9ee0d157e5e72da521430c4b784abcbe81 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed @@ -400,13 +417,14 @@ F ext/misc/regexp.c 4bdd0045912f81c84908bd535ec5ad3b1c8540b4287c70ab840709636240 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 80692f675de989629ee20796f75010ce87ca826cb383131040e84f7e1bea5d7a +F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac +F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea -F ext/misc/sqlar.c 53e7d48f68d699a24f1a92e68e71eca8b3a9ff991fe9588c2a05bde103c6e7b7 +F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 +F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b F ext/misc/totype.c 75ed9827d19cc3b434fc2aeb60725d4d46e1534373615612a4d1cfdcc3d60922 F ext/misc/uint.c 053fed3bce2e89583afcd4bf804d75d659879bbcedac74d0fa9ed548839a030b @@ -415,10 +433,10 @@ F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d -F ext/misc/vtablog.c f2c9d41afe00b51b2c8307b79f508eb0c2dcd57bae074807944e73376fcf15b7 +F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 -F ext/misc/zipfile.c 64cb3d98b6316586e6056d182051aa9d28fdedfbf4b908e6b7a7d70209b1db11 +F ext/misc/zipfile.c 5a3bf1b9cccb8e0da2389fe9e39e9c7f2b1351474b7e5090635f817d495eee3f F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 @@ -464,26 +482,28 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9 +F ext/rbu/sqlite3rbu.c 4a3376c0fb9a844a799ac529fb81260523f6b13c9f629bc270c632dbae5fc1f8 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 -F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949 -F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf +F ext/rbu/test_rbu.c b9727c3394307d058e806c1da0f8bb7b24daf3c6bb94cb10cca88ea4d5c806c0 +F ext/recover/dbdata.c 5295f4f922b60d7035b6b9fd5846b13071b9d97ed7fad8496837bb7640d24771 +F ext/recover/recover1.test e16d78e94183562abff569967b18b7c77451d7044365516cd0fe14713a284851 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 -F ext/recover/recovercorrupt2.test b9b974f006340a1300b5a7e687bd6097c5238498181c6f92d87406a77ece430b +F ext/recover/recovercorrupt2.test 1418f1710debc24ff38276cedfcea234beb37a34205708e7e3e6d76cc4a979db +F ext/recover/recovercorrupt3.test 2e7b9a1b528ca23ed382cec6f64e3fcbbd0f8e852add7562397fd8df83f335d5 +F ext/recover/recovercorrupt4.test 3e2794145dad2517c018cb68b96f59d4d55b18b3d6271e1d37852cfd7a30b50c F ext/recover/recoverfault.test 9d9f88eeb222615a25e7514f234c950d46bee20d24cd8db49d8fff8d650dcfe1 F ext/recover/recoverfault2.test 730e7371bcda769554d15460cb23126abba1be8eca9539ccabf63623e7bb7e09 F ext/recover/recoverold.test 68db3d6f85dd2b98e785b6c4da4f5eea4bbe52ccf6674d9a94c7506dc92596aa -F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1be847182410ffc58 +F ext/recover/recoverpgsz.test 88766fcb810e52ee05335c456d4e5fb06d02b73d3ccb48c52bf293434305e2b1 F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c e6eb20c469bcdb96f297f2241860bccabf9f036bfa7f3d47bcc6ca1191b108dc +F ext/recover/sqlite3recover.c 2dcf6b56c5e0e2b43fc4c6115b689ab194c374ced7f7f8380ad9a24d8ef24ac9 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 -F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade +F ext/recover/test_recover.c 072260d7452a3b81aba995b2b3269e7ec2aa7f06725544ba4c25b1b0a1dbc61a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c af5c66463f51462d8a6f796b2c44ef8cfa1116bbdc35a15da07c67a705388bfd @@ -495,9 +515,9 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/geopoly.c 0dd4775e896cee6067979d67aff7c998e75c2c9d9cd8d62a1a790c09cde7adca -F ext/rtree/rtree.c d0134bb75bc92b18a1dc011ec10419642f055c67af8ff44fc4a07c5fa9f189cb +F ext/rtree/rtree.c b6133dba5ae331fa6c1fc34df6aa623eba951b05ac35116f954a0bf7ab550436 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 -F ext/rtree/rtree1.test 2b5b8c719c6a4abe377f57766f428a49af36a93061cb146cccfdc3b30000c0a4 +F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d F ext/rtree/rtree3.test 272594f88c344e973864008bbe4c71fd3a41a264c097d568593ee7886d83d409 F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b @@ -515,6 +535,7 @@ F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd F ext/rtree/rtreeG.test 1b9ca6e3effb48f4161edaa463ddeaa8fca4b2526d084f9cbf5dbe4e0184939c F ext/rtree/rtreeH.test 0885151ee8429242625600ae47142cca935332c70a06737f35af53a7bd7aaf90 F ext/rtree/rtreeI.test 608e77f7fde9be5a12eae316baef640fffaafcfa90a3d67443e78123e19c4ca4 +F ext/rtree/rtreeJ.test 93227ccd4d6c328f5ac46a902b8880041509dd2d68f6ce71560f0d8ab5bb507a F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 202ca70df1f0645ef9d5a2170e62d378a28098d9407f0569e85c9c1cf1bd020a F ext/rtree/rtreecheck.test 934546ad9b563e090ee0c5cbdc69ad014189ad76e5df7320526797a9a345661f @@ -525,18 +546,18 @@ F ext/rtree/rtreedoc2.test 194ebb7d561452dcdc10bf03f44e30c082c2f0c14efeb07f5e02c F ext/rtree/rtreedoc3.test 555a878c4d79c4e37fa439a1c3b02ee65d3ebaf75d9e8d96a9c55d66db3efbf8 F ext/rtree/rtreefuzz001.test 44f680a23dbe00d1061dbde381d711119099846d166580c4381e402b9d62cb74 F ext/rtree/sqlite3rtree.h 03c8db3261e435fbddcfc961471795cbf12b24e03001d0015b2636b0f3881373 -F ext/rtree/test_rtreedoc.c de76b3472bc74b788d079342fdede22ff598796dd3d97acffe46e09228af83a3 +F ext/rtree/test_rtreedoc.c d20f51d1ad69c72947a4ac72194e5a12e70b3464e7492538fcef66fa871c5081 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/util/randomshape.tcl 54ee03d0d4a1c621806f7f44d5b78d2db8fac26e0e8687c36c4bd0203b27dbff F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/rtree/visual01.txt e9c2564083bcd30ec51b07f881bffbf0e12b50a3f6fced0c222c5c1d2f94ac66 F ext/session/changeset.c 7a1e6a14c7e92d36ca177e92e88b5281acd709f3b726298dc34ec0fb58869cb5 F ext/session/changesetfuzz.c 227076ab0ae4447d742c01ee88a564da6478bbf26b65108bf8fac9cd8b0b24aa -F ext/session/changesetfuzz1.test 2e1b90d888fbf0eea5e1bd2f1e527a48cc85f8e0ff75df1ec4e320b21f580b3a +F ext/session/changesetfuzz1.test 15b629004e58d5ffcc852e6842a603775bb64b1ce51254831f3d12b113b616cd F ext/session/session1.test e94f764fbfb672147c0ef7026b195988133b371dc8cf9e52423eba6cad69717e F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f F ext/session/session3.test 2cc1629cfb880243aec1a7251145e07b78411d851b39b2aa1390704550db8e6a -F ext/session/session4.test 6778997065b44d99c51ff9cece047ff9244a32856b328735ae27ddef68979c40 +F ext/session/session4.test 823f6f018fcbb8dacf61e2960f8b3b848d492b094f8b495eae1d9407d9ab7219 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 35279f2ec45448cd2e24a61688219dc6cf7871757716063acf4a8b5455e1e926 F ext/session/session8.test 326f3273abf9d5d2d7d559eee8f5994c4ea74a5d935562454605e6607ee29904 @@ -554,8 +575,10 @@ F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8 F ext/session/sessionalter.test 460bdac2832a550519f6bc32e5db2c0cee94f335870aaf25a3a403a81ab20e17 F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf -F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec -F ext/session/sessionfault.test 573bf027fb870d57bd4e7cf50822a3e4b17b2b923407438747aaa918dec57a09 +F ext/session/sessionchange.test 77c4702050f24270b58070e2cf01c95c3d232a3ef164b70f31974b386ce69903 +F ext/session/sessionconflict.test 8b8cbd98548e2e636ddc17d0986276f60e833fb865617dd4f88ea5bbe3a16b96 +F ext/session/sessiondiff.test e89f7aedcdd89e5ebac3a455224eb553a171e9586fc3e1e6a7b3388d2648ba8d +F ext/session/sessionfault.test c2b43d01213b389a3f518e90775fca2120812ba51e50444c4066962263e45c11 F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c F ext/session/sessionfault3.test 7c7547202775de268f3fe6f074c4d0d165151829710b4e64f90d4a01645ba9e7 F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 @@ -566,44 +589,49 @@ F ext/session/sessionnoop2.test de4672dce88464396ec9f30ed08c6c01643a69c53ae540fa F ext/session/sessionrebase.test 702378bdcb5062f1106e74457beca8797d09c113a81768734a58b197b5b334e2 F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a80600a44396f7363 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 -F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c +F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c 7b0f9bf6479aebd49e5f34cdaeb1479cde843cf5189bca6f32db4f36c130b414 -F ext/session/sqlite3session.h 4cf19a51975746d7cff2fdd74db8b769c570958e1c3639ac150d824ac1553b3e -F ext/session/test_session.c 7b94ad945cd4afe6c73ee935aeb3d44b4446186e1729362af616c7695a5283d9 +F ext/session/sqlite3session.c c7473aafbd88f796391a8c25aa90975a8f3729ab7f4f8cf74ab9d3b014e10abe +F ext/session/sqlite3session.h 683ccbf16e2c2521661fc4c1cf918ce57002039efbcabcd8097fa4bca569104b +F ext/session/test_session.c 6acbe67db80ab0806147eb62a12f9e3a44930f4a740b68b0a4340dddda2c10d7 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 -F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 +F ext/userauth/user-auth.txt ca7e9ee82ca4e1c1744295f8184dd70edfae1992865d26c64303f539eb6c084c F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 99aad6d6a28c43573f80825e986427c1a024a3298aaf0c69c56a0c6b336f12c8 +F ext/wasm/GNUmakefile d4f6586d9a36ee2869a8c7f77227a8b7f42b6c4623f3be594beafb1554ab20d9 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff -F ext/wasm/SQLTester/SQLTester.mjs ec2f6ba63a0f2f0562941a0fb8e46b7dc55589711513f1952349785966edfe50 +F ext/wasm/SQLTester/SQLTester.mjs ce765c0ad7d57f93553d12ef4dca574deb00300134a26d472daacab49031e1fb F ext/wasm/SQLTester/SQLTester.run.mjs c72b7fe2072d05992f7a3d8c6a1d34e95712513ceabe40849784e24e41c84638 F ext/wasm/SQLTester/index.html 3f8a016df0776be76605abf20e815ecaafbe055abac0e1fe5ea080e7846b760d F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api f442460ed9a109e637dd3ea1caa4489553ad9414e8988118b208bb7a4bbece6b +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth 7ac80cc3b6a6d52e041bb295e85555ce797be78c15ef2008a64ae58815014080 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 w ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate d1d62a2212099f2c0782d730beb8cb84a7a52d99c15ead2cb9b1411fff5fd6b1 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session 213b6c04267cb9bd760172db011eb1650732805fb3d01f9395478a8ceec18eb0 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-vtab fd57af1f4502a052be27d8402df74be1dc60fcb6a687d372972abd90e424120a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134 +F ext/wasm/api/README.md 34fe11466f9c1d81b10a0469e1114e5f1c5a6365c73d80a1a6ca639a1a358b73 F ext/wasm/api/extern-post-js.c-pp.js c4154a7f90c2d7e51fd6738273908152036c3457fdc0b6523f1be3ef51105aac F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 -F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 +F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4734472d8e29c1c122 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e -F ext/wasm/api/sqlite3-api-glue.js 119b91c8a7ce6648679eb66fcdd1ed07ef7fd892eb501d658fbfefcc962012d9 -F ext/wasm/api/sqlite3-api-oo1.js 7f3bcf0549ac44cde4b9da0b642d771916738d3f6781fb8a1757c50a91e506c0 -F ext/wasm/api/sqlite3-api-prologue.js 9aeba7b45cf41b3a26d34d7fb2525633cd1adfc544888c1ea8dbb077496f4ce9 -F ext/wasm/api/sqlite3-api-worker1.js fd46628ef147dd5856c88f63a9a279a40f744f1fdfddd55251ad8fbc3d8200ae +F ext/wasm/api/sqlite3-api-glue.c-pp.js 54b32b5321105a72d6f3d3e8b77f28f162d0367b08c63184263d3f85f3d7dbed w ext/wasm/api/sqlite3-api-glue.js +F ext/wasm/api/sqlite3-api-oo1.c-pp.js f3a8e2004c6625d17946c11f2fb32008be78bc5207bf746fc77d59848813225f w ext/wasm/api/sqlite3-api-oo1.js +F ext/wasm/api/sqlite3-api-prologue.js 6f1257e04885632ed9f44d43aba200b86e0bc16709ffdba29abbbeb1bc8e8b76 +F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d27567e63e8bc8b832d74371c352d w ext/wasm/api/sqlite3-api-worker1.js F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 -F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 595953994aa3ae2287c889c4da39ab3d6f17b6461ecf4bec334b7a3faafddb02 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91ccfafc38b400fe36e90bdda88e -F ext/wasm/api/sqlite3-wasm.c dfd1f1a225b267e8fd641dcd6c7d579fbe2b731aeaa123324135efac830a2bcf -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f234191fe6bf41a5a1e59c9f43ed816e74a522b3d60d3f556f66c3085c448503 +F ext/wasm/api/sqlite3-opfs-async-proxy.js e8f1df56e97a29004a95a2eddd26778f52c33b3e797d32d4b1b668a38e6493dd +F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d w ext/wasm/api/sqlite3-v-helper.js +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js e529a99b7d5a088284821e2902b20d3404b561126969876997d5a73a656c9199 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e99e3d99f736937914527070f00ab13e9391d3f1cef884ab99a64cbcbee8d675 +F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 +F ext/wasm/api/sqlite3-wasm.c 09a938fc570f282e602acd111147c7b74b5332da72540c512a79b916ab57882a +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 46f303ba8ddd1b2f0a391798837beddfa72e8c897038c8047eda49ce7d5ed46b F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd @@ -613,25 +641,24 @@ F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4 +F ext/wasm/common/whwasmutil.js 6181f8cd958700f3723350bd4d76c7cc797db331a9aa14b25b42d121f12d6fee F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 -F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f -F ext/wasm/demo-worker1-promiser.js 5e5c7d7c91cd7aae9cc733afd02569ba9c6928292db413b550e8b842f4b75e87 +F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 w ext/wasm/demo-worker1-promiser.html +F ext/wasm/demo-worker1-promiser.c-pp.js fcc628cb42fcfaf07d250477801de1e6deb1e319d003976612a0db8d76b9fccc w ext/wasm/demo-worker1-promiser.js F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js 836bece8615b17b1b572584f7b15912236a5947fe8c68b98d2737d7e287447ef -F ext/wasm/dist.make 3a851858aad72e246a5d9c5aaf6b6a144305f1bf898ac1846760ea7bab95c9a3 +F ext/wasm/dist.make 653e212c1e84aa3be168d62a10616ccea45ee9585b0192745d2706707a5248ce F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make fa2ba6e90457ba2a71cb745f200d409caf773076df75ae5b177cc225d7627a11 -F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d -F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 -F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 -F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263 -F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9 +F ext/wasm/fiddle.make 2406b02473878a99fb6a2eaff0923277017adc45eb041b2afb2d7707bf7b375c +F ext/wasm/fiddle/fiddle-worker.js 850e66fce39b89d59e161d1abac43a181a4caa89ddeea162765d660277cd84ce +F ext/wasm/fiddle/fiddle.js b444a5646a9aac9f3fc06c53d78af5e1912eb235d69a8e6010723e4eb0e9d4a1 +F ext/wasm/fiddle/index.html 739e0b75bc592679665d25e2f7649d2b8b2db678f3b41a772a8720b609b8482d +F ext/wasm/index-dist.html 564b5ec5669676482c5a25dea9e721d8eafed426ecb155f93d29aeff8507511f +F ext/wasm/index.html 4337f495416756802669f69f9f9f3df9f87ee4c1918e6718719b4b5718e4713a F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 59a20df389abcc3606eb4eaea7fb7ba14504beb3e345dbea9b99a0618ba3bec8 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 @@ -640,7 +667,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852 -F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd +F ext/wasm/speedtest1-worker.js 95e549e13a4d35863a9a7fc66122b5f546c0130d3be7b06dfcc556eb66d24bde F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -649,15 +676,15 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js a92dc256738dbd1b50f142d1fd0c835294ba09b7bb6526650360e942f88cb63f -F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 -F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d +F ext/wasm/tester1.c-pp.js a88b9c669715adc1c5e76750ca8c0994ae33d04572e3bf295b6f4f5870f3bdf3 +F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e +F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966f1bc16a F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk ef8d6b0e76b27d2b32d7b71ea30a2b2626b668f998a4f32f6171c9623a310a53 +F main.mk 9541ffdce424ddddb463e2480c3f1cb4067bbd3a6d2c84b3f083cb128a2fa721 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -669,45 +696,45 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c 30c2333b8bb3af71e4eb9adeadee8aa20edb15917ed44b8422e5cd15f3dfcddc -F src/analyze.c 0f15753308c3bca7674f31fa7e0807ffcb8b120c36eef7d00b62b33079ddc854 -F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 -F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 +F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532 +F src/analyze.c 30bf40ec4208ead9e977bec017bccc8a9681820936e38ca5a4a7443100a6d5c5 +F src/attach.c 08235ab62ed5ccc93c22bf36e640d19effcd632319615851bccf724ec9341333 +F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c dee25e097b749275333b55d64a5ffc079249576f8e88a2ee476468cf67510f4b -F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 -F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 -F src/build.c e7d9044592eeeea8e78d8ae53ca8d31fd6e92ca0d4f53e2f2e8ccf7352e0b04b +F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333 +F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0 +F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 +F src/build.c 3a1840d9d171ce2d24f4c1f7acda7266ab796c664290c1acba65ff98ce2bd01e F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b -F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 +F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3 +F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 -F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 -F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 3381ee4c9aa7ccde22a2a7f35ce343925a7a25d96bdc943649131f9decdebad2 +F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c +F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 +F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 -F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 -F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52 +F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f +F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f +F src/global.c 61a419dd9e993b9be0f91de4c4ccf322b053eb829868e089f0321dd669be3b90 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4913fd22c4f0fa30643afb93a4d78d289cd490620e782b31016c3d4b2049b1cc +F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70 +F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c 438b95162acfa17b7d218f586f5bde11d6ae82bcf030c9611fc537556870ad6b -F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc +F src/main.c a520c325c7400b249242945f2c602acb19662c40f5db1c04d88664cfaa8b85ac +F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff -F src/memdb.c 559c42e61eb70cd6d4bc692b042497133c6d96c09a3d514d92f3dac72268e223 +F src/memdb.c 16679def118b5fd75292a253166d3feba3ec9c6189205bf209643ecdb2174ecc F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 F src/mutex.c 1b4c7e5e3621b510e0c18397210be27cd54c8084141144fbbafd003fde948e88 @@ -721,112 +748,113 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 5dc41030cd5dfd99b907976b1725a4ed695566405d33744e4824c3d6aff245a3 -F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 +F src/os_unix.c 6e3e4fc75904ff85184091dbab996e6e35c1799e771788961cc3b4fcbe8f852c +F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd +F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a -F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec -F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 +F src/parse.y 5972b7d00af4c8d96fdad781af1ea1d5d51fc3b907ad61bda60e49503274e5ed +F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 -F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781 +F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 +F src/pragma.c 52bfbf6dfd668b69b5eb9bd1186e3a67367c8453807150d6e75239229924f684 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 -F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c -F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 +F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce +F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c e25f51a473a5f30a0d978e4df2aaa98aeec84eac29ecae1ad4708a6c3e669345 +F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c f1a81ff4f8e9e76c224e2ab3a4baa799add0db22158c7fcede65d8cc4a6fa2da -F src/shell.c.in aff59dbd24dcee008dc64a3b0aa8b58d5fa43c5fd0fd9893a81ed6fd538ba6ac -F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3 +F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe +F src/shell.c.in 40de636c1d90fb8a9ca7f49dc8f50d930f1b60736e73aca5eb37c4c7d0e47f9d +F src/sqlite.h.in f07bff4225a1244efd604a0ffef81ed69f29d3dbaed7e22f906f26229ba3ca9e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 73800d73e21180e6b3df8d0fe7d11758dc24367fd2b0b0075b48fc116de406bb +F src/sqliteInt.h 28c878bdf528879afefe1994ac007d094f8061f2fdacdc55d6055d7e9341151e F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c ecbc3c99c0d0c3ed122a913f143026c26d38d57f33e06bb71185dd5c1efe37cd -F src/test1.c ac6542cddd1f405e332d869946b977b2ce8b4dc28b9f7cc61df38abe1fe49bc3 -F src/test2.c 54520d0565ef2b9bf0f8f1dcac43dc4d06baf4ffe13d10905f8d8c3ad3e4b9ab -F src/test3.c e5178558c41ff53236ae0271e9acb3d6885a94981d2eb939536ee6474598840e -F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 -F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d -F src/test6.c e53bc69dc3cb3815fb74df74f38159ec05ba6dd5273216062e26bc797f925530 -F src/test8.c 303c2e3bcf7795e888810a7ef03809602b851f0ebec8d6e06a429ed85cafd9a2 -F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 -F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a -F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 -F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 -F src/test_bestindex.c f6af1e41cb7901edafb065a8198e4a0192dd42432b642d038965be5e628dec12 -F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce -F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c f0cc1f517deaa96dd384822ae2bb91534fa56aa458528b439830d709941d3932 +F src/tclsqlite.c c6888598f08dee3d9112a38ef42c8f5c89ca7f3190f4694744d0b84250f4bf8c +F src/tclsqlite.h c6af51f31a2b2172d674608763a4b98fdf5cd587e4025053e546fb8077757262 +F src/test1.c 3f18399557d954bc85f4564aec8ea1777d2161a81d98a3ff6c9e9046bf3554c1 +F src/test2.c 7ebc518e6735939d8979273a6f7b1d9b5702babf059f6ad62499f7f60a9eb9a3 +F src/test3.c e7573aa0f78ee4e070a4bc8c3493941c1aa64d5c66d4825c74c0f055451f432b +F src/test4.c 13e57ae7ec7a959ee180970aef09deed141252fe9bb07c61054f0dfa4f1dfd5d +F src/test5.c bb87279ed12e199486894e6c83e58dc8cd1de9524ace171d59219d3ab696a0c1 +F src/test6.c 763b92489f11f4a77b773f0d3b8369ab0edd5292ac794043062c337019f12d8a +F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 +F src/test9.c 7a708ad27f8fda79113e5e15de66632710958c401e64c2f22bc04e2f5a7a1b62 +F src/test_async.c 0101173cf8137ba5473a84a695281fa9dedc2a1d155998c68623f2978017ad98 +F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f +F src/test_backup.c bd901e3c116c7f3b3bbbd4aae4ce87d99b400c9cbb0a9e7b4610af451d9719a7 +F src/test_bestindex.c 3401bee51665cbf7f9ed2552b5795452a8b86365e4c9ece745b54155a55670c6 +F src/test_blob.c bcdf6a6c22d0bcc13c41479d63692ef413add2a4d30e1e26b9f74ab85b9fb4d5 +F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 +F src/test_config.c 46eaf39842cace9d540aeefb50fe24dd3204a622893a97952cbb49c20b2f8b21 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f -F src/test_demovfs.c 38a459d1c78fd9afa770445b224c485e079018d6ac07332ff9bd07b54d2b8ce9 +F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c942383 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 -F src/test_fs.c 56cc17e4fdc57efa61695026e2ba96e910b17060d7ee01d775ec048791522e2f -F src/test_func.c 4d2dc7e3e0946e55091784ddaf0302294f2ee300614f6f3e19a4b38df77d5167 -F src/test_hexio.c 9478e56a0f08e07841a014a93b20e4ba2709ab56d039d1ca8020e26846aa19bd -F src/test_init.c f2cc4774b7c9140f76e45ecbb2ae219f68e3acbbe248c0179db666a70eae9f08 -F src/test_intarray.c 26ffba666beb658d73cd925d9b4fb56913a3ca9aaeac122b3691436abb192b92 +F src/test_fs.c c411c40baba679536fc34e2679349f59d8225570aed3488b5b3ef1908525a3d5 +F src/test_func.c 8c0e89192f70fac307822d1ac2911ee51751288780b3db0c5ab5ca75fa0fe851 +F src/test_hexio.c 0f777bf9fbb2684bb4978372bacc90ef7337d5d9e3cebe067a9409941e88bacf +F src/test_init.c 17313332d58e90defc527129d5eda4a08bd6b6e8de7207a231523c8d98fb445e +F src/test_intarray.c e4216aadee9df2de7d1aee7e70f6b22c80ee79ece72a63d57105db74217639e5 F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293 F src/test_journal.c a0b9709b2f12b1ec819eea8a1176f283bca6d688a6d4a502bd6fd79786f4e287 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd -F src/test_malloc.c 21121ea85b49ec0bdb69995847cef9036ef9beca3ce63bbb776e4ea2ecc44b97 -F src/test_md5.c 0472c86d561f7f9e4ff94080100c2783196f50e583bb83375b759450c5b81802 -F src/test_multiplex.c 70479161239d65af2a231550b270e9d11ece717ad7bf0e13ef42206586e9dd7f +F src/test_malloc.c a0295e022103b14a1bc5e0660cc2af7fbec05e0d029098782e326e50612e69d9 +F src/test_md5.c 811a45330c9391933360f998156a8907ee29909c828ab83ac05d329942cbea8f +F src/test_multiplex.c b99d7f43ec859e6b93a40aaa5455420b3ad133053cce3db739498d29ea30735f F src/test_multiplex.h f0ff5b6f4462bfd46dac165d6375b9530d08089b7bcbe75e88e0926110db5363 -F src/test_mutex.c cd5bac43f2fd168f43c4326b1febe0966439217fac52afb270a6b8215f94cb40 +F src/test_mutex.c f10fcbc2086b19c7b0ddf2752caf2095e42be74d8d7f6093619445b43b1f777b F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 -F src/test_osinst.c 8e11faf10f5d4df10d3450ecee0b8f4cfa2b62e0f341fafbeb480a08cefeaec4 -F src/test_pcache.c 3960cd2c1350adc992c4bf7adcfb0d1ac0574733012bd1a5f94e195928577599 -F src/test_quota.c ea44c05f29b995bdb71c55eb0c602604884e55681d59b7736e604bbcc68b0464 +F src/test_osinst.c 7aa3feaa3a1da1b5f75bde2ce958dbfe14ec484f065bb2b5b9727d8851fa089b +F src/test_pcache.c 496da3f7e2ca66aefbc36bbf22138b1eff43ba0dff175c228b760fa020a37bd0 +F src/test_quota.c 07369655d24c3f3fbdbd8fd8f42e856a054a7497846ca1c83ed4be68152a251f F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d -F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b -F src/test_schema.c cbfd7a9a9b6b40d4377d0c76a6c5b2a58387385977f26edab4e77eb5f90a14ce +F src/test_rtree.c d844d746a3cc027247318b970025a927f14772339c991f40e7911583ea5ed0d9 +F src/test_schema.c b06d3ddc3edc173c143878f3edb869dd200d57d918ae2f38820534f9a5e3d7d9 F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a -F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e -F src/test_syscall.c 9fdb13b1df05e639808d44fcb8f6064aaded32b6565c00b215cfd05a060d1aca -F src/test_tclsh.c 3ff5d188a72f00807425954ea3b493dfd3a4b890ecc6700ea83bad2fd1332ecf -F src/test_tclvar.c 3273f9d59395b336e381b53cfc68ec6ebdaada4e93106a2e976ffb0550504e1c -F src/test_thread.c 7ddcf0c8b79fa3c1d172f82f322302c963d923cdb503c6171f3c8081586d0b01 -F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43 -F src/test_vfs.c 193c18da3dbf62a0e33ae7a240bbef938a50846672ee947664512b77d853fe81 +F src/test_superlock.c 18355ca274746aa6909e3744163e5deb1196a85d5bc64b9cd377273cef626da7 +F src/test_syscall.c 9ad7ab39910c16d29411678d91b0d27a7a996a718df5ee93dcd635e846d0275c +F src/test_tclsh.c 6077f2bdc6b4ea2bace2a0cd6ea48e0a4651007ae7382c13efc0c495eb0c6956 +F src/test_tclvar.c ae873248a0188459b1c16ca7cc431265dacce524399e8b46725c2b3b7e048424 +F src/test_thread.c d7a8bcea7445f37cc2a1f7f81dd6059634f45e0c61bfe80182b02872fb0328bb +F src/test_vdbecov.c 5c426d9cd2b351f5f9ceb30cabf8c64a63bfcad644c507e0bd9ce2f6ae1a3bf3 +F src/test_vfs.c f298475e468c7e14945b20af885917181090c265aa3c4ade897849c9fbd396f2 F src/test_vfstrace.c a2ea82df2ed8927e9eba49bdba1aa1aeb1dcb13dbf6558cff036da813031de9a F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb775ebc50 -F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f +F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 23d9f4539880b40226254ad9072f4ecf12eb1902e62aea47aac29928afafcfd5 -F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee -F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 -F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 -F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 +F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 +F src/treeview.c 88aa39b754f5ef7214385c1bbbdd2f3dc20efafeed0cf590e8d1199b9c6e44aa +F src/trigger.c 0bb986a5b96047fd597c6aac28588853df56064e576e6b81ba777ef2ccaac461 +F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 +F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e -F src/util.c 078f040366d5bd5f47658d045f901c768c1c636c6eaea121f3a1cbd63c3edb5b -F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 96ac876e57f480bd35ec8d74ed992bca6ae9deebe8b527a3a718e7b4714d6c2e -F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb +F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375 +F src/vacuum.c b763b6457bd058d2072ef9364832351fd8d11e8abf70cbb349657360f7d55c40 +F src/vdbe.c be5f58bc29f60252e041a618eae59e8d57d460ba136c5403cf0abf955560c457 +F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c -F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b -F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea +F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df +F src/vdbeaux.c 25d685cafe119ff890c94345e884ea558a6b5d823bfa52ba708eb8ff3c70aa71 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5 -F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0 -F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547 +F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89 +F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 -F src/vdbevtab.c 2143db7db0ceed69b21422581f434baffc507a08d831565193a7a02882a1b6a7 -F src/vtab.c 11948e105f56e84099ca17f1f434b1944539ea84de26d0d767eadfbc670ce1ea +F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 +F src/vtab.c 5fb499d20494b7eecaadb7584634af9afcb374cb0524912b475fcb1712458a1b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c bf41478461ca8a2758617691152befd9d526921e2fb59d5647638679cff98381 +F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 -F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 217fe82a26c0fb6a3c7fd01865d821e752f9c01fb72f114af3f0b77ce234d1fb -F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 -F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 -F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 -F src/window.c 5b1387d59df30d481ed14cceef5f4d1dab1f8752aa106ba72c8b62777bd139d2 +F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 +F src/where.c c046dd58c3410f7b7528e1e6317cb876398557bad346d568ed8562321a7d002d +F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 +F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf +F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa +F src/window.c 499d48f315a09242dc68f2fac635ed27dcf6bbb0d9ab9084857898c64489e975 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 @@ -837,7 +865,7 @@ F test/aggorderby.test cc3abf5de64d46ff66395ca8c2346b66c2576d5aedb7bffc5b0742508 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 F test/alter.test 3c00eff1e2036b9f93e9cd0f3d3e63750ac87ecb5bc71b9d7bd07cbf2ac4c494 -F test/alter2.test a966ccfcddf9ce0a4e0e6ff1aca9e6e7948e0e242cd7e43fc091948521807687 +F test/alter2.test 7e3d26ab409df52df887b366a63902c3429b935c41cb962fd58ffc25784f2f19 F test/alter3.test ffc4ab29ce78a3517a66afd69b2730667e3471622509c283b2bd4c46f680fba3 F test/alter4.test 716caa071dd8a3c6d57225778d15d3c3cbf5e34b2e84ae44199aeb2bbf50a707 F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959 @@ -853,8 +881,8 @@ F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584 F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd436f076e81 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 -F test/altertab2.test 62597b6fd08feaba1b6bfe7d31dac6117c67e06dc9ce9c478a3abe75b5926de0 -F test/altertab3.test 6c432fbb9963e0bd6549bf1422f6861d744ee5a80cb3298564e81e556481df16 +F test/altertab2.test fff90e3f01e8eb0e09282f538b8ec7cfeb035dbedbe570fe1983440f4613ad0e +F test/altertab3.test b331ae34e69594e19605e3297805202d6156fcc8f75379dfd972a2e51cae8721 F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b @@ -890,17 +918,17 @@ F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 997d6f185f138229dc4251583a1d04816423dddc2fc034871a01aeb1d728cb39 -F test/autoindex1.test d34caffb0384003ee28eae87679214c029e9be4b332d9649a79e0b94ab70502c +F test/autoindex1.test 714cac6e60beeb5a26ed346dd46505ba60b5a5597e9122c9ed3a55f89a922aa4 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df -F test/autoindex3.test dcd6b2f8bed2be67b131e2e671f892e971d934e24fd00988952d0e0a67e24aa7 +F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 F test/autoindex5.test 2ee94f033b87ca0160e08d81034c507aff8e230df2627f0304fa309b2fee19a3 F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df F test/autovacuum2.test 76f7eb4fe6a6bf6d33a196a7141dba98886d2fb53a268d7feca285d5da4759d7 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 0c3a38e03cccb0fc3127838462dc05dc3f4c1480d770c084b388304c25de3652 +F test/avfs.test 76f59743dc1f5fa533840d1818b420fe1ee45e21c0fd6bbac7942ba677903128 F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e -F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d +F test/backcompat.test f2431465ed668f09fc3f6998e56e893a1506ccea6e8b6f409f085f759f431b48 F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f F test/backup2.test 8facb54df1388419d34b362ab1f7e233310ff3a3af64e8ad5ec47ba3c2bbe5cf F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 @@ -918,10 +946,12 @@ F test/bestindex4.test 3039894f2dad50f3a68443dffad1b44c9b067ac03870102df1ce3d9a4 F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb75dc98d F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e -F test/bestindex8.test 333ad8c6a554b885a49b68c019166eda92b05f493a92b36b0acdf7f766d04dad +F test/bestindex8.test b63a4f171a2c83d481bb14c431a8b72e85d27b2ffdaa0435a95d58ca941678f9 F test/bestindex9.test 1a4b93db117fd8abe74ae9be982f86aa72f01e60cd4ac541e6ede39673a451a0 F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572eef44c7f F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce +F test/bestindexC.test 2df6ada16d8f00d9bb6a9664d9c323560aeed0e0ebc7a32b99d85d70037fd250 +F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 F test/between.test b9a65fb065391980119e8a781a7409d3fcf059d89968279c750e190a9a1d5263 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -945,25 +975,26 @@ F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/btree01.test fef17d9e999ac4f04095948e3438fbe674f4e07bb2c63bb1cad41d87baee077f F test/btree02.test 7555a5440453d900410160a52554fe6478af4faf53098f7235f1f443d5a1d6cc F test/btreefault.test a82a23b0578bc587afbf9a622c8f54a54f63762f062ba8a35613cfee38ab42f9 -F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb7834727 +F test/busy.test caff7164c16ce06a53af51f9e4c2753d4cc64250e00790a5e48b9c4f4be37597 F test/busy2.test 20823a5d7c42fb257d9f108c66312d90b1bb4ec3d80ba6b4e371073727560f98 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9 F test/capi2.test 4ee545824adc3eb33bf57ef89f77440b28188ec3da72e5425ff0fcdba32e8d5a -F test/capi3.test 3910a73c38ac76d69778dd9eb481ab7cd6ed59117fc047b4f6056a5c72529de1 +F test/capi3.test 4892b5e53d2a6941edc9d204a0ab174dd66e8689282d9a15e4384561c3965945 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a490f7 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/carray01.test 23ed7074307c4a829ba5ff2970993a9d87db7c5cdbbe1a2cbef672d0df6d6e31 -F test/cast.test af2286fdd28f3470b7dcad23977282b8cc117747ad55acff74a770dad3b19398 +F test/cast.test 42f7d79d88ab5e8080e96c650c52fcf72eef3e6476aaaee2c9f6e074396cfdfc F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/changes.test 4377d202a487f66fc2822c1bf57c46798c8b2caf7446f4f701723b1dbb6b86f6 F test/changes2.test 07949edcc732af28cb54276bfb7d99723bccc1e905a423648bf57ac5cb0dc792 F test/check.test 56e4ed457e9f8683b9fc56f5b964f461f6e8a8dd5a13f3d495408215d66419ed F test/checkfault.test da6cb3d50247169efcb20bdf57863a3ccfa1d27d9e55cd324f0680096970f014 F test/chunksize.test 427d87791743486cbf0c3b8c625002f3255cb3a89c6eba655a98923b1387b760 +F test/cksumvfs.test 6f05dc95847c06a3dc10eee6b5ab1351d78314a52d0db15717c9388f4cb96646 F test/close.test eccbad8ecd611d974cbf47278c3d4e5874faf02d811338d5d348af42d56d647c F test/closure01.test 9905883f1b171a4638f98fc764879f154e214a306d3d8daf412a15e7f3a9b1e0 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -979,16 +1010,16 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test 87ad5458bb8709312dac0d6755fd30e8e4ca83298d0a9ef6e5c24277a3c3390e +F test/colname.test 387e880eeac0889900f7b3e9703c375515f5d323f71fd4f2bb5428a4ac8e2023 F test/columncount.test 6fe99c2f35738b0129357a1cf3fa483f76140f4cd8a89014c88c33c876d2638f F test/conflict.test b705cddf025a675d3c13d62fa78ab1e2696fb8e07a3d7cccce1596ff8b301492 F test/conflict2.test 5557909ce683b1073982f5d1b61dfb1d41e369533bfdaf003180c5bc87282dd1 F test/conflict3.test 81865d9599609aca394fb3b9cd5f561d4729ea5b176bece3644f6ecb540f88ac F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 -F test/corrupt.test d7cb0300e4a297147b6a05e92a1684bc8973635c3bcaa3d66e983c9cbdbf47a3 -F test/corrupt2.test 9745c55b3ff2d84d9b6dc4f7365f91a99e70d90f3127ebc97ff0549c418e4d3d +F test/corrupt.test 54509b182b1927663e0a425b681b0935a08a01b11d8153a4a9545ed36760ebe2 +F test/corrupt2.test 4ce5eadd51baa1aedb48e141dd885d155946f5c3677bb032547e350ce91b17f4 F test/corrupt3.test 6a982535d52c8165654cbc79a043cfd0bf02495a5efbf4754295e056fc548539 -F test/corrupt4.test b5ae41607e8d17d9c1f3e94fdb572ce061ed3beeebdb46fb3a348181b8c8a097 +F test/corrupt4.test 5fa4559bcfd14afbb99670d463546ba75fb4975c710b7f6dfa592ae90471cce7 F test/corrupt5.test 387be3250795e2a86e6234745558b80efb248a357d0cd8e53bce75c7463f545d F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568 F test/corrupt7.test ffa86896fe63a3d00b0a131e1e64f402e4da9f7e5d89609d6501c851e511d73a @@ -996,23 +1027,23 @@ F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec -F test/corruptC.test 9cf32275dae3ca33f645afe5d1d3f5ba5ac2af2b0833dfb5282f9dccb6fb81bb -F test/corruptD.test a828c788535946a372a56a750b242cd96287cd823657abe5a73c5e51b91bdd28 +F test/corruptC.test 7d6d9e907334ea3ccb7111a0656cafa30a28f8a5f2aaf1c45ad712236302856a +F test/corruptD.test 614320aa519f6bf6c7dd2f581f9513ff7b6826954180cca1a606d0e25ea084a3 F test/corruptE.test 4143791f2dfb443aec5b7fabfa5821e6063eccc3b49b06f212c2f014715fd476 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 9d8cbf6214e492abe9e822e759b9751ae336cec0a6fe3ff3b37bfbd8ff9c22ca F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 -F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af -F test/corruptL.test 504d90502d9993440226edc355d2275524b89064ea3df5ee5c27f7028ec59d07 +F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc23dcb +F test/corruptL.test 652fc8ac0763a6fd3eb28b951d481924167b2d9936083bcc68253b2274a0c8fe F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 -F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 +F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b -F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f +F test/crash.test f699152b8ae759bdf1c19c278b135f5d43fa4b6466e63489cd02edbc94aebad0 F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 F test/crash3.test 8f5de9d32ab9ab95475a9efe7f47a940aa889418 F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc @@ -1029,10 +1060,11 @@ F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c47 F test/cursorhint.test 05cf0febe5c5f8a31f199401fd1c9322249e753950d55f26f9d5aca61408a270 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 -F test/date.test ff2341a1ef71b9a27979494d299222f9a293aa22cb9ff6e9c38d88a895317ebf +F test/date.test c8ff835023f2107b57ce7a45c92265d51c98a23fc93231e998f12d850831aad6 F test/date2.test 7e12ec14aaf4d5e6294b4ba140445b0eca06ea50062a9c3a69c4ee13d0b6f8b1 F test/date3.test a1b77abf05c6772fe5ca2337cac1398892f2a41e62bce7e6be0f4a08a0e64ae5 -F test/date4.test 8aeb3de5b5e9fda968baa9357e4c0fae573724b7904943410195a19e96e31b6a +F test/date4.test 75dc8401e8c0639a228cd26a6eaa4ff5ea8ccda912b9853d1c9462c476670e17 +F test/date5.test 14ba189bc4d03efc371dd5302e035764f6633355a3e13acb4a45e7b33530231e F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 @@ -1043,7 +1075,7 @@ F test/dbpagefault.test 35f06cfb2ef100a9b19d25754e8141b9cba9b7daabd4c60fa5af93fc F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test ef731887b43ee32ef86e1c8fddb61a40789f988332c029c601dcf2c319277e9e -F test/default.test 9687cfb16717e4b8238c191697c98be88c0b16e568dd5368cd9284154097ef50 +F test/default.test c7124864cded213a3f118bc7e2e26f34b7c36dfa26cf6945cc8b7f5db1191277 F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab @@ -1054,8 +1086,8 @@ F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 -F test/distinct2.test bb71cc7b5e58e895787f9910a788c254f679928d324732d063fe9bc202ecbe71 -F test/distinctagg.test ad2b4cf1483cd4cf24867dfafbfa0abb61184d92085fcc9784cea0592b278d64 +F test/distinct2.test 4d6316b6487a0aa5a90bee111575c957e2a5ba5a9be9156febe9533ce78876e8 +F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50 @@ -1069,7 +1101,7 @@ F test/e_expr.test b950818a48269506d75a41c819003bd77a0893bc4a4f2fdee191bc74109c1 F test/e_fkey.test feeba6238aeff9d809fb6236b351da8df4ae9bda89e088e54526b31a0cbfeec5 F test/e_fts3.test 17ba7c373aba4d4f5696ba147ee23fd1a1ef70782af050e03e262ca187c5ee07 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e -F test/e_reindex.test 2b0e29344497d9a8a999453a003cb476b6b1d2eef2d6c120f83c2d3a429f3164 +F test/e_reindex.test 027bb13d2c7e9e865886eed6349f126a273f8037899b636bf5fb53c7fc815921 F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 F test/e_select.test 327a15f14068bbd6f647cedc67210f8680fcb2f05e481a0a855fccd2abfa1292 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f @@ -1086,16 +1118,18 @@ F test/enc.test 9a7be5479da985381d740b15f432800f65e2c87029ee57a318f42cb2eb43763a F test/enc2.test 848bf05f15b011719f478dddb7b5e9aea35e39e457493cba4c4eef75d849a5ec F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test f3f7548d2f3df03e2f23ecaf35c7c2cc7b89848bd7c3606d94a010521b7ea4f6 +F test/eqp.test 82f221e8cd588434d7f3bba9a0f4c78cbe7a541615a41632e12f50608bfb4a99 +F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/eval.test 73969a2d43a511bf44080c44485a8c4d796b6a4f038d19e491867081155692c0 F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650bf0747 -F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 +F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac F test/expr.test 5c06696478212e5a04e04b043f993373f6f8e5ce5a80f5548a84703b123b6caa F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/exprfault.test da33606d799718e2f8e34efd0e5858884a1ad87f608774c552a7f5517cc27181 +F test/exprfault2.test c49e84273898969af5dbc4fe6a3f4335f14639799f343590336c9ddf84425965 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/external_reader.test c7d34694f1b25c32d866f56ac80c1e29edddc42b4ef90cad589263ffac2cde0c F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 @@ -1167,7 +1201,7 @@ F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1 F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3 F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45 F test/fts3fault2.test 7b2741e5095367238380b0fcdb837f36c24484c7a5f353659b387df63cf039ec -F test/fts3fault3.test 4a39a1618546776255dc1de306213b600aef87eca589ca8428a70c00fd11961b +F test/fts3fault3.test ccdd2292dd2d4e21e30fc5f4c8e064f79e516087eec5ff57ab6bc4f6a7714097 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3fuzz001.test c78afcd8ad712ea0b8d2ed50851a8aab3bc9dc52c64a536291e07112f519357c F test/fts3integrity.test 0c6fe7353d7b24d78862f4272ee9df4da2f32b3ff30fa3396945cda8119580a8 @@ -1184,8 +1218,8 @@ F test/fts3query.test 45806a302921b245a9dba5d85c9d51fb98b3f137eea6e6bf6eae4883e0 F test/fts3rank.test cd99bc83a3c923c8d52afd90d86979cf05fc41849f892faeac3988055ef37b99 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 57e26a801f21027b7530da77db54286a6fe4997e -F test/fts3snippet.test 0887196d67cffbe365edde535b95ecc642a532ce8551ccd9a73aab5999c3ffae -F test/fts3snippet2.test e79afeb1f673713f96d7fc5655726081975399d11e659d15553207be43301dc4 +F test/fts3snippet.test 560c7f38c5fa591d88e367eac1313b64e503625616708ff61da9d5f52cbf75e5 +F test/fts3snippet2.test 03f6738ab3897bea2ba6be424a0613872e167acbf37a66200d655d737b470f65 F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test a663f4cac22a9505400bc22aacb818d7055240409c28729669ea7d4cc2120d15 F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d @@ -1197,7 +1231,7 @@ F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 289833c34ad45a5e6e6133b53b6a71647231fb89d36ddcb8d9c87211b6721d7f F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d -F test/fts4intck1.test 43774c641fdf6607c6ee90c3db8af065a37434d55d6eaf13bafe515e8b0c5729 +F test/fts4intck1.test 54e7f28e34b72fb0c614d414bb1f568154d463c5a00b20944e893df858372ed4 F test/fts4langid.test 4be912f42454998e239a2e877600263e0394afbaba03e06cedcc5a08693a345a F test/fts4lastrowid.test 185835895948d5325c7710649824042373b2203149abe8024a9319d25234dfd7 F test/fts4merge.test 57d093660a5093ae6e9fbd2d17592a88b45bbd66db2703c4b640b28828dbe38b @@ -1215,23 +1249,23 @@ F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f75 F test/fts4unicode.test 82a9c16b68ba2f358a856226bb2ee02f81583797bc4744061c54401bf1a0f4c9 F test/fts4upfrom.test f25835162c989dffd5e2ef91ec24c4848cc9973093e2d492d1c7b32afac1b49d F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test 3a29323b640c0552f6e9f1577407ced3a68e7d8c0bc04b61dd6040fa593a3a02 -F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f +F test/func.test b56905748ce0567c01d60005f3e6ad1af19453d224ba4730ee687d048fd09ef9 +F test/func2.test 69f6ae3751b4ec765bdc3b803c0a255aa0f693f28f44805bef03e6b4a3fd242f F test/func3.test 600a632c305a88f3946d38f9a51efe145c989b2e13bd2b2a488db47fe76bab6a -F test/func4.test e8ef9b2bd6a192a213cbd5cf31a3b35e25cd6ff2fdaeea0b58d63be31b03d220 +F test/func4.test a3f9062487dbd826776f54f4e0e9517fe8c3cf689af92735308965774d51fac5 F test/func5.test 863e6d1bd0013d09c17236f8a13ea34008dd857d87d85a13a673960e4c25d82a -F test/func6.test 9cc9b1f43b435af34fe1416eb1e318c8920448ea7a6962f2121972f5215cb9b0 -F test/func7.test adbfc910385a6ffd15dc47be3c619ef070c542fcb7488964badb17b2d9a4d080 +F test/func6.test 3bc89ec0f2605736d3a118f43d25ef58115a7db4dba8ae939a363917d815c0bb +F test/func7.test 7e009275f52c52954c8c028fdb62f8bc16cc47276fcc8753c1d2b22c6e074598 F test/func8.test c4e2ecacf9f16e47a245e7a25fbabcc7e78f9c7c41a80f158527cdfdc6dd299d F test/func9.test b32d313f679aa9698d52f39519d301c3941823cb72b4e23406c210eadd82c824 F test/fuzz-oss1.test 514dcabb24687818ea949fa6760229eaacad74ca70157743ef36d35bbe01ffb0 F test/fuzz.test 4608c1310cff4c3014a84bcced6278139743e080046e5f6784b0de7b069371d8 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 -F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c31 +F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce313 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 7e1bcc242dc4b42e43e4708c8140fe268db83a6901fbc681d150c77aa185e328 +F test/fuzzcheck.c 6e87c27df3d95c556870187987dff6efdc712b5cea60abedc8ab9215f471907a F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1243,7 +1277,7 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c b34530e8431f2cf3591eff588fc7684d6fdef466916fb46141c8c5374a3d8099 +F test/fuzzinvariants.c 81167c9a7e82c0539a1d704aeb3384046d01f4108cda160a2447cb2a149d6362 F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -1252,15 +1286,16 @@ F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 F test/hook.test 18cae9140fa7f9a6f346e892a3fe3e31b2ca0be1494cd01b918adb74281016a6 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 -F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1 +F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 F test/in.test d1cad4ededd425568b2e39fb0c31fa9a3772311dd595801ff13ba3912b69bba6 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test fdd1d8134da8376985c2edba6035a2de1f6c731524d2ffa651419e8fe2cd1c5a -F test/in5.test b32ce7f4a93f44c5dee94af16886d922cc16ebe33c8e1765c73d4049d0f4b40f +F test/in4.test bb767ec1cfd1730256f0a83219f0acda36bc251b63f8b8bb7d8c7cff17875a4f +F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b +F test/in7.test 9256cdb30dc487f2078bb4bb30f43f2c1ff4d277a9c7c9a14bd1c9510c9c8cae F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f F test/incrblob3.test 67621a04b3084113bf38ce03797d70eca012d9d8f948193b8f655df577b0da6f @@ -1270,7 +1305,7 @@ F test/incrblobfault.test de274b1e329169c2c3438f9528994807ea8201ebf38ae9f157d34b F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test 3fa6145f5e71f603554fd7b8ec3da4290b1341029682313285cb5f9e1893d6ba F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d -F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a +F test/incrvacuum3.test 0bf0ffe7f2cbc87ba1d471e4bbadabbf10dacf8d4ee26b3a072708d575d637a9 F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 F test/index.test d866054c88b394fd42cbf2825628f127ca24dfac525fa019069a936674d92cbe F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 @@ -1283,7 +1318,7 @@ F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a91 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexA.test 11d84f6995e6e5b9d8315953fb1b6d29772ee7c7803ee9112715e7e4dd3e4974 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test 833f511213a5e26549186813f0566bd72f978177a7e6e98a2d2dd695de3c670d +F test/indexexpr1.test 24fa85a12da384dd1d56f7b24e593c51a8a54b4c5e2e8bbb9e5fdf1099427faf F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1301,18 +1336,18 @@ F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d F test/intpkey.test aee694afed1a65c86c4e69ad030224b3fc268113d00685234d40079fca16bad3 F test/intreal.test 68829a8bb073ee1610ca3f8f9e0f99b0371fb36e0fa64862dd5ced4ef03c2343 F test/io.test f138f3fe696d1ed8c51dfea5b325910d319a1b29e1d25ea57231a02092f02cca -F test/ioerr.test 530d05801ff1b6327018b2e140da34a74effa2773a844ddb8dc79c32e9567318 +F test/ioerr.test c94eef1cd8bfc36f9aa493e41e151e9160281ac8e2d960cc9dcdcc8e6aa99ab3 F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c -F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 +F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d520 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 F test/join.test f7abfef3faeaf2800308872e33a57e5b6e4a2b44fb8c6b90c6068412e71a6cf4 F test/join2.test 8561fe82ce434ac96de91544072e578dc2cadddf2d9bc9cd802f866a9b92502e F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 7cc3f3595bb41e60f3f96d5cb6dd8cfcbc31212f0136bba6fc081c082994a94a +F test/join5.test 380d12a9350f99f0cc681a4f1fea999886f18b3fe0d71a9b3065bcaead1e007f F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c F test/join7.test 2268dcbb54b724391dda3748ea95c60d960607ffeed67885675998e7117697f6 F test/join8.test d384d63985e3991c404afccadaf3efd1cdf9cd72680167f80e3cb80b95c18c68 @@ -1323,7 +1358,7 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test f69e5b53b7d887914e854b6a131efbed4ea9f5ca52bdab81788bfc3e79299f43 +F test/joinH.test 55f69e64da74d4eca2235237f3acb657aef181e22e45daa228e35bba865e0255 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e @@ -1335,32 +1370,37 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh 912ee03e700a65c827ee0c7b4752c21ec5ef69cf7679d2f482ca817042bead52 x F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 -F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547 -F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef +F test/json101.test 30db5b055b103ccabc53a29cfe6cda3345d07e171aeb25403dafa04f19e98b19 +F test/json102.test 4b3a0f94535f033239b67c13dbee8b47d2b5ee467e0f2fdab5eadf370bbe5fd3 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 -F test/json106.test 1d46a9294e2ced35c7f87cebbcb9626d01abab04f1969d7ded7b6f6a1d9be0f2 -F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 +F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8abef +F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 +F test/json108.test 0a5f1e2d4b35a1bc33052563d2a5ede03052e2099e58cb424547656c898e0f49 +F test/json501.test b95e2d14988b682a5cadf079dd6162f0f85fb74cd59c6b1f1624110104a974eb F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b -F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 +F test/jsonb01.test f4cdfb4cf5a0c940091b17675ed9583f45add0c938f07d65b0de0e19d3a9a101 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 +F test/lemon-test01.y 70110eff607ab137ccc851edb2bc7e14a6d4f246b5d2d25f82a60b69d87a9ff2 F test/like.test 242ee7f5d08a031144c0daf63bbd7e7710c847ccf387a83347e0b61b3aa69526 F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3 F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e +F test/literal.test a65dca9fef86e51b8e45544268e37abbd4bb94ba35fd65f6fdcab2f288cd8f79 +F test/literal2.tcl 1499037beaf661aeecdbe48801220a181d805372a64c6128d5f26bb6a4a8f0ce +F test/literal2.test b149e16b5fc9ee6249069a8858ed41052f222014fe0ba7ad43c2fb989c2dada2 F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37b8 -F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 +F test/lock5.test 24693e40a805f71d80836f720d1f2034684a39b64f1e1990989002c7968c11ee F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413 @@ -1394,32 +1434,33 @@ F test/malloctraceviewer.tcl 3e3ddf11e30d2b20f53aa16aa6615082fb24a100bea61cca721 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 -F test/memdb1.test 2c4e9cc10d21c6bf4e217d72b7f6b8ba9b2605971bb2c5e6df76018e189f98f5 +F test/memdb1.test c737ac9aa5895092332b1dde24fae7ae494b7fcbcd346d22d600891096a3836d F test/memdb2.test 4ba1fc09e2f51df80d148a540e4a3fa66d0462e91167b27497084de4d1f6b5b4 F test/memjournal.test 70f3a00c7f84ee2978ad14e831231caa1e7f23915a2c54b4f775a021d5740c6c F test/memjournal2.test dbc2c5cb5f7b38950f4f6dc3e73fcecf0fcbed3fc32c7ce913bba164d288da1e F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 F test/memsubsys1.test 86b8158752af9188ed5b32a30674a1ef71183e6bc4e6808e815cd658ca9058a6 F test/memsubsys2.test 774b93cb09ca50d1b759bb7c645baa2a9ce172edc3a3da67d5150a26a9fc2a08 -F test/merge1.test 2de6d6ef8d25402764b1aab49d8f9d7f89208c89a6674e437f76de4c812157b8 +F test/merge1.test 7dd9dc6838bcd0623a069485fe3a8dd498a051c16e1877cf84f506c0d6a29b43 F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 272ca395257f05937dc96441c9dde4bc9fbf116a8d4fa02baeb0d13d50e36c87 -F test/misc1.test 8d138a4926ab90617c1aa29ce26e7785ae2b83a4d3a195d543b7374e05589dd1 -F test/misc2.test 71e746af479119386ac2ed7ab7d81d99970e75b49ffd3e8efffee100b4b5f350 -F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d +F test/misc1.test e3e36262aff1bd9b8b9bf1eeb3af04adb3fc1e23f0a92dbff708bba9e939ace1 +F test/misc2.test a1a3573cc02662becd967766021d6f16c54684d56df5f227481c7ef0d9df0bd0 +F test/misc3.test 651b88bca19b8ff6a7b6af73dae00c3fd5b3ea5bee0c0d1d91abd4c4b4748718 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e -F test/misc5.test c4aeaa0fa28faa08f2485309c38db4719e6cd1364215d5687a5b96d340a3fa58 +F test/misc5.test 027cf0ac10314ea534173f335a33bb4059907ddabbac2c16786766d6f26c8923 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test d912f3d45c2989191b797504a220ca225d6be80b21acad22ba0d35f4a9ee4579 -F test/misc8.test 4db9f8be59834cea08c87e9658014080efa02678ef54a088f84fa5647e81fee0 +F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7 F test/mjournal.test 28a08d5cb5fb5b5702a46e19176e45e964e0800d1f894677169e79f34030e152 -F test/mmap1.test 5c1f768828094b0dd94e55ae7f10489a1ded74772682be2c4c78679d0acaf7ef +F test/mmap1.test 18de3fd7b70a777af6004ca2feecfcdd3d0be17fa04058e808baf530c94b1a1d F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 +F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 @@ -1428,7 +1469,7 @@ F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f9 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test 42cb5e244c3a77bb0ef2b967e06fa5e7ba7d32d90a9b20bed98f6f5ede185a25 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 -F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 +F test/nan.test 73ea63ab43668313e2f8cc9ef9e9a966672c7934f3ce76926fbe991235d07d91 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/normalize.test f23b6c5926c59548635fcf39678ac613e726121e073dd902a3062fbb83903b72 @@ -1436,7 +1477,7 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test 1ee4acbd614d3cf5f1c4a52f5af7fc771b82352f1a51a86afeaa02c9df1d82ef +F test/notnull2.test 2ac7b4e04917148c7a1a9ed36df20150175ce942f07f5714375b29acbaca7106 F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 @@ -1447,7 +1488,7 @@ F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 F test/optfuzz.c 690430a0bf0ad047d5a168bf52b05b2ee97aedaad8c14337e9eb5050faa64994 -F test/orderby1.test 02cfd870127a7342170b829175c5c53e9e7405744451ac1aeb2f7e2b0c18ca76 +F test/orderby1.test 7d0e4ee692a3e808c1026b3c483594ad1e468b68b50dcefa0d678a8c05274ceb F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -1457,6 +1498,7 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/orderbyA.test df608e59efc2ef50c1eddf1a773b272de3252e9401bfec86d04b52fd973866d5 +F test/orderbyB.test 32576c7b138105bc72f7fbf33bd320ca3a7d303641fc939e0e56af6cba884b3d F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 F test/ossfuzz.c 9636dad2092a05a32110df0ca06713038dd0c43dd89a77dabe4b8b0d71096715 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 @@ -1473,52 +1515,54 @@ F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035ce4b3 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 -F test/pendingrace.test 6aa33756b950c4529f79c4f3817a9a1e4025bd0d9961571a05c0279bd183d9c6 -F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test f7caf8dd5c7b1da74842a48df116f7f193399c656d4ffc805cd0d9658568c675 +F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 +F test/percentile.test 74e383216a075251512d6ba98beb9dccd6da465e3f73817fc438379e3a628de7 +F test/permutations.test 405542f1d659942994a6b38a9e024cf5cfd23eaa68c806aeb24a72d7c9186e80 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f -F test/pragma.test 57a36226218c03cfb381019fe43234b2cefbd8a1f12825514f906a17ccf7991e +F test/pragma.test 11cb9310c42f921918f7f563e3c0b6e70f9f9c3a6a1cf12af8fccb6c574f3882 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 92a46bbea12322dd94a404f49edcfbfc913a2c98115f0d030a7459bb4712ef31 -F test/pragma4.test ca5e4dfc46adfe490f75d73734f70349d95a199e6510973899e502eef2c8b1f8 +F test/pragma4.test f93f317693e90ece4ed0f5505d9035cae03f9fc684b718278433f2e48703f693 F test/pragma5.test 7b33fc43e2e41abf17f35fb73f71b49671a380ea92a6c94b6ce530a25f8d9102 +F test/pragma6.test c5ec577ba087954b4dfa619a3cbe97b155b60a0af487527abe89b10fc17e6512 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/prefixes.test b524a1c44bffec225b9aec98bd728480352aa8532ac4c15771fb85e8beef65d9 -F test/printf.test 512152dca7f2f578f045a5a732e7bee08e4f47a8a212f83ce46791b518eba70f +F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b05333de F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 1495a09837a1cedfc0adf07ba42dc6b83be05a2c15de331b67c39a0e22078238 +F test/pushdown.test 84b525767442b3695d671f9df59dd91cf0ed8fb24cbbcdc55959f0dadeee8b39 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 -F test/quote.test ffb40f0eb7a25c1d8cfe11ee2fe67f8e85fbf3fed348810834114be1fdada142 +F test/quote.test 7b01b2a261bc26d9821aea9f4941ce1e08191d62fc55ba8862440fb3a59197a4 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test fd5199f928757cb308661b5fdca1abc19398a798ff7f24b57c3071e9f8e0471e +F test/readonly.test 69a7ccec846cad2e000b3539d56360d02f327061dc5e41f7f9a3e01f19719952 +F test/recover.test a163a156ea9f2beea63fa83c4dcd8dea6e57b8a569fc647155e3d2754eaac1b5 F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test db532cde29d6aebbc48c6ddc3149b30476f8e69ca7a2c4b53986c7635e6fd8ec +F test/returning1.test 212cd4111bb941a60abf608f20250db666c21eb1bc4d49217e96c87ff3ab9d1a F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 -F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa +F test/rollback.test 952c4d805bca96adc2be76f621ea22115fe40b330015af36fcc8028c8547fcee F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190cfd4 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test e29025be95baf6b32f0d5edef59a7633028325896a98f1caa8019559ca910350 +F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e F test/rowvalue.test baf4fa3ec1a8c1c920c3faa5fd25959cb454bbd99ac8960397c34549d9fc4abe F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 1347e25ca11c547c5a6ff0cc5626f95aa9740e9275bfaec096029f57cb2130ce -F test/rowvalue4.test 441e7e366ac6d939a3a95a574031c56ec2a854077a91d66eee5ff1d86cb5be58 +F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test c1cbdbf407029db01f87764097c6ac02a1c5a37efd2776eff32a9cdfdf6f2dba @@ -1537,7 +1581,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e -F test/scanstatus2.test 317670daf7f3eef48a9598cb7800ba8eccab51949cf52bca3f7da3b83a0c1c8c +F test/scanstatus2.test d85d17f2b0b4c013dde95232f7beab749f11f0ef847f5ecffb9486d2f5ecf9f9 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -1554,10 +1598,10 @@ F test/select3.test 180223af31e1ca5537dd395ef9708ae18e651a233777fd366fd0d75469fc F test/select4.test f0684d3da3bccacbe2a1ebadf6fb49d9df6f53acb4c6ebc228a88d0d6054cc7b F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f390ae F test/select6.test 9b2fb4ffedf52e1b5703cfcae1212e7a4a063f014c0458d78d29aca3db766d1f -F test/select7.test f659f231489349e8c5734e610803d7654207318f +F test/select7.test b825420da8a0b5722fdb77f3369f6396a3d198c46e8787eb26ff9425d4ac9d27 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 -F test/selectA.test 6aef8b2136a4ac7a3e2e4161d2b8ca7bc6ebe2779de084f9bb66ca9e2323a937 +F test/selectA.test 1da8ce3884c326e11d2855baffb76436b0d7e044404af8a2a70d1399a4ff7e29 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d32189bfb @@ -1567,7 +1611,7 @@ F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae F test/selectH.test 0b54599f1917d99568c9b929df22ec6261ed7b6d2f02a46b5945ef81b7871aac F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb -F test/sessionfuzz.c 666b47e177c7b25f01ba645d41fb9131d2d54ae673f0d81c08f5af2b3e6ecbda +F test/sessionfuzz.c f693b8827034a3bed7616d89c65fb4fe8b7ff3c0f000c6ea6beda69b7f1aced3 F test/shared.test f022874d9d299fe913529dc10f52ad5a386e4e7ff709270b9b1111b3a0f3420a F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 F test/shared3.test f8cd07c1a2b7cdb315c01671a0b2f8e3830b11ef31da6baa9a9cd8da88965403 @@ -1580,21 +1624,21 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test c7127a5e780ffc9e14c476773127fdf292c6db226529c44c1676f37b3793123f -F test/shell2.test 35226c070a8c7f64fd016dfac2a0db2a40f709b3131f61daacd9dad61536c9cb -F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a -F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f -F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b -F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 -F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f -F test/shell8.test 3fd093d481aaa94dc77fb73f1044c1f19c7efe3477a395cc4f7450133bc54915 -F test/shell9.test fe41f890a89f57bdde7351d7a828a1d4b503ebe239d413137369f20faf7f8865 +F test/shell1.test 490bf9d0c7c9564fea318c46d49369f4690b825b584c9a544dbdccf61bc0babc +F test/shell2.test 56da24128304c9ab67da2964cc80beff7b35761c446ec6e6e98bff2775b15026 +F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8 +F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807 +F test/shell5.test bafa4c0b67b7a8027e729970a625c9225cb7ef854acc4e52624c45074faaaddf +F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 +F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 +F test/shell8.test aea51ecbcd4494c746b096aeff51d841d04d5f0dc4b62eb42427f16109b87acd +F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209 F test/shmlock.test 3dbf017d34ab0c60abe6a44e447d3552154bd0c87b41eaf5ceacd408dd13fda5 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 2668e607dcdfa19c52828c09b69685b38da793856582ae31debf79d90c7bbbdc F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test e03ba5b977da6fd71662a4b0a668f04053bda4b187ff3214db7533e28c732279 +F test/skipscan1.test 9cbbb6575517b15292bd87ee85b853bbd3cd4b4735d69b0f083020cec16ff304 F test/skipscan2.test b032ed3e0ba5caa4df6c43ef22c31566aac67783bc031869155989a7ccdb5bd5 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 0672103fd2c8f96bd114133f356192b35ece45c794fe3677e1d9e5e3104a608e @@ -1622,28 +1666,30 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c f9505ad6f9c2c3c488a370a2d193e9603a030e51126ef3ecfeb056d21f0e7ad5 +F test/speedtest1.c 19c9b60908d25502d2831f97efee8b81006c356ab8c08327e25d24a4144f2131 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb F test/sqldiff1.test 1b7ab4f312442c5cc6b3a5f299fa8ca051416d1dd173cb1126fd51bf64f2c3fb -F test/sqllimits1.test b28e5cc8d337aaf290614d96a47e8fbfb720bb7ad35620c9d5432996fd413ac4 +F test/sqllimits1.test dee96a51b83ef866d06ec3c687d4c951d97b02549facc5be88c9dfcb215b98bf F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a +F test/starschema1.test a84205f97fe278a015ac39546c86b97228d22043af28f3a2ef809e8d5637ce1d F test/startup.c 1beb5ca66fcc0fce95c3444db9d1674f90fc605499a574ae2434dcfc10d22805 F test/stat.test 123212a20ceb496893d5254a5f6c76442ce549fdc08d1702d8288a2bbaac8408 F test/statfault.test 064f43379e4992b5221b7d9ac887c313b3191f85cce605d78e416fc4045da64e F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75 +F test/stmtrand.test 340e2ea4841c5cdc02d36e33739769c5d907ab529b12bb535407def0e413ca17 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5 F test/strict1.test 4d2b492152b984fd7e8196d23eb88e2ccb0ef9e46ca2f96c2ce7147ceef9d168 F test/strict2.test b22c7a98b5000aef937f1990776497f0e979b1a23bc4f63e2d53b00e59b20070 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49 -F test/subquery.test 312c5d26304b0e93a56ba2cb9d4480b8a1c8217e3b2b8f8be2bfb0b2458ac3a7 +F test/subquery.test 903abf41049f8404256f7be24b3151328304a5b25162e17ab0079460237382fc F test/subquery2.test 90cf944b9de8204569cf656028391e4af1ccc8c0cc02d4ef38ee3be8de1ffb12 F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test a673e3763e247e9b5e497a6cacbaf3da2bd8ec8921c0677145c109f2e633f36b F test/subtype1.test 7a9c55ed84d4ce551203d18046f925e293d75f69da81bff71aaf2696e4a2a748 -F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 +F test/superlock.test 85256830339a6871ce36a2ef591c3f67716a701b5497788fb2068b90159c2442 F test/swarmvtab.test 250231404fcac88f61a6c147bb0e3a118ed879278cd3ccb0ae2d3a729e1e8e26 F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c F test/swarmvtab3.test 41a3ab47cb7a834d4e5336425103b617410a67bb95d335ef536f887587ece073 @@ -1652,9 +1698,9 @@ F test/symlink.test 4368af0e213dd6e726a6240a16f2bb96a5a58f83f2d5d60652f27547b28c F test/symlink2.test 9531f475a53d8781c4f81373f87faf2e2aff4f5fb2102ec6386e0c827916a670 F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d4333092 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 -F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d +F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 54f27eacd054aa528a8b6e3331192c484104f30aaee351ad035f2b39a00f87c4 +F test/tabfunc01.test 6002a5f37b76355f173c75c2b3b03173b19d6a8b078c5baaa4c78bbcd0fa6323 F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1666,9 +1712,9 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 6f6e53981b4bdd42ef088f52e23236bc1ba0ca41ed395cbd7f33cbcff7d74d3c -F test/testrunner.tcl 8e2a5c7550b78d3283eee6103104ae2bcf56aa1df892dbd1608f27b93ebf4de8 -F test/testrunner_data.tcl 7ffd951527bbc614e723fd8d123b6834321878530696adecfdf6035100bac64e +F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 +F test/testrunner.tcl 5d02deeba7a53baeadae6aa7641d90aac58fdfa3a7bcac85cfcfd752b1aab87c +F test/testrunner_data.tcl c5ae2b1f9a99210b0600d002fb3af1fee350997cee9416551e83b93501360ebf F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1689,7 +1735,7 @@ F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-18458b1a.test 6a62cb1ee50fa3c620da59e3a6f531eb38fceaf7e2166203816b724524e6f1d6 F test/tkt-26ff0c2d1e.test c15bec890c4d226c0da2f35ff30f9e84c169cfef90e73a8cb5cec11d723dfa96 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 -F test/tkt-2d1a5c67d.test f143872a2102c62e777be3486b38ac2744c18ece31585ed3d0afcb573ca3b4f5 +F test/tkt-2d1a5c67d.test 92bf2a2de5757d2d24ef554f8a6a38476a6735074e32dc28c775b5b9a23f96a3 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 F test/tkt-313723c356.test 4b306ad45c736cedf2f5221f6155b92143244b6d @@ -1713,7 +1759,7 @@ F test/tkt-7a31705a7e6.test 9e9c057b6a9497c8f7ba7b16871029414ccf6550e7345d9085d6 F test/tkt-7bbfb7d442.test e87b59e620700b5a52ecd92f05d56686c1cad9e1aa17456eada55e0bb821b698 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 F test/tkt-80e031a00f.test 7c93af53f43527f50020983a4bcc39b077e77c7362d7af8e04a5fd155fe5e829 -F test/tkt-8454a207b9.test aff2e76143cfa443ddce6f7d85968a2e9b57a3deb0b881b730120740555f9e2f +F test/tkt-8454a207b9.test ead80b7a01438ca1436cee029694a96c821346cf1e24f06de12f8172e139ddbb F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 @@ -1730,7 +1776,7 @@ F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-b75a9ca6b0.test dc6a853c242f7d0326564ae32e9e5eb462b5e8d2bc5b01ea3b18fd24f8e5894b F test/tkt-ba7cbfaedc.test b4c0deccc12aeb55cfdb57935b16b5d67c5a9877 -F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 +F test/tkt-bd484a090c.test e6af3e3a4242cd8f1c91c736364f09075d8e33e3b86f6492a1ee36278ea71b61 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 F test/tkt-c694113d5.test 82c461924ada5c14866c47e85535b0b0923ba16a2e907e370061a5ca77f65d77 @@ -1795,7 +1841,7 @@ F test/tkt3357.test 77c37c6482b526fe89941ce951c22d011f5922ed F test/tkt3419.test 1bbf36d7ea03b638c15804251287c2391f5c1f6b F test/tkt3424.test 61f831bd2b071bd128fa5d00fbda57e656ca5812 F test/tkt3442.test c9d95b4c8f4f35a51b523f35d2afd0ce124937812af296545ad551ff763504fd -F test/tkt3457.test 5651e2cbb94645b677ec663160b9e192b87b7d365aecdfb24e19f749575a6fc2 +F test/tkt3457.test 5b9cc2b6cbbf896e9b973db83f6520f43f326f4d08604372a7b0379625e28412 F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d @@ -1838,7 +1884,7 @@ F test/trans.test 45f6f9ab6f66a7b5744f1caac06b558f95da62501916906cf55586a896f9f4 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test f8ee983600b33d167da1885657f064aec404e1c0d0bc8765fdf163f4c749237a -F test/trigger1.test 02cc64dc98278816c1c1ed8e472e18db8edbad88f37018bf46223e9614831963 +F test/trigger1.test 2834f8830a1ae338d95c2e3ea0c2a7bc4cda126cdeb715004cf0fd071892e44f F test/trigger2.test 30fcb3a6aa6782020d47968735ee6086ed795f73a7affa9406c8d5a36e7b5265 F test/trigger3.test aa640bb2bbb03edd5ff69c055117ea088f121945 F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 @@ -1846,7 +1892,7 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test e7ce54bfda67a88d778aea42544e151c465547a7e617127b6914c2221a6d53c1 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test fd49aff8b724ae1a6238989ecf84320d88c0e8f8d9142a891b93f826dfe37b59 +F test/trigger9.test 1724b595661da3dd3c8d79f0ebae818132a39e65c241bad2049f66952b1dc29d F test/triggerA.test 837be862d8721f903dba3f3ceff05b32e0bee5214cf6ea3da5fadf12d3650e9d F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 29f5a28d0fe39e6e2c01f6e1f53f08c0955170ae10a63ad023e33cb0a1682a51 @@ -1864,9 +1910,9 @@ F test/tt3_stress.c f9a769ca8b026ecc76ee93ca8c9700a5619f8e51c581107c4053ba6ac97f F test/tt3_vacuum.c 71b254cde1fc49d6c8c44efd54f4668f3e57d7b3a8f4601ade069f75a999ba39 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac -F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a +F test/types3.test c60e89c4d6babe44b23a2ea0090f3044e549403b20648b1c6bb65a69fea5f1ed F test/unhex.test b7f1b806207cb77fa31c3e434fe92fba524464e3e9356809bfcc28f15af1a8b7 -F test/unionall.test 5b1c4186a661e4bf762875caf4c61d8fda3dd04a6fa9005187f6ba8900c2913f +F test/unionall.test 04d30726c5056f84f92b3a12bf8d8a1dbbe807d1ddc8af95def09e6ef2dd91e3 F test/unionall2.test 71e8fa08d5699d50dc9f9dc0c9799c2e7a6bb7931a330d369307a4df7f157fa1 F test/unionallfault.test 652bfbb630e6c43135965dc1e8f0a9a791da83aec885d626a632fe1909c56f73 F test/unionvtab.test e1704ab1b4c1bb3ffc9da4681f8e85a0b909fd80b937984fc94b27415ac8e5a4 @@ -1883,17 +1929,17 @@ F test/upfrom2.test 66f3ebf721b3cebd922faee5c386bf244f816d416b57c000753ff51af623 F test/upfrom3.test 6130f24ebf97f5ea865e5d2a14a2d543fe5428a62e87cc60f62d875e45c1f5f0 F test/upfrom4.test 78f742a6577c91a7a55c64edb8811004e7c6aa99b8d57b2320f70a918c357807 F test/upfromfault.test 3a10075a0043f0c4fad6614b2c371f88a8ba5a4acab68b907438413865d6a8d6 -F test/upsert1.test a512e2f884d3a36159fce2e45108c236f78ae38e35bda55f4050db580ceb25d3 +F test/upsert1.test beba4316fbd4b7b9d76784313f6129a548cfe7abea04d46db33e2efce1ab0ac2 F test/upsert2.test 720e94d09f7362a282bc69b3c6b83d51daeaaf0440eb4920a08b86518b8c7496 F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3179c F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5 -F test/upsert5.test fff0dcfce73c649204543088d8e5bde01172676063ec9b8f8fc7f195abc386fe +F test/upsert5.test 9953b180d02d1369cdbb6c73c900834e5fef8cb78e98e07511c8762ec21cc176 F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35 F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da -F test/vacuum-into.test 35dc6f79b563f91c61822f61797363e97fed1bf28f1f722688b98d43f1980d76 +F test/vacuum-into.test 77845cee98770c416dae9b0da6bb3229753861f2da65c11b4f9715d081712d8a F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520 F test/vacuum3.test d9d9a04ee58c485b94694fd4f68cffaba49c32234fdefe1ac1a622c5e17d4ce3 @@ -1901,9 +1947,11 @@ F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb +F test/values.test 0eda08a6ce6545f1ab012dff4cc72a7dd0fee2510f42444136bb2b2b5ed84bc0 +F test/valuesfault.test 2ef23ed965e3bd08e268cdc38a0d11653390ddbbe1e8e2e98d16f55edd30f6e8 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test d4c4281e1679245829db35597817282f60dc513fc39cc5439078f009bd118487 +F test/view.test 3c23d7a068e9e4a0c4e6907498042772adea725f0630c3d9638ffd4e5a08b92b F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 F test/vt02.c 5b44ac67b1a283fedecf2d6e2ceda61e7a157f01d44dcb4490dcb1e87d057060 @@ -1922,18 +1970,19 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test 2a143fe75a11275781d1fd1988d86b66a3f69cb98f4add62e3da8fd0f637b45f F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test 2efb5a24b0bb50796b21eca23032cfb77abfa4b0c03938e38ce5897abac404ca +F test/vtabH.test 8e338acba32207085b6fe9cb2a58f7b408e74c8e1a2964cbdaca903ac82213cc F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtabJ.test a6aef49d558af90fae10565b29501f82a95781cb4f797f2d13e2d19f9b6bc77b F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3afbf +F test/vtabL.test c7b7f537978005d063fa2f53a3cd5a46ecf651ecd19970cb9ed4203698398deb F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c -F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089 -F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046 +F test/wal.test 519c550255c78f55959e9159b93ebbfad2b4e9f36f5b76284da41f572f9d27da +F test/wal2.test 44fe1cb4935dbbddfa0a34c2c4fd90f0ba8654d59b83c4136eb90fb327fd264f F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1942,11 +1991,11 @@ F test/wal64k.test 2a525c0f45d709bae3765c71045ccec5df7d100ccbd3a7860fdba46c9addb F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 -F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe +F test/wal_common.tcl 4589f701d5527ace2eba43823c96c2177e1f9dd2a6098256ee2203a0a313c13a F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d -F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278 +F test/walcksum.test ba02b4fe6d22cb42e57a323003cbae62f77a740983e1355b2b520e019ae261c7 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af @@ -1966,7 +2015,7 @@ F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cf F test/walseh1.test bae700eb99519b6d5cd3f893c04759accc5a59c391d4189fe4dd6995a533442b F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370dbc7a3 -F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f +F test/walslow.test 0c51843836c9dcf40a5ac05aa781bfb977b396ee2c872d92bd48b79d5dd9aa23 F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/where.test 59abb854eee24f166b5f7ba9d17eb250abc59ce0a66c48912ffb10763648196d @@ -1989,18 +2038,20 @@ F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a -F test/whereL.test 9d7c8a9f4e5e82d6859e61cf8758c3856c7e0a7fd8be11c92cac8c3ec39228fd +F test/whereL.test 438a397fa883b77bb6361c08a8befa41b52e9cfbe15a2a43715d122f8cfa8649 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 +F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb0e837a7 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31 F test/wherelimit2.test b9e4bfe7b4d7c2f85f99cf2bd2c51369378d04b1f3d1b60557423752003bfd90 +F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b3667c1fae6 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c -F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 +F test/win32longpath.test 42210789bcfc5c0ac202643d6d0237db08df2c9218f2070d9a69e8af1eccf7d7 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 5e8abe56a7d667eeddbba6de180086dcf69ed528d046447a25464f945ece101f +F test/window1.test 79dc3b9a2226f622d7e104a1fc750d1c4c3c08d6147b59085bdbe05352947ffa F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2018,17 +2069,17 @@ F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af F test/windowB.test aad7c31739999f68a98a813cfd78390918fc70f56d2d925317a1523cab548ecf F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b -F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316684c +F test/windowE.test c98507e0b0d95980ad25845db758557757be2d7054198ccf522c7d277057a3df F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c -F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df +F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d -F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df +F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 -F test/with6.test e097a03e5c898a8cd8f3a2d6a994ec510ea4376b5d484c2b669a41001e7758c8 +F test/with6.test 9ff3503c3ff7cd459dc4852a02aaefa998dccace53f4142a0eb726174ad5984a F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test a5210b8770dc4736bca4e74bc96588f43025ad03ad6a80f885afd36d9890e217 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -2042,14 +2093,15 @@ F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501d F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b1d99e F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 416adb0ca9bb54f978fe2e77978b1b964ce5e1c0199f45e381caa771e9f8cfc1 -F test/zipfile2.test 9903388a602a3834189857a985106ff95c3bba6a3969e0134127df991889db5d +F test/zipfile.test a36327c5697a03150a313ba06ab45842facef8b0c21be19d73a3a4fee58bc54c +F test/zipfile2.test 6df5f5ef9d247756f7200066f43e7f3f52cffff47f0c02cbefe4ce9c3284cb10 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x F tool/build-shell.sh f193b5e3eb4afcb4abbf96bf1475be6cfb74763ee2e50c82bc7ca105e8a136c5 +F tool/buildtclext.tcl b64d250517b148e644d26fcbc097851867a0df52cd4bafe9bcd94b8421e1428a F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629cca @@ -2061,6 +2113,7 @@ F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651e F tool/extract-sqlite3h.tcl 069ceab0cee26cba99952bfa08c0b23e35941c837acabe143f0c355d96c9e2eb x F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c c129ae2924a48310c7b766810391da9e8fda532b9f6bd3f9a9e3a799a1b42af9 +F tool/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/fuzzershell.c 41480c8a1e4749351f381431ecfdfceba645396c5d836f8d26b51a33c4a21b33 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 @@ -2068,8 +2121,8 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 19e368bc8e97ff4071115119a7911ca3b0c56eba7926d8ada8b4a86fcc69a176 -F tool/lempar.c 57478ea48420da05faa873c6d1616321caa5464644588c97fbe8e0ea04450748 +F tool/lemon.c d048516b2c3ad4119b1c1154a73f4f9435b275fea076318959f817effe23b827 +F tool/lempar.c e6b649778e5c027c8365ff01d7ef39297cd7285fa1f881cce31792689541e79f F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 @@ -2077,29 +2130,29 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkautoconfamal.sh cbdcf993fa83dccbef7fb77b39cdeb31ef9f77d9d88c9e343b58d35ca3898a6a F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x -F tool/mkctimec.tcl a16682eae5f01f85e5861b2aa215ca0d46b4230658ee25977e02b4508566fb75 x +F tool/mkctimec.tcl 060e9785e9503bf51f8b1b11b542bdeef90fd0ceb0738154f6762acec0c61e5f x F tool/mkkeywordhash.c b9faa0ae7e14e4dbbcd951cddd786bf46b8a65bb07b129ba8c0cfade723aaffd -F tool/mkmsvcmin.tcl 8897d515ef7f94772322db95a3b6fce6c614d84fe0bdd06ba5a1c786351d5a1d +F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef -F tool/mkopcodeh.tcl 769d9e6a8b462323150dc13a8539d6064664b72974f7894befe2491cc73e05cd +F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 32e359ccb21011958a821955254bd7a5fa7915d01a8c16fed91ffc8b40cb4adf F tool/mkshellc.tcl b7adf08b82de60811d2cb6af05ff59fc17e5cd6f3e98743c14eaaa3f8971fed0 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f -F tool/mksqlite3c.tcl 2c760ab786cb509b47f00c96fea82994866cb99f5e046df81c768288f57897b4 +F tool/mksqlite3c.tcl c6acfdf4e4ef93478ff3ce3cd593e17abb03f446036ce710c3156bcfa18665e0 F tool/mksqlite3h.tcl d391cff7cad0a372ee1406faee9ccc7dad9cb80a0c95cae0f73d10dd26e06762 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mktoolzip.tcl c7a9b685f5131d755e7d941cec50cee7f34178b9e34c9a89811eeb08617f8423 -F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 +F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e540a F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 F tool/omittest.tcl e99c9fecc3f7a8ca2fa75d8ec8bdbb5acce33dc69f0c280aae53064693387f65 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b -F tool/replace.tcl 937c931ad560688e85bdd6258bdc754371bb1e2732e1fb28ef441e44c9228fce -F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a +F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a +F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 F tool/showdb.c 0f74b54cc67076c76cba9b2b7f54d3e05b78d130c70ffc394eb84c5b41bab017 @@ -2109,8 +2162,8 @@ F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a80 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl 70c87c04cfd2e77b3e6f21c33ca768296aa8f67d4ab4874786ac8fbb28433477 -F tool/speed-check.sh 72dc85b2c0484af971ee3e7d10775f72b4e771e27e162c2099b3bf25517c25fb +F tool/spaceanal.tcl 3568b2b15b83dcaf789e787a4db0980da81eb6fa0e1e00783e4a927fdf584550 +F tool/speed-check.sh e8d20cc2eb9c85ec1ba562226de144435456dcdff4ee618de49603c6958f6116 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -2118,7 +2171,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 -F tool/sqldiff.c 985452ffc8554baee0f945d078859d393a22abc0bbf553a9f12acae1b6e1fdd0 +F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2153,12 +2206,12 @@ F vsixtest/Package.appxmanifest 6b6db1eb7df3a315c5d681059754d5f0e0c47a93 F vsixtest/pch.cpp cb823cfac36f1a39a7eb0acbd7e9a0b0de8f23af F vsixtest/pch.h 9cab7980f2ac4baa40807d8b5e52af32a21cf78c F vsixtest/vsixtest.sln 77cadbe4e96c1fe1bf51cd77de9e9b0a12ada547 -F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 +F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080bb302912 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6be68be17927e72b1f1c4b93e781b7e9235876b3ad9512fe168e5b9d0b7b763a -R 2f77944a93e4bc36153f9450a52f3c42 -U larrybr -Z 5b64f41a30f93cac02d6a6331bee0468 +P 2e74a938f771a889dd8c49648b5a125ff748835ba1925388cc6b379a0abde9ac 886545a8d0e9cbf4911ecf747da0ea8aad702d79cb065dad01a04aa4f51da12b +R 79479d743cc98e2dac5001e42c2a12ce +U drh +Z d5c2cd4bb2868922ff61d70f26a944a8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9db802f7c..179264e0e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e74a938f771a889dd8c49648b5a125ff748835ba1925388cc6b379a0abde9ac
\ No newline at end of file +2d52db98f47fbcda0622c034b21c2fb19bf4345b88c5c565ae9e6f7128642e43 diff --git a/src/alter.c b/src/alter.c index ec45e1433..ff2075758 100644 --- a/src/alter.c +++ b/src/alter.c @@ -1320,7 +1320,7 @@ static int renameResolveTrigger(Parse *pParse){ /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ @@ -1366,8 +1366,9 @@ static int renameResolveTrigger(Parse *pParse){ int i; for(i=0; i<pStep->pFrom->nSrc && rc==SQLITE_OK; i++){ SrcItem *p = &pStep->pFrom->a[i]; - if( p->pSelect ){ - sqlite3SelectPrep(pParse, p->pSelect, 0); + if( p->fg.isSubquery ){ + assert( p->u4.pSubq!=0 ); + sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); } } } @@ -1435,8 +1436,12 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ } if( pStep->pFrom ){ int i; - for(i=0; i<pStep->pFrom->nSrc; i++){ - sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect); + SrcList *pFrom = pStep->pFrom; + for(i=0; i<pFrom->nSrc; i++){ + if( pFrom->a[i].fg.isSubquery ){ + assert( pFrom->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + } } } } @@ -1683,7 +1688,7 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ } for(i=0; i<pSrc->nSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->pTab==p->pTab ){ + if( pItem->pSTab==p->pTab ){ renameTokenFind(pWalker->pParse, p, pItem->zName); } } @@ -2262,7 +2267,12 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } diff --git a/src/analyze.c b/src/analyze.c index 59e3d9837..fc8680d78 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -872,7 +872,7 @@ static void statGet( if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); #ifdef SQLITE_ENABLE_STAT4 - assert( p->current.anEq[i] ); + assert( p->current.anEq[i] || p->nRow==0 ); #endif } sqlite3ResultStrAccum(context, &sStat); @@ -1057,7 +1057,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -1081,9 +1081,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -1122,41 +1127,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: - ** + /* Implementation of the following: + ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -1263,6 +1263,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -1286,6 +1292,13 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + if( doOnce ){ int mxCol = nCol; Index *pX; @@ -1338,7 +1351,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } @@ -1784,12 +1797,13 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int nSample; /* Number of samples */ - int nByte; /* Bytes of space required */ - int i; /* Bytes of space required */ - tRowcnt *pSpace; + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -1809,7 +1823,7 @@ static int loadStatTbl( } pIdx->nSampleCol = nIdxCol; pIdx->mxSample = nSample; - nByte = sizeof(IndexSample) * nSample; + nByte = ROUND8(sizeof(IndexSample) * nSample); nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -1818,7 +1832,10 @@ static int loadStatTbl( sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; } - pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pPtr = (u8*)pIdx->aSample; + pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); pIdx->aAvgEq = pSpace; pSpace += nIdxCol; pIdx->pTable->tabFlags |= TF_HasStat4; for(i=0; i<nSample; i++){ diff --git a/src/attach.c b/src/attach.c index 4a6a25bf0..76476685f 100644 --- a/src/attach.c +++ b/src/attach.c @@ -479,20 +479,21 @@ static int fixSelectCb(Walker *p, Select *pSelect){ if( NEVER(pList==0) ) return WRC_Continue; for(i=0, pItem=pList->a; i<pList->nSrc; i++, pItem++){ - if( pFix->bTemp==0 ){ - if( pItem->zDatabase ){ - if( iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + if( pFix->bTemp==0 && pItem->fg.isSubquery==0 ){ + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( iDb!=sqlite3FindDbName(db, pItem->u4.zDatabase) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); + pFix->zType, pFix->pName, pItem->u4.zDatabase); return WRC_Abort; } - sqlite3DbFree(db, pItem->zDatabase); - pItem->zDatabase = 0; + sqlite3DbFree(db, pItem->u4.zDatabase); pItem->fg.notCte = 1; + pItem->fg.hadSchema = 1; } - pItem->pSchema = pFix->pSchema; + pItem->u4.pSchema = pFix->pSchema; pItem->fg.fromDDL = 1; + pItem->fg.fixedSchema = 1; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( pList->a[i].fg.isUsing==0 diff --git a/src/auth.c b/src/auth.c index fa6c82d85..fba2c0990 100644 --- a/src/auth.c +++ b/src/auth.c @@ -165,7 +165,7 @@ void sqlite3AuthRead( assert( pTabList ); for(iSrc=0; iSrc<pTabList->nSrc; iSrc++){ if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ - pTab = pTabList->a[iSrc].pTab; + pTab = pTabList->a[iSrc].pSTab; break; } } diff --git a/src/btree.c b/src/btree.c index 907e37f1e..c752b0771 100644 --- a/src/btree.c +++ b/src/btree.c @@ -151,8 +151,47 @@ int corruptPageError(int lineno, MemPage *p){ # define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) #endif +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** @@ -229,6 +268,8 @@ static int hasSharedCacheTableLock( iTab = iRoot; } + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ @@ -362,6 +403,8 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); @@ -429,6 +472,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -467,6 +512,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -5080,9 +5128,12 @@ static int accessPayload( if( pCur->aOverflow==0 || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -5092,6 +5143,12 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. @@ -5573,6 +5630,23 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; ii<pCur->iPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -5601,18 +5675,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; ii<pCur->iPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } @@ -5665,6 +5728,7 @@ int sqlite3BtreeTableMoveto( } if( pCur->info.nKey<intKey ){ if( (pCur->curFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } @@ -6131,10 +6195,10 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; @@ -6280,7 +6344,10 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ } pPage = pCur->pPage; - assert( pPage->isInit ); + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } if( !pPage->leaf ){ int idx = pCur->ix; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); @@ -6953,7 +7020,10 @@ static int fillInCell( n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; assert( nSrc<=nPayload ); testcase( nSrc<nPayload ); @@ -7497,7 +7567,8 @@ static int rebuildPage( if( j>(u32)usableSize ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); - for(k=0; ALWAYS(k<NB*2) && pCArray->ixNx[k]<=i; k++){} + assert( pCArray->ixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i; k++){} pSrcEnd = pCArray->apEnd[k]; pData = pEnd; @@ -7580,7 +7651,8 @@ static int pageInsertArray( u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ if( iEnd<=iFirst ) return 0; - for(k=0; ALWAYS(k<NB*2) && pCArray->ixNx[k]<=i ; k++){} + assert( pCArray->ixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i ; k++){} pEnd = pCArray->apEnd[k]; while( 1 /*Exit by break*/ ){ int sz, rc; @@ -7865,6 +7937,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ b.szCell = &szCell; b.apEnd[0] = pPage->aDataEnd; b.ixNx[0] = 2; + b.ixNx[NB*2-1] = 0x7fffffff; rc = rebuildPage(&b, 0, 1, pNew); if( NEVER(rc) ){ releasePage(pNew); @@ -8100,7 +8173,9 @@ static int balance_nonroot( CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); - memset(&b, 0, sizeof(b)); + assert( sizeof(b) - sizeof(b.ixNx) == offsetof(CellArray,ixNx) ); + memset(&b, 0, sizeof(b)-sizeof(b.ixNx[0])); + b.ixNx[NB*2-1] = 0x7fffffff; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -8259,7 +8334,7 @@ static int balance_nonroot( ** table-interior, index-leaf, or index-interior). */ if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } @@ -8283,7 +8358,7 @@ static int balance_nonroot( memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ if( NEVER(limit<pOld->aiOvfl[0]) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } limit = pOld->aiOvfl[0]; @@ -8691,7 +8766,8 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - for(k=0; ALWAYS(k<NB*2) && b.ixNx[k]<=j; k++){} + assert( b.ixNx[NB*2-1]>j ); + for(k=0; b.ixNx[k]<=j; k++){} pSrcEnd = b.apEnd[k]; if( SQLITE_OVERFLOW(pSrcEnd, pCell, pCell+sz) ){ rc = SQLITE_CORRUPT_BKPT; @@ -8926,7 +9002,7 @@ static int anotherValidCursor(BtCursor *pCur){ && pOther->eState==CURSOR_VALID && pOther->pPage==pCur->pPage ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pCur->pPage); } } return SQLITE_OK; @@ -8986,7 +9062,7 @@ static int balance(BtCursor *pCur){ /* The page being written is not a root page, and there is currently ** more than one reference to it. This only happens if the page is one ** of its own ancestor pages. Corruption. */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -9150,7 +9226,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -9178,7 +9254,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd || pCur->info.pPayload < pPage->aData + pPage->cellOffset ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCur->info.nLocal==nTotal ){ /* The entire cell is local */ @@ -9259,7 +9335,7 @@ int sqlite3BtreeInsert( ** Which can only happen if the SQLITE_NoSchemaError flag was set when ** the schema was loaded. This cannot be asserted though, as a user might ** set the flag, load the schema, and then unset the flag. */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } @@ -9382,7 +9458,7 @@ int sqlite3BtreeInsert( if( pPage->nFree<0 ){ if( NEVER(pCur->eState>CURSOR_INVALID) ){ /* ^^^^^--- due to the moveToRoot() call above */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ rc = btreeComputeFreeSpace(pPage); } @@ -9399,7 +9475,10 @@ int sqlite3BtreeInsert( if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; szNew = p->pBt->nPreformatSize; - if( szNew<4 ) szNew = 4; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); @@ -9421,7 +9500,7 @@ int sqlite3BtreeInsert( CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -9448,10 +9527,10 @@ int sqlite3BtreeInsert( ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( oldCell+szNew > pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } memcpy(oldCell, newCell, szNew); return SQLITE_OK; @@ -9461,7 +9540,7 @@ int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } @@ -9491,7 +9570,7 @@ int sqlite3BtreeInsert( */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -9553,7 +9632,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } nRem = pSrc->info.nPayload; if( nIn==nRem && nIn<pDest->pPage->maxLocal ){ @@ -9578,7 +9657,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ if( nRem>nIn ){ if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } ovflIn = get4byte(&pSrc->info.pPayload[nIn]); } @@ -9674,7 +9753,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); if( rc || pCur->eState!=CURSOR_VALID ) return rc; }else{ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } assert( pCur->eState==CURSOR_VALID ); @@ -9683,14 +9762,14 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ iCellIdx = pCur->ix; pPage = pCur->pPage; if( pPage->nCell<=iCellIdx ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } pCell = findCell(pPage, iCellIdx); if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCell<&pPage->aCellIdx[pPage->nCell] ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must @@ -9781,7 +9860,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ n = pCur->pPage->pgno; } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; @@ -9897,7 +9976,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); if( pgnoRoot>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgnoRoot); } pgnoRoot++; @@ -9945,7 +10024,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -10035,14 +10114,14 @@ static int clearDatabasePage( assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno); } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; } hdr = pPage->hdrOffset; @@ -10146,7 +10225,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); if( iTable>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(iTable); } rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -10740,6 +10819,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -10851,6 +10933,7 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header @@ -10950,6 +11033,7 @@ int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -10963,7 +11047,9 @@ int sqlite3BtreeIntegrityCheck( int bPartial = 0; /* True if not checking all btrees */ int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); /* aRoot[0]==0 means this is a partial check */ if( aRoot[0]==0 ){ @@ -11036,15 +11122,18 @@ int sqlite3BtreeIntegrityCheck( testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)i<nRoot && sCheck.mxErr; i++){ - i64 notUsed; - if( aRoot[i]==0 ) continue; + sCheck.nRow = 0; + if( aRoot[i] ){ + i64 notUsed; #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); - } + if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } #endif - sCheck.v0 = aRoot[i]; - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; diff --git a/src/btree.h b/src/btree.h index b45ace7e1..9731b8f2d 100644 --- a/src/btree.h +++ b/src/btree.h @@ -331,6 +331,7 @@ int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ diff --git a/src/btreeInt.h b/src/btreeInt.h index 67a7db25c..121329725 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -707,6 +707,7 @@ struct IntegrityCk { StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* diff --git a/src/build.c b/src/build.c index a2553da9f..943d862e9 100644 --- a/src/build.c +++ b/src/build.c @@ -189,7 +189,7 @@ void sqlite3FinishCoding(Parse *pParse){ } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION +#if SQLITE_USER_AUTHENTICATION && !defined(SQLITE_OMIT_SHARED_CACHE) if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevel<UAUTH_User ){ @@ -497,12 +497,12 @@ Table *sqlite3LocateTableItem( SrcItem *p ){ const char *zDb; - assert( p->pSchema==0 || p->zDatabase==0 ); - if( p->pSchema ){ - int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + if( p->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); zDb = pParse->db->aDb[iDb].zDbSName; }else{ - zDb = p->zDatabase; + assert( !p->fg.isSubquery ); + zDb = p->u4.zDatabase; } return sqlite3LocateTable(pParse, flags, p->zName, zDb); } @@ -2828,20 +2828,20 @@ void sqlite3EndTable( int regRowid; /* Rowid of the next row to insert */ int addrInsLoop; /* Top of the loop for inserting rows */ Table *pSelTab; /* A table that describes the SELECT results */ + int iCsr; /* Write cursor on the new table */ if( IN_SPECIAL_PARSE ){ pParse->rc = SQLITE_ERROR; pParse->nErr++; return; } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; @@ -2862,11 +2862,11 @@ void sqlite3EndTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -2923,13 +2923,10 @@ void sqlite3EndTable( /* Test for cycles in generated columns and illegal expressions ** in CHECK constraints and in DEFAULT clauses. */ if( p->tabFlags & TF_HasGenerated ){ - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, - sqlite3MPrintf(db, "PRAGMA \"%w\".integrity_check(%Q)", - db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } /* Add the table to the in-memory representation of the database. @@ -3006,9 +3003,12 @@ void sqlite3CreateView( ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -3064,8 +3064,9 @@ create_view_fail: #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ @@ -3188,7 +3189,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; } int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable!=0 ); @@ -3486,6 +3487,8 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; assert( isView==0 || isView==LOCATE_VIEW ); @@ -3494,7 +3497,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ if( pTab==0 ){ if( noErr ){ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } goto exit_drop_table; @@ -4585,15 +4588,17 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ } assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; } - pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].u4.zDatabase); if( pIndex==0 ){ if( !ifExists ){ sqlite3ErrorMsg(pParse, "no such index: %S", pName->a); }else{ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } pParse->checkSchema = 1; @@ -4890,12 +4895,14 @@ SrcList *sqlite3SrcListAppend( if( pDatabase && pDatabase->z==0 ){ pDatabase = 0; } + assert( pItem->fg.fixedSchema==0 ); + assert( pItem->fg.isSubquery==0 ); if( pDatabase ){ pItem->zName = sqlite3NameFromToken(db, pDatabase); - pItem->zDatabase = sqlite3NameFromToken(db, pTable); + pItem->u4.zDatabase = sqlite3NameFromToken(db, pTable); }else{ pItem->zName = sqlite3NameFromToken(db, pTable); - pItem->zDatabase = 0; + pItem->u4.zDatabase = 0; } return pList; } @@ -4911,14 +4918,41 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ for(i=0, pItem=pList->a; i<pList->nSrc; i++, pItem++){ if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; - if( pItem->pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + if( pItem->fg.isSubquery ){ + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + assert( pItem->u4.pSubq->pSelect->pSrc!=0 ); + sqlite3SrcListAssignCursors(pParse, pItem->u4.pSubq->pSelect->pSrc); } } } } /* +** Delete a Subquery object and its substructure. +*/ +void sqlite3SubqueryDelete(sqlite3 *db, Subquery *pSubq){ + assert( pSubq!=0 && pSubq->pSelect!=0 ); + sqlite3SelectDelete(db, pSubq->pSelect); + sqlite3DbFree(db, pSubq); +} + +/* +** Remove a Subquery from a SrcItem. Return the associated Select object. +** The returned Select becomes the responsibility of the caller. +*/ +Select *sqlite3SubqueryDetach(sqlite3 *db, SrcItem *pItem){ + Select *pSel; + assert( pItem!=0 ); + assert( pItem->fg.isSubquery ); + pSel = pItem->u4.pSubq->pSelect; + sqlite3DbFree(db, pItem->u4.pSubq); + pItem->u4.pSubq = 0; + pItem->fg.isSubquery = 0; + return pSel; +} + +/* ** Delete an entire SrcList including all its substructure. */ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ @@ -4927,13 +4961,24 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; i<pList->nSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + + /* Check invariants on SrcItem */ + assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc ); + assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy ); + assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery ); + assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && + pItem->u4.pSubq->pSelect!=0) ); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); + if( pItem->fg.isSubquery ){ + sqlite3SubqueryDelete(db, pItem->u4.pSubq); + }else if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3DbNNFreeNN(db, pItem->u4.zDatabase); + } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); - sqlite3DeleteTable(db, pItem->pTab); - if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); + sqlite3DeleteTable(db, pItem->pSTab); if( pItem->fg.isUsing ){ sqlite3IdListDelete(db, pItem->u3.pUsing); }else if( pItem->u3.pOn ){ @@ -4944,6 +4989,54 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ } /* +** Attach a Subquery object to pItem->uv.pSubq. Set the +** pSelect value but leave all the other values initialized +** to zero. +** +** A copy of the Select object is made if dupSelect is true, and the +** SrcItem takes responsibility for deleting the copy. If dupSelect is +** false, ownership of the Select passes to the SrcItem. Either way, +** the SrcItem will take responsibility for deleting the Select. +** +** When dupSelect is zero, that means the Select might get deleted right +** away if there is an OOM error. Beware. +** +** Return non-zero on success. Return zero on an OOM error. +*/ +int sqlite3SrcItemAttachSubquery( + Parse *pParse, /* Parsing context */ + SrcItem *pItem, /* Item to which the subquery is to be attached */ + Select *pSelect, /* The subquery SELECT. Must be non-NULL */ + int dupSelect /* If true, attach a copy of pSelect, not pSelect itself.*/ +){ + Subquery *p; + assert( pSelect!=0 ); + assert( pItem->fg.isSubquery==0 ); + if( pItem->fg.fixedSchema ){ + pItem->u4.pSchema = 0; + pItem->fg.fixedSchema = 0; + }else if( pItem->u4.zDatabase!=0 ){ + sqlite3DbFree(pParse->db, pItem->u4.zDatabase); + pItem->u4.zDatabase = 0; + } + if( dupSelect ){ + pSelect = sqlite3SelectDup(pParse->db, pSelect, 0); + if( pSelect==0 ) return 0; + } + p = pItem->u4.pSubq = sqlite3DbMallocRawNN(pParse->db, sizeof(Subquery)); + if( p==0 ){ + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + } + pItem->fg.isSubquery = 1; + p->pSelect = pSelect; + assert( offsetof(Subquery, pSelect)==0 ); + memset(((char*)p)+sizeof(p->pSelect), 0, sizeof(*p)-sizeof(p->pSelect)); + return 1; +} + + +/* ** This routine is called by the parser to add a new term to the ** end of a growing FROM clause. The "p" parameter is the part of ** the FROM clause that has already been constructed. "p" is NULL @@ -4992,10 +5085,12 @@ SrcList *sqlite3SrcListAppendFromTerm( if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); } + assert( pSubquery==0 || pDatabase==0 ); if( pSubquery ){ - pItem->pSelect = pSubquery; - if( pSubquery->selFlags & SF_NestedFrom ){ - pItem->fg.isNestedFrom = 1; + if( sqlite3SrcItemAttachSubquery(pParse, pItem, pSubquery, 0) ){ + if( pSubquery->selFlags & SF_NestedFrom ){ + pItem->fg.isNestedFrom = 1; + } } } assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 ); diff --git a/src/ctime.c b/src/ctime.c index cf761299f..0ffe2a5bd 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -65,6 +65,9 @@ static const char * const sqlite3azCompileOpt[] = { "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), # endif #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", +#endif #ifdef SQLITE_ALLOW_URI_AUTHORITY "ALLOW_URI_AUTHORITY", #endif diff --git a/src/date.c b/src/date.c index e493542db..8c48a81fa 100644 --- a/src/date.c +++ b/src/date.c @@ -71,13 +71,14 @@ struct DateTime { int tz; /* Timezone offset in minutes */ double s; /* Seconds */ char validJD; /* True (1) if iJD is valid */ - char rawS; /* Raw numeric value stored in s */ char validYMD; /* True (1) if Y,M,D are valid */ char validHMS; /* True (1) if h,m,s are valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ - char isError; /* An overflow has occurred */ - char useSubsec; /* Display subsecond precision */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -175,6 +176,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -187,7 +190,6 @@ static int parseTimezone(const char *zDate, DateTime *p){ p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -231,7 +233,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -270,24 +271,49 @@ static void computeJD(DateTime *p){ Y--; M += 12; } - A = Y/100; - B = 2 - A + (A/4); + A = (Y+4800)/100; + B = 38 - A + (A/4); X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); - if( p->validTZ ){ + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } /* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<<p->M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + +/* ** Parse dates of the form ** ** YYYY-MM-DD HH:MM:SS.FFF @@ -325,12 +351,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -340,6 +370,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; @@ -423,7 +456,7 @@ static int validJulianDay(sqlite3_int64 iJD){ ** Compute the Year, Month, and Day from the julian day number. */ static void computeYMD(DateTime *p){ - int Z, A, B, C, D, E, X1; + int Z, alpha, A, B, C, D, E, X1; if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; @@ -434,8 +467,8 @@ static void computeYMD(DateTime *p){ return; }else{ Z = (int)((p->iJD + 43200000)/86400000); - A = (int)((Z - 1867216.25)/36524.25); - A = Z + 1 + A - (A/4); + alpha = (int)((Z + 32044.75)/36524.25) - 52; + A = Z + 1 + alpha - ((alpha+100)/4) + 25; B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; @@ -478,7 +511,7 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME @@ -610,7 +643,7 @@ static int toLocaltime( p->validHMS = 1; p->validJD = 0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->isError = 0; return SQLITE_OK; } @@ -630,12 +663,12 @@ static const struct { float rLimit; /* Maximum NNN value for this transform */ float rXform; /* Constant used for this transform */ } aXformType[] = { - { 6, "second", 4.6427e+14, 1.0 }, - { 6, "minute", 7.7379e+12, 60.0 }, - { 4, "hour", 1.2897e+11, 3600.0 }, - { 3, "day", 5373485.0, 86400.0 }, - { 5, "month", 176546.0, 2592000.0 }, - { 4, "year", 14713.0, 31536000.0 }, + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 2592000.0 }, + /* 5 */ { 4, "year", 14713.0, 31536000.0 }, }; /* @@ -667,14 +700,20 @@ static void autoAdjustDate(DateTime *p){ ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written @@ -705,6 +744,37 @@ static int parseModifier( } break; } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } case 'j': { /* ** julianday @@ -731,7 +801,9 @@ static int parseModifier( ** show local time. */ if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ - rc = toLocaltime(p, pCtx); + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -756,7 +828,7 @@ static int parseModifier( } #ifndef SQLITE_OMIT_LOCALTIME else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ - if( p->tzSet==0 ){ + if( p->isUtc==0 ){ i64 iOrigJD; /* Original localtime */ i64 iGuess; /* Guess at the corresponding utc time */ int cnt = 0; /* Safety to prevent infinite loop */ @@ -779,7 +851,8 @@ static int parseModifier( memset(p, 0, sizeof(*p)); p->iJD = iGuess; p->validJD = 1; - p->tzSet = 1; + p->isUtc = 1; + p->isLocal = 0; } rc = SQLITE_OK; } @@ -799,7 +872,7 @@ static int parseModifier( && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -839,7 +912,7 @@ static int parseModifier( p->h = p->m = 0; p->s = 0.0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; @@ -910,6 +983,7 @@ static int parseModifier( x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); computeJD(p); p->validHMS = 0; p->validYMD = 0; @@ -956,11 +1030,12 @@ static int parseModifier( z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; + if( n<3 || n>10 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; + p->nFloor = 0; for(i=0; i<ArraySize(aXformType); i++){ if( aXformType[i].nName==n && sqlite3_strnicmp(aXformType[i].zName, z, n)==0 @@ -968,21 +1043,24 @@ static int parseModifier( ){ switch( i ){ case 4: { /* Special processing to add months */ - assert( strcmp(aXformType[i].zName,"month")==0 ); + assert( strcmp(aXformType[4].zName,"month")==0 ); computeYMD_HMS(p); p->M += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } case 5: { /* Special processing to add years */ int y = (int)r; - assert( strcmp(aXformType[i].zName,"year")==0 ); + assert( strcmp(aXformType[5].zName,"year")==0 ); computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); p->Y += y; + computeFloor(p); p->validJD = 0; r -= (int)r; break; @@ -1237,21 +1315,82 @@ static void dateFunc( } /* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + +/* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "am" or "pm" +** %P "AM" or "PM" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 Sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -1288,7 +1427,7 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='d' ? "%02d" : "%2d", x.D); break; } - case 'f': { + case 'f': { /* Fractional seconds. (Non-standard) */ double s = x.s; if( s>59.999 ) s = 59.999; sqlite3_str_appendf(&sRes, "%06.3f", s); @@ -1298,6 +1437,21 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); break; } + case 'G': /* Fall thru */ + case 'g': { + DateTime y = x; + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); + }else{ + sqlite3_str_appendf(&sRes, "%04d", y.Y); + } + break; + } case 'H': case 'k': { sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); @@ -1311,25 +1465,11 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); break; } - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ - DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( cf=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); - }else{ - sqlite3_str_appendf(&sRes,"%03d",nDay+1); - } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); break; } - case 'J': { + case 'J': { /* Julian day number. (Non-standard) */ sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); break; } @@ -1372,13 +1512,33 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); break; } - case 'u': /* Fall thru */ - case 'w': { - char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; if( c=='0' && cf=='u' ) c = '7'; sqlite3_str_appendchar(&sRes, 1, c); break; } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); + break; + } case 'Y': { sqlite3_str_appendf(&sRes,"%04d",x.Y); break; @@ -1525,9 +1685,7 @@ static void timediffFunc( d1.iJD = d2.iJD - d1.iJD; d1.iJD += (u64)1486995408 * (u64)100000; } - d1.validYMD = 0; - d1.validHMS = 0; - d1.validTZ = 0; + clearYMD_HMS_TZ(&d1); computeYMD_HMS(&d1); sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", @@ -1596,6 +1754,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -1611,6 +1799,9 @@ void sqlite3RegisterDateTimeFunctions(void){ PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), diff --git a/src/dbstat.c b/src/dbstat.c index c70d80637..d635a8297 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -279,6 +279,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; pIdxInfo->idxNum |= 0x08; } + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_HEX; return SQLITE_OK; } diff --git a/src/delete.c b/src/delete.c index 2baff5b3d..4cdb3946e 100644 --- a/src/delete.c +++ b/src/delete.c @@ -24,8 +24,8 @@ ** ** The following fields are initialized appropriate in pSrc: ** -** pSrc->a[0].pTab Pointer to the Table object -** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** pSrc->a[0].spTab Pointer to the Table object +** pSrc->a[0].u2.pIBIndex Pointer to the INDEXED BY index, if there is one ** */ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ @@ -33,8 +33,8 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - if( pItem->pTab ) sqlite3DeleteTable(pParse->db, pItem->pTab); - pItem->pTab = pTab; + if( pItem->pSTab ) sqlite3DeleteTable(pParse->db, pItem->pSTab); + pItem->pSTab = pTab; pItem->fg.notCte = 1; if( pTab ){ pTab->nTabRef++; @@ -156,7 +156,8 @@ void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].fg.fixedSchema==0 && pFrom->a[0].fg.isSubquery==0 ); + pFrom->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].fg.isUsing==0 ); assert( pFrom->a[0].u3.pOn==0 ); } @@ -218,7 +219,7 @@ Expr *sqlite3LimitWhere( ** ); */ - pTab = pSrc->a[0].pTab; + pTab = pSrc->a[0].pSTab; if( HasRowid(pTab) ){ pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); pEList = sqlite3ExprListAppend( @@ -251,9 +252,9 @@ Expr *sqlite3LimitWhere( /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab = 0; pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); - pSrc->a[0].pTab = pTab; + pSrc->a[0].pSTab = pTab; if( pSrc->a[0].fg.isIndexedBy ){ assert( pSrc->a[0].fg.isCte==0 ); pSrc->a[0].u2.pIBIndex = 0; diff --git a/src/expr.c b/src/expr.c index f9b280bbc..1b18828dd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -85,7 +85,9 @@ char sqlite3ExprAffinity(const Expr *pExpr){ op = pExpr->op; continue; } - if( op!=TK_REGISTER || (op = pExpr->op2)==TK_REGISTER ) break; + if( op!=TK_REGISTER ) break; + op = pExpr->op2; + if( NEVER( op==TK_REGISTER ) ) break; } return pExpr->affExpr; } @@ -218,9 +220,10 @@ Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } } return pExpr; @@ -914,11 +917,12 @@ void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ Expr *sqlite3ExprAlloc( sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ @@ -934,7 +938,7 @@ Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -1366,6 +1370,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); assert( db!=0 ); +exprDeleteRestart: assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -1381,7 +1386,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); @@ -1396,6 +1400,19 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } #endif } + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } + } } if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbNNFreeNN(db, p); @@ -1428,11 +1445,11 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** ** The pExpr might be deleted immediately on an OOM error. ** -** The deferred delete is (currently) implemented by adding the -** pExpr to the pParse->pConstExpr list with a register number of 0. +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. */ -void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); +int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -1860,30 +1877,46 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcItem *pNewItem = &pNew->a[i]; const SrcItem *pOldItem = &p->a[i]; Table *pTab; - pNewItem->pSchema = pOldItem->pSchema; - pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->fg = pOldItem->fg; + if( pOldItem->fg.isSubquery ){ + Subquery *pNewSubq = sqlite3DbMallocRaw(db, sizeof(Subquery)); + if( pNewSubq==0 ){ + assert( db->mallocFailed ); + pNewItem->fg.isSubquery = 0; + }else{ + memcpy(pNewSubq, pOldItem->u4.pSubq, sizeof(*pNewSubq)); + pNewSubq->pSelect = sqlite3SelectDup(db, pNewSubq->pSelect, flags); + if( pNewSubq->pSelect==0 ){ + sqlite3DbFree(db, pNewSubq); + pNewSubq = 0; + pNewItem->fg.isSubquery = 0; + } + } + pNewItem->u4.pSubq = pNewSubq; + }else if( pOldItem->fg.fixedSchema ){ + pNewItem->u4.pSchema = pOldItem->u4.pSchema; + }else{ + pNewItem->u4.zDatabase = sqlite3DbStrDup(db, pOldItem->u4.zDatabase); + } pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; - pNewItem->addrFillSub = pOldItem->addrFillSub; - pNewItem->regReturn = pOldItem->regReturn; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } - pTab = pNewItem->pTab = pOldItem->pTab; + pTab = pNewItem->pSTab = pOldItem->pSTab; if( pTab ){ pTab->nTabRef++; } - pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); if( pOldItem->fg.isUsing ){ assert( pNewItem->fg.isUsing ); pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); @@ -1957,7 +1990,6 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pp = &pNew->pPrior; pNext = pNew; } - return pRet; } #else @@ -2344,6 +2376,54 @@ Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ return pExpr; } +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -2372,6 +2452,7 @@ Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from ** the ON or USING clauses of an outer join disqualifies the expression @@ -2391,6 +2472,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ){ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; @@ -2419,9 +2502,11 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_IF_NULL_ROW: case TK_REGISTER: case TK_DOT: + case TK_RAISE: testcase( pExpr->op==TK_REGISTER ); testcase( pExpr->op==TK_IF_NULL_ROW ); testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; case TK_VARIABLE: @@ -2443,15 +2528,15 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -2463,9 +2548,15 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* @@ -2481,8 +2572,24 @@ int sqlite3ExprIsConstant(Expr *p){ ** can be added to the pParse->pConstExpr list and evaluated once when ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -2490,9 +2597,26 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* @@ -2510,7 +2634,10 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** -** (2) pExpr cannot use subqueries or non-deterministic functions. +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. ** ** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. ** (Is there some way to relax this constraint?) @@ -2539,7 +2666,8 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){ int sqlite3ExprIsSingleTableConstraint( Expr *pExpr, /* The constraint */ const SrcList *pSrcList, /* Complete FROM clause */ - int iSrc /* Which element of pSrcList to use */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ ){ const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ @@ -2564,7 +2692,8 @@ int sqlite3ExprIsSingleTableConstraint( } } } - return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } @@ -2649,7 +2778,7 @@ int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -2675,8 +2804,12 @@ int sqlite3ExprContainsSubquery(Expr *p){ ** to fit in a 32-bit integer, return 1 and put the value of the integer ** in *pValue. If the expression is not an integer or if it is too big ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** +** If the pParse pointer is provided, then allow the expression p to be +** a parameter (TK_VARIABLE) that is bound to an integer. +** But if pParse is NULL, then p must be a pure integer literal. */ -int sqlite3ExprIsInteger(const Expr *p, int *pValue){ +int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){ int rc = 0; if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ @@ -2691,18 +2824,38 @@ int sqlite3ExprIsInteger(const Expr *p, int *pValue){ } switch( p->op ){ case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue); + rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0); break; } case TK_UMINUS: { int v = 0; - if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){ assert( ((unsigned int)v)!=0x80000000 ); *pValue = -v; rc = 1; } break; } + case TK_VARIABLE: { + sqlite3_value *pVal; + if( pParse==0 ) break; + if( NEVER(pParse->pVdbe==0) ) break; + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break; + sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn); + pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn, + SQLITE_AFF_BLOB); + if( pVal ){ + if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){ + sqlite3_int64 vv = sqlite3_value_int64(pVal); + if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */ + *pValue = (int)vv; + rc = 1; + } + } + sqlite3ValueFree(pVal); + } + break; + } default: break; } return rc; @@ -2739,9 +2892,12 @@ int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ && ALWAYS(p->iColumn<p->y.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); @@ -2853,8 +3009,8 @@ static Select *isCandidateForInOpt(const Expr *pX){ pSrc = p->pSrc; assert( pSrc!=0 ); if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ - pTab = pSrc->a[0].pTab; + if( pSrc->a[0].fg.isSubquery) return 0;/* FROM is not a subquery or view */ + pTab = pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ @@ -2894,13 +3050,13 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -3037,7 +3193,7 @@ int sqlite3FindInIndex( assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; /* Code an OP_Transaction and OP_TableLock for <table>. */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -3169,7 +3325,7 @@ int sqlite3FindInIndex( if( eType==0 && (inFlags & IN_INDEX_NOOP_OK) && ExprUseXList(pX) - && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ pParse->nTab--; /* Back out the allocation of the unused cursor */ iTab = -1; /* Cursor is not allocated */ @@ -3279,6 +3435,49 @@ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ #ifndef SQLITE_OMIT_SUBQUERY /* +** Scan all previously generated bytecode looking for an OP_BeginSubrtn +** that is compatible with pExpr. If found, add the y.sub values +** to pExpr and return true. If not found, return false. +*/ +static int findCompatibleInRhsSubrtn( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* IN operator with RHS that we want to reuse */ + SubrtnSig *pNewSig /* Signature for the IN operator */ +){ + VdbeOp *pOp, *pEnd; + SubrtnSig *pSig; + Vdbe *v; + + if( pNewSig==0 ) return 0; + if( (pParse->mSubrtnSig & (1<<(pNewSig->selId&7)))==0 ) return 0; + assert( pExpr->op==TK_IN ); + assert( !ExprUseYSub(pExpr) ); + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( (pExpr->x.pSelect->selFlags & SF_All)==0 ); + v = pParse->pVdbe; + assert( v!=0 ); + pOp = sqlite3VdbeGetOp(v, 1); + pEnd = sqlite3VdbeGetLastOp(v); + for(; pOp<pEnd; pOp++){ + if( pOp->p4type!=P4_SUBRTNSIG ) continue; + assert( pOp->opcode==OP_BeginSubrtn ); + pSig = pOp->p4.pSubrtnSig; + assert( pSig!=0 ); + if( pNewSig->selId!=pSig->selId ) continue; + if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue; + pExpr->y.sub.iAddr = pSig->iAddr; + pExpr->y.sub.regReturn = pSig->regReturn; + pExpr->iTable = pSig->iTable; + ExprSetProperty(pExpr, EP_Subrtn); + return 1; + } + return 0; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* ** Generate code that will construct an ephemeral table containing all terms ** in the RHS of an IN operator. The IN operator can be in either of two ** forms: @@ -3326,11 +3525,28 @@ void sqlite3CodeRhsOfIN( ** and reuse it many names. */ if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ - /* Reuse of the RHS is allowed */ - /* If this routine has already been coded, but the previous code - ** might not have been invoked yet, so invoke it now as a subroutine. + /* Reuse of the RHS is allowed + ** + ** Compute a signature for the RHS of the IN operator to facility + ** finding and reusing prior instances of the same IN operator. */ - if( ExprHasProperty(pExpr, EP_Subrtn) ){ + SubrtnSig *pSig = 0; + assert( !ExprUseXSelect(pExpr) || pExpr->x.pSelect!=0 ); + if( ExprUseXSelect(pExpr) && (pExpr->x.pSelect->selFlags & SF_All)==0 ){ + pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0])); + if( pSig ){ + pSig->selId = pExpr->x.pSelect->selId; + pSig->zAff = exprINAffinity(pParse, pExpr); + } + } + + /* Check to see if there is a prior materialization of the RHS of + ** this IN operator. If there is, then make use of that prior + ** materialization rather than recomputing it. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) + || findCompatibleInRhsSubrtn(pParse, pExpr, pSig) + ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); if( ExprUseXSelect(pExpr) ){ ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", @@ -3342,6 +3558,10 @@ void sqlite3CodeRhsOfIN( assert( iTab!=pExpr->iTable ); sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); sqlite3VdbeJumpHere(v, addrOnce); + if( pSig ){ + sqlite3DbFree(pParse->db, pSig->zAff); + sqlite3DbFree(pParse->db, pSig); + } return; } @@ -3352,7 +3572,13 @@ void sqlite3CodeRhsOfIN( pExpr->y.sub.regReturn = ++pParse->nMem; pExpr->y.sub.iAddr = sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; - + if( pSig ){ + pSig->iAddr = pExpr->y.sub.iAddr; + pSig->regReturn = pExpr->y.sub.regReturn; + pSig->iTable = iTab; + pParse->mSubrtnSig = 1 << (pSig->selId&7); + sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG); + } addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } @@ -3393,19 +3619,34 @@ void sqlite3CodeRhsOfIN( SelectDest dest; int i; int rc; + int addrBloom = 0; sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; + if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + int regBloom = ++pParse->nMem; + addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); + VdbeComment((v, "Bloom filter")); + dest.iSDParm2 = regBloom; + } testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ pCopy = sqlite3SelectDup(pParse->db, pSelect, 0); rc = pParse->db->mallocFailed ? 1 :sqlite3Select(pParse, pCopy, &dest); sqlite3SelectDelete(pParse->db, pCopy); sqlite3DbFree(pParse->db, dest.zAffSdst); + if( addrBloom ){ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + if( dest.iSDParm2==0 ){ + sqlite3VdbeChangeToNoop(v, addrBloom); + }else{ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + } + } if( rc ){ sqlite3KeyInfoUnref(pKeyInfo); return; - } + } assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ assert( pEList!=0 ); assert( pEList->nExpr>0 ); @@ -3452,7 +3693,7 @@ void sqlite3CodeRhsOfIN( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ sqlite3VdbeChangeToNoop(v, addrOnce-1); sqlite3VdbeChangeToNoop(v, addrOnce); ExprClearProperty(pExpr, EP_Subrtn); @@ -3699,9 +3940,7 @@ static void sqlite3ExprCodeIN( if( sqlite3ExprCheckIN(pParse, pExpr) ) return; zAff = exprINAffinity(pParse, pExpr); nVector = sqlite3ExprVectorSize(pExpr->pLeft); - aiMap = (int*)sqlite3DbMallocZero( - pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 - ); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, nVector*sizeof(int)); if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; /* Attempt to compute the RHS. After this step, if anything other than @@ -3844,6 +4083,15 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); if( destIfFalse==destIfNull ){ /* Combine Step 3 and Step 5 into a single opcode */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); + assert( pOp->opcode==OP_Once || pParse->nErr ); + if( pOp->opcode==OP_Once && pOp->p3>0 ){ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + } + } sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, rLhs, nVector); VdbeCoverage(v); goto sqlite3ExprCodeIN_finished; @@ -4126,13 +4374,17 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ ** register iReg. The caller must ensure that iReg already contains ** the correct value for the expression. */ -static void exprToRegister(Expr *pExpr, int iReg){ +void sqlite3ExprToRegister(Expr *pExpr, int iReg){ Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); if( NEVER(p==0) ) return; - p->op2 = p->op; - p->op = TK_REGISTER; - p->iTable = iReg; - ExprClearProperty(p, EP_Skip); + if( p->op==TK_REGISTER ){ + assert( p->iTable==iReg ); + }else{ + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); + } } /* @@ -4616,12 +4868,6 @@ expr_code_doover: assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); - pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ - sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); - } return target; } case TK_REGISTER: { @@ -4795,7 +5041,9 @@ expr_code_doover: } #endif - if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ /* SQL functions can be expensive. So try to avoid running them ** multiple times if we know they always give the same result */ return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -4826,7 +5074,7 @@ expr_code_doover: } for(i=0; i<nFarg; i++){ - if( i<32 && sqlite3ExprIsConstant(pFarg->a[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -4968,8 +5216,9 @@ expr_code_doover: if( !ExprHasProperty(pExpr, EP_Collate) ){ /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called ** "SOFT-COLLATE" that is added to constraints that are pushed down - ** from outer queries into sub-queries by the push-down optimization. - ** Clear subtypes as subtypes may not cross a subquery boundary. + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. */ assert( pExpr->pLeft ); sqlite3ExprCode(pParse, pExpr->pLeft, target); @@ -5138,7 +5387,7 @@ expr_code_doover: break; } testcase( pX->op==TK_COLUMN ); - exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); testcase( regFree1==0 ); memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; @@ -5192,15 +5441,14 @@ expr_code_doover: } assert( !ExprHasProperty(pExpr, EP_IntValue) ); if( pExpr->affExpr==OE_Ignore ){ - sqlite3VdbeAddOp4( - v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, OE_Ignore); VdbeCoverage(v); }else{ - sqlite3HaltConstraint(pParse, + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp3(v, OP_Halt, pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, - pExpr->affExpr, pExpr->u.zToken, 0, 0); + pExpr->affExpr, r1); } - break; } #endif @@ -5293,7 +5541,7 @@ int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ if( ConstFactorOk(pParse) && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ *pReg = 0; r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -5357,7 +5605,7 @@ void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ sqlite3ExprCodeCopy(pParse, pExpr, target); @@ -5416,7 +5664,7 @@ int sqlite3ExprCodeExprList( sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ @@ -5489,7 +5737,7 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = pDel; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); if( xJump ){ xJump(pParse, &exprAnd, dest, jumpIfNull); }else{ @@ -6567,9 +6815,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } }else{ @@ -6578,9 +6825,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } } diff --git a/src/fkey.c b/src/fkey.c index bace1ae5e..f1117a884 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1043,9 +1043,9 @@ void sqlite3FkCheck( pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ SrcItem *pItem = pSrc->a; - pItem->pTab = pFKey->pFrom; + pItem->pSTab = pFKey->pFrom; pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nTabRef++; + pItem->pSTab->nTabRef++; pItem->iCursor = pParse->nTab++; if( regNew!=0 ){ @@ -1328,7 +1328,8 @@ static Trigger *fkActionTrigger( SrcList *pSrc; Expr *pRaise; - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + pRaise = sqlite3Expr(db, TK_STRING, "FOREIGN KEY constraint failed"), + pRaise = sqlite3PExpr(pParse, TK_RAISE, pRaise, 0); if( pRaise ){ pRaise->affExpr = OE_Abort; } @@ -1336,7 +1337,8 @@ static Trigger *fkActionTrigger( if( pSrc ){ assert( pSrc->nSrc==1 ); pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); + pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), diff --git a/src/func.c b/src/func.c index 58ef4fef9..8fcda11dc 100644 --- a/src/func.c +++ b/src/func.c @@ -1101,13 +1101,13 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ double r1, r2; const char *zVal; r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!.15g", r1); + sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); if( r1!=r2 ){ sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!.20e", r1); + sqlite3_str_appendf(pStr, "%!0.20e", r1); } } break; @@ -1409,7 +1409,7 @@ static void replaceFunc( } if( zPattern[0]==0 ){ assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); - sqlite3_result_value(context, argv[0]); + sqlite3_result_text(context, (const char*)zStr, nStr, SQLITE_TRANSIENT); return; } nPattern = sqlite3_value_bytes(argv[1]); @@ -1892,7 +1892,7 @@ static void sumFinalize(sqlite3_context *context){ if( p->approx ){ if( p->ovrfl ){ sqlite3_result_error(context,"integer overflow",-1); - }else if( !sqlite3IsNaN(p->rErr) ){ + }else if( !sqlite3IsOverflow(p->rErr) ){ sqlite3_result_double(context, p->rSum+p->rErr); }else{ sqlite3_result_double(context, p->rSum); @@ -1909,7 +1909,7 @@ static void avgFinalize(sqlite3_context *context){ double r; if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -1923,7 +1923,7 @@ static void totalFinalize(sqlite3_context *context){ if( p ){ if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -2206,6 +2206,8 @@ static void groupConcatValue(sqlite3_context *context){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); @@ -2545,6 +2547,7 @@ static void fpdecodeFunc( x = sqlite3_value_double(argv[0]); y = sqlite3_value_int(argv[1]); z = sqlite3_value_int(argv[2]); + if( z<=0 ) z = 1; sqlite3FpDecode(&s, x, y, z); if( s.isSpecial==2 ){ sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); diff --git a/src/global.c b/src/global.c index 7f27d91d1..121b3f6d6 100644 --- a/src/global.c +++ b/src/global.c @@ -289,6 +289,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ +#endif 0, /* bLocaltimeFault */ 0, /* xAltLocaltime */ 0x7ffffffe, /* iOnceResetThreshold */ diff --git a/src/insert.c b/src/insert.c index 1c31ca233..d380281be 100644 --- a/src/insert.c +++ b/src/insert.c @@ -577,6 +577,210 @@ void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + assert( (pItem->fg.isSubquery && pItem->u4.pSubq!=0) || pParse->nErr ); + if( pItem->fg.isSubquery ){ + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); + } + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; ii<pRow->nExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; ii<pRow->nExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + Subquery *pSubq; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->fg.viaCoroutine = 1; + p->iCursor = -1; + assert( !p->fg.isIndexedBy && !p->fg.isTabFunc ); + p->u1.nRow = 2; + if( sqlite3SrcItemAttachSubquery(pParse, p, pLeft, 0) ){ + pSubq = p->u4.pSubq; + pSubq->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, + pSubq->regReturn, 0, pSubq->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + pSubq->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + } + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + Subquery *pSubq; + assert( p!=0 ); + assert( p->fg.isSubquery ); + pSubq = p->u4.pSubq; + assert( pSubq!=0 ); + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + if( pSubq->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, pSubq->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, pSubq->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, pSubq->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -913,25 +1117,45 @@ void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - assert( db->pParse==pParse ); - if( rc || pParse->nErr ) goto insert_cleanup; - assert( db->mallocFailed==0 ); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + Subquery *pSubq; + assert( pItem->fg.isSubquery ); + pSubq = pItem->u4.pSubq; + dest.iSDParm = pSubq->regReturn; + regFromSelect = pSubq->regResult; + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + nColumn = pSubq->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -1086,7 +1310,7 @@ void sqlite3Insert( pNx->iDataCur = iDataCur; pNx->iIdxCur = iIdxCur; if( pNx->pUpsertTarget ){ - if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ goto insert_cleanup; } } @@ -2827,7 +3051,7 @@ static int xferOptimization( if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } - if( pSelect->pSrc->a[0].pSelect ){ + if( pSelect->pSrc->a[0].fg.isSubquery ){ return 0; /* FROM clause cannot contain a subquery */ } if( pSelect->pWhere ){ @@ -2978,7 +3202,10 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif diff --git a/src/json.c b/src/json.c index 682f30597..7361c46d8 100644 --- a/src/json.c +++ b/src/json.c @@ -563,7 +563,6 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ } } - /* Append formatted text (not to exceed N bytes) to the JsonString. */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ @@ -589,6 +588,16 @@ static void jsonAppendChar(JsonString *p, char c){ } } +/* Remove a single character from the end of the string +*/ +static void jsonStringTrimOneChar(JsonString *p){ + if( p->eErr==0 ){ + assert( p->nUsed>0 ); + p->nUsed--; + } +} + + /* Make sure there is a zero terminator on p->zBuf[] ** ** Return true on success. Return false if an OOM prevents this @@ -596,7 +605,7 @@ static void jsonAppendChar(JsonString *p, char c){ */ static int jsonStringTerminate(JsonString *p){ jsonAppendChar(p, 0); - p->nUsed--; + jsonStringTrimOneChar(p); return p->eErr==0; } @@ -611,6 +620,40 @@ static void jsonAppendSeparator(JsonString *p){ jsonAppendChar(p, ','); } +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. +*/ +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && c<sizeof(aSpecial) ); + assert( p->nUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; + } +} + /* Append the N-byte string in zIn to the end of the JsonString string ** under construction. Enclose the string in double-quotes ("...") and ** escape any double-quotes or backslash characters contained within the @@ -670,35 +713,14 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ } c = z[0]; if( c=='"' || c=='\\' ){ - json_simple_escape: if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ p->zBuf[p->nUsed++] = c; }else{ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - assert( c>=0 && c<sizeof(aSpecial) ); - if( aSpecial[c] ){ - c = aSpecial[c]; - goto json_simple_escape; - } if( (p->nUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; + jsonAppendControlChar(p, c); } z++; N--; @@ -1399,7 +1421,10 @@ static u32 jsonbValidityCheck( if( !jsonIsOk[z[j]] && z[j]!='\'' ){ if( z[j]=='"' ){ if( x==JSONB_TEXTJ ) return j+1; - }else if( z[j]!='\\' || j+1>=k ){ + }else if( z[j]<=0x1f ){ + /* Control characters in JSON5 string literals are ok */ + if( x==JSONB_TEXTJ ) return j+1; + }else if( NEVER(z[j]!='\\') || j+1>=k ){ return j+1; }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ j++; @@ -1597,6 +1622,7 @@ json_parse_restart: case '[': { /* Parse array */ iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); iStart = pParse->nBlob; if( pParse->oom ) return -1; @@ -1693,9 +1719,14 @@ json_parse_restart: return -1; } }else if( c<=0x1f ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; + if( c==0 ){ + pParse->iErr = j; + return -1; + } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; }else if( c=='"' ){ opcode = JSONB_TEXT5; } @@ -1911,6 +1942,7 @@ json_parse_restart: return i+4; } /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through } default: { u32 k; @@ -1995,6 +2027,10 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; memset(&px, 0, sizeof(px)); jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); @@ -2062,8 +2098,8 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; n = 9; } - if( i+sz+n > pParse->nBlob - && i+sz+n > pParse->nBlob-pParse->delta + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta ){ sz = 0; n = 0; @@ -2113,6 +2149,7 @@ static u32 jsonTranslateBlobToText( } case JSONB_INT: case JSONB_FLOAT: { + if( sz==0 ) goto malformed_jsonb; jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); break; } @@ -2121,6 +2158,7 @@ static u32 jsonTranslateBlobToText( sqlite3_uint64 u = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; int bOverflow = 0; + if( sz==0 ) goto malformed_jsonb; if( zIn[0]=='-' ){ jsonAppendChar(pOut, '-'); k++; @@ -2143,6 +2181,7 @@ static u32 jsonTranslateBlobToText( case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ u32 k = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( sz==0 ) goto malformed_jsonb; if( zIn[0]=='-' ){ jsonAppendChar(pOut, '-'); k++; @@ -2172,7 +2211,7 @@ static u32 jsonTranslateBlobToText( zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); while( sz2>0 ){ - for(k=0; k<sz2 && zIn[k]!='\\' && zIn[k]!='"'; k++){} + for(k=0; k<sz2 && (jsonIsOk[(u8)zIn[k]] || zIn[k]=='\''); k++){} if( k>0 ){ jsonAppendRawNZ(pOut, zIn, k); if( k>=sz2 ){ @@ -2187,6 +2226,13 @@ static u32 jsonTranslateBlobToText( sz2--; continue; } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } assert( zIn[0]=='\\' ); assert( sz2>=1 ); if( sz2<2 ){ @@ -2256,11 +2302,12 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '['); j = i+n; iEnd = j+sz; - while( j<iEnd ){ + while( j<iEnd && pOut->eErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, ','); } - if( sz>0 ) pOut->nUsed--; + if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, ']'); break; } @@ -2269,17 +2316,18 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '{'); j = i+n; iEnd = j+sz; - while( j<iEnd ){ + while( j<iEnd && pOut->eErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } - if( x & 1 ) pOut->eErr |= JSTRING_MALFORMED; - if( sz>0 ) pOut->nUsed--; + if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, '}'); break; } default: { + malformed_jsonb: pOut->eErr |= JSTRING_MALFORMED; break; } @@ -2287,6 +2335,112 @@ static u32 jsonTranslateBlobToText( return i+n+sz; } +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jj<pPretty->nIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } +} + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( j<iEnd ){ + jsonAppendChar(pOut, '\n'); + pPretty->nIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, ']'); + i = iEnd; + break; + } + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( j<iEnd ){ + jsonAppendChar(pOut, '\n'); + pPretty->nIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, '}'); + i = iEnd; + break; + } + default: { + i = jsonTranslateBlobToText(pParse, i, pOut); + break; + } + } + return i; +} + + /* Return true if the input pJson ** ** For performance reasons, this routine does not do a detailed check of the @@ -3207,6 +3361,38 @@ jsonInsertIntoBlob_patherror: } /* +** If pArg is a blob that seems like a JSONB blob, then initialize +** p to point to that JSONB and return TRUE. If pArg does not seem like +** a JSONB blob, then return FALSE; +** +** This routine is only called if it is already known that pArg is a +** blob. The only open question is whether or not the blob appears +** to be a JSONB blob. +*/ +static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ + u32 n, sz = 0; + p->aBlob = (u8*)sqlite3_value_blob(pArg); + p->nBlob = (u32)sqlite3_value_bytes(pArg); + if( p->nBlob==0 ){ + p->aBlob = 0; + return 0; + } + if( NEVER(p->aBlob==0) ){ + return 0; + } + if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + && (n = jsonbPayloadSize(p, 0, &sz))>0 + && sz+n==p->nBlob + && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + ){ + return 1; + } + p->aBlob = 0; + p->nBlob = 0; + return 0; +} + +/* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new ** JsonParse object. @@ -3262,34 +3448,30 @@ rebuild_from_cache: return p; } if( eType==SQLITE_BLOB ){ - u32 n, sz = 0; - p->aBlob = (u8*)sqlite3_value_blob(pArg); - p->nBlob = (u32)sqlite3_value_bytes(pArg); - if( p->nBlob==0 ){ - goto json_pfa_malformed; - } - if( NEVER(p->aBlob==0) ){ - goto json_pfa_oom; - } - if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){ - goto json_pfa_malformed; - } - n = jsonbPayloadSize(p, 0, &sz); - if( n==0 - || sz+n!=p->nBlob - || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0) - ){ - goto json_pfa_malformed; - } - if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ - goto json_pfa_oom; + if( jsonArgIsJsonb(pArg,p) ){ + if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ + goto json_pfa_oom; + } + return p; } - return p; + /* If the blob is not valid JSONB, fall through into trying to cast + ** the blob into text which is then interpreted as JSON. (tag-20240123-a) + ** + ** This goes against all historical documentation about how the SQLite + ** JSON functions were suppose to work. From the beginning, blob was + ** reserved for expansion and a blob value should have raised an error. + ** But it did not, due to a bug. And many applications came to depend + ** upon this buggy behavior, espeically when using the CLI and reading + ** JSON text using readfile(), which returns a blob. For this reason + ** we will continue to support the bug moving forward. + ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d + */ } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; if( p->nJson==0 ) goto json_pfa_malformed; - if( NEVER(p->zJson==0) ) goto json_pfa_oom; + assert( p->zJson!=0 ); if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; @@ -3455,10 +3637,10 @@ static void jsonDebugPrintBlob( if( sz==0 && x<=JSONB_FALSE ){ sqlite3_str_append(pOut, "\n", 1); }else{ - u32 i; + u32 j; sqlite3_str_appendall(pOut, ": \""); - for(i=iStart+n; i<iStart+n+sz; i++){ - u8 c = pParse->aBlob[i]; + for(j=iStart+n; j<iStart+n+sz; j++){ + u8 c = pParse->aBlob[j]; if( c<0x20 || c>=0x7f ) c = '.'; sqlite3_str_append(pOut, (char*)&c, 1); } @@ -3509,11 +3691,12 @@ static void jsonParseFunc( if( p==0 ) return; if( argc==1 ){ jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); - sqlite3_result_text64(ctx, out.zText, out.nChar, SQLITE_DYNAMIC, SQLITE_UTF8); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); }else{ jsonShowParse(p); } jsonParseFree(p); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -3612,13 +3795,6 @@ static void jsonArrayLengthFunc( jsonParseFree(p); } -/* True if the string is all digits */ -static int jsonAllDigits(const char *z, int n){ - int i; - for(i=0; i<n && sqlite3Isdigit(z[i]); i++){} - return i==n; -} - /* True if the string is all alphanumerics and underscores */ static int jsonAllAlphanum(const char *z, int n){ int i; @@ -3681,10 +3857,16 @@ static void jsonExtractFunc( ** NUMBER ==> $[NUMBER] // PG compatible ** LABEL ==> $.LABEL // PG compatible ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + ** + ** Updated 2024-05-27: If the NUMBER is negative, then PG counts from + ** the right of the array. Hence for negative NUMBER: + ** + ** NUMBER ==> $[#NUMBER] // PG compatible */ jsonStringInit(&jx, ctx); - if( jsonAllDigits(zPath, nPath) ){ + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ jsonAppendRawNZ(&jx, "[", 1); + if( zPath[0]=='-' ) jsonAppendRawNZ(&jx,"#",1); jsonAppendRaw(&jx, zPath, nPath); jsonAppendRawNZ(&jx, "]", 2); }else if( jsonAllAlphanum(zPath, nPath) ){ @@ -4178,6 +4360,40 @@ json_type_done: } /* +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. +** +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). +*/ +static void jsonPrettyFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; + }else{ + x.szIndent = (u32)strlen(x.zIndent); + } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); +} + +/* ** json_valid(JSON) ** json_valid(JSON, FLAGS) ** @@ -4260,12 +4476,12 @@ static void jsonValidFunc( return; } case SQLITE_BLOB: { - if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ + if( jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; - }else{ + }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; @@ -4276,8 +4492,11 @@ static void jsonValidFunc( iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); res = iErr==0; } + break; } - break; + /* Fall through into interpreting the input as text. See note + ** above at tag-20240123-a. */ + /* no break */ deliberate_fall_through } default: { JsonParse px; @@ -4402,7 +4621,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ if( isFinal ){ if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } return; }else if( isFinal ){ @@ -4412,7 +4631,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); @@ -4522,7 +4741,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ if( isFinal ){ if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); }else{ - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } return; }else if( isFinal ){ @@ -4532,7 +4751,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); @@ -4863,6 +5082,9 @@ static int jsonEachColumn( case JEACH_VALUE: { u32 i = jsonSkipLabel(p); jsonReturnFromBlob(&p->sParse, i, ctx, 1); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } break; } case JEACH_TYPE: { @@ -4909,9 +5131,9 @@ static int jsonEachColumn( case JEACH_JSON: { if( p->sParse.zJson==0 ){ sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); + SQLITE_TRANSIENT); }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); } break; } @@ -5013,13 +5235,9 @@ static int jsonEachFilter( memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; - if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); - }else{ - goto json_each_malformed_input; - } + if( jsonFuncArgMightBeBinary(argv[0]) ){ + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); @@ -5189,6 +5407,8 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), diff --git a/src/main.c b/src/main.c index 03429983d..5d6212208 100644 --- a/src/main.c +++ b/src/main.c @@ -765,6 +765,18 @@ int sqlite3_config(int op, ...){ } #endif /* SQLITE_OMIT_DESERIALIZE */ + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + default: { rc = SQLITE_ERROR; break; @@ -4381,6 +4393,18 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, sqlite3 *db, int *N) + ** + ** Write the current optimization settings into *N. A zero bit means that + ** the optimization is on, and a 1 bit means that the optimization is off. + */ + case SQLITE_TESTCTRL_GETOPT: { + sqlite3 *db = va_arg(ap, sqlite3*); + int *pN = va_arg(ap, int*); + *pN = db->dbOptFlags; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt); ** ** If parameter onoff is 1, subsequent calls to localtime() fail. diff --git a/src/malloc.c b/src/malloc.c index 356750682..9a635e716 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -221,6 +221,24 @@ static void sqlite3MallocAlarm(int nByte){ sqlite3_mutex_enter(mem0.mutex); } +#ifdef SQLITE_DEBUG +/* +** This routine is called whenever an out-of-memory condition is seen, +** It's only purpose to to serve as a breakpoint for gdb or similar +** code debuggers when working on out-of-memory conditions, for example +** caused by PRAGMA hard_heap_limit=N. +*/ +static SQLITE_NOINLINE void test_oom_breakpoint(u64 n){ + static u64 nOomFault = 0; + nOomFault += n; + /* The assert() is never reached in a human lifetime. It is here mostly + ** to prevent code optimizers from optimizing out this function. */ + assert( (nOomFault>>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -247,6 +265,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); *pp = 0; return; } @@ -535,6 +554,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); return 0; } } diff --git a/src/memdb.c b/src/memdb.c index 657cb9ca6..d83a51d54 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -799,6 +799,14 @@ unsigned char *sqlite3_serialize( pOut = 0; }else{ sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } if( piSize ) *piSize = sz; if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ pOut = 0; diff --git a/src/os_unix.c b/src/os_unix.c index 80e6f6ad9..5d1dc9ac6 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -322,7 +322,7 @@ static pid_t randomnessPid = 0; #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ #define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ -#ifndef SQLITE_DISABLE_DIRSYNC +#if !defined(SQLITE_DISABLE_DIRSYNC) && !defined(_AIX) # define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ #else # define UNIXFILE_DIRSYNC 0x00 @@ -1295,8 +1295,12 @@ static int unixLogErrorAtLine( ** available, the error message will often be an empty string. Not a ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -2275,26 +2279,22 @@ static int nolockClose(sqlite3_file *id) { /* ** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. -** -** In dotfile locking, either a lock exists or it does not. So in this -** variation of CheckReservedLock(), *pResOut is set to true if any lock -** is held on the file and false if the file is unlocked. +** file by this or any other process. If the caller holds a SHARED +** or greater lock when it is called, then it is assumed that no other +** client may hold RESERVED. Or, if the caller holds no lock, then it +** is assumed another client holds RESERVED if the lock-file exists. */ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { - int rc = SQLITE_OK; - int reserved = 0; unixFile *pFile = (unixFile*)id; - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - assert( pFile ); - reserved = osAccess((const char*)pFile->lockingContext, 0)==0; - OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); - *pResOut = reserved; - return rc; + if( pFile->eFileLock>=SHARED_LOCK ){ + *pResOut = 0; + }else{ + *pResOut = osAccess((const char*)pFile->lockingContext, 0)==0; + } + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, 0, *pResOut)); + return SQLITE_OK; } /* @@ -3983,7 +3983,7 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ /* Forward declaration */ static int unixGetTempname(int nBuf, char *zBuf); -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) static int unixFcntlExternalReader(unixFile*, int*); #endif @@ -4110,7 +4110,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ case SQLITE_FCNTL_EXTERNAL_READER: { -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) return unixFcntlExternalReader((unixFile*)id, (int*)pArg); #else *(int*)pArg = 0; @@ -4283,7 +4283,7 @@ static int unixGetpagesize(void){ #endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) /* ** Object used to represent an shared memory buffer. @@ -5441,11 +5441,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -6389,12 +6394,19 @@ static int unixOpen( rc = SQLITE_READONLY_DIRECTORY; }else if( errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } } } if( fd<0 ){ diff --git a/src/os_win.c b/src/os_win.c index dc16c08b5..442c108e9 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -4521,6 +4521,11 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ){ @@ -4529,7 +4534,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return rc; } } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; diff --git a/src/pager.c b/src/pager.c index 37588f0b2..1ac858a07 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4064,6 +4064,7 @@ static int pagerAcquireMapPage( return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; + assert( EIGHT_BYTE_ALIGNMENT( p->pExtra ) ); p->flags = PGHDR_MMAP; p->nRef = 1; p->pPager = pPager; @@ -7087,7 +7088,7 @@ sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; diff --git a/src/parse.y b/src/parse.y index 19491192e..926dd6e7d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -21,6 +21,10 @@ */ } +// Function used to enlarge the parser stack, if needed +%realloc parserStackRealloc +%free sqlite3_free + // All token codes are small integers with #defines that begin with "TK_" %token_prefix TK_ @@ -45,7 +49,7 @@ } } %stack_overflow { - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); } // The name of the generated procedure that implements the parser @@ -232,11 +236,13 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} // improve performance and reduce the executable size. The goal here is // to get the "jump" operations in ISNULL through ESCAPE to have numeric // values that are early enough so that all jump operations are clustered -// at the beginning. +// at the beginning. Also, operators like NE and EQ need to be adjacent, +// and all of the comparison operators need to be clustered together. +// Various assert() statements throughout the code enforce these restrictions. // %token ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST. %token CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL. -%token OR AND NOT IS MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. +%token OR AND NOT IS ISNOT MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. %token GT LE LT GE ESCAPE. // The following directive causes tokens ABORT, AFTER, ASC, etc. to @@ -526,9 +532,9 @@ cmd ::= select(X). { break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } @@ -547,12 +553,21 @@ cmd ::= select(X). { } return pSelect; } + + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. + */ + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } } %ifndef SQLITE_OMIT_CTE select(A) ::= WITH wqlist(W) selectnowith(X). {A = attachWithToSelect(pParse,X,W);} select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). {A = attachWithToSelect(pParse,X,W);} + %endif /* SQLITE_OMIT_CTE */ select(A) ::= selectnowith(A). { Select *p = A; @@ -610,24 +625,27 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) %endif -oneselect(A) ::= values(A). - +// Single row VALUES clause. +// %type values {Select*} +oneselect(A) ::= values(A). %destructor values {sqlite3SelectDelete(pParse->db, $$);} values(A) ::= VALUES LP nexprlist(X) RP. { A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0); } -values(A) ::= values(A) COMMA LP nexprlist(Y) RP. { - Select *pRight, *pLeft = A; - pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - A = pRight; - }else{ - A = pLeft; - } + +// Multiple row VALUES clause. +// +%type mvalues {Select*} +oneselect(A) ::= mvalues(A). { + sqlite3MultiValuesEnd(pParse, A); +} +%destructor mvalues {sqlite3SelectDelete(pParse->db, $$);} +mvalues(A) ::= values(A) COMMA LP nexprlist(Y) RP. { + A = sqlite3MultiValues(pParse, A, Y); +} +mvalues(A) ::= mvalues(A) COMMA LP nexprlist(Y) RP. { + A = sqlite3MultiValues(pParse, A, Y); } // The "distinct" nonterminal is true (1) if the DISTINCT keyword is @@ -722,11 +740,21 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N if( A ){ SrcItem *pNew = &A->a[A->nSrc-1]; SrcItem *pOld = F->a; + assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; - pNew->pSelect = pOld->pSelect; - if( pNew->pSelect && (pNew->pSelect->selFlags & SF_NestedFrom)!=0 ){ - pNew->fg.isNestedFrom = 1; + assert( pOld->fg.fixedSchema==0 ); + if( pOld->fg.isSubquery ){ + pNew->fg.isSubquery = 1; + pNew->u4.pSubq = pOld->u4.pSubq; + pOld->u4.pSubq = 0; + pOld->fg.isSubquery = 0; + assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 ); + if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){ + pNew->fg.isNestedFrom = 1; + } + }else{ + pNew->u4.zDatabase = pOld->u4.zDatabase; + pOld->u4.zDatabase = 0; } if( pOld->fg.isTabFunc ){ pNew->u1.pFuncArg = pOld->u1.pFuncArg; @@ -734,8 +762,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N pOld->fg.isTabFunc = 0; pNew->fg.isTabFunc = 1; } - pOld->zName = pOld->zDatabase = 0; - pOld->pSelect = 0; + pOld->zName = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ @@ -1266,8 +1293,17 @@ expr(A) ::= NOT(B) expr(X). expr(A) ::= BITNOT(B) expr(X). {A = sqlite3PExpr(pParse, @B, X, 0);/*A-overwrites-B*/} expr(A) ::= PLUS|MINUS(B) expr(X). [BITNOT] { - A = sqlite3PExpr(pParse, @B==TK_PLUS ? TK_UPLUS : TK_UMINUS, X, 0); - /*A-overwrites-B*/ + Expr *p = X; + u8 op = @B + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + A = p; + }else{ + A = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } } expr(A) ::= expr(B) PTR(C) expr(D). { @@ -1309,7 +1345,7 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { if( A ) sqlite3ExprIdToTrueFalse(A); }else{ Expr *pRHS = Y->a[0].pExpr; - if( Y->nExpr==1 && sqlite3ExprIsConstant(pRHS) && A->op!=TK_VECTOR ){ + if( Y->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && A->op!=TK_VECTOR ){ Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); @@ -1633,8 +1669,8 @@ expr(A) ::= RAISE LP IGNORE RP. { A->affExpr = OE_Ignore; } } -expr(A) ::= RAISE LP raisetype(T) COMMA nm(Z) RP. { - A = sqlite3ExprAlloc(pParse->db, TK_RAISE, &Z, 1); +expr(A) ::= RAISE LP raisetype(T) COMMA expr(Z) RP. { + A = sqlite3PExpr(pParse, TK_RAISE, Z, 0); if( A ) { A->affExpr = (char)T; } @@ -1749,9 +1785,10 @@ with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W, 1); } wqas(A) ::= AS. {A = M10d_Any;} wqas(A) ::= AS MATERIALIZED. {A = M10d_Yes;} wqas(A) ::= AS NOT MATERIALIZED. {A = M10d_No;} -wqitem(A) ::= nm(X) eidlist_opt(Y) wqas(M) LP select(Z) RP. { +wqitem(A) ::= withnm(X) eidlist_opt(Y) wqas(M) LP select(Z) RP. { A = sqlite3CteNew(pParse, &X, Y, Z, M); /*A-overwrites-X*/ } +withnm(A) ::= nm(A). {pParse->bHasWith = 1;} wqlist(A) ::= wqitem(X). { A = sqlite3WithAdd(pParse, 0, X); /*A-overwrites-X*/ } @@ -1912,8 +1949,8 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } TRUEFALSE /* True or false keyword */ ISNOT /* Combination of IS and NOT */ FUNCTION /* A function invocation */ - UMINUS /* Unary minus */ UPLUS /* Unary plus */ + UMINUS /* Unary minus */ TRUTH /* IS TRUE or IS FALSE or IS NOT TRUE or IS NOT FALSE */ REGISTER /* Reference to a VDBE register */ VECTOR /* Vector */ @@ -1923,6 +1960,12 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } SPAN /* The span operator */ ERROR /* An expression containing an error */ . + +term(A) ::= QNUMBER(X). { + A=tokenExpr(pParse,@X,X); + sqlite3DequoteNumber(pParse, A); +} + /* There must be no more than 255 tokens defined above. If this grammar ** is extended with new rules and tokens, they must either be so few in ** number that TK_SPAN is no more than 255, or else the new tokens must diff --git a/src/pcache.c b/src/pcache.c index 2974b0810..3429284dc 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -512,6 +512,7 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, 8); + assert( EIGHT_BYTE_ALIGNMENT( pPgHdr->pExtra ) ); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; pPgHdr->flags = PGHDR_CLEAN; diff --git a/src/pcache1.c b/src/pcache1.c index 1591f014c..a0a8c7e28 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -320,7 +320,8 @@ static int pcache1InitBulk(PCache1 *pCache){ do{ PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage]; pX->page.pBuf = zBulk; - pX->page.pExtra = &pX[1]; + pX->page.pExtra = (u8*)pX + ROUND8(sizeof(*pX)); + assert( EIGHT_BYTE_ALIGNMENT( pX->page.pExtra ) ); pX->isBulkLocal = 1; pX->isAnchor = 0; pX->pNext = pCache->pFree; @@ -457,7 +458,8 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ if( pPg==0 ) return 0; p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; p->page.pBuf = pPg; - p->page.pExtra = &p[1]; + p->page.pExtra = (u8*)p + ROUND8(sizeof(*p)); + assert( EIGHT_BYTE_ALIGNMENT( p->page.pExtra ) ); p->isBulkLocal = 0; p->isAnchor = 0; p->pLruPrev = 0; /* Initializing this saves a valgrind error */ diff --git a/src/pragma.c b/src/pragma.c index 4c9057418..a8045aab1 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -31,6 +31,34 @@ #include "pragma.h" /* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen emperically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + +/* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or ** unrecognized string argument. The FULL and EXTRA option is disallowed @@ -1675,7 +1703,7 @@ void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - if( sqlite3GetInt32(zRight, &mxErr) ){ + if( sqlite3GetInt32(pValue->z, &mxErr) ){ if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -1692,7 +1720,6 @@ void sqlite3Pragma( Hash *pTbls; /* Set of all tables in the schema */ int *aRoot; /* Array of root page numbers of all btrees */ int cnt = 0; /* Number of entries in aRoot[] */ - int mxIdx = 0; /* Maximum number of indexes for any table */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -1714,7 +1741,6 @@ void sqlite3Pragma( if( pObjTab && pObjTab!=pTab ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } if( cnt==0 ) continue; if( pObjTab ) cnt++; @@ -1734,11 +1760,11 @@ void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - sqlite3TouchRegister(pParse, 8+mxIdx); + sqlite3TouchRegister(pParse, 8+cnt); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, @@ -1748,6 +1774,36 @@ void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ @@ -1762,31 +1818,7 @@ void sqlite3Pragma( int mxCol; /* Maximum non-virtual column number */ if( pObjTab && pObjTab!=pTab ) continue; - if( !IsOrdinaryTable(pTab) ){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_vtab *pVTab; - int a1; - if( !IsVirtual(pTab) ) continue; - if( pTab->nCol<=0 ){ - const char *zMod = pTab->u.vtab.azArg[0]; - if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; - } - sqlite3ViewGetColumnNames(pParse, pTab); - if( pTab->u.vtab.p==0 ) continue; - pVTab = pTab->u.vtab.p->pVtab; - if( NEVER(pVTab==0) ) continue; - if( NEVER(pVTab->pModule==0) ) continue; - if( pVTab->pModule->iVersion<4 ) continue; - if( pVTab->pModule->xIntegrity==0 ) continue; - sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); - pTab->nTabRef++; - sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); - a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, a1); -#endif - continue; - } + if( !IsOrdinaryTable(pTab) ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; r2 = 0; @@ -1921,6 +1953,7 @@ void sqlite3Pragma( ** is REAL, we have to load the actual data using OP_Column ** to reliably determine if the value is a NULL. */ sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); VdbeCoverage(v); } @@ -2094,23 +2127,43 @@ void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); - if( !isQuick ){ - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, addr); - } - if( pPk ){ - sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); - } + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); + } + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( pObjTab && pObjTab!=pTab ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); @@ -2374,44 +2427,63 @@ void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. + ** + ** (2) The table is an ordinary table, not a virtual table or view. ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (3) The table name does not begin with "sqlite_". ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more - ** since the last time ANALYZE was run. + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. + ** + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -2423,6 +2495,10 @@ void sqlite3Pragma( LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2430,6 +2506,14 @@ void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimit<SQLITE_DEFAULT_OPTIMIZE_LIMIT ){ + nLimit = 0; + }else{ + nLimit = SQLITE_DEFAULT_OPTIMIZE_LIMIT; + } iTabCur = pParse->nTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -2438,23 +2522,61 @@ void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; - /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ - break; + szThreshold = -1; /* Always analyze if any index lacks statistics */ } } - if( szThreshold ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", @@ -2464,11 +2586,27 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddr<iEnd; iAddr++){ + if( aOp[iAddr].opcode==OP_SqlExec ) aOp[iAddr].p2 = nLimit; + } + } break; } @@ -2732,9 +2870,9 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ seen[0] = 0; seen[1] = 0; for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; j = pConstraint->iColumn - pTab->iHidden; assert( j < 2 ); seen[j] = i+1; @@ -2747,12 +2885,13 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; - j = seen[1]-1; - pIdxInfo->aConstraintUsage[j].argvIndex = 2; - pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } return SQLITE_OK; } @@ -2772,6 +2911,7 @@ static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ int i; sqlite3_finalize(pCsr->pPragma); pCsr->pPragma = 0; + pCsr->iRowid = 0; for(i=0; i<ArraySize(pCsr->azArg); i++){ sqlite3_free(pCsr->azArg[i]); pCsr->azArg[i] = 0; diff --git a/src/prepare.c b/src/prepare.c index 87569ee91..df9c98f74 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -633,7 +633,13 @@ void *sqlite3ParserAddCleanup( void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ - ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } if( pCleanup ){ pCleanup->pNext = pParse->pCleanup; pParse->pCleanup = pCleanup; diff --git a/src/printf.c b/src/printf.c index c6b3803ca..a14056514 100644 --- a/src/printf.c +++ b/src/printf.c @@ -498,6 +498,7 @@ void sqlite3_str_vappendf( if( xtype==etFLOAT ){ iRound = -precision; }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; iRound = precision; }else{ iRound = precision+1; @@ -533,13 +534,14 @@ void sqlite3_str_vappendf( } exp = s.iDP-1; - if( xtype==etGENERIC && precision>0 ) precision--; /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -846,18 +848,25 @@ void sqlite3_str_vappendf( if( pItem->zAlias && !flag_altform2 ){ sqlite3_str_appendall(pAccum, pItem->zAlias); }else if( pItem->zName ){ - if( pItem->zDatabase ){ - sqlite3_str_appendall(pAccum, pItem->zDatabase); + if( pItem->fg.fixedSchema==0 + && pItem->fg.isSubquery==0 + && pItem->u4.zDatabase!=0 + ){ + sqlite3_str_appendall(pAccum, pItem->u4.zDatabase); sqlite3_str_append(pAccum, ".", 1); } sqlite3_str_appendall(pAccum, pItem->zName); }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); - }else{ - Select *pSel = pItem->pSelect; - assert( pSel!=0 ); + }else if( ALWAYS(pItem->fg.isSubquery) ){/* Because of tag-20240424-1 */ + Select *pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } diff --git a/src/resolve.c b/src/resolve.c index b4f03fe7e..b755cc864 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -79,6 +79,8 @@ static void resolveAlias( assert( iCol>=0 && iCol<pEList->nExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( db->mallocFailed ){ @@ -213,7 +215,7 @@ static void extendFJMatch( if( pNew ){ pNew->iTable = pMatch->iCursor; pNew->iColumn = iColumn; - pNew->y.pTab = pMatch->pTab; + pNew->y.pTab = pMatch->pSTab; assert( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ); ExprSetProperty(pNew, EP_CanBeNull); *ppList = sqlite3ExprListAppend(pParse, *ppList, pNew); @@ -226,7 +228,7 @@ static void extendFJMatch( static SQLITE_NOINLINE int isValidSchemaTableName( const char *zTab, /* Name as it appears in the SQL */ Table *pTab, /* The schema table we are trying to match */ - Schema *pSchema /* non-NULL if a database qualifier is present */ + const char *zDb /* non-NULL if a database qualifier is present */ ){ const char *zLegacy; assert( pTab!=0 ); @@ -237,7 +239,7 @@ static SQLITE_NOINLINE int isValidSchemaTableName( if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ return 1; } - if( pSchema==0 ) return 0; + if( zDb==0 ) return 0; if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; }else{ @@ -277,7 +279,7 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ @@ -294,6 +296,7 @@ static int lookupName( Table *pTab = 0; /* Table holding the row */ Column *pCol; /* A column of pTab */ ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ @@ -343,10 +346,10 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; i<pSrcList->nSrc; i++, pItem++){ u8 hCol; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem)); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: @@ -355,8 +358,12 @@ static int lookupName( ** This pItem -------------^ */ int hit = 0; - assert( pItem->pSelect!=0 ); - pEList = pItem->pSelect->pEList; + Select *pSel; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); + pEList = pSel->pEList; assert( pEList!=0 ); assert( pEList->nExpr==pTab->nCol ); for(j=0; j<pEList->nExpr; j++){ @@ -419,7 +426,7 @@ static int lookupName( } }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ if( pTab->tnum!=1 ) continue; - if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ @@ -466,14 +473,43 @@ static int lookupName( } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && ALWAYS(pMatch!=0) + && ALWAYS(pMatch->pSTab!=0) + && (pMatch->pSTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pMatch->pTab; + pExpr->y.pTab = pMatch->pSTab; if( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } @@ -496,7 +532,8 @@ static int lookupName( if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 && ALWAYS(zTab==0 - || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -514,7 +551,7 @@ static int lookupName( if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ Upsert *pUpsert = pNC->uNC.pUpsert; if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ - pTab = pUpsert->pUpsertSrc->a[0].pTab; + pTab = pUpsert->pUpsertSrc->a[0].pSTab; pExpr->iTable = EXCLUDED_TABLE_NUMBER; } } @@ -593,13 +630,18 @@ static int lookupName( ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) + && ALWAYS(VisibleRowid(pMatch->pSTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; + cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pSTab!=0 && IsView(pMatch->pSTab) ){ + eNewExprOp = TK_NULL; + } +#endif if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -753,6 +795,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } @@ -786,8 +832,12 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && cnt==1 && pMatch!=0 ){ - pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; + } } pExpr->op = eNewExprOp; @@ -825,7 +875,7 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ SrcItem *pItem = &pSrc->a[iSrc]; Table *pTab; assert( ExprUseYTab(p) ); - pTab = p->y.pTab = pItem->pTab; + pTab = p->y.pTab = pItem->pSTab; p->iTable = pItem->iCursor; if( p->y.pTab->iPKey==iCol ){ p->iColumn = -1; @@ -944,7 +994,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pItem = pSrcList->a; pExpr->op = TK_COLUMN; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pItem->pTab; + pExpr->y.pTab = pItem->pSTab; pExpr->iTable = pItem->iCursor; pExpr->iColumn--; pExpr->affExpr = SQLITE_AFF_INTEGER; @@ -964,6 +1014,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** resolved. This prevents "column" from being counted as having been ** referenced, which might prevent a SELECT from being erroneously ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ case TK_NOTNULL: case TK_ISNULL: { @@ -974,19 +1037,36 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ anRef[i] = p->nRef; } sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ - testcase( ExprHasProperty(pExpr, EP_OuterON) ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->u.iValue = (pExpr->op==TK_NOTNULL); - pExpr->flags |= EP_IntValue; - pExpr->op = TK_INTEGER; + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; + } - for(i=0, p=pNC; p && i<ArraySize(anRef); p=p->pNext, i++){ - p->nRef = anRef[i]; + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ } - sqlite3ExprDelete(pParse->db, pExpr->pLeft); - pExpr->pLeft = 0; } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && i<ArraySize(anRef); p=p->pNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; return WRC_Prune; } @@ -1000,7 +1080,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ */ case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; @@ -1009,7 +1088,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zDb = 0; zTable = 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zColumn = pExpr->u.zToken; + pRight = pExpr; }else{ Expr *pLeft = pExpr->pLeft; testcase( pNC->ncFlags & NC_IdxExpr ); @@ -1028,14 +1107,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); zTable = pLeft->u.zToken; - zColumn = pRight->u.zToken; assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names @@ -1211,11 +1289,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif } } -#ifndef SQLITE_OMIT_WINDOWFUNC - else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ is_agg = 1; } -#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ if( pExpr->pLeft ){ @@ -1224,9 +1300,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); } #ifndef SQLITE_OMIT_WINDOWFUNC - if( pWin ){ + if( pWin && pParse->nErr==0 ){ Select *pSel = pNC->pWinSelect; - assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) ); + assert( ExprUseYWin(pExpr) && pWin==pExpr->y.pWin ); if( IN_RENAME_OBJECT==0 ){ sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); if( pParse->db->mallocFailed ) break; @@ -1285,6 +1361,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -1293,6 +1370,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pExpr->x.pSelect->selFlags |= SF_Correlated; } pNC->ncFlags |= NC_Subquery; } @@ -1431,7 +1509,7 @@ static int resolveOrderByTermToExprList( int rc; /* Return code from subprocedures */ u8 savedSuppErr; /* Saved value of db->suppressErr */ - assert( sqlite3ExprIsInteger(pE, &i)==0 ); + assert( sqlite3ExprIsInteger(pE, &i, 0)==0 ); pEList = pSelect->pEList; /* Resolve all names in the ORDER BY term expression @@ -1530,7 +1608,7 @@ static int resolveCompoundOrderBy( if( pItem->fg.done ) continue; pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( NEVER(pE==0) ) continue; - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( sqlite3ExprIsInteger(pE, &iCol, 0) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr, pE); return 1; @@ -1715,7 +1793,7 @@ static int resolveOrderGroupBy( continue; } } - if( sqlite3ExprIsInteger(pE2, &iCol) ){ + if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ @@ -1806,7 +1884,11 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** moves the pOrderBy down to the sub-query. It will be moved back ** after the names have been resolved. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + assert( p->pSrc->a[0].u4.pSubq!=0 ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); assert( p->pSrc->nSrc==1 && p->pOrderBy ); assert( pSub->pPrior && pSub->pOrderBy==0 ); pSub->pOrderBy = p->pOrderBy; @@ -1818,12 +1900,16 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; i<p->pSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; - if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ + assert( pItem->zName!=0 + || pItem->fg.isSubquery ); /* Test of tag-20240424-1*/ + if( pItem->fg.isSubquery + && (pItem->u4.pSubq->pSelect->selFlags & SF_Resolved)==0 + ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + sqlite3ResolveSelectNames(pParse, pItem->u4.pSubq->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr ) return WRC_Abort; assert( db->mallocFailed==0 ); @@ -1886,7 +1972,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; i<p->pSrc->nSrc; i++){ @@ -1923,7 +2011,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** These integers will be replaced by copies of the corresponding result ** set expressions by the call to resolveOrderGroupBy() below. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); p->pOrderBy = pSub->pOrderBy; pSub->pOrderBy = 0; } @@ -2077,6 +2168,9 @@ int sqlite3ResolveExprNames( ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ @@ -2085,7 +2179,7 @@ int sqlite3ResolveExprListNames( int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -2099,7 +2193,7 @@ int sqlite3ResolveExprListNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif sqlite3WalkExprNN(&w, pExpr); @@ -2116,10 +2210,10 @@ int sqlite3ResolveExprListNames( (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* @@ -2187,7 +2281,7 @@ int sqlite3ResolveSelfReference( if( pTab ){ sSrc.nSrc = 1; sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pTab = pTab; + sSrc.a[0].pSTab = pTab; sSrc.a[0].iCursor = -1; if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP diff --git a/src/select.c b/src/select.c index 121572779..9fcf30ff4 100644 --- a/src/select.c +++ b/src/select.c @@ -332,11 +332,13 @@ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; - assert( pItem->pSelect!=0 ); - pResults = pItem->pSelect->pEList; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + pResults = pItem->u4.pSubq->pSelect->pEList; assert( pResults!=0 ); assert( iCol>=0 && iCol<pResults->nExpr ); pResults->a[iCol].fg.bUsed = 1; @@ -370,9 +372,9 @@ static int tableAndColumnIndex( assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ for(i=iStart; i<=iEnd; i++){ - iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol); + iCol = sqlite3ColumnIndex(pSrc->a[i].pSTab, zCol); if( iCol>=0 - && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pSTab->aCol[iCol])==0) ){ if( piTab ){ sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol); @@ -501,10 +503,10 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pLeft = &pSrc->a[0]; pRight = &pLeft[1]; for(i=0; i<pSrc->nSrc-1; i++, pRight++, pLeft++){ - Table *pRightTab = pRight->pTab; + Table *pRightTab = pRight->pSTab; u32 joinType; - if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; + if( NEVER(pLeft->pSTab==0 || pRightTab==0) ) continue; joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; /* If this is a NATURAL join, synthesize an appropriate USING clause @@ -1377,12 +1379,18 @@ static void selectInnerLoop( ** case the order does matter */ pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm2 = 0; /* Signal that any Bloom filter is unpopulated */ }else{ int r1 = sqlite3GetTempReg(pParse); assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, r1, pDest->zAffSdst, nResultCol); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + if( pDest->iSDParm2 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + regResult, nResultCol); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); } break; @@ -1673,9 +1681,16 @@ static void generateSortTail( int addrExplain; /* Address of OP_Explain instruction */ #endif - ExplainQueryPlan2(addrExplain, (pParse, 0, - "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") - ); + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); @@ -1713,7 +1728,6 @@ static void generateSortTail( regRow = sqlite3GetTempRange(pParse, nColumn); } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; @@ -1918,8 +1932,12 @@ static const char *columnTypeImpl( SrcList *pTabList = pNC->pSrcList; for(j=0;j<pTabList->nSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( j<pTabList->nSrc ){ - pTab = pTabList->a[j].pTab; - pS = pTabList->a[j].pSelect; + pTab = pTabList->a[j].pSTab; + if( pTabList->a[j].fg.isSubquery ){ + pS = pTabList->a[j].u4.pSubq->pSelect; + }else{ + pS = 0; + } }else{ pNC = pNC->pNext; } @@ -1953,11 +1971,7 @@ static const char *columnTypeImpl( ** data for the result-set column of the sub-select. */ if( iCol<pS->pEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif + && (!ViewCanHaveRowid || iCol>=0) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see @@ -2322,8 +2336,7 @@ void sqlite3SubqueryColumnTypes( NameContext sNC; assert( pSelect!=0 ); - testcase( (pSelect->selFlags & SF_Resolved)==0 ); - assert( (pSelect->selFlags & SF_Resolved)!=0 || IN_RENAME_OBJECT ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed || IN_RENAME_OBJECT ) return; @@ -2334,17 +2347,22 @@ void sqlite3SubqueryColumnTypes( for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){ const char *zType; i64 n; + int m = 0; + Select *pS2 = pSelect; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } if( pCol->affinity<=SQLITE_AFF_NONE ){ pCol->affinity = aff; } - if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ - int m = 0; - Select *pS2; - for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); } if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ @@ -2374,12 +2392,12 @@ void sqlite3SubqueryColumnTypes( } } if( zType ){ - i64 m = sqlite3Strlen30(zType); + const i64 k = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; } } @@ -2486,7 +2504,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -2966,7 +2984,7 @@ static int multiSelect( p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit) + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ p->nSelectRow = sqlite3LogEst((u64)nLimit); @@ -3310,6 +3328,11 @@ static int generateOutputSubroutine( r1, pDest->zAffSdst, pIn->nSdst); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, pIn->iSdst, pIn->nSdst); + if( pDest->iSDParm2>0 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + pIn->iSdst, pIn->nSdst); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); break; } @@ -3966,7 +3989,9 @@ static void substSelect( pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(pSubst, pItem->pSelect, 1); + if( pItem->fg.isSubquery ){ + substSelect(pSubst, pItem->u4.pSubq->pSelect, 1); + } if( pItem->fg.isTabFunc ){ substExprList(pSubst, pItem->u1.pFuncArg); } @@ -3997,7 +4022,7 @@ static void recomputeColumnsUsed( SrcItem *pSrcItem /* Which FROM clause item to recompute */ ){ Walker w; - if( NEVER(pSrcItem->pTab==0) ) return; + if( NEVER(pSrcItem->pSTab==0) ) return; memset(&w, 0, sizeof(w)); w.xExprCallback = recomputeColumnsUsedExpr; w.xSelectCallback = sqlite3SelectWalkNoop; @@ -4037,8 +4062,10 @@ static void srclistRenumberCursors( aCsrMap[pItem->iCursor+1] = pParse->nTab++; } pItem->iCursor = aCsrMap[pItem->iCursor+1]; - for(p=pItem->pSelect; p; p=p->pPrior){ - srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + if( pItem->fg.isSubquery ){ + for(p=pItem->u4.pSubq->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } } } } @@ -4349,7 +4376,8 @@ static int flattenSubquery( assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; - pSub = pSubitem->pSelect; + assert( pSubitem->fg.isSubquery ); + pSub = pSubitem->u4.pSubq->pSelect; assert( pSub!=0 ); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -4402,7 +4430,7 @@ static int flattenSubquery( */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ + || IsVirtual(pSubSrc->a[0].pSTab) /* (3b) */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ @@ -4488,14 +4516,18 @@ static int flattenSubquery( pParse->zAuthContext = zSavedAuthContext; /* Delete the transient structures associated with the subquery */ - pSub1 = pSubitem->pSelect; - sqlite3DbFree(db, pSubitem->zDatabase); + + if( ALWAYS(pSubitem->fg.isSubquery) ){ + pSub1 = sqlite3SubqueryDetach(db, pSubitem); + }else{ + pSub1 = 0; + } + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->fg.fixedSchema==0 ); sqlite3DbFree(db, pSubitem->zName); sqlite3DbFree(db, pSubitem->zAlias); - pSubitem->zDatabase = 0; pSubitem->zName = 0; pSubitem->zAlias = 0; - pSubitem->pSelect = 0; assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -4536,8 +4568,8 @@ static int flattenSubquery( ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; Select *pPrior = p->pPrior; - Table *pItemTab = pSubitem->pTab; - pSubitem->pTab = 0; + Table *pItemTab = pSubitem->pSTab; + pSubitem->pSTab = 0; p->pOrderBy = 0; p->pPrior = 0; p->pLimit = 0; @@ -4545,7 +4577,7 @@ static int flattenSubquery( p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->op = TK_ALL; - pSubitem->pTab = pItemTab; + pSubitem->pSTab = pItemTab; if( pNew==0 ){ p->pPrior = pPrior; }else{ @@ -4560,11 +4592,14 @@ static int flattenSubquery( TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } - assert( pSubitem->pSelect==0 ); + assert( pSubitem->fg.isSubquery==0 ); } sqlite3DbFree(db, aCsrMap); if( db->mallocFailed ){ - pSubitem->pSelect = pSub1; + assert( pSubitem->fg.fixedSchema==0 ); + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->u4.zDatabase==0 ); + sqlite3SrcItemAttachSubquery(pParse, pSubitem, pSub1, 0); return 1; } @@ -4575,8 +4610,8 @@ static int flattenSubquery( ** ** pSubitem->pTab is always non-NULL by test restrictions and tests above. */ - if( ALWAYS(pSubitem->pTab!=0) ){ - Table *pTabToDel = pSubitem->pTab; + if( ALWAYS(pSubitem->pSTab!=0) ){ + Table *pTabToDel = pSubitem->pSTab; if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); @@ -4584,7 +4619,7 @@ static int flattenSubquery( }else{ pTabToDel->nTabRef--; } - pSubitem->pTab = 0; + pSubitem->pSTab = 0; } /* The following loop runs once for each term in a compound-subquery @@ -4640,8 +4675,11 @@ static int flattenSubquery( */ for(i=0; i<nSubSrc; i++){ SrcItem *pItem = &pSrc->a[i+iFrom]; - if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); assert( pItem->fg.isTabFunc==0 ); + assert( pItem->fg.isSubquery + || pItem->fg.fixedSchema + || pItem->u4.zDatabase==0 ); + if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); *pItem = pSubSrc->a[i]; pItem->fg.jointype |= ltorj; iNewParent = pSubSrc->a[i].iCursor; @@ -4776,7 +4814,7 @@ static void constInsert( ){ int i; assert( pColumn->op==TK_COLUMN ); - assert( sqlite3ExprIsConstant(pValue) ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; @@ -4834,10 +4872,10 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ pLeft = pExpr->pLeft; assert( pRight!=0 ); assert( pLeft!=0 ); - if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ constInsert(pConst,pRight,pLeft,pExpr); } - if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ constInsert(pConst,pLeft,pRight,pExpr); } } @@ -5058,6 +5096,19 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization" +** or sometimes the "predicate push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** ** (1) (** This restriction was removed on 2017-09-29. We used to @@ -5123,15 +5174,19 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING ** clause and the subquery. ** -** Without this restriction, the push-down optimization might move -** the ON/USING filter expression from the left side of a RIGHT JOIN -** over to the right side, which leads to incorrect answers. See -** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). ** ** (10) The inner query is not the right-hand table of a RIGHT JOIN. ** ** (11) The subquery is not a VALUES clause ** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -5242,7 +5297,19 @@ static int pushDownWhereTerms( } #endif - if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -5296,10 +5363,10 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ if( pItem->fg.isCorrelated || pItem->fg.isCte ){ return 0; } - assert( pItem->pTab!=0 ); - pTab = pItem->pTab; - assert( pItem->pSelect!=0 ); - pSub = pItem->pSelect; + assert( pItem->pSTab!=0 ); + pTab = pItem->pSTab; + assert( pItem->fg.isSubquery ); + pSub = pItem->u4.pSubq->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); for(pX=pSub; pX; pX=pX->pPrior){ if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ @@ -5428,13 +5495,13 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( p->pWhere || p->pEList->nExpr!=1 || p->pSrc->nSrc!=1 - || p->pSrc->a[0].pSelect + || p->pSrc->a[0].fg.isSubquery || pAggInfo->nFunc!=1 || p->pHaving ){ return 0; } - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); if( !IsOrdinaryTable(pTab) ) return 0; @@ -5459,7 +5526,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; assert( pTab!=0 ); @@ -5536,7 +5603,11 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ if( pNew==0 ) return WRC_Abort; memset(&dummy, 0, sizeof(dummy)); pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0); - if( pNewSrc==0 ) return WRC_Abort; + assert( pNewSrc!=0 || pParse->nErr ); + if( pParse->nErr ){ + sqlite3SrcListDelete(db, pNewSrc); + return WRC_Abort; + } *pNew = *p; p->pSrc = pNewSrc; p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); @@ -5591,7 +5662,7 @@ static struct Cte *searchWith( ){ const char *zName = pItem->zName; With *p; - assert( pItem->zDatabase==0 ); + assert( pItem->fg.fixedSchema || pItem->u4.zDatabase==0 ); assert( zName!=0 ); for(p=pWith; p; p=p->pOuter){ int i; @@ -5661,7 +5732,7 @@ static int resolveFromTermToCte( Cte *pCte; /* Matched CTE (or NULL if no match) */ With *pWith; /* The matching WITH */ - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( pParse->pWith==0 ){ /* There are no WITH clauses in the stack. No match is possible */ return 0; @@ -5671,7 +5742,8 @@ static int resolveFromTermToCte( ** go no further. */ return 0; } - if( pFrom->zDatabase!=0 ){ + assert( pFrom->fg.hadSchema==0 || pFrom->fg.notCte!=0 ); + if( pFrom->fg.fixedSchema==0 && pFrom->u4.zDatabase!=0 ){ /* The FROM term contains a schema qualifier (ex: main.t1) and so ** it cannot possibly be a CTE reference. */ return 0; @@ -5707,7 +5779,7 @@ static int resolveFromTermToCte( } if( cannotBeFunction(pParse, pFrom) ) return 2; - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return 2; pCteUse = pCte->pUse; @@ -5721,26 +5793,29 @@ static int resolveFromTermToCte( } pCteUse->eM10d = pCte->eM10d; } - pFrom->pTab = pTab; + pFrom->pSTab = pTab; pTab->nTabRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; - pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pCte->pSelect, 1); if( db->mallocFailed ) return 2; - pFrom->pSelect->selFlags |= SF_CopyCte; - assert( pFrom->pSelect ); + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + pSel = pFrom->u4.pSubq->pSelect; + assert( pSel!=0 ); + pSel->selFlags |= SF_CopyCte; if( pFrom->fg.isIndexedBy ){ sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); return 2; } + assert( !pFrom->fg.isIndexedBy ); pFrom->fg.isCte = 1; pFrom->u2.pCteUse = pCteUse; pCteUse->nUse++; /* Check if this is a recursive CTE. */ - pRecTerm = pSel = pFrom->pSelect; + pRecTerm = pSel; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; @@ -5748,11 +5823,13 @@ static int resolveFromTermToCte( assert( pRecTerm->pPrior!=0 ); for(i=0; i<pSrc->nSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->zDatabase==0 - && pItem->zName!=0 + if( pItem->zName!=0 + && !pItem->fg.hadSchema + && ALWAYS( !pItem->fg.isSubquery ) + && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0) && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ - pItem->pTab = pTab; + pItem->pSTab = pTab; pTab->nTabRef++; pItem->fg.isRecursive = 1; if( pRecTerm->selFlags & SF_Recursive ){ @@ -5854,11 +5931,14 @@ void sqlite3SelectPopWith(Walker *pWalker, Select *p){ ** SQLITE_NOMEM. */ int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ - Select *pSel = pFrom->pSelect; + Select *pSel; Table *pTab; + assert( pFrom->fg.isSubquery ); + assert( pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; assert( pSel ); - pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + pFrom->pSTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); if( pTab==0 ) return SQLITE_NOMEM; pTab->nTabRef = 1; if( pFrom->zAlias ){ @@ -5869,12 +5949,14 @@ int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } @@ -5976,33 +6058,35 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); - if( pFrom->pTab ) continue; + assert( pFrom->fg.isRecursive==0 || pFrom->pSTab!=0 ); + if( pFrom->pSTab ) continue; assert( pFrom->fg.isRecursive==0 ); if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->pSelect; + Select *pSel; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; #endif #ifndef SQLITE_OMIT_CTE }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ if( rc>1 ) return WRC_Abort; - pTab = pFrom->pTab; + pTab = pFrom->pSTab; assert( pTab!=0 ); #endif }else{ /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + assert( pFrom->pSTab==0 ); + pFrom->pSTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; if( pTab->nTabRef>=0xffff ){ sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", pTab->zName); - pFrom->pTab = 0; + pFrom->pSTab = 0; return WRC_Abort; } pTab->nTabRef++; @@ -6014,7 +6098,7 @@ static int selectExpander(Walker *pWalker, Select *p){ i16 nCol; u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->pSelect==0 ); + assert( pFrom->fg.isSubquery==0 ); if( IsView(pTab) ){ if( (db->flags & SQLITE_EnableView)==0 && pTab->pSchema!=db->aDb[1].pSchema @@ -6022,7 +6106,7 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", pTab->zName); } - pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pTab->u.view.pSelect, 1); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) @@ -6038,7 +6122,9 @@ static int selectExpander(Walker *pWalker, Select *p){ nCol = pTab->nCol; pTab->nCol = -1; pWalker->eCode = 1; /* Turn on Select.selId renumbering */ - sqlite3WalkSelect(pWalker, pFrom->pSelect); + if( pFrom->fg.isSubquery ){ + sqlite3WalkSelect(pWalker, pFrom->u4.pSubq->pSelect); + } pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } @@ -6125,7 +6211,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ int nAdd; /* Number of cols including rowid */ - Table *pTab = pFrom->pTab; /* Table for this data source */ + Table *pTab = pFrom->pSTab; /* Table for this data source */ ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ char *zTabName; /* AS name for this data source */ const char *zSchemaName = 0; /* Schema name for this data source */ @@ -6136,13 +6222,14 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom) ); if( pFrom->fg.isNestedFrom ){ - assert( pFrom->pSelect!=0 ); - pNestedFrom = pFrom->pSelect->pEList; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + assert( pFrom->u4.pSubq->pSelect!=0 ); + pNestedFrom = pFrom->u4.pSubq->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); - assert( VisibleRowid(pTab)==0 ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; @@ -6174,7 +6261,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pUsing = 0; } - nAdd = pTab->nCol + (VisibleRowid(pTab) && (selFlags&SF_NestedFrom)); + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; for(j=0; j<nAdd; j++){ const char *zName; struct ExprList_item *pX; /* Newly added ExprList term */ @@ -6256,7 +6344,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pX = &pNew->a[pNew->nExpr-1]; assert( pX->zEName==0 ); if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - if( pNestedFrom ){ + if( pNestedFrom && (!ViewCanHaveRowid || j<pNestedFrom->nExpr) ){ + assert( j<pNestedFrom->nExpr ); pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ @@ -6373,18 +6462,15 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; - testcase( (p->selFlags & SF_Resolved)==0 ); - assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT ); + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + if( (pTab->tabFlags & TF_Ephemeral)!=0 && pFrom->fg.isSubquery ){ /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); - } + Select *pSel = pFrom->u4.pSubq->pSelect; + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -6444,6 +6530,8 @@ void sqlite3SelectPrep( */ static void printAggInfo(AggInfo *pAggInfo){ int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); for(ii=0; ii<pAggInfo->nColumn; ii++){ struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; sqlite3DebugPrintf( @@ -6696,6 +6784,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); + if( pParse->nErr ) return; pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs @@ -6905,6 +6994,7 @@ static void updateAccumulator( if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } + if( pParse->nErr ) return; } if( regHit==0 && pAggInfo->nAccumulator ){ regHit = regAcc; @@ -6914,6 +7004,7 @@ static void updateAccumulator( } for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); + if( pParse->nErr ) return; } pAggInfo->directMode = 0; @@ -7029,25 +7120,28 @@ static SrcItem *isSelfJoinView( int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; - assert( pThis->pSelect!=0 ); - if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + Select *pSel; + assert( pThis->fg.isSubquery ); + pSel = pThis->u4.pSubq->pSelect; + assert( pSel!=0 ); + if( pSel->selFlags & SF_PushDown ) return 0; while( iFirst<iEnd ){ Select *pS1; pItem = &pTabList->a[iFirst++]; - if( pItem->pSelect==0 ) continue; + if( !pItem->fg.isSubquery ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; - assert( pItem->pTab!=0 ); - assert( pThis->pTab!=0 ); - if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; + assert( pItem->pSTab!=0 ); + assert( pThis->pSTab!=0 ); + if( pItem->pSTab->pSchema!=pThis->pSTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; - pS1 = pItem->pSelect; - if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + pS1 = pItem->u4.pSubq->pSelect; + if( pItem->pSTab->pSchema==0 && pSel->selId!=pS1->selId ){ /* The query flattener left two different CTE tables with identical ** names in the same FROM clause. */ continue; } - if( pItem->pSelect->selFlags & SF_PushDown ){ + if( pS1->selFlags & SF_PushDown ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ continue; @@ -7091,6 +7185,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ Expr *pExpr; Expr *pCount; sqlite3 *db; + SrcItem *pFrom; if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; @@ -7105,8 +7200,9 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ - pSub = p->pSrc->a[0].pSelect; - if( pSub==0 ) return 0; /* The FROM is a subquery */ + pFrom = p->pSrc->a; + if( pFrom->fg.isSubquery==0 ) return 0; /* FROM is a subquery */ + pSub = pFrom->u4.pSubq->pSelect; if( pSub->pPrior==0 ) return 0; /* Must be a compound */ if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ @@ -7115,7 +7211,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ assert( pSub->pHaving==0 ); /* Due to the previous */ - pSub = pSub->pPrior; /* Repeat over compound */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -7123,8 +7219,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ db = pParse->db; pCount = pExpr; pExpr = 0; - pSub = p->pSrc->a[0].pSelect; - p->pSrc->a[0].pSelect = 0; + pSub = sqlite3SubqueryDetach(db, pFrom); sqlite3SrcListDelete(db, p->pSrc); p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); while( pSub ){ @@ -7169,12 +7264,12 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ for(i=0; i<pSrc->nSrc; i++){ SrcItem *p1 = &pSrc->a[i]; if( p1==p0 ) continue; - if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + if( p0->pSTab==p1->pSTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ return 1; } - if( p1->pSelect - && (p1->pSelect->selFlags & SF_NestedFrom)!=0 - && sameSrcAlias(p0, p1->pSelect->pSrc) + if( p1->fg.isSubquery + && (p1->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->u4.pSubq->pSelect->pSrc) ){ return 1; } @@ -7239,13 +7334,13 @@ static int fromClauseTermCanBeCoroutine( if( i==0 ) break; i--; pItem--; - if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + if( pItem->fg.isSubquery ) return 0; /* (1c-i) */ } return 1; } /* -** Generate code for the SELECT statement given in the p argument. +** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. ** See comments in sqliteInt.h for further information. @@ -7256,6 +7351,40 @@ static int fromClauseTermCanBeCoroutine( ** ** This routine does NOT free the Select structure passed in. The ** calling function needs to do that. +** +** This is a long function. The following is an outline of the processing +** steps, with tags referencing various milestones: +** +** * Resolve names and similar preparation tag-select-0100 +** * Scan of the FROM clause tag-select-0200 +** + OUTER JOIN strength reduction tag-select-0220 +** + Sub-query ORDER BY removal tag-select-0230 +** + Query flattening tag-select-0240 +** * Separate subroutine for compound-SELECT tag-select-0300 +** * WHERE-clause constant propagation tag-select-0330 +** * Count()-of-VIEW optimization tag-select-0350 +** * Scan of the FROM clause again tag-select-0400 +** + Authorize unreferenced tables tag-select-0410 +** + Predicate push-down optimization tag-select-0420 +** + Omit unused subquery columns optimization tag-select-0440 +** + Generate code to implement subqueries tag-select-0480 +** - Co-routines tag-select-0482 +** - Reuse previously computed CTE tag-select-0484 +** - REuse previously computed VIEW tag-select-0486 +** - Materialize a VIEW or CTE tag-select-0488 +** * DISTINCT ORDER BY -> GROUP BY optimization tag-select-0500 +** * Set up for ORDER BY tag-select-0600 +** * Create output table tag-select-0630 +** * Prepare registers for LIMIT tag-select-0650 +** * Setup for DISTINCT tag-select-0680 +** * Generate code for non-aggregate and non-GROUP BY tag-select-0700 +** * Generate code for aggregate and/or GROUP BY tag-select-0800 +** + GROUP BY queries tag-select-0810 +** + non-GROUP BY queries tag-select-0820 +** - Special case of count() w/o GROUP BY tag-select-0821 +** - General case of non-GROUP BY aggregates tag-select-0822 +** * Sort results, as needed tag-select-0900 +** * Internal self-checks tag-select-1000 */ int sqlite3Select( Parse *pParse, /* The parser context */ @@ -7299,6 +7428,7 @@ int sqlite3Select( } #endif + /* tag-select-0100 */ assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); @@ -7350,7 +7480,7 @@ int sqlite3Select( if( sameSrcAlias(p0, p->pSrc) ){ sqlite3ErrorMsg(pParse, "target object/alias may not appear in FROM clause: %s", - p0->zAlias ? p0->zAlias : p0->pTab->zName + p0->zAlias ? p0->zAlias : p0->pSTab->zName ); goto select_end; } @@ -7385,12 +7515,13 @@ int sqlite3Select( /* Try to do various optimizations (flattening subqueries, and strength ** reduction of join operators) in the FROM clause up into the main query + ** tag-select-0200 */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && i<pTabList->nSrc; i++){ SrcItem *pItem = &pTabList->a[i]; - Select *pSub = pItem->pSelect; - Table *pTab = pItem->pTab; + Select *pSub = pItem->fg.isSubquery ? pItem->u4.pSubq->pSelect : 0; + Table *pTab = pItem->pSTab; /* The expander should have already created transient Table objects ** even for FROM clause elements such as subqueries that do not correspond @@ -7407,6 +7538,7 @@ int sqlite3Select( ** way that the i-th table cannot be the NULL row of a join, then ** perform the appropriate simplification. This is called ** "OUTER JOIN strength reduction" in the SQLite documentation. + ** tag-select-0220 */ if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, @@ -7477,7 +7609,8 @@ int sqlite3Select( if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; assert( pSub->pGroupBy==0 ); - /* If a FROM-clause subquery has an ORDER BY clause that is not + /* tag-select-0230: + ** If a FROM-clause subquery has an ORDER BY clause that is not ** really doing anything, then delete it now so that it does not ** interfere with query flattening. See the discussion at ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a @@ -7496,13 +7629,16 @@ int sqlite3Select( ** (a) The outer query has a different ORDER BY clause ** (b) The subquery is part of a join ** See forum post 062d576715d277c8 + ** (6) The subquery is not a recursive CTE. ORDER BY has a different + ** meaning for recursive CTEs and this optimization does not + ** apply. ** ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. */ if( pSub->pOrderBy!=0 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ && pSub->pLimit==0 /* Condition (1) */ - && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */ + && (pSub->selFlags & (SF_OrderByReqd|SF_Recursive))==0 /* (2) and (6) */ && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ @@ -7540,6 +7676,7 @@ int sqlite3Select( continue; } + /* tag-select-0240 */ if( flattenSubquery(pParse, p, i, isAgg) ){ if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ @@ -7555,7 +7692,7 @@ int sqlite3Select( #ifndef SQLITE_OMIT_COMPOUND_SELECT /* Handle compound SELECT statements using the separate multiSelect() - ** procedure. + ** procedure. tag-select-0300 */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); @@ -7571,9 +7708,9 @@ int sqlite3Select( #endif /* Do the WHERE-clause constant propagation optimization if this is - ** a join. No need to speed time on this operation for non-join queries + ** a join. No need to spend time on this operation for non-join queries ** as the equivalent optimization will be handled by query planner in - ** sqlite3WhereBegin(). + ** sqlite3WhereBegin(). tag-select-0330 */ if( p->pWhere!=0 && p->pWhere->op==TK_AND @@ -7590,6 +7727,7 @@ int sqlite3Select( TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } + /* tag-select-0350 */ if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ @@ -7597,20 +7735,26 @@ int sqlite3Select( pTabList = p->pSrc; } - /* For each term in the FROM clause, do two things: - ** (1) Authorized unreferenced tables - ** (2) Generate code for all sub-queries + /* Loop over all terms in the FROM clause and do two things for each term: + ** + ** (1) Authorize unreferenced tables + ** (2) Generate code for all sub-queries + ** + ** tag-select-0400 */ for(i=0; i<pTabList->nSrc; i++){ SrcItem *pItem = &pTabList->a[i]; SrcItem *pPrior; SelectDest dest; + Subquery *pSubq; Select *pSub; #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) const char *zSavedAuthContext; #endif - /* Issue SQLITE_READ authorizations with a fake column name for any + /* Authorized unreferenced tables. tag-select-0410 + ** + ** Issue SQLITE_READ authorizations with a fake column name for any ** tables that are referenced but from which no values are extracted. ** Examples of where these kinds of null SQLITE_READ authorizations ** would occur: @@ -7627,17 +7771,28 @@ int sqlite3Select( ** string for the fake column name seems safer. */ if( pItem->colUsed==0 && pItem->zName!=0 ){ - sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); + const char *zDb; + if( pItem->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pItem->u4.pSchema); + zDb = db->aDb[iDb].zDbSName; + }else if( pItem->fg.isSubquery ){ + zDb = 0; + }else{ + zDb = pItem->u4.zDatabase; + } + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", zDb); } #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Generate code for all sub-queries in the FROM clause */ - pSub = pItem->pSelect; - if( pSub==0 ) continue; + if( pItem->fg.isSubquery==0 ) continue; + pSubq = pItem->u4.pSubq; + assert( pSubq!=0 ); + pSub = pSubq->pSelect; /* The code for a subquery should only be generated once. */ - assert( pItem->addrFillSub==0 ); + if( pSubq->addrFillSub!=0 ) continue; /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -7650,6 +7805,7 @@ int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. + ** This is the "predicate push-down optimization". tag-select-0420 */ if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 @@ -7663,13 +7819,14 @@ int sqlite3Select( sqlite3TreeViewSelect(0, p, 0); } #endif - assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); + assert( pSubq->pSelect && (pSub->selFlags & SF_PushDown)!=0 ); }else{ - TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); } /* Convert unused result columns of the subquery into simple NULL ** expressions, to avoid unneeded searching and computation. + ** tag-select-0440 */ if( OptimizationEnabled(db, SQLITE_NullUnusedCols) && disableUnusedSubqueryResultColumns(pItem) @@ -7687,32 +7844,33 @@ int sqlite3Select( zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; - /* Generate code to implement the subquery + /* Generate byte-code to implement the subquery tag-select-0480 */ if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result - ** set on each invocation. + ** set on each invocation. tag-select-0482 */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; - pItem->regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pSubq->regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); - pItem->addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + pSubq->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; - pItem->regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->regReturn); + pSubq->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pSubq->regReturn); + VdbeComment((v, "end %!S", pItem)); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, - ** the make the pItem->iCursor be a copy of the ephemeral table that - ** holds the result of the materialization. */ + ** then make the pItem->iCursor be a copy of the ephemeral table that + ** holds the result of the materialization. tag-select-0484 */ CteUse *pCteUse = pItem->u2.pCteUse; sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); if( pItem->iCursor!=pCteUse->iCur ){ @@ -7722,25 +7880,30 @@ int sqlite3Select( pSub->nSelectRow = pCteUse->nRowEst; }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in - ** this same FROM clause. Reuse it. */ - if( pPrior->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + ** this same FROM clause. Reuse it. tag-select-0486 */ + Subquery *pPriorSubq; + assert( pPrior->fg.isSubquery ); + pPriorSubq = pPrior->u4.pSubq; + assert( pPriorSubq!=0 ); + if( pPriorSubq->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPriorSubq->regReturn, + pPriorSubq->addrFillSub); } sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - pSub->nSelectRow = pPrior->pSelect->nSelectRow; + pSub->nSelectRow = pPriorSubq->pSelect->nSelectRow; }else{ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of - ** the same view can reuse the materialization. */ + ** the same view can reuse the materialization. tag-select-0488 */ int topAddr; int onceAddr = 0; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int addrExplain; #endif - pItem->regReturn = ++pParse->nMem; + pSubq->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); - pItem->addrFillSub = topAddr+1; + pSubq->addrFillSub = topAddr+1; pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of @@ -7755,17 +7918,17 @@ int sqlite3Select( ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); + sqlite3VdbeAddOp2(v, OP_Return, pSubq->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; - pCteUse->addrM9e = pItem->addrFillSub; - pCteUse->regRtn = pItem->regReturn; + pCteUse->addrM9e = pSubq->addrFillSub; + pCteUse->regRtn = pSubq->regReturn; pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } @@ -7791,7 +7954,9 @@ int sqlite3Select( } #endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + /* tag-select-0500 + ** + ** If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: ** @@ -7808,12 +7973,18 @@ int sqlite3Select( */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 #endif ){ p->selFlags &= ~SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + if( pGroupBy ){ + for(i=0; i<pGroupBy->nExpr; i++){ + pGroupBy->a[i].u.x.iOrderByCol = i+1; + } + } p->selFlags |= SF_Aggregate; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the @@ -7835,7 +8006,7 @@ int sqlite3Select( ** If that is the case, then the OP_OpenEphemeral instruction will be ** changed to an OP_Noop once we figure out that the sorting index is ** not needed. The sSort.addrSortIndex variable is used to facilitate - ** that change. + ** that change. tag-select-0600 */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; @@ -7852,6 +8023,7 @@ int sqlite3Select( } /* If the output is destined for a temporary table, open that table. + ** tag-select-0630 */ if( pDest->eDest==SRT_EphemTab ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); @@ -7869,7 +8041,7 @@ int sqlite3Select( } } - /* Set the limiter. + /* Set the limiter. tag-select-0650 */ iEnd = sqlite3VdbeMakeLabel(pParse); if( (p->selFlags & SF_FixedLimit)==0 ){ @@ -7881,7 +8053,7 @@ int sqlite3Select( sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open an ephemeral index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. tag-select-0680 */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; @@ -7896,7 +8068,7 @@ int sqlite3Select( } if( !isAgg && pGroupBy==0 ){ - /* No aggregate functions and no GROUP BY clause */ + /* No aggregate functions and no GROUP BY clause. tag-select-0700 */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) | (p->selFlags & SF_FixedLimit); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -7969,8 +8141,8 @@ int sqlite3Select( sqlite3WhereEnd(pWInfo); } }else{ - /* This case when there exist aggregate functions or a GROUP BY clause - ** or both */ + /* This case is for when there exist aggregate functions or a GROUP BY + ** clause or both. tag-select-0800 */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -8089,7 +8261,7 @@ int sqlite3Select( /* Processing for aggregates with GROUP BY is very different and - ** much more complex than aggregates without a GROUP BY. + ** much more complex than aggregates without a GROUP BY. tag-select-0810 */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ @@ -8276,12 +8448,25 @@ int sqlite3Select( sortOut, sortPTab); } for(j=0; j<pGroupBy->nExpr; j++){ + int iOrderByCol = pGroupBy->a[j].u.x.iOrderByCol; + if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); }else{ pAggInfo->directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } + + if( iOrderByCol ){ + Expr *pX = p->pEList->a[iOrderByCol-1].pExpr; + Expr *pBase = sqlite3ExprSkipCollateAndLikely(pX); + if( ALWAYS(pBase!=0) + && pBase->op!=TK_AGG_COLUMN + && pBase->op!=TK_REGISTER + ){ + sqlite3ExprToRegister(pX, iAMem+j); + } + } } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); @@ -8297,9 +8482,9 @@ int sqlite3Select( ** and resets the aggregate accumulator registers in preparation ** for the next GROUP BY batch. */ - sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); VdbeComment((v, "output one row")); + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); @@ -8373,9 +8558,12 @@ int sqlite3Select( } } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { + /* Aggregate functions without GROUP BY. tag-select-0820 */ Table *pTab; if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ - /* If isSimpleCount() returns a pointer to a Table structure, then + /* tag-select-0821 + ** + ** If isSimpleCount() returns a pointer to a Table structure, then ** the SQL statement is of the form: ** ** SELECT count(*) FROM <tbl> @@ -8434,6 +8622,8 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ + /* The general case of an aggregate query without GROUP BY + ** tag-select-0822 */ int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; u16 distFlag = 0; @@ -8522,7 +8712,7 @@ int sqlite3Select( } /* If there is an ORDER BY clause, then we need to sort the results - ** and send them to the callback one by one. + ** and send them to the callback one by one. tag-select-0900 */ if( sSort.pOrderBy ){ assert( p->pEList==pEList ); @@ -8545,7 +8735,14 @@ select_end: assert( db->mallocFailed==0 || pParse->nErr!=0 ); sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG + /* Internal self-checks. tag-select-1000 */ if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } +#endif for(i=0; i<pAggInfo->nColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; if( pExpr==0 ) continue; diff --git a/src/shell.c.in b/src/shell.c.in index 33f2ea9a7..b8f932794 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -269,6 +269,9 @@ INCLUDE ../ext/consio/console_io.c * setOutputStream(FILE *pf) * This is normally the stream that CLI normal output goes to. * For the stand-alone CLI, it is stdout with no .output redirect. + * + * The ?putz(z) forms are required for the Fiddle builds for string literal + * output, in aid of enforcing format string to argument correspondence. */ # define sputz(s,z) fPutsUtf8(z,s) # define sputf fPrintfUtf8 @@ -281,12 +284,18 @@ INCLUDE ../ext/consio/console_io.c #else /* For Fiddle, all console handling and emit redirection is omitted. */ -# define sputz(fp,z) fputs(z,fp) +/* These next 3 macros are for emitting formatted output. When complaints + * from the WASM build are issued for non-formatted output, when a mere + * string literal is to be emitted, the ?putz(z) forms should be used. + * (This permits compile-time checking of format string / argument mismatch.) + */ +# define oputf(fmt, ...) printf(fmt,__VA_ARGS__) +# define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) # define sputf(fp,fmt, ...) fprintf(fp,fmt,__VA_ARGS__) +/* These next 3 macros are for emitting simple string literals. */ # define oputz(z) fputs(z,stdout) -# define oputf(fmt, ...) printf(fmt,__VA_ARGS__) # define eputz(z) fputs(z,stderr) -# define eputf(fmt, ...) fprintf(stderr,fmt,__VA_ARGS__) +# define sputz(fp,z) fputs(z,fp) # define oputb(buf,na) fwrite(buf,1,na,stdout) # undef fflush #endif @@ -529,6 +538,14 @@ static char *shell_strncpy(char *dest, const char *src, size_t n){ } /* +** strcpy() workalike to squelch an unwarranted link-time warning +** from OpenBSD. +*/ +static void shell_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} + +/* ** Optionally disable dynamic continuation prompt. ** Unless disabled, the continuation prompt shows open SQL lexemes if any, ** or open parentheses level if non-zero, or continuation prompt as set. @@ -593,7 +610,7 @@ static char *dynamicContinuePrompt(void){ size_t ncp = strlen(continuePrompt); size_t ndp = strlen(dynPrompt.zScannerAwaits); if( ndp > ncp-3 ) return continuePrompt; - strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); @@ -1194,6 +1211,7 @@ INCLUDE ../ext/misc/pcachetrace.c INCLUDE ../ext/misc/shathree.c INCLUDE ../ext/misc/uint.c INCLUDE ../ext/misc/decimal.c +INCLUDE ../ext/misc/percentile.c #undef sqlite3_base_init #define sqlite3_base_init sqlite3_base64_init INCLUDE ../ext/misc/base64.c @@ -1215,6 +1233,9 @@ INCLUDE ../ext/misc/sqlar.c #endif INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +INCLUDE ../ext/intck/sqlite3intck.h +INCLUDE ../ext/intck/sqlite3intck.c +INCLUDE ../ext/misc/stmtrand.c #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 @@ -2411,7 +2432,7 @@ static int shell_callback( case MODE_Explain: { static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; - static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; const int *aWidth = aExplainWidth; @@ -3421,7 +3442,13 @@ static void display_scanstats( if( pArg->scanstatsOn==3 ){ const char *zSql = " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," - " round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles" + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" " FROM bytecode(?)"; int rc = SQLITE_OK; @@ -3529,6 +3556,15 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ sqlite3_bind_double(pStmt, i, INFINITY); #endif + }else if( strncmp(zVar, "$int_", 5)==0 ){ + sqlite3_bind_int(pStmt, i, atoi(&zVar[5])); + }else if( strncmp(zVar, "$text_", 6)==0 ){ + size_t szVar = strlen(zVar); + char *zBuf = sqlite3_malloc64( szVar-5 ); + if( zBuf ){ + memcpy(zBuf, &zVar[6], szVar-5); + sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); + } }else{ sqlite3_bind_null(pStmt, i); } @@ -3767,6 +3803,7 @@ static void exec_prepared_stmt_columnar( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ) return; nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; nAlloc = nColumn*4; if( nAlloc<=0 ) nAlloc = 1; azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); @@ -3852,7 +3889,6 @@ static void exec_prepared_stmt_columnar( if( n>p->actualWidth[j] ) p->actualWidth[j] = n; } if( seenInterrupt ) goto columnar_end; - if( nColumn==0 ) goto columnar_end; switch( p->cMode ){ case MODE_Column: { colSep = " "; @@ -4244,6 +4280,7 @@ static int shell_exec( sqlite3_reset(pExplain); rc = sqlite3_stmt_explain(pExplain, 2); if( rc==SQLITE_OK ){ + bind_prepared_stmt(pArg, pExplain); while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); int iEqpId = sqlite3_column_int(pExplain, 0); @@ -4261,6 +4298,7 @@ static int shell_exec( if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; assert( sqlite3_stmt_isexplain(pExplain)==1 ); + bind_prepared_stmt(pArg, pExplain); explain_data_prepare(pArg, pExplain); exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); @@ -4735,6 +4773,7 @@ static const char *(azHelp[]) = { ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", + ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif @@ -5354,7 +5393,9 @@ static void open_db(ShellState *p, int openFlags){ #endif sqlite3_shathree_init(p->db, 0, 0); sqlite3_uint_init(p->db, 0, 0); + sqlite3_stmtrand_init(p->db, 0, 0); sqlite3_decimal_init(p->db, 0, 0); + sqlite3_percentile_init(p->db, 0, 0); sqlite3_base64_init(p->db, 0, 0); sqlite3_base85_init(p->db, 0, 0); sqlite3_regexp_init(p->db, 0, 0); @@ -5507,15 +5548,18 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){ #elif HAVE_LINENOISE /* -** Linenoise completion callback +** Linenoise completion callback. Note that the 3rd argument is from +** the "msteveb" version of linenoise, not the "antirez" version. */ -static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ +static void linenoise_completion(const char *zLine, linenoiseCompletions *lc, + void *pUserData){ i64 nLine = strlen(zLine); i64 i, iStart; sqlite3_stmt *pStmt = 0; char *zSql; char zBuf[1000]; + UNUSED_PARAMETER(pUserData); if( nLine>(i64)sizeof(zBuf)-30 ) return; if( zLine[0]=='.' || zLine[0]=='#') return; for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){} @@ -6312,11 +6356,16 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ #endif /* SQLITE_SHELL_HAVE_RECOVER */ /* +** Print the given string as an error message. +*/ +static void shellEmitError(const char *zErr){ + eputf("Error: %s\n", zErr); +} +/* ** Print the current sqlite3_errmsg() value to stderr and return 1. */ static int shellDatabaseError(sqlite3 *db){ - const char *zErr = sqlite3_errmsg(db); - eputf("Error: %s\n", zErr); + shellEmitError(sqlite3_errmsg(db)); return 1; } @@ -6861,7 +6910,7 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); - eputf("Error: %s\n", z); + shellEmitError(z); if( pAr->fromCmdLine ){ eputz("Use \"-A\" for more help\n"); }else{ @@ -7644,6 +7693,40 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } #endif /* SQLITE_SHELL_HAVE_RECOVER */ +/* +** Implementation of ".intck STEPS_PER_UNLOCK" command. +*/ +static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ + sqlite3_intck *p = 0; + int rc = SQLITE_OK; + + rc = sqlite3_intck_open(pState->db, "main", &p); + if( rc==SQLITE_OK ){ + i64 nStep = 0; + i64 nError = 0; + const char *zErr = 0; + while( SQLITE_OK==sqlite3_intck_step(p) ){ + const char *zMsg = sqlite3_intck_message(p); + if( zMsg ){ + oputf("%s\n", zMsg); + nError++; + } + nStep++; + if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){ + sqlite3_intck_unlock(p); + } + } + rc = sqlite3_intck_error(p, &zErr); + if( zErr ){ + eputf("%s\n", zErr); + } + sqlite3_intck_close(p); + + oputf("%lld steps, %lld errors\n", nStep, nError); + } + + return rc; +} /* * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it. @@ -7888,6 +7971,45 @@ static int outputDumpWarning(ShellState *p, const char *zLike){ } /* +** Fault-Simulator state and logic. +*/ +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; + +/* +** This is the fault-sim callback +*/ +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; + } + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + oputf("FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); + } + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + oputf("FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; + } + return faultsim_state.iErr; +} + +/* ** If an input line begins with "." then invoke this routine to ** process that line. ** @@ -8019,7 +8141,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb); if( pBackup==0 ){ - eputf("Error: %s\n", sqlite3_errmsg(pDest)); + shellDatabaseError(pDest); close_db(pDest); return 1; } @@ -8028,7 +8150,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc==SQLITE_DONE ){ rc = 0; }else{ - eputf("Error: %s\n", sqlite3_errmsg(pDest)); + shellDatabaseError(pDest); rc = 1; } close_db(pDest); @@ -8204,7 +8326,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); rc = 1; }else{ while( sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -8378,7 +8500,8 @@ static int do_meta_command(char *zLine, ShellState *p){ zSql = sqlite3_mprintf( "SELECT sql FROM sqlite_schema AS o " "WHERE (%s) AND sql NOT NULL" - " AND type IN ('index','trigger','view')", + " AND type IN ('index','trigger','view') " + "ORDER BY type COLLATE NOCASE DESC", zLike ); run_table_dump_query(p, zSql); @@ -8701,16 +8824,15 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* within this schema (may default to "main") */ + char *zSchema = 0; /* Schema of zTable */ char *zFile = 0; /* Name of file to extra content from */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ - int nByte; /* Number of bytes in an SQL string */ + i64 nByte; /* Number of bytes in an SQL string */ int i, j; /* Loop counters */ int needCommit; /* True to COMMIT or ROLLBACK at end */ int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql; /* An SQL statement */ - char *zFullTabName; /* Table name with schema if applicable */ + char *zSql = 0; /* An SQL statement */ ImportCtx sCtx; /* Reader context */ char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ int eVerbose = 0; /* Larger for more console output */ @@ -8844,24 +8966,14 @@ static int do_meta_command(char *zLine, ShellState *p){ while( (nSkip--)>0 ){ while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} } - if( zSchema!=0 ){ - zFullTabName = sqlite3_mprintf("\"%w\".\"%w\"", zSchema, zTable); - }else{ - zFullTabName = sqlite3_mprintf("\"%w\"", zTable); - } - zSql = sqlite3_mprintf("SELECT * FROM %s", zFullTabName); - if( zSql==0 || zFullTabName==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - nByte = strlen30(zSql); - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) ){ + /* Table does not exist. Create it. */ sqlite3 *dbCols = 0; char *zRenames = 0; char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE %s", zFullTabName); + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); while( xRead(&sCtx) ){ zAutoColumn(sCtx.z, &dbCols, 0); if( sCtx.cTerm!=sCtx.cColSep ) break; @@ -8876,43 +8988,69 @@ static int do_meta_command(char *zLine, ShellState *p){ assert(dbCols==0); if( zColDefs==0 ){ eputf("%s: empty file\n", sCtx.zFile); - import_fail: - sqlite3_free(zCreate); - sqlite3_free(zSql); - sqlite3_free(zFullTabName); import_cleanup(&sCtx); rc = 1; + sqlite3_free(zCreate); goto meta_command_exit; } zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } if( eVerbose>=1 ){ oputf("%s\n", zCreate); } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + zCreate = 0; if( rc ){ eputf("%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zCreate); - zCreate = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ if (pStmt) sqlite3_finalize(pStmt); - eputf("Error: %s\n", sqlite3_errmsg(p->db)); - goto import_fail; + shellDatabaseError(p->db); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; } - sqlite3_free(zSql); - nCol = sqlite3_column_count(pStmt); sqlite3_finalize(pStmt); pStmt = 0; if( nCol==0 ) return 0; /* no columns, no error */ - zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 ); + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); if( zSql==0 ){ import_cleanup(&sCtx); shell_out_of_memory(); } - sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zFullTabName); + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } j = strlen30(zSql); for(i=1; i<nCol; i++){ zSql[j++] = ','; @@ -8920,17 +9058,20 @@ static int do_meta_command(char *zLine, ShellState *p){ } zSql[j++] = ')'; zSql[j] = 0; + assert( j<nByte ); if( eVerbose>=2 ){ oputf("Insert using: %s\n", zSql); } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; if( rc ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); if (pStmt) sqlite3_finalize(pStmt); - goto import_fail; + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; } - sqlite3_free(zSql); - sqlite3_free(zFullTabName); needCommit = sqlite3_get_autocommit(p->db); if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); do{ @@ -9101,6 +9242,21 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ + i64 iArg = 0; + if( nArg==2 ){ + iArg = integerValue(azArg[1]); + if( iArg==0 ) iArg = -1; + } + if( (nArg!=1 && nArg!=2) || iArg<0 ){ + eputf("%s","Usage: .intck STEPS_PER_UNLOCK\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + rc = intckDatabaseCmd(p, iArg); + }else + #ifdef SQLITE_ENABLE_IOTRACE if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); @@ -9204,7 +9360,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); if( rc!=SQLITE_OK ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); sqlite3_free(zErrMsg); rc = 1; } @@ -9826,7 +9982,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main"); if( pBackup==0 ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); close_db(pSrc); return 1; } @@ -9844,7 +10000,7 @@ static int do_meta_command(char *zLine, ShellState *p){ eputz("Error: source database is busy\n"); rc = 1; }else{ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); rc = 1; } close_db(pSrc); @@ -9941,7 +10097,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", -1, &pStmt, 0); if( rc ){ - eputf("Error: %s\n", sqlite3_errmsg(p->db)); + shellDatabaseError(p->db); sqlite3_finalize(pStmt); rc = 1; goto meta_command_exit; @@ -10010,7 +10166,7 @@ static int do_meta_command(char *zLine, ShellState *p){ freeText(&sSelect); } if( zErrMsg ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); sqlite3_free(zErrMsg); rc = 1; }else if( rc != SQLITE_OK ){ @@ -10774,18 +10930,18 @@ static int do_meta_command(char *zLine, ShellState *p){ /*{"bitvec_test", SQLITE_TESTCTRL_BITVEC_TEST, 1, "" },*/ {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, - /*{"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ + {"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..." }, {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, {"json_selfcheck", SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN" }, {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, {"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" }, - {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" }, + {"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK ..."}, #ifdef YYCOVERAGE {"parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE,0,"" }, #endif - {"pending_byte", SQLITE_TESTCTRL_PENDING_BYTE,0, "OFFSET " }, + {"pending_byte", SQLITE_TESTCTRL_PENDING_BYTE,1, "OFFSET " }, {"prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE,0, "" }, {"prng_save", SQLITE_TESTCTRL_PRNG_SAVE, 0, "" }, {"prng_seed", SQLITE_TESTCTRL_PRNG_SEED, 0, "SEED ?db?" }, @@ -10845,8 +11001,118 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ switch(testctrl){ + /* Special processing for .testctrl opt MASK ... + ** Each MASK argument can be one of: + ** + ** +LABEL Enable the named optimization + ** + ** -LABEL Disable the named optimization + ** + ** INTEGER Mask of optimizations to disable + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + static const struct { + unsigned int mask; /* Mask for this optimization */ + unsigned int bDsply; /* Display this on output */ + const char *zLabel; /* Name of optimization */ + } aLabel[] = { + { 0x00000001, 1, "QueryFlattener" }, + { 0x00000001, 0, "Flatten" }, + { 0x00000002, 1, "WindowFunc" }, + { 0x00000004, 1, "GroupByOrder" }, + { 0x00000008, 1, "FactorOutConst" }, + { 0x00000010, 1, "DistinctOpt" }, + { 0x00000020, 1, "CoverIdxScan" }, + { 0x00000040, 1, "OrderByIdxJoin" }, + { 0x00000080, 1, "Transitive" }, + { 0x00000100, 1, "OmitNoopJoin" }, + { 0x00000200, 1, "CountOfView" }, + { 0x00000400, 1, "CurosrHints" }, + { 0x00000800, 1, "Stat4" }, + { 0x00001000, 1, "PushDown" }, + { 0x00002000, 1, "SimplifyJoin" }, + { 0x00004000, 1, "SkipScan" }, + { 0x00008000, 1, "PropagateConst" }, + { 0x00010000, 1, "MinMaxOpt" }, + { 0x00020000, 1, "SeekScan" }, + { 0x00040000, 1, "OmitOrderBy" }, + { 0x00080000, 1, "BloomFilter" }, + { 0x00100000, 1, "BloomPulldown" }, + { 0x00200000, 1, "BalancedMerge" }, + { 0x00400000, 1, "ReleaseReg" }, + { 0x00800000, 1, "FlttnUnionAll" }, + { 0x01000000, 1, "IndexedEXpr" }, + { 0x02000000, 1, "Coroutines" }, + { 0x04000000, 1, "NullUnusedCols" }, + { 0x08000000, 1, "OnePass" }, + { 0x10000000, 1, "OrderBySubq" }, + { 0xffffffff, 0, "All" }, + }; + unsigned int curOpt; + unsigned int newOpt; + int ii; + sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, p->db, &curOpt); + newOpt = curOpt; + for(ii=2; ii<nArg; ii++){ + const char *z = azArg[ii]; + int useLabel = 0; + const char *zLabel = 0; + if( (z[0]=='+'|| z[0]=='-') && !IsDigit(z[1]) ){ + useLabel = z[0]; + zLabel = &z[1]; + }else if( !IsDigit(z[0]) && z[0]!=0 && !IsDigit(z[1]) ){ + useLabel = '+'; + zLabel = z; + }else{ + newOpt = (unsigned int)strtol(z,0,0); + } + if( useLabel ){ + int jj; + for(jj=0; jj<ArraySize(aLabel); jj++){ + if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break; + } + if( jj>=ArraySize(aLabel) ){ + eputf("Error: no such optimization: \"%s\"\n", zLabel); + eputz("Should be one of:"); + for(jj=0; jj<ArraySize(aLabel); jj++){ + eputf(" %s", aLabel[jj].zLabel); + } + eputz("\n"); + rc = 1; + goto meta_command_exit; + } + if( useLabel=='+' ){ + newOpt &= ~aLabel[jj].mask; + }else{ + newOpt |= aLabel[jj].mask; + } + } + } + if( curOpt!=newOpt ){ + sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,p->db,newOpt); + }else if( nArg<3 ){ + curOpt = ~newOpt; + } + if( newOpt==0 ){ + oputz("+All\n"); + }else if( newOpt==0xffffffff ){ + oputz("-All\n"); + }else{ + int jj; + for(jj=0; jj<ArraySize(aLabel); jj++){ + unsigned int m = aLabel[jj].mask; + if( !aLabel[jj].bDsply ) continue; + if( (curOpt&m)!=(newOpt&m) ){ + oputf("%c%s\n", (newOpt & m)==0 ? '+' : '-', + aLabel[jj].zLabel); + } + } + } + rc2 = isOk = 3; + break; + } + /* sqlite3_test_control(int, db, int) */ - case SQLITE_TESTCTRL_OPTIMIZATIONS: case SQLITE_TESTCTRL_FK_NO_ACTION: if( nArg==3 ){ unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0); @@ -11007,6 +11273,76 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_test_control(testctrl, &rc2); break; + case SQLITE_TESTCTRL_FAULT_INSTALL: { + int kk; + int bShowHelp = nArg<=2; + isOk = 3; + for(kk=2; kk<nArg; kk++){ + const char *z = azArg[kk]; + if( z[0]=='-' && z[1]=='-' ) z++; + if( cli_strcmp(z,"off")==0 ){ + sqlite3_test_control(testctrl, 0); + }else if( cli_strcmp(z,"on")==0 ){ + faultsim_state.iCnt = faultsim_state.nSkip; + if( faultsim_state.iErr==0 ) faultsim_state.iErr = 1; + faultsim_state.nHit = 0; + sqlite3_test_control(testctrl, faultsim_callback); + }else if( cli_strcmp(z,"reset")==0 ){ + faultsim_state.iCnt = faultsim_state.nSkip; + faultsim_state.nHit = 0; + sqlite3_test_control(testctrl, faultsim_callback); + }else if( cli_strcmp(z,"status")==0 ){ + oputf("faultsim.iId: %d\n", faultsim_state.iId); + oputf("faultsim.iErr: %d\n", faultsim_state.iErr); + oputf("faultsim.iCnt: %d\n", faultsim_state.iCnt); + oputf("faultsim.nHit: %d\n", faultsim_state.nHit); + oputf("faultsim.iInterval: %d\n", faultsim_state.iInterval); + oputf("faultsim.eVerbose: %d\n", faultsim_state.eVerbose); + oputf("faultsim.nRepeat: %d\n", faultsim_state.nRepeat); + oputf("faultsim.nSkip: %d\n", faultsim_state.nSkip); + }else if( cli_strcmp(z,"-v")==0 ){ + if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++; + }else if( cli_strcmp(z,"-q")==0 ){ + if( faultsim_state.eVerbose>0 ) faultsim_state.eVerbose--; + }else if( cli_strcmp(z,"-id")==0 && kk+1<nArg ){ + faultsim_state.iId = atoi(azArg[++kk]); + }else if( cli_strcmp(z,"-errcode")==0 && kk+1<nArg ){ + faultsim_state.iErr = atoi(azArg[++kk]); + }else if( cli_strcmp(z,"-interval")==0 && kk+1<nArg ){ + faultsim_state.iInterval = atoi(azArg[++kk]); + }else if( cli_strcmp(z,"-repeat")==0 && kk+1<nArg ){ + faultsim_state.nRepeat = atoi(azArg[++kk]); + }else if( cli_strcmp(z,"-skip")==0 && kk+1<nArg ){ + faultsim_state.nSkip = atoi(azArg[++kk]); + }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){ + bShowHelp = 1; + }else{ + eputf("Unrecognized fault_install argument: \"%s\"\n", + azArg[kk]); + rc = 1; + bShowHelp = 1; + break; + } + } + if( bShowHelp ){ + oputz( + "Usage: .testctrl fault_install ARGS\n" + "Possible arguments:\n" + " off Disable faultsim\n" + " on Activate faultsim\n" + " reset Reset the trigger counter\n" + " status Show current status\n" + " -v Increase verbosity\n" + " -q Decrease verbosity\n" + " --errcode N When triggered, return N as error code\n" + " --id ID Trigger only for the ID specified\n" + " --interval N Trigger only after every N-th call\n" + " --repeat N Turn off after N hits. 0 means never\n" + " --skip N Skip the first N encounters\n" + ); + } + break; + } } } if( isOk==0 && iCtrl>=0 ){ @@ -11427,61 +11763,72 @@ static int line_is_complete(char *zSql, int nSql){ ** ** 0: Have not seen any SQL. ** 1: Have seen "PRAGMA foreign_keys=OFF;". -** 2: Currently assuming we are parsing ".dump" restore, defensive mode -** should be disabled following the current transaction. -** 3: Nothing left to do. +** 2-6: Currently running .dump transaction. If the "2" bit is set, +** disable DEFENSIVE when done. If "4" is set, disable DQS_DDL. +** 7: Nothing left to do. This function becomes a no-op. */ static int doAutoDetectRestore(ShellState *p, const char *zSql){ int rc = SQLITE_OK; - switch( p->eRestoreState ){ - case 0: { - int bDefense = 0; /* True if in defensive mode */ - const char *zExpect = "PRAGMA foreign_keys=OFF;"; - assert( strlen(zExpect)==24 ); - sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense); - if( p->bSafeMode==0 && bDefense && memcmp(zSql, zExpect, 25)==0 ){ - p->eRestoreState = 1; - }else{ - p->eRestoreState = 3; - } - break; - }; - - case 1: { - const char *zExpect = "BEGIN TRANSACTION;"; - assert( strlen(zExpect)==18 ); - if( memcmp(zSql, zExpect, 19)==0 ){ - /* Now check if the database is empty. */ - const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1"; - sqlite3_stmt *pStmt = 0; - int bEmpty = 1; - - shellPrepare(p->db, &rc, zQuery, &pStmt); - if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - bEmpty = 0; + if( p->eRestoreState<7 ){ + switch( p->eRestoreState ){ + case 0: { + const char *zExpect = "PRAGMA foreign_keys=OFF;"; + assert( strlen(zExpect)==24 ); + if( p->bSafeMode==0 + && strlen(zSql)>=24 + && memcmp(zSql, zExpect, 25)==0 + ){ + p->eRestoreState = 1; + }else{ + p->eRestoreState = 7; } - shellFinalize(&rc, pStmt); - if( bEmpty && rc==SQLITE_OK ){ + break; + }; + + case 1: { + int bIsDump = 0; + const char *zExpect = "BEGIN TRANSACTION;"; + assert( strlen(zExpect)==18 ); + if( memcmp(zSql, zExpect, 19)==0 ){ + /* Now check if the database is empty. */ + const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1"; + sqlite3_stmt *pStmt = 0; + + bIsDump = 1; + shellPrepare(p->db, &rc, zQuery, &pStmt); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + bIsDump = 0; + } + shellFinalize(&rc, pStmt); + } + if( bIsDump && rc==SQLITE_OK ){ + int bDefense = 0; + int bDqsDdl = 0; + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, -1, &bDqsDdl); sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 1, 0); + p->eRestoreState = (bDefense ? 2 : 0) + (bDqsDdl ? 4 : 0); }else{ - p->eRestoreState = 3; + p->eRestoreState = 7; } + break; } - break; - } - - case 2: { - if( sqlite3_get_autocommit(p->db) ){ - sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); - p->eRestoreState = 3; + + default: { + if( sqlite3_get_autocommit(p->db) ){ + if( (p->eRestoreState & 2) ){ + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0); + } + if( (p->eRestoreState & 4) ){ + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 0, 0); + } + p->eRestoreState = 7; + } + break; } - break; } - - default: /* Nothing to do */ - assert( p->eRestoreState==3 ); - break; } return rc; @@ -11767,27 +12114,29 @@ static char *find_home_dir(int clearFlag){ /* ** On non-Windows platforms, look for $XDG_CONFIG_HOME. ** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return -** the path to it, else return 0. The result is cached for -** subsequent calls. +** the path to it. If there is no $(XDG_CONFIG_HOME) then +** look for $(HOME)/.config/sqlite3/sqliterc and if found +** return that. If none of these are found, return 0. +** +** The string returned is obtained from sqlite3_malloc() and +** should be freed by the caller. */ -static const char *find_xdg_config(void){ +static char *find_xdg_config(void){ #if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \ || defined(__RTP__) || defined(_WRS_KERNEL) return 0; #else - static int alreadyTried = 0; - static char *zConfig = 0; + char *zConfig = 0; const char *zXdgHome; - if( alreadyTried!=0 ){ - return zConfig; - } - alreadyTried = 1; zXdgHome = getenv("XDG_CONFIG_HOME"); if( zXdgHome==0 ){ - return 0; + const char *zHome = getenv("HOME"); + if( zHome==0 ) return 0; + zConfig = sqlite3_mprintf("%s/.config/sqlite3/sqliterc", zHome); + }else{ + zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); } - zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome); shell_check_oom(zConfig); if( access(zConfig,0)!=0 ){ sqlite3_free(zConfig); @@ -11815,7 +12164,7 @@ static void process_sqliterc( int savedLineno = p->lineno; if( sqliterc == NULL ){ - sqliterc = find_xdg_config(); + sqliterc = zBuf = find_xdg_config(); } if( sqliterc == NULL ){ home_dir = find_home_dir(0); @@ -11888,6 +12237,7 @@ static const char zOptions[] = " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" " -nonce STRING set the safe-mode escape nonce\n" + " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -pcachetrace trace all page cache operations\n" @@ -11920,7 +12270,7 @@ static void usage(int showDetail){ }else{ eputz("Use the -help option for additional information\n"); } - exit(1); + exit(0); } /* @@ -12178,6 +12528,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 0; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + int val = 0; + sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW, &val); + assert( val==0 ); }else if( cli_strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) const char *zSize; @@ -12453,6 +12807,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ /* already handled */ + }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ + /* already handled */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ @@ -12499,7 +12855,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ open_db(&data, 0); rc = shell_exec(&data, z, &zErrMsg); if( zErrMsg!=0 ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); if( bail_on_error ) return rc!=0 ? rc : 1; }else if( rc!=0 ){ eputf("Error: unable to process SQL \"%s\"\n", z); @@ -12544,8 +12900,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( azCmd[i][0]=='.' ){ rc = do_meta_command(azCmd[i], &data); if( rc ){ - free(azCmd); - return rc==2 ? 0 : rc; + if( rc==2 ) rc = 0; + goto shell_main_exit; } }else{ open_db(&data, 0); @@ -12553,13 +12909,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg || rc ){ if( zErrMsg!=0 ){ - eputf("Error: %s\n", zErrMsg); + shellEmitError(zErrMsg); }else{ eputf("Error: unable to process SQL: %s\n", azCmd[i]); } sqlite3_free(zErrMsg); - free(azCmd); - return rc!=0 ? rc : 1; + if( rc==0 ) rc = 1; + goto shell_main_exit; } } } @@ -12597,7 +12953,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if HAVE_READLINE || HAVE_EDITLINE rl_attempted_completion_function = readline_completion; #elif HAVE_LINENOISE - linenoiseSetCompletionCallback(linenoise_completion); + linenoiseSetCompletionCallback(linenoise_completion, NULL); #endif data.in = 0; rc = process_input(&data); @@ -12614,6 +12970,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( data.expert.pExpert ){ + expertFinish(&data, 1, 0); + } +#endif + shell_main_exit: free(azCmd); set_table_name(&data, 0); if( data.db ){ @@ -12646,7 +13008,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif -#endif /* !SQLITE_SHELL_FIDDLE */ +#else /* SQLITE_SHELL_FIDDLE... */ + shell_main_exit: +#endif return rc; } @@ -12680,7 +13044,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - printf("fiddle_db_arg(%p)\n", (const void*)arg); + oputf("fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -12706,12 +13070,22 @@ const char * fiddle_db_filename(const char * zDbName){ /* ** Completely wipes out the contents of the currently-opened database -** but leaves its storage intact for reuse. +** but leaves its storage intact for reuse. If any transactions are +** active, they are forcibly rolled back. */ void fiddle_reset_db(void){ if( globalDb ){ - int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); - if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); + int rc; + while( sqlite3_txn_state(globalDb,0)>0 ){ + /* + ** Resolve problem reported in + ** https://sqlite.org/forum/forumpost/0b41a25d65 + */ + oputz("Rolling back in-progress transaction.\n"); + sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); + } + rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) sqlite3_exec(globalDb, "VACUUM", 0, 0, 0); sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 4a19fe918..810ccecc9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -420,6 +420,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. ** <li> The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +** <li> The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** </ul> */ int sqlite3_exec( @@ -762,16 +764,16 @@ struct sqlite3_file { ** </ul> ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, -** PENDING, or EXCLUSIVE lock on the file. It returns true -** if such a lock exists and false otherwise. +** PENDING, or EXCLUSIVE lock on the file. It returns, via its output +** pointer parameter, true if such a lock exists and false otherwise. ** ** The xFileControl() method is a generic interface that allows custom ** VFS implementations to directly control an open file using the @@ -2141,6 +2143,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +** <dt>SQLITE_CONFIG_ROWID_IN_VIEW +** <dd>The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** </dl> */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2172,6 +2190,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -3286,8 +3305,8 @@ int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -3551,8 +3570,8 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ** [[OPEN_EXRESCODE]] ^(<dt>[SQLITE_OPEN_EXRESCODE]</dt> ** <dd>The database connection comes up in "extended result code mode". -** In other words, the database behaves has if -** [sqlite3_extended_result_codes(db,1)] where called on the database +** In other words, the database behaves as if +** [sqlite3_extended_result_codes(db,1)] were called on the database ** connection as soon as the connection is created. In addition to setting ** the extended result code mode, this flag also causes [sqlite3_open_v2()] ** to return an extended result code.</dd> @@ -5801,7 +5820,7 @@ int sqlite3_value_encoding(sqlite3_value*); ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. ** -** Every [application-defined SQL function] that invoke this interface +** Every [application-defined SQL function] that invokes this interface ** should include the [SQLITE_SUBTYPE] property in the text ** encoding argument when the function is [sqlite3_create_function|registered]. ** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() @@ -6868,6 +6887,12 @@ int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -7402,9 +7427,11 @@ struct sqlite3_module { ** will be returned by the strategy. ** ** The xBestIndex method may optionally populate the idxFlags field with a -** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - -** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite -** assumes that the strategy may visit at most one row. +** mask of SQLITE_INDEX_SCAN_* flags. One such flag is +** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN] +** output to show the idxNum has hex instead of as decimal. Another flag is +** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will +** return at most one row. ** ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then ** SQLite also assumes that if a call to the xUpdate() method is made as @@ -7468,7 +7495,9 @@ struct sqlite3_index_info { ** [sqlite3_index_info].idxFlags field to some combination of ** these bits. */ -#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ +#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */ +#define SQLITE_INDEX_SCAN_HEX 0x00000002 /* Display idxNum as hex */ + /* in EXPLAIN QUERY PLAN */ /* ** CAPI3REF: Virtual Table Constraint Operator Codes @@ -8305,6 +8334,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_GETOPT 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ #define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 @@ -8338,7 +8368,7 @@ int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -9917,24 +9947,45 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** <li value="2"><p> ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. ** <li value="3"><p> -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** </ol> ** +** <p>The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** <table border=1 cellspacing=0 cellpadding=10 width="90%"> +** <tr> +** <td valign="top">sqlite3_vtab_distinct() return value +** <td valign="top">Rows are returned in aOrderBy order +** <td valign="top">Rows with the same value in all aOrderBy columns are adjacent +** <td valign="top">Duplicates over all colUsed columns may be omitted +** <tr><td>0<td>yes<td>yes<td>no +** <tr><td>1<td>no<td>yes<td>no +** <tr><td>2<td>no<td>yes<td>yes +** <tr><td>3<td>yes<td>yes<td>yes +** </table> +** ** ^For the purposes of comparing virtual table output values to see if the ** values are same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" @@ -10786,8 +10837,6 @@ int sqlite3_deserialize( #if defined(__wasi__) # undef SQLITE_WASI # define SQLITE_WASI 1 -# undef SQLITE_OMIT_WAL -# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */ # ifndef SQLITE_OMIT_LOAD_EXTENSION # define SQLITE_OMIT_LOAD_EXTENSION # endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 79a36e060..d1ffce6d8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -143,10 +143,13 @@ /* ** Macro to disable warnings about missing "break" at the end of a "case". */ -#if GCC_VERSION>=7000000 -# define deliberate_fall_through __attribute__((fallthrough)); -#else -# define deliberate_fall_through +#if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through #endif /* @@ -609,6 +612,8 @@ # define SQLITE_OMIT_ALTERTABLE #endif +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -886,7 +891,7 @@ typedef INT16_TYPE LogEst; # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ - (defined(__APPLE__) && defined(__POWERPC__)) || \ + (defined(__APPLE__) && defined(__ppc__)) || \ (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else @@ -1123,6 +1128,7 @@ extern u32 sqlite3TreeTrace; ** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated +** 0x00080000 NOT NULL strength reduction */ /* @@ -1153,7 +1159,7 @@ extern u32 sqlite3WhereTrace; ** 0x00000010 Display sqlite3_index_info xBestIndex calls ** 0x00000020 Range an equality scan metrics ** 0x00000040 IN operator decisions -** 0x00000080 WhereLoop cost adjustements +** 0x00000080 WhereLoop cost adjustments ** 0x00000100 ** 0x00000200 Covering index decisions ** 0x00000400 OR optimization @@ -1329,6 +1335,7 @@ typedef struct Savepoint Savepoint; typedef struct Select Select; typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; +typedef struct Subquery Subquery; typedef struct SrcItem SrcItem; typedef struct SrcList SrcList; typedef struct sqlite3_str StrAccum; /* Internal alias for sqlite3_str */ @@ -1596,6 +1603,10 @@ struct FuncDefHash { }; #define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) +#if defined(SQLITE_USER_AUTHENTICATION) +# warning "The SQLITE_USER_AUTHENTICATION extension is deprecated. \ + See ext/userauth/user-auth.txt for details." +#endif #ifdef SQLITE_USER_AUTHENTICATION /* ** Information held in the "sqlite3" database connection object and used @@ -1899,7 +1910,7 @@ struct sqlite3 { #define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ #define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ #define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ @@ -1917,6 +1928,7 @@ struct sqlite3 { #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ +#define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -2472,8 +2484,7 @@ struct Table { #define TF_HasStored 0x00000040 /* Has one or more STORED columns */ #define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ #define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ -#define TF_StatsUsed 0x00000100 /* Query planner decisions affected by - ** Index.aiRowLogEst[] values */ +#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */ #define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ #define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ @@ -2529,6 +2540,15 @@ struct Table { #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -2896,9 +2916,15 @@ struct AggInfo { ** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. ** The assert()s that are part of this macro verify that constraint. */ +#ifndef NDEBUG #define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) #define AggInfoFuncReg(A,I) \ (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) +#else +#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + ((A)->iFirstReg+(A)->nColumn+(I)) +#endif /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. @@ -3251,6 +3277,16 @@ struct IdList { #define EU4_EXPR 2 /* Uses IdList.a.u4.pExpr -- NOT CURRENTLY USED */ /* +** Details of the implementation of a subquery. +*/ +struct Subquery { + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to initialize a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ +}; + +/* ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. ** @@ -3262,27 +3298,40 @@ struct IdList { ** In the colUsed field, the high-order bit (bit 63) is set if the table ** contains more than 63 columns and the 64-th or later column is used. ** -** Union member validity: +** Aggressive use of "union" helps keep the size of the object small. This +** has been shown to boost performance, in addition to saving memory. +** Access to union elements is gated by the following rules which should +** always be checked, either by an if-statement or by an assert(). ** -** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc -** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy -** u2.pIBIndex fg.isIndexedBy && !fg.isCte -** u2.pCteUse fg.isCte && !fg.isIndexedBy +** Field Only access if this is true +** --------------- ----------------------------------- +** u1.zIndexedBy fg.isIndexedBy +** u1.pFuncArg fg.isTabFunc +** u1.nRow !fg.isTabFunc && !fg.isIndexedBy +** +** u2.pIBIndex fg.isIndexedBy +** u2.pCteUse fg.isCte +** +** u3.pOn !fg.isUsing +** u3.pUsing fg.isUsing +** +** u4.zDatabase !fg.fixedSchema && !fg.isSubquery +** u4.pSchema fg.fixedSchema +** u4.pSubq fg.isSubquery +** +** See also the sqlite3SrcListDelete() routine for assert() statements that +** check invariants on the fields of this object, especially the flags +** inside the fg struct. */ struct SrcItem { - Schema *pSchema; /* Schema to which this item is fixed */ - char *zDatabase; /* Name of database holding this table */ char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ - Table *pTab; /* An SQL table corresponding to zName */ - Select *pSelect; /* A SELECT statement used in place of a table name */ - int addrFillSub; /* Address of subroutine to manifest a subquery */ - int regReturn; /* Register holding return address of addrFillSub */ - int regResult; /* Registers holding results of a co-routine */ + Table *pSTab; /* Table object for zName. Mnemonic: Srcitem-TABle */ struct { u8 jointype; /* Type of join between this table and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isSubquery :1; /* True if this term is a subquery */ unsigned isTabFunc :1; /* True if table-valued-function syntax */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned isMaterialized:1; /* This is a materialized view */ @@ -3295,21 +3344,30 @@ struct SrcItem { unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ + unsigned rowidUsed :1; /* The ROWID of this table is referenced */ + unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ + unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ - union { - Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ - IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ - } u3; Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY <zIndex>" clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ + u32 nRow; /* Number of rows in a VALUES clause */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; + union { + Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ + IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ + } u3; + union { + Schema *pSchema; /* Schema to which this item is fixed */ + char *zDatabase; /* Name of database holding this table */ + Subquery *pSubq; /* Description of a subquery */ + } u4; }; /* @@ -3369,7 +3427,7 @@ struct SrcList { #define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ #define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ - /* 0x2000 not currently used */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ @@ -3441,13 +3499,14 @@ struct NameContext { #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ #define NC_UBaseReg 0x000400 /* True if uNC.iBaseReg is used */ #define NC_MinMaxAgg 0x001000 /* min/max aggregates seen. See note above */ -#define NC_Complex 0x002000 /* True if a function or subquery seen */ +/* 0x002000 // available for reuse */ #define NC_AllowWin 0x004000 /* Window functions are allowed here */ #define NC_HasWin 0x008000 /* One or more window functions seen */ #define NC_IsDDL 0x010000 /* Resolving names in a CREATE statement */ #define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ #define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ #define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */ #define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ /* @@ -3471,6 +3530,7 @@ struct Upsert { Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + u8 isDup; /* True if 2nd or later with same pUpsertIdx */ /* Above this point is the parse tree for the ON CONFLICT clauses. ** The next group of fields stores intermediate data. */ void *pToFree; /* Free memory when deleting the Upsert object */ @@ -3560,14 +3620,17 @@ struct Select { #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ -#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ -/* True if S exists and has SF_NestedFrom */ -#define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) +/* True if SrcItem X is a subquery that has SF_NestedFrom */ +#define IsNestedFrom(X) \ + ((X)->fg.isSubquery && \ + ((X)->u4.pSubq->pSelect->selFlags&SF_NestedFrom)!=0) /* ** The results of a SELECT can be distributed in several ways, as defined @@ -3597,7 +3660,11 @@ struct Select { ** SRT_Set The result must be a single column. Store each ** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". +** results. if pDest->iSDParm2 is positive, then it is +** a regsiter holding a Bloom filter for the IN operator +** that should be populated in addition to the +** pDest->iSDParm table. This SRT is used to +** implement "IN (SELECT ...)". ** ** SRT_EphemTab Create an temporary table pDest->iSDParm and store ** the result there. The cursor is left open after @@ -3804,6 +3871,8 @@ struct Parse { u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasWith; /* True if statement contains WITH */ + u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -4099,7 +4168,7 @@ struct Returning { }; /* -** An objected used to accumulate the text of a string where we +** An object used to accumulate the text of a string where we ** do not necessarily know how big the string will be in the end. */ struct sqlite3_str { @@ -4113,7 +4182,7 @@ struct sqlite3_str { }; #define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ #define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ -#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ +#define SQLITE_PRINTF_MALLOCED 0x04 /* True if zText is allocated space */ #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) @@ -4242,6 +4311,11 @@ struct Sqlite3Config { #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ +#endif int bLocaltimeFault; /* True to fail localtime() calls */ int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ int iOnceResetThreshold; /* When to reset OP_Once counters */ @@ -4478,6 +4552,9 @@ struct Window { ** due to the SQLITE_SUBTYPE flag */ }; +Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow); +void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal); + #ifndef SQLITE_OMIT_WINDOWFUNC void sqlite3WindowDelete(sqlite3*, Window*); void sqlite3WindowUnlinkFromSelect(Window*); @@ -4697,10 +4774,13 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex*); # define EXP754 (((u64)0x7ff)<<52) # define MAN754 ((((u64)1)<<52)-1) # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) int sqlite3IsNaN(double); + int sqlite3IsOverflow(double); #else -# define IsNaN(X) 0 -# define sqlite3IsNaN(X) 0 +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* @@ -4792,6 +4872,7 @@ int sqlite3ErrorToParser(sqlite3*,int); void sqlite3Dequote(char*); void sqlite3DequoteExpr(Expr*); void sqlite3DequoteToken(Token*); +void sqlite3DequoteNumber(Parse*, Expr*); void sqlite3TokenInit(Token*,char*); int sqlite3KeywordCode(const unsigned char*, int); int sqlite3RunParser(Parse*, const char*); @@ -4822,7 +4903,7 @@ void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*); void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); void sqlite3ExprDelete(sqlite3*, Expr*); void sqlite3ExprDeleteGeneric(sqlite3*,void*); -void sqlite3ExprDeferredDelete(Parse*, Expr*); +int sqlite3ExprDeferredDelete(Parse*, Expr*); void sqlite3ExprUnmapAndDelete(Parse*, Expr*); ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); @@ -4941,6 +5022,9 @@ int sqlite3IdListIndex(IdList*,const char*); SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2); SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); +void sqlite3SubqueryDelete(sqlite3*,Subquery*); +Select *sqlite3SubqueryDetach(sqlite3*,SrcItem*); +int sqlite3SrcItemAttachSubquery(Parse*, SrcItem*, Select*, int); SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, OnOrUsing*); void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); @@ -4990,6 +5074,7 @@ void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); +void sqlite3ExprToRegister(Expr *pExpr, int iReg); void sqlite3ExprCode(Parse*, Expr*, int); #ifndef SQLITE_OMIT_GENERATED_COLUMNS void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int); @@ -5045,16 +5130,14 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*); u32 sqlite3IsTrueOrFalse(const char*); int sqlite3ExprIdToTrueFalse(Expr*); int sqlite3ExprTruthValue(const Expr*); -int sqlite3ExprIsConstant(Expr*); -int sqlite3ExprIsConstantNotJoin(Expr*); +int sqlite3ExprIsConstant(Parse*,Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -int sqlite3ExprIsTableConstant(Expr*,int); -int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); +int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS int sqlite3ExprContainsSubquery(Expr*); #endif -int sqlite3ExprIsInteger(const Expr*, int*); +int sqlite3ExprIsInteger(const Expr*, int*, Parse*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); @@ -5235,7 +5318,9 @@ void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); void sqlite3Error(sqlite3*,int); void sqlite3ErrorClear(sqlite3*); void sqlite3SystemError(sqlite3*,int); +#if !defined(SQLITE_OMIT_BLOB_LITERAL) void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +#endif u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -5546,7 +5631,7 @@ const char *sqlite3JournalModename(int); Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); void sqlite3UpsertDelete(sqlite3*,Upsert*); Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); - int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); + int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*); void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); int sqlite3UpsertNextIsIPK(Upsert*); diff --git a/src/tclsqlite.c b/src/tclsqlite.c index d91b2fa3f..0c8888fd4 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -35,14 +35,23 @@ # include "msvc.h" #endif +/****** Copy of tclsqlite.h ******/ #if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" +# include "sqlite_tcl.h" /* Special case for Windows using STDCALL */ #else -# include "tcl.h" +# include <tcl.h> /* All normal cases */ # ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI +# define SQLITE_TCLAPI # endif #endif +/* Compatability between Tcl8.6 and Tcl9.0 */ +#if TCL_MAJOR_VERSION==9 +# define CONST const +#else + typedef int Tcl_Size; +#endif +/**** End copy of tclsqlite.h ****/ + #include <errno.h> /* @@ -209,7 +218,8 @@ struct SqliteDb { struct IncrblobChannel { sqlite3_blob *pBlob; /* sqlite3 blob handle */ SqliteDb *pDb; /* Associated database connection */ - int iSeek; /* Current seek offset */ + sqlite3_int64 iSeek; /* Current seek offset */ + unsigned int isClosed; /* TCL_CLOSE_READ or TCL_CLOSE_WRITE */ Tcl_Channel channel; /* Channel identifier */ IncrblobChannel *pNext; /* Linked list of all open incrblob channels */ IncrblobChannel *pPrev; /* Linked list of all open incrblob channels */ @@ -249,14 +259,23 @@ static void closeIncrblobChannels(SqliteDb *pDb){ /* ** Close an incremental blob channel. */ -static int SQLITE_TCLAPI incrblobClose( +static int SQLITE_TCLAPI incrblobClose2( ClientData instanceData, - Tcl_Interp *interp + Tcl_Interp *interp, + int flags ){ IncrblobChannel *p = (IncrblobChannel *)instanceData; - int rc = sqlite3_blob_close(p->pBlob); + int rc; sqlite3 *db = p->pDb->db; + if( flags ){ + p->isClosed |= flags; + return TCL_OK; + } + + /* If we reach this point, then we really do need to close the channel */ + rc = sqlite3_blob_close(p->pBlob); + /* Remove the channel from the SqliteDb.pIncrblob list. */ if( p->pNext ){ p->pNext->pPrev = p->pPrev; @@ -277,6 +296,13 @@ static int SQLITE_TCLAPI incrblobClose( } return TCL_OK; } +static int SQLITE_TCLAPI incrblobClose( + ClientData instanceData, + Tcl_Interp *interp +){ + return incrblobClose2(instanceData, interp, 0); +} + /* ** Read data from an incremental blob channel. @@ -288,9 +314,9 @@ static int SQLITE_TCLAPI incrblobInput( int *errorCodePtr ){ IncrblobChannel *p = (IncrblobChannel *)instanceData; - int nRead = bufSize; /* Number of bytes to read */ - int nBlob; /* Total size of the blob */ - int rc; /* sqlite error code */ + sqlite3_int64 nRead = bufSize; /* Number of bytes to read */ + sqlite3_int64 nBlob; /* Total size of the blob */ + int rc; /* sqlite error code */ nBlob = sqlite3_blob_bytes(p->pBlob); if( (p->iSeek+nRead)>nBlob ){ @@ -300,7 +326,7 @@ static int SQLITE_TCLAPI incrblobInput( return 0; } - rc = sqlite3_blob_read(p->pBlob, (void *)buf, nRead, p->iSeek); + rc = sqlite3_blob_read(p->pBlob, (void *)buf, (int)nRead, (int)p->iSeek); if( rc!=SQLITE_OK ){ *errorCodePtr = rc; return -1; @@ -320,9 +346,9 @@ static int SQLITE_TCLAPI incrblobOutput( int *errorCodePtr ){ IncrblobChannel *p = (IncrblobChannel *)instanceData; - int nWrite = toWrite; /* Number of bytes to write */ - int nBlob; /* Total size of the blob */ - int rc; /* sqlite error code */ + sqlite3_int64 nWrite = toWrite; /* Number of bytes to write */ + sqlite3_int64 nBlob; /* Total size of the blob */ + int rc; /* sqlite error code */ nBlob = sqlite3_blob_bytes(p->pBlob); if( (p->iSeek+nWrite)>nBlob ){ @@ -333,7 +359,7 @@ static int SQLITE_TCLAPI incrblobOutput( return 0; } - rc = sqlite3_blob_write(p->pBlob, (void *)buf, nWrite, p->iSeek); + rc = sqlite3_blob_write(p->pBlob, (void*)buf,(int)nWrite, (int)p->iSeek); if( rc!=SQLITE_OK ){ *errorCodePtr = EIO; return -1; @@ -343,12 +369,19 @@ static int SQLITE_TCLAPI incrblobOutput( return nWrite; } +/* The datatype of Tcl_DriverWideSeekProc changes between tcl8.6 and tcl9.0 */ +#if TCL_MAJOR_VERSION==9 +# define WideSeekProcType long long +#else +# define WideSeekProcType Tcl_WideInt +#endif + /* ** Seek an incremental blob channel. */ -static int SQLITE_TCLAPI incrblobSeek( +static WideSeekProcType SQLITE_TCLAPI incrblobWideSeek( ClientData instanceData, - long offset, + WideSeekProcType offset, int seekMode, int *errorCodePtr ){ @@ -370,6 +403,14 @@ static int SQLITE_TCLAPI incrblobSeek( return p->iSeek; } +static int SQLITE_TCLAPI incrblobSeek( + ClientData instanceData, + long offset, + int seekMode, + int *errorCodePtr +){ + return incrblobWideSeek(instanceData,offset,seekMode,errorCodePtr); +} static void SQLITE_TCLAPI incrblobWatch( @@ -388,7 +429,7 @@ static int SQLITE_TCLAPI incrblobHandle( static Tcl_ChannelType IncrblobChannelType = { "incrblob", /* typeName */ - TCL_CHANNEL_VERSION_2, /* version */ + TCL_CHANNEL_VERSION_5, /* version */ incrblobClose, /* closeProc */ incrblobInput, /* inputProc */ incrblobOutput, /* outputProc */ @@ -397,11 +438,11 @@ static Tcl_ChannelType IncrblobChannelType = { 0, /* getOptionProc */ incrblobWatch, /* watchProc (this is a no-op) */ incrblobHandle, /* getHandleProc (always returns error) */ - 0, /* close2Proc */ + incrblobClose2, /* close2Proc */ 0, /* blockModeProc */ 0, /* flushProc */ 0, /* handlerProc */ - 0, /* wideSeekProc */ + incrblobWideSeek, /* wideSeekProc */ }; /* @@ -433,8 +474,9 @@ static int createIncrblobChannel( } p = (IncrblobChannel *)Tcl_Alloc(sizeof(IncrblobChannel)); - p->iSeek = 0; + memset(p, 0, sizeof(*p)); p->pBlob = pBlob; + if( (flags & TCL_WRITABLE)==0 ) p->isClosed |= TCL_CLOSE_WRITE; sqlite3_snprintf(sizeof(zChannel), zChannel, "incrblob_%d", ++count); p->channel = Tcl_CreateChannel(&IncrblobChannelType, zChannel, p, flags); @@ -474,7 +516,7 @@ static int safeToUseEvalObjv(Tcl_Interp *interp, Tcl_Obj *pCmd){ ** characters appear in pCmd, we will report the string as unsafe. */ const char *z; - int n; + Tcl_Size n; z = Tcl_GetStringFromObj(pCmd, &n); while( n-- > 0 ){ int c = *(z++); @@ -981,7 +1023,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ ** be preserved and reused on the next invocation. */ Tcl_Obj **aArg; - int nArg; + Tcl_Size nArg; if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); return; @@ -1044,7 +1086,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); }else{ Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); - int n; + Tcl_Size n; u8 *data; const char *zType = (pVar->typePtr ? pVar->typePtr->name : ""); char c = zType[0]; @@ -1455,7 +1497,7 @@ static int dbPrepareAndBind( } } if( pVar ){ - int n; + Tcl_Size n; u8 *data; const char *zType = (pVar->typePtr ? pVar->typePtr->name : ""); c = zType[0]; @@ -1469,8 +1511,9 @@ static int dbPrepareAndBind( Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; }else if( c=='b' && strcmp(zType,"boolean")==0 ){ - Tcl_GetIntFromObj(interp, pVar, &n); - sqlite3_bind_int(pStmt, i, n); + int nn; + Tcl_GetIntFromObj(interp, pVar, &nn); + sqlite3_bind_int(pStmt, i, nn); }else if( c=='d' && strcmp(zType,"double")==0 ){ double r; Tcl_GetDoubleFromObj(interp, pVar, &r); @@ -2034,7 +2077,7 @@ static int SQLITE_TCLAPI DbObjCmd( } }else{ char *zAuth; - int len; + Tcl_Size len; if( pDb->zAuth ){ Tcl_Free(pDb->zAuth); } @@ -2137,7 +2180,7 @@ static int SQLITE_TCLAPI DbObjCmd( } }else{ char *zCallback; - int len; + Tcl_Size len; if( pDb->zBindFallback ){ Tcl_Free(pDb->zBindFallback); } @@ -2167,7 +2210,7 @@ static int SQLITE_TCLAPI DbObjCmd( } }else{ char *zBusy; - int len; + Tcl_Size len; if( pDb->zBusy ){ Tcl_Free(pDb->zBusy); } @@ -2274,7 +2317,7 @@ static int SQLITE_TCLAPI DbObjCmd( SqlCollate *pCollate; char *zName; char *zScript; - int nScript; + Tcl_Size nScript; if( objc!=4 ){ Tcl_WrongNumArgs(interp, 2, objv, "NAME SCRIPT"); return TCL_ERROR; @@ -2333,7 +2376,7 @@ static int SQLITE_TCLAPI DbObjCmd( } }else{ const char *zCommit; - int len; + Tcl_Size len; if( pDb->zCommit ){ Tcl_Free(pDb->zCommit); } @@ -2653,7 +2696,8 @@ static int SQLITE_TCLAPI DbObjCmd( Tcl_Obj *pValue = 0; unsigned char *pBA; unsigned char *pData; - int len, xrc; + Tcl_Size len; + int xrc; sqlite3_int64 mxSize = 0; int i; int isReadonly = 0; @@ -3024,7 +3068,7 @@ deserialize_error: return TCL_ERROR; } if( objc==3 ){ - int len; + Tcl_Size len; char *zNull = Tcl_GetStringFromObj(objv[2], &len); if( pDb->zNull ){ Tcl_Free(pDb->zNull); @@ -3078,7 +3122,7 @@ deserialize_error: #endif }else if( objc==4 ){ char *zProgress; - int len; + Tcl_Size len; int N; if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){ return TCL_ERROR; @@ -3124,7 +3168,7 @@ deserialize_error: } }else{ char *zProfile; - int len; + Tcl_Size len; if( pDb->zProfile ){ Tcl_Free(pDb->zProfile); } @@ -3335,7 +3379,7 @@ deserialize_error: } }else{ char *zTrace; - int len; + Tcl_Size len; if( pDb->zTrace ){ Tcl_Free(pDb->zTrace); } @@ -3375,7 +3419,7 @@ deserialize_error: } }else{ char *zTraceV2; - int len; + Tcl_Size len; Tcl_WideInt wMask = 0; if( objc==4 ){ static const char *TTYPE_strs[] = { @@ -3961,14 +4005,25 @@ EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_ERROR; } EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){return TCL_ERROR;} +/* +** Versions of all of the above entry points that omit the "3" at the end +** of the name. Years ago (circa 2004) the "3" was necessary to distinguish +** SQLite version 3 from Sqlite version 2. But two decades have elapsed. +** SQLite2 is not longer a conflict. So it is ok to omit the "3". +** +** Omitting the "3" helps TCL find the entry point. +*/ +EXTERN int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);} +EXTERN int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } +EXTERN int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } +EXTERN int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } +EXTERN int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_ERROR; } +EXTERN int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){return TCL_ERROR;} +/* Also variants with a lowercase "s" */ +EXTERN int sqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);} +EXTERN int sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);} -#ifndef SQLITE_3_SUFFIX_ONLY -int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); } -int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; } -#endif /* ** If the TCLSH macro is defined, add code to make a stand-alone program. diff --git a/src/tclsqlite.h b/src/tclsqlite.h new file mode 100644 index 000000000..b9a948ef1 --- /dev/null +++ b/src/tclsqlite.h @@ -0,0 +1,42 @@ +/* +** 2024-07-30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface to TCL as used by SQLite. +** SQLite subcomponents that use TCL (the libsqlite3.c interface library +** and various test*.c pieces) should #include this file rather than +** including tcl.h directly. +*/ +/****** Any edits to this file must mirrored in tclsqlite.c ***********/ + +/* When compiling for Windows using STDCALL instead of CDECL calling +** conventions, the MSVC makefile has to build a customized version of +** the "tcl.h" header that specifies the calling conventions for each +** interface. That customized "tcl.h" is named "sqlite_tcl.h". +*/ +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" /* Special case for Windows using STDCALL */ +#else +# include <tcl.h> /* All normal cases */ +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif + +/****** Any edits to this file must mirrored in tclsqlite.c ***********/ + +/* Compatability between Tcl8.6 and Tcl9.0 */ +#if TCL_MAJOR_VERSION==9 +# define CONST const +#else + typedef int Tcl_Size; +#endif + +/****** Any edits to this file must mirrored in tclsqlite.c ***********/ diff --git a/src/test1.c b/src/test1.c index 9c28259b4..177091a0e 100644 --- a/src/test1.c +++ b/src/test1.c @@ -26,11 +26,7 @@ #endif #include "vdbeInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> @@ -992,6 +988,39 @@ static void intrealFunction( } /* +** These SQL functions attempt to return a value (their first argument) +** that has been modified to have multiple datatypes. For example both +** TEXT and INTEGER. +*/ +static void addTextTypeFunction( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)sqlite3_value_text(argv[0]); + (void)argc; + sqlite3_result_value(context, argv[0]); +} +static void addIntTypeFunction( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)sqlite3_value_int64(argv[0]); + (void)argc; + sqlite3_result_value(context, argv[0]); +} +static void addRealTypeFunction( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)sqlite3_value_double(argv[0]); + (void)argc; + sqlite3_result_value(context, argv[0]); +} + +/* ** SQL function: strtod(X) ** ** Use the C-library strtod() function to convert string X into a double. @@ -1103,6 +1132,22 @@ static int SQLITE_TCLAPI test_create_function( 0, intrealFunction, 0, 0); } + /* The add_text_type(), add_int_type(), and add_real_type() functions + ** attempt to return a value that has multiple datatypes. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "add_text_type", 1, SQLITE_UTF8, + 0, addTextTypeFunction, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "add_int_type", 1, SQLITE_UTF8, + 0, addIntTypeFunction, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "add_real_type", 1, SQLITE_UTF8, + 0, addRealTypeFunction, 0, 0); + } + /* Functions strtod() and dtostr() work as in the shell. These routines ** use the standard C library to convert between floating point and ** text. This is used to compare SQLite's internal conversion routines @@ -1723,7 +1768,7 @@ static int SQLITE_TCLAPI blobHandleFromObj( sqlite3_blob **ppBlob ){ char *z; - int n; + Tcl_Size n; z = Tcl_GetStringFromObj(pObj, &n); if( n==0 ){ @@ -2290,13 +2335,13 @@ static int SQLITE_TCLAPI test_stmt_scanstatus( }; Tcl_Obj **aFlag = 0; - int nFlag = 0; + Tcl_Size nFlag = 0; int ii; if( Tcl_ListObjGetElements(interp, objv[2], &nFlag, &aFlag) ){ return TCL_ERROR; } - for(ii=0; ii<nFlag; ii++){ + for(ii=0; ii<(int)nFlag; ii++){ int iVal = 0; int res = Tcl_GetIndexFromObjStruct( interp, aFlag[ii], aTbl, sizeof(aTbl[0]), "flag", 0, &iVal @@ -2717,7 +2762,7 @@ static int SQLITE_TCLAPI test_snapshot_open_blob( sqlite3 *db; char *zName; unsigned char *pBlob; - int nBlob; + Tcl_Size nBlob; if( objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME SNAPSHOT"); @@ -2752,8 +2797,8 @@ static int SQLITE_TCLAPI test_snapshot_cmp_blob( int res; unsigned char *p1; unsigned char *p2; - int n1; - int n2; + Tcl_Size n1; + Tcl_Size n2; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "SNAPSHOT1 SNAPSHOT2"); @@ -4053,7 +4098,7 @@ static int SQLITE_TCLAPI test_bind_text( ){ sqlite3_stmt *pStmt; int idx; - int trueLength = 0; + Tcl_Size trueLength = 0; int bytes; char *value; int rc; @@ -4111,7 +4156,7 @@ static int SQLITE_TCLAPI test_bind_text16( char *value; char *toFree = 0; int rc; - int trueLength = 0; + Tcl_Size trueLength = 0; void (*xDel)(void*) = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT); Tcl_Obj *oStmt = objv[objc-4]; @@ -4165,7 +4210,8 @@ static int SQLITE_TCLAPI test_bind_blob( Tcl_Obj *CONST objv[] ){ sqlite3_stmt *pStmt; - int len, idx; + Tcl_Size len; + int idx; int bytes; char *value; int rc; @@ -4191,7 +4237,7 @@ static int SQLITE_TCLAPI test_bind_blob( if( bytes>len ){ char zBuf[200]; sqlite3_snprintf(sizeof(zBuf), zBuf, - "cannot use %d blob bytes, have %d", bytes, len); + "cannot use %d blob bytes, have %d", bytes, (int)len); Tcl_AppendResult(interp, zBuf, (char*)0); return TCL_ERROR; } @@ -4489,9 +4535,9 @@ static int SQLITE_TCLAPI test_carray_bind( struct iovec *a = sqlite3_malloc( sizeof(struct iovec)*nData ); if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; } for(j=0; j<nData; j++){ - int n = 0; + Tcl_Size n = 0; unsigned char *v = Tcl_GetByteArrayFromObj(objv[i+i], &n); - a[j].iov_len = n; + a[j].iov_len = (size_t)n; a[j].iov_base = sqlite3_malloc64( n ); if( a[j].iov_base==0 ){ a[j].iov_len = 0; @@ -5067,7 +5113,7 @@ static int SQLITE_TCLAPI test_prepare16( char zBuf[50]; int rc; int bytes; /* The integer specified as arg 3 */ - int objlen; /* The byte-array length of arg 2 */ + Tcl_Size objlen; /* The byte-array length of arg 2 */ if( objc!=5 && objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -5127,7 +5173,7 @@ static int SQLITE_TCLAPI test_prepare16_v2( char zBuf[50]; int rc; int bytes; /* The integer specified as arg 3 */ - int objlen; /* The byte-array length of arg 2 */ + Tcl_Size objlen; /* The byte-array length of arg 2 */ if( objc!=5 && objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -5207,9 +5253,9 @@ static int SQLITE_TCLAPI test_open_v2( int rc; char zBuf[100]; - int nFlag; + Tcl_Size nFlag; Tcl_Obj **apFlag; - int i; + Tcl_Size i; if( objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "FILENAME FLAGS VFS"); @@ -5918,6 +5964,145 @@ static int SQLITE_TCLAPI tcl_variable_type( return TCL_OK; } +#include <ctype.h> + +/* +** Usage: fpnum_compare STRING1 STRING2 +** +** Compare two strings. Return true if the strings are the same and +** false if they differ. +** +** For this comparison, the strings are analyzed as a sequenced of +** whitespace separated tokens. The whitespace is ignored. Only the +** tokens are compared. Comparison rules: +** +** A. Tokens that are not floating-point numbers must match exactly. +** +** B. Floating point number must have exactly the same digits before +** the decimal point. +** +** C. Digits must match after the decimal point up to 15 digits, +** taking rounding into consideration. +** +** D. An exponent on a floating point of the form "e+NN" will +** match "e+N" if NN==N. Likewise for the negative exponent. +** +** This routine is used for comparing results that might involve floating +** point values. Tcl9.0 and Tcl8.6 differ in the number of significant +** digits that they show, so there is no way to write a portable test result +** without this routine. +** +** This routine is only called after [string compare] fails, which is seldom, +** so performance is not a pressing concern. Better to get the correct answer +** slowly. +*/ +static int SQLITE_TCLAPI fpnum_compare( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const unsigned char *zA; + const unsigned char *zB; + int i, j; + int nDigit; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STRING1 STRING2"); + return TCL_ERROR; + } + zA = (const unsigned char*)Tcl_GetString(objv[1]); + zB = (const unsigned char*)Tcl_GetString(objv[2]); + i = j = 0; + while( 1 ){ + /* Skip whitespace before and after tokens */ + while( isspace(zA[i]) ){ i++; } + while( isspace(zB[j]) ){ j++; } + + if( zA[i]!=zB[j] ) break; /* First character must match */ + if( zA[i]=='-' && isdigit(zA[i+1]) ){ i++; j++; } /* Skip initial '-' */ + if( !isdigit(zA[i]) ){ + /* Not a number. Must match exactly */ + while( !isspace(zA[i]) && zA[i] && zA[i]==zB[j] ){ i++; j++; } + if( zA[i]!=zB[j] ) break; + if( isspace(zA[i]) ) continue; + break; + } + + /* At this point we know we are dealing with a number zA[i] and zB[j] + ** are both digits (leading "-" have been skipped). See if they are + ** the same number. Start by matching digits before the decimal + ** point, which must all be the same. */ + nDigit = 0; + while( zA[i]==zB[j] && isdigit(zA[i]) ){ i++; j++; nDigit++; } + if( zA[i]!=zB[j] ) break; + if( zA[i]==0 ) break; + if( zA[i]=='.' && zB[j]=='.' ){ + /* Count more matching digits after the decimal point */ + i++; + j++; + while( zA[i]==zB[j] && isdigit(zA[i]) ){ i++; j++; nDigit++; } + if( zA[i]==0 ){ + while( zB[j]=='0' || (isdigit(zB[j]) && nDigit>=15) ){ j++; nDigit++; } + break; + } + if( zB[j]==0 ){ + while( zA[i]=='0' || (isdigit(zA[i]) && nDigit>=15) ){ i++; nDigit++; } + break; + } + if( isspace(zA[i]) && isspace(zB[j]) ) continue; + + if( isdigit(zA[i]) && isdigit(zB[j]) ){ + /* A and B are both digits, but different digits */ + if( zA[i]==zB[j]+1 && !isdigit(zA[i+1]) && isdigit(zB[j+1]) ){ + /* Is A a rounded up version of B? */ + j++; + while( zB[j]=='9' ){ j++; nDigit++; } + if( nDigit<14 && (!isdigit(zB[j]) || zB[j]<5) ) break; + while( isdigit(zB[j]) ){ j++; } + i++; + }else if( zB[j]==zA[i]+1 && !isdigit(zB[j+1]) && isdigit(zA[i+1]) ){ + /* Is B a rounded up version of A? */ + i++; + while( zA[i]=='9' ){ i++; nDigit++; } + if( nDigit<14 && (!isdigit(zA[i]) || zA[i]<5) ) break; + while( isdigit(zA[i]) ){ i++; } + j++; + }else{ + break; + } + }else if( !isdigit(zA[i]) && isdigit(zB[j]) ){ + while( zB[j]=='0' ){ j++; nDigit++; } + if( nDigit<15 ) break; + while( isdigit(zB[j]) ){ j++; } + }else if( !isdigit(zB[j]) && isdigit(zA[i]) ){ + while( zA[i]=='0' ){ i++; nDigit++; } + if( nDigit<15 ) break; + while( isdigit(zA[i]) ){ i++; } + }else{ + break; + } + } + if( zA[i]=='e' && zB[j]=='e' ){ + i++; + j++; + if( (zA[i]=='+' || zA[i]=='-') && zB[j]==zA[i] ){ i++; j++; } + if( zA[i]!=zB[j] ){ + if( zA[i]=='0' && zA[i+1]==zB[j] ){ i++; } + if( zB[j]=='0' && zB[j+1]==zA[i] ){ j++; } + } + while( zA[i]==zB[j] && isdigit(zA[i]) ){ i++; j++; } + if( zA[i]!=zB[j] ) break; + if( zA[i]==0 ) break; + continue; + } + } + while( isspace(zA[i]) ){ i++; } + while( isspace(zB[j]) ){ j++; } + Tcl_SetObjResult(interp, Tcl_NewIntObj(zA[i]==0 && zB[j]==0)); + return TCL_OK; +} + /* ** Usage: sqlite3_release_memory ?N? ** @@ -6606,7 +6791,7 @@ static int SQLITE_TCLAPI file_control_lockproxy_test( { char *testPath; int rc; - int nPwd; + Tcl_Size nPwd; const char *zPwd; char proxyPath[400]; @@ -7876,145 +8061,6 @@ static int SQLITE_TCLAPI win32_file_lock( CloseHandle(ev); return TCL_OK; } - -/* -** exists_win32_path PATH -** -** Returns non-zero if the specified path exists, whose fully qualified name -** may exceed 260 characters if it is prefixed with "\\?\". -*/ -static int SQLITE_TCLAPI win32_exists_path( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "PATH"); - return TCL_ERROR; - } - Tcl_SetObjResult(interp, Tcl_NewBooleanObj( - GetFileAttributesW( Tcl_GetUnicode(objv[1]))!=INVALID_FILE_ATTRIBUTES )); - return TCL_OK; -} - -/* -** find_win32_file PATTERN -** -** Returns a list of entries in a directory that match the specified pattern, -** whose fully qualified name may exceed 248 characters if it is prefixed with -** "\\?\". -*/ -static int SQLITE_TCLAPI win32_find_file( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - HANDLE hFindFile = INVALID_HANDLE_VALUE; - WIN32_FIND_DATAW findData; - Tcl_Obj *listObj; - DWORD lastErrno; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "PATTERN"); - return TCL_ERROR; - } - hFindFile = FindFirstFileW(Tcl_GetUnicode(objv[1]), &findData); - if( hFindFile==INVALID_HANDLE_VALUE ){ - Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); - return TCL_ERROR; - } - listObj = Tcl_NewObj(); - Tcl_IncrRefCount(listObj); - do { - Tcl_ListObjAppendElement(interp, listObj, Tcl_NewUnicodeObj( - findData.cFileName, -1)); - Tcl_ListObjAppendElement(interp, listObj, Tcl_NewWideIntObj( - findData.dwFileAttributes)); - } while( FindNextFileW(hFindFile, &findData) ); - lastErrno = GetLastError(); - if( lastErrno!=NO_ERROR && lastErrno!=ERROR_NO_MORE_FILES ){ - FindClose(hFindFile); - Tcl_DecrRefCount(listObj); - Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); - return TCL_ERROR; - } - FindClose(hFindFile); - Tcl_SetObjResult(interp, listObj); - return TCL_OK; -} - -/* -** delete_win32_file FILENAME -** -** Deletes the specified file, whose fully qualified name may exceed 260 -** characters if it is prefixed with "\\?\". -*/ -static int SQLITE_TCLAPI win32_delete_file( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "FILENAME"); - return TCL_ERROR; - } - if( !DeleteFileW(Tcl_GetUnicode(objv[1])) ){ - Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); - return TCL_ERROR; - } - Tcl_ResetResult(interp); - return TCL_OK; -} - -/* -** make_win32_dir DIRECTORY -** -** Creates the specified directory, whose fully qualified name may exceed 248 -** characters if it is prefixed with "\\?\". -*/ -static int SQLITE_TCLAPI win32_mkdir( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DIRECTORY"); - return TCL_ERROR; - } - if( !CreateDirectoryW(Tcl_GetUnicode(objv[1]), NULL) ){ - Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); - return TCL_ERROR; - } - Tcl_ResetResult(interp); - return TCL_OK; -} - -/* -** remove_win32_dir DIRECTORY -** -** Removes the specified directory, whose fully qualified name may exceed 248 -** characters if it is prefixed with "\\?\". -*/ -static int SQLITE_TCLAPI win32_rmdir( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DIRECTORY"); - return TCL_ERROR; - } - if( !RemoveDirectoryW(Tcl_GetUnicode(objv[1])) ){ - Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); - return TCL_ERROR; - } - Tcl_ResetResult(interp); - return TCL_OK; -} #endif @@ -8054,6 +8100,7 @@ static int SQLITE_TCLAPI optimization_control( { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, { "order-by-idx-join", SQLITE_OrderByIdxJoin }, + { "order-by-subquery", SQLITE_OrderBySubq }, { "transitive", SQLITE_Transitive }, { "omit-noop-join", SQLITE_OmitNoopJoin }, { "stat4", SQLITE_Stat4 }, @@ -8125,6 +8172,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( extern int sqlite3_remember_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_stmtrand_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -8158,6 +8206,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( { "remember", sqlite3_remember_init }, { "series", sqlite3_series_init }, { "spellfix", sqlite3_spellfix_init }, + { "stmtrand", sqlite3_stmtrand_init }, { "totype", sqlite3_totype_init }, { "unionvtab", sqlite3_unionvtab_init }, { "wholenumber", sqlite3_wholenumber_init }, @@ -8276,7 +8325,7 @@ static int SQLITE_TCLAPI sorter_test_sort4_helper( for(iStep=0; iStep<nStep && SQLITE_ROW==sqlite3_step(pStmt); iStep++){ int a = sqlite3_column_int(pStmt, 0); if( a!=sqlite3_column_int(pStmt, iB) ){ - Tcl_AppendResult(interp, "data error: (a!=b)", 0); + Tcl_AppendResult(interp, "data error: (a!=b)", (void*)0); return TCL_ERROR; } @@ -8295,13 +8344,13 @@ static int SQLITE_TCLAPI sorter_test_sort4_helper( if( rc!=SQLITE_OK ) goto sql_error; if( iCksum1!=iCksum2 ){ - Tcl_AppendResult(interp, "checksum mismatch", 0); + Tcl_AppendResult(interp, "checksum mismatch", (void*)0); return TCL_ERROR; } return TCL_OK; sql_error: - Tcl_AppendResult(interp, "sql error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "sql error: ", sqlite3_errmsg(db), (void*)0); return TCL_ERROR; } @@ -8319,7 +8368,7 @@ static int SQLITE_TCLAPI test_user_authenticate( ){ char *zUser = 0; char *zPasswd = 0; - int nPasswd = 0; + Tcl_Size nPasswd = 0; sqlite3 *db; int rc; @@ -8332,7 +8381,7 @@ static int SQLITE_TCLAPI test_user_authenticate( } zUser = Tcl_GetString(objv[2]); zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd); - rc = sqlite3_user_authenticate(db, zUser, zPasswd, nPasswd); + rc = sqlite3_user_authenticate(db, zUser, zPasswd, (int)nPasswd); Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); return TCL_OK; } @@ -8350,7 +8399,7 @@ static int SQLITE_TCLAPI test_user_add( ){ char *zUser = 0; char *zPasswd = 0; - int nPasswd = 0; + Tcl_Size nPasswd = 0; int isAdmin = 0; sqlite3 *db; int rc; @@ -8365,7 +8414,7 @@ static int SQLITE_TCLAPI test_user_add( zUser = Tcl_GetString(objv[2]); zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd); Tcl_GetBooleanFromObj(interp, objv[4], &isAdmin); - rc = sqlite3_user_add(db, zUser, zPasswd, nPasswd, isAdmin); + rc = sqlite3_user_add(db, zUser, zPasswd, (int)nPasswd, isAdmin); Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); return TCL_OK; } @@ -8383,7 +8432,7 @@ static int SQLITE_TCLAPI test_user_change( ){ char *zUser = 0; char *zPasswd = 0; - int nPasswd = 0; + Tcl_Size nPasswd = 0; int isAdmin = 0; sqlite3 *db; int rc; @@ -8398,7 +8447,7 @@ static int SQLITE_TCLAPI test_user_change( zUser = Tcl_GetString(objv[2]); zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd); Tcl_GetBooleanFromObj(interp, objv[4], &isAdmin); - rc = sqlite3_user_change(db, zUser, zPasswd, nPasswd, isAdmin); + rc = sqlite3_user_change(db, zUser, zPasswd, (int)nPasswd, isAdmin); Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); return TCL_OK; } @@ -8682,7 +8731,7 @@ static int SQLITE_TCLAPI test_write_db( sqlite3 *db = 0; Tcl_WideInt iOff = 0; const unsigned char *aData = 0; - int nData = 0; + Tcl_Size nData = 0; sqlite3_file *pFile = 0; int rc; @@ -8695,7 +8744,7 @@ static int SQLITE_TCLAPI test_write_db( aData = Tcl_GetByteArrayFromObj(objv[3], &nData); sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFile); - rc = pFile->pMethods->xWrite(pFile, aData, nData, iOff); + rc = pFile->pMethods->xWrite(pFile, aData, (int)(nData&0x7fffffff), iOff); Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; @@ -9107,11 +9156,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "optimization_control", optimization_control,0}, #if SQLITE_OS_WIN { "lock_win32_file", win32_file_lock, 0 }, - { "exists_win32_path", win32_exists_path, 0 }, - { "find_win32_file", win32_find_file, 0 }, - { "delete_win32_file", win32_delete_file, 0 }, - { "make_win32_dir", win32_mkdir, 0 }, - { "remove_win32_dir", win32_rmdir, 0 }, #endif { "tcl_objproc", runAsObjProc, 0 }, @@ -9186,6 +9230,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif { "sqlite3_test_errstr", test_errstr, 0 }, { "tcl_variable_type", tcl_variable_type, 0 }, + { "fpnum_compare", fpnum_compare, 0 }, #ifndef SQLITE_OMIT_SHARED_CACHE { "sqlite3_enable_shared_cache", test_enable_shared, 0 }, { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, diff --git a/src/test2.c b/src/test2.c index c75fa2eba..a9549aa7f 100644 --- a/src/test2.c +++ b/src/test2.c @@ -14,11 +14,7 @@ ** testing of the SQLite library. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <ctype.h> diff --git a/src/test3.c b/src/test3.c index 7fd766247..f1b2b0168 100644 --- a/src/test3.c +++ b/src/test3.c @@ -15,11 +15,7 @@ */ #include "sqliteInt.h" #include "btreeInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> @@ -623,6 +619,7 @@ static int SQLITE_TCLAPI btree_insert( BtCursor *pCur; int rc; BtreePayload x; + Tcl_Size n; if( objc!=4 && objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE"); @@ -633,10 +630,11 @@ static int SQLITE_TCLAPI btree_insert( if( objc==4 ){ if( Tcl_GetIntFromObj(interp, objv[2], &rc) ) return TCL_ERROR; x.nKey = rc; - x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &x.nData); + x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &n); + x.nData = (int)n; }else{ - x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &rc); - x.nKey = rc; + x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &n); + x.nKey = (int)n; } pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); diff --git a/src/test4.c b/src/test4.c index 2043a3383..8a68f7d3e 100644 --- a/src/test4.c +++ b/src/test4.c @@ -12,11 +12,7 @@ ** Code for testing the SQLite library in a multithreaded environment. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #if SQLITE_OS_UNIX && SQLITE_THREADSAFE #include <stdlib.h> #include <string.h> diff --git a/src/test5.c b/src/test5.c index 0d9242862..334b5d07f 100644 --- a/src/test5.c +++ b/src/test5.c @@ -17,11 +17,7 @@ */ #include "sqliteInt.h" #include "vdbeInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> @@ -36,7 +32,7 @@ static int SQLITE_TCLAPI binarize( int objc, Tcl_Obj *CONST objv[] ){ - int len; + Tcl_Size len; char *bytes; Tcl_Obj *pRet; assert(objc==2); @@ -133,7 +129,7 @@ static int SQLITE_TCLAPI test_translate( sqlite3_value *pVal; char *z; - int len; + Tcl_Size len; void (*xDel)(void *p) = SQLITE_STATIC; if( objc!=4 && objc!=5 ){ @@ -164,7 +160,7 @@ static int SQLITE_TCLAPI test_translate( z = (char*)Tcl_GetByteArrayFromObj(objv[1], &len); if( objc==5 ){ char *zTmp = z; - z = sqlite3_malloc(len); + z = sqlite3_malloc64(len); memcpy(z, zTmp, len); } sqlite3ValueSetStr(pVal, -1, z, enc_from, xDel); diff --git a/src/test6.c b/src/test6.c index 5d8e6b9be..76db640c4 100644 --- a/src/test6.c +++ b/src/test6.c @@ -16,11 +16,7 @@ */ #if SQLITE_TEST /* This file is used for testing only */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #ifndef SQLITE_OMIT_DISKIO /* This file is a no-op if disk I/O is disabled */ @@ -751,7 +747,7 @@ static int processDevSymArgs( int setDeviceChar = 0; for(i=0; i<objc; i+=2){ - int nOpt; + Tcl_Size nOpt; char *zOpt = Tcl_GetStringFromObj(objv[i], &nOpt); if( (nOpt>11 || nOpt<2 || strncmp("-sectorsize", zOpt, nOpt)) @@ -776,11 +772,11 @@ static int processDevSymArgs( }else{ int j; Tcl_Obj **apObj; - int nObj; + Tcl_Size nObj; if( Tcl_ListObjGetElements(interp, objv[i+1], &nObj, &apObj) ){ return TCL_ERROR; } - for(j=0; j<nObj; j++){ + for(j=0; j<(int)nObj; j++){ int rc; int iChoice; Tcl_Obj *pFlag = Tcl_DuplicateObj(apObj[j]); @@ -925,7 +921,8 @@ static int SQLITE_TCLAPI crashParamsObjCmd( ){ int iDelay; const char *zCrashFile; - int nCrashFile, iDc, iSectorSize; + Tcl_Size nCrashFile; + int iDc, iSectorSize; iDc = -1; iSectorSize = -1; diff --git a/src/test8.c b/src/test8.c index 4aeb555c7..8a13f5d55 100644 --- a/src/test8.c +++ b/src/test8.c @@ -14,11 +14,7 @@ ** testing of the SQLite library. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> diff --git a/src/test9.c b/src/test9.c index 5b139e8a5..b5362adb7 100644 --- a/src/test9.c +++ b/src/test9.c @@ -15,11 +15,7 @@ ** as there is not much point in binding to Tcl. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> diff --git a/src/test_async.c b/src/test_async.c index c32c74c66..afe401ac6 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -14,15 +14,8 @@ ** (defined in ext/async/sqlite3async.h) to Tcl. */ -#define TCL_THREADS -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#define TCL_THREADS +#include "tclsqlite.h" #ifdef SQLITE_ENABLE_ASYNCIO diff --git a/src/test_autoext.c b/src/test_autoext.c index e23e41a08..74ca55879 100644 --- a/src/test_autoext.c +++ b/src/test_autoext.c @@ -11,14 +11,7 @@ ************************************************************************* ** Test extension for testing the sqlite3_auto_extension() function. */ -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #include "sqlite3ext.h" #ifndef SQLITE_OMIT_LOAD_EXTENSION diff --git a/src/test_backup.c b/src/test_backup.c index 9b684a28f..8051888ee 100644 --- a/src/test_backup.c +++ b/src/test_backup.c @@ -13,14 +13,7 @@ ** */ -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #include "sqlite3.h" #include <assert.h> diff --git a/src/test_bestindex.c b/src/test_bestindex.c index 8128530b4..2f9203d85 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -93,11 +93,7 @@ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -215,6 +211,9 @@ static int tclConnect( rc = SQLITE_ERROR; }else{ rc = sqlite3_declare_vtab(db, Tcl_GetStringResult(interp)); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("declare_vtab: %s", sqlite3_errmsg(db)); + } } if( rc!=SQLITE_OK ){ @@ -226,7 +225,7 @@ static int tclConnect( } sqlite3_free(zCmd); - *ppVtab = &pTab->base; + *ppVtab = pTab ? &pTab->base : 0; return rc; } @@ -302,11 +301,9 @@ static int tclFilter( Tcl_IncrRefCount(pScript); Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("xFilter", -1)); Tcl_ListObjAppendElement(interp, pScript, Tcl_NewIntObj(idxNum)); - if( idxStr ){ - Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj(idxStr, -1)); - }else{ - Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("", -1)); - } + Tcl_ListObjAppendElement( + interp, pScript, Tcl_NewStringObj(idxStr ? idxStr : "", -1) + ); pArg = Tcl_NewObj(); Tcl_IncrRefCount(pArg); @@ -351,14 +348,14 @@ static int tclFilter( */ Tcl_Obj *pRes = Tcl_GetObjResult(interp); Tcl_Obj **apElem = 0; - int nElem; + Tcl_Size nElem; rc = Tcl_ListObjGetElements(interp, pRes, &nElem, &apElem); if( rc!=TCL_OK ){ const char *zErr = Tcl_GetStringResult(interp); rc = SQLITE_ERROR; pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr); }else{ - for(ii=0; rc==SQLITE_OK && ii<nElem; ii+=2){ + for(ii=0; rc==SQLITE_OK && ii<(int)nElem; ii+=2){ const char *zCmd = Tcl_GetString(apElem[ii]); Tcl_Obj *p = apElem[ii+1]; if( sqlite3_stricmp("sql", zCmd)==0 ){ @@ -527,6 +524,7 @@ static int SQLITE_TCLAPI testBestIndexObj( "distinct", /* 3 */ "in", /* 4 */ "rhs_value", /* 5 */ + "collation", /* 6 */ 0 }; int ii; @@ -607,6 +605,17 @@ static int SQLITE_TCLAPI testBestIndexObj( Tcl_SetObjResult(interp, Tcl_NewStringObj(zVal, -1)); break; } + + case 6: assert( sqlite3_stricmp(azSub[ii], "collation")==0 ); { + int iCons = 0; + const char *zColl = ""; + if( Tcl_GetIntFromObj(interp, objv[2], &iCons) ){ + return TCL_ERROR; + } + zColl = sqlite3_vtab_collation(pIdxInfo, iCons); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zColl, -1)); + break; + } } return TCL_OK; @@ -651,7 +660,7 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ */ Tcl_Obj *pRes = Tcl_GetObjResult(interp); Tcl_Obj **apElem = 0; - int nElem; + Tcl_Size nElem; rc = Tcl_ListObjGetElements(interp, pRes, &nElem, &apElem); if( rc!=TCL_OK ){ const char *zErr = Tcl_GetStringResult(interp); @@ -660,7 +669,7 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ }else{ int ii; int iArgv = 1; - for(ii=0; rc==SQLITE_OK && ii<nElem; ii+=2){ + for(ii=0; rc==SQLITE_OK && ii<(int)nElem; ii+=2){ const char *zCmd = Tcl_GetString(apElem[ii]); Tcl_Obj *p = apElem[ii+1]; if( sqlite3_stricmp("cost", zCmd)==0 ){ @@ -697,6 +706,10 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->aConstraintUsage[iCons].omit = bOmit; } } + }else + if( sqlite3_stricmp("constraint", zCmd)==0 ){ + rc = SQLITE_CONSTRAINT; + pTab->base.zErrMsg = sqlite3_mprintf("%s", Tcl_GetString(p)); }else{ rc = SQLITE_ERROR; pTab->base.zErrMsg = sqlite3_mprintf("unexpected: %s", zCmd); diff --git a/src/test_blob.c b/src/test_blob.c index cbdf9f069..bddad240c 100644 --- a/src/test_blob.c +++ b/src/test_blob.c @@ -12,11 +12,7 @@ ** */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <assert.h> @@ -58,7 +54,7 @@ static int blobHandleFromObj( sqlite3_blob **ppBlob ){ char *z; - int n; + Tcl_Size n; z = Tcl_GetStringFromObj(pObj, &n); if( n==0 ){ @@ -88,7 +84,7 @@ static int blobHandleFromObj( ** NULL Pointer is returned. */ static char *blobStringFromObj(Tcl_Obj *pObj){ - int n; + Tcl_Size n; char *z; z = Tcl_GetStringFromObj(pObj, &n); return (n ? z : 0); @@ -112,7 +108,7 @@ static int SQLITE_TCLAPI test_blob_open( Tcl_WideInt iRowid; int flags; const char *zVarname; - int nVarname; + Tcl_Size nVarname; sqlite3_blob *pBlob = (sqlite3_blob*)&flags; /* Non-zero initialization */ int rc; @@ -281,7 +277,8 @@ static int SQLITE_TCLAPI test_blob_write( int rc; unsigned char *zBuf; - int nBuf; + Tcl_Size nBuf; + int n; if( objc!=4 && objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "HANDLE OFFSET DATA ?NDATA?"); @@ -294,10 +291,11 @@ static int SQLITE_TCLAPI test_blob_write( } zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); - if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ + n = (int)(nBuf & 0x7fffffff); + if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &n) ){ return TCL_ERROR; } - rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); + rc = sqlite3_blob_write(pBlob, zBuf, n, iOffset); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } diff --git a/src/test_btree.c b/src/test_btree.c index 03b8b207c..168a10f1f 100644 --- a/src/test_btree.c +++ b/src/test_btree.c @@ -14,11 +14,7 @@ ** testing of the SQLite library. */ #include "btreeInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" /* ** Usage: sqlite3_shared_cache_report diff --git a/src/test_config.c b/src/test_config.c index ee766a26d..58de5a462 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -24,11 +24,7 @@ # include "os_win.h" #endif -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> @@ -59,6 +55,14 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "rowid32", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + Tcl_SetVar2( + interp, "sqlite_options", "allow_rowid_in_view", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2( + interp, "sqlite_options", "allow_rowid_in_view", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_CASE_SENSITIVE_LIKE Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","1",TCL_GLOBAL_ONLY); #else diff --git a/src/test_demovfs.c b/src/test_demovfs.c index e990e98f2..e92fd5613 100644 --- a/src/test_demovfs.c +++ b/src/test_demovfs.c @@ -645,14 +645,7 @@ sqlite3_vfs *sqlite3_demovfs(void){ #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #if SQLITE_OS_UNIX static int SQLITE_TCLAPI register_demovfs( diff --git a/src/test_fs.c b/src/test_fs.c index f88f3a942..d821a83b9 100644 --- a/src/test_fs.c +++ b/src/test_fs.c @@ -62,12 +62,7 @@ ** SELECT * FROM fstree WHERE path LIKE '/home/dan/sqlite/%' */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif - +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <sys/types.h> diff --git a/src/test_func.c b/src/test_func.c index 80df48828..8c06705ae 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -13,11 +13,7 @@ ** implements new SQL functions used by the test scripts. */ #include "sqlite3.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <assert.h> diff --git a/src/test_hexio.c b/src/test_hexio.c index 61a41d5b1..3c856b330 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -18,11 +18,7 @@ ** easier and safer to build our own mechanism. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <assert.h> @@ -155,7 +151,8 @@ static int SQLITE_TCLAPI hexio_write( Tcl_Obj *CONST objv[] ){ int offset; - int nIn, nOut, written; + Tcl_Size nIn; + int nOut, written; const char *zFile; const unsigned char *zIn; unsigned char *aOut; @@ -168,11 +165,11 @@ static int SQLITE_TCLAPI hexio_write( if( Tcl_GetIntFromObj(interp, objv[2], &offset) ) return TCL_ERROR; zFile = Tcl_GetString(objv[1]); zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[3], &nIn); - aOut = sqlite3_malloc( 1 + nIn/2 ); + aOut = sqlite3_malloc64( 1 + nIn/2 ); if( aOut==0 ){ return TCL_ERROR; } - nOut = sqlite3TestHexToBin(zIn, nIn, aOut); + nOut = sqlite3TestHexToBin(zIn, (int)nIn, aOut); out = fopen(zFile, "r+b"); if( out==0 ){ out = fopen(zFile, "r+"); @@ -203,7 +200,8 @@ static int SQLITE_TCLAPI hexio_get_int( Tcl_Obj *CONST objv[] ){ int val; - int nIn, nOut; + Tcl_Size nIn; + int nOut; const unsigned char *zIn; unsigned char *aOut; unsigned char aNum[4]; @@ -213,11 +211,11 @@ static int SQLITE_TCLAPI hexio_get_int( return TCL_ERROR; } zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn); - aOut = sqlite3_malloc( 1 + nIn/2 ); + aOut = sqlite3_malloc64( 1 + nIn/2 ); if( aOut==0 ){ return TCL_ERROR; } - nOut = sqlite3TestHexToBin(zIn, nIn, aOut); + nOut = sqlite3TestHexToBin(zIn, (int)nIn, aOut); if( nOut>=4 ){ memcpy(aNum, aOut, 4); }else{ @@ -300,7 +298,7 @@ static int SQLITE_TCLAPI utf8_to_utf8( Tcl_Obj *CONST objv[] ){ #ifdef SQLITE_DEBUG - int n; + Tcl_Size n; int nOut; const unsigned char *zOrig; unsigned char *z; @@ -309,8 +307,8 @@ static int SQLITE_TCLAPI utf8_to_utf8( return TCL_ERROR; } zOrig = (unsigned char *)Tcl_GetStringFromObj(objv[1], &n); - z = sqlite3_malloc( n+4 ); - n = sqlite3TestHexToBin(zOrig, n, z); + z = sqlite3_malloc64( n+4 ); + n = sqlite3TestHexToBin(zOrig, (int)n, z); z[n] = 0; nOut = sqlite3Utf8To8(z); sqlite3TestBinToHex(z,nOut); @@ -361,7 +359,7 @@ static int SQLITE_TCLAPI read_fts3varint( int objc, Tcl_Obj *CONST objv[] ){ - int nBlob; + Tcl_Size nBlob; unsigned char *zBlob; sqlite3_int64 iVal; int nVal; @@ -388,10 +386,10 @@ static int SQLITE_TCLAPI make_fts3record( Tcl_Obj *CONST objv[] ){ Tcl_Obj **aArg = 0; - int nArg = 0; + Tcl_Size nArg = 0; unsigned char *aOut = 0; - int nOut = 0; - int nAlloc = 0; + sqlite3_int64 nOut = 0; + sqlite3_int64 nAlloc = 0; int i; if( objc!=2 ){ @@ -402,8 +400,8 @@ static int SQLITE_TCLAPI make_fts3record( return TCL_ERROR; } - for(i=0; i<nArg; i++){ - sqlite3_int64 iVal; + for(i=0; i<(int)nArg; i++){ + Tcl_WideInt iVal; if( TCL_OK==Tcl_GetWideIntFromObj(0, aArg[i], &iVal) ){ if( nOut+10>nAlloc ){ int nNew = nAlloc?nAlloc*2:128; @@ -417,11 +415,11 @@ static int SQLITE_TCLAPI make_fts3record( } nOut += putFts3Varint((char*)&aOut[nOut], iVal); }else{ - int nVal = 0; + Tcl_Size nVal = 0; char *zVal = Tcl_GetStringFromObj(aArg[i], &nVal); while( (nOut + nVal)>nAlloc ){ - int nNew = nAlloc?nAlloc*2:128; - unsigned char *aNew = sqlite3_realloc(aOut, nNew); + sqlite3_int64 nNew = nAlloc?nAlloc*2:128; + unsigned char *aNew = sqlite3_realloc64(aOut, nNew); if( aNew==0 ){ sqlite3_free(aOut); return TCL_ERROR; diff --git a/src/test_init.c b/src/test_init.c index 400ab9a2b..f7b85875b 100644 --- a/src/test_init.c +++ b/src/test_init.c @@ -27,11 +27,7 @@ #include "sqliteInt.h" #include <string.h> -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" static struct Wrapped { sqlite3_pcache_methods2 pcache; diff --git a/src/test_intarray.c b/src/test_intarray.c index a978ed585..16c1df2e9 100644 --- a/src/test_intarray.c +++ b/src/test_intarray.c @@ -279,14 +279,7 @@ SQLITE_API int sqlite3_intarray_bind( ** Everything below is interface for testing this module. */ #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" /* ** Routines to encode and decode pointers diff --git a/src/test_malloc.c b/src/test_malloc.c index 8146501c9..21faa0d29 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -14,11 +14,7 @@ ** memory allocation subsystem. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <assert.h> @@ -387,7 +383,8 @@ static int SQLITE_TCLAPI test_memset( Tcl_Obj *CONST objv[] ){ void *p; - int size, n, i; + int size, i; + Tcl_Size n; char *zHex; char *zOut; char zBin[100]; @@ -409,7 +406,7 @@ static int SQLITE_TCLAPI test_memset( } zHex = Tcl_GetStringFromObj(objv[3], &n); if( n>sizeof(zBin)*2 ) n = sizeof(zBin)*2; - n = sqlite3TestHexToBin(zHex, n, zBin); + n = sqlite3TestHexToBin(zHex, (int)n, zBin); if( n==0 ){ Tcl_AppendResult(interp, "no data", (char*)0); return TCL_ERROR; @@ -624,7 +621,7 @@ static int SQLITE_TCLAPI test_memdebug_fail( if( Tcl_GetIntFromObj(interp, objv[1], &iFail) ) return TCL_ERROR; for(ii=2; ii<objc; ii+=2){ - int nOption; + Tcl_Size nOption; char *zOption = Tcl_GetStringFromObj(objv[ii], &nOption); char *zErr = 0; diff --git a/src/test_md5.c b/src/test_md5.c index 7903797db..a09f1ad6c 100644 --- a/src/test_md5.c +++ b/src/test_md5.c @@ -16,14 +16,7 @@ #include <stdlib.h> #include <string.h> #include "sqlite3.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" /* * This code implements the MD5 message-digest algorithm. diff --git a/src/test_multiplex.c b/src/test_multiplex.c index d06ed2f79..e5b43f4cc 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -1219,14 +1219,7 @@ int sqlite3_multiplex_shutdown(int eForce){ /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" extern const char *sqlite3ErrName(int); diff --git a/src/test_mutex.c b/src/test_mutex.c index a203208ab..e60a06df3 100644 --- a/src/test_mutex.c +++ b/src/test_mutex.c @@ -11,12 +11,7 @@ ************************************************************************* ** This file contains test logic for the sqlite3_mutex interfaces. */ - -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include "sqlite3.h" #include "sqliteInt.h" #include <stdlib.h> diff --git a/src/test_osinst.c b/src/test_osinst.c index 062e83159..2d03d2bbc 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -1109,14 +1109,7 @@ int sqlite3_vfslog_register(sqlite3 *db){ #if defined(SQLITE_TEST) || defined(TCLSH) -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" static int SQLITE_TCLAPI test_vfslog( void *clientData, diff --git a/src/test_pcache.c b/src/test_pcache.c index 5266d6769..ceefa13e5 100644 --- a/src/test_pcache.c +++ b/src/test_pcache.c @@ -99,7 +99,7 @@ static void testpcacheShutdown(void *pArg){ */ typedef struct testpcache testpcache; struct testpcache { - int szPage; /* Size of each page. Multiple of 8. */ + sqlite3_int64 szPage; /* Size of each page. Multiple of 8. */ int szExtra; /* Size of extra data that accompanies each page */ int bPurgeable; /* True if the page cache is purgeable */ int nFree; /* Number of unused slots in a[] */ @@ -141,6 +141,7 @@ static sqlite3_pcache *testpcacheCreate( int i; assert( testpcacheGlobal.pDummy!=0 ); szPage = (szPage+7)&~7; + szExtra = (szPage+7)&~7; nMem = sizeof(testpcache) + TESTPCACHE_NPAGE*(szPage+szExtra); p = sqlite3_malloc( nMem ); if( p==0 ) return 0; diff --git a/src/test_quota.c b/src/test_quota.c index b436de466..1bfc5ce11 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -1278,14 +1278,7 @@ int sqlite3_quota_remove(const char *zFilename){ /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" /* ** Argument passed to a TCL quota-over-limit callback. @@ -1420,7 +1413,7 @@ static int SQLITE_TCLAPI test_quota_set( Tcl_Obj *pScript; /* Tcl script to invoke to increase quota */ int rc; /* Value returned by quota_set() */ TclQuotaCallback *p; /* Callback object */ - int nScript; /* Length of callback script */ + Tcl_Size nScript; /* Length of callback script */ void (*xDestroy)(void*); /* Optional destructor for pArg */ void (*xCallback)(const char *, sqlite3_int64 *, sqlite3_int64, void *); diff --git a/src/test_rtree.c b/src/test_rtree.c index 0c6dbf3cd..53af6e5cf 100644 --- a/src/test_rtree.c +++ b/src/test_rtree.c @@ -14,11 +14,7 @@ */ #include "sqlite3.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" /* Solely for the UNUSED_PARAMETER() macro. */ #include "sqliteInt.h" @@ -357,11 +353,7 @@ static int bfs_query_func(sqlite3_rtree_query_info *p){ *************************************************************************/ #include <assert.h> -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" typedef struct Cube Cube; struct Cube { diff --git a/src/test_schema.c b/src/test_schema.c index 2cbc18e2b..660d21ea4 100644 --- a/src/test_schema.c +++ b/src/test_schema.c @@ -36,11 +36,7 @@ */ #ifdef SQLITE_TEST # include "sqliteInt.h" -# if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -# else -# include "tcl.h" -# endif +# include "tclsqlite.h" #else # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 diff --git a/src/test_superlock.c b/src/test_superlock.c index 45d0d623a..7f3bf163a 100644 --- a/src/test_superlock.c +++ b/src/test_superlock.c @@ -256,14 +256,7 @@ int sqlite3demo_superlock( #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" struct InterpAndScript { Tcl_Interp *interp; diff --git a/src/test_syscall.c b/src/test_syscall.c index 3cd1034d3..af2ae1001 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -76,11 +76,7 @@ #include "sqliteInt.h" #include "sqlite3.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> #include <assert.h> @@ -197,7 +193,7 @@ static int tsIsFail(void){ */ static int tsErrno(const char *zFunc){ int i; - int nFunc = strlen(zFunc); + size_t nFunc = strlen(zFunc); for(i=0; aSyscall[i].zName; i++){ if( strlen(aSyscall[i].zName)!=nFunc ) continue; if( memcmp(aSyscall[i].zName, zFunc, nFunc) ) continue; @@ -429,7 +425,7 @@ static int SQLITE_TCLAPI test_syscall_install( Tcl_Obj *CONST objv[] ){ sqlite3_vfs *pVfs; - int nElem; + Tcl_Size nElem; int i; Tcl_Obj **apElem; @@ -442,7 +438,7 @@ static int SQLITE_TCLAPI test_syscall_install( } pVfs = sqlite3_vfs_find(0); - for(i=0; i<nElem; i++){ + for(i=0; i<(int)nElem; i++){ int iCall; int rc = Tcl_GetIndexFromObjStruct(interp, apElem[i], aSyscall, sizeof(aSyscall[0]), "system-call", 0, &iCall @@ -502,7 +498,7 @@ static int SQLITE_TCLAPI test_syscall_reset( rc = pVfs->xSetSystemCall(pVfs, 0, 0); for(i=0; aSyscall[i].zName; i++) aSyscall[i].xOrig = 0; }else{ - int nFunc; + Tcl_Size nFunc; char *zFunc = Tcl_GetStringFromObj(objv[2], &nFunc); rc = pVfs->xSetSystemCall(pVfs, Tcl_GetString(objv[2]), 0); for(i=0; rc==SQLITE_OK && aSyscall[i].zName; i++){ diff --git a/src/test_tclsh.c b/src/test_tclsh.c index 32aee4267..db362049e 100644 --- a/src/test_tclsh.c +++ b/src/test_tclsh.c @@ -20,14 +20,7 @@ ** in an effort to keep the tclsqlite.c file pure. */ #include "sqlite3.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" /* Needed for the setrlimit() system call on unix */ #if defined(unix) @@ -108,6 +101,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ extern int Sqlitetest_window_Init(Tcl_Interp *); extern int Sqlitetestvdbecov_Init(Tcl_Interp *); extern int TestRecover_Init(Tcl_Interp*); + extern int Sqlitetestintck_Init(Tcl_Interp*); Tcl_CmdInfo cmdInfo; @@ -175,6 +169,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ Sqlitetest_window_Init(interp); Sqlitetestvdbecov_Init(interp); TestRecover_Init(interp); + Sqlitetestintck_Init(interp); Tcl_CreateObjCommand( interp, "load_testfixture_extensions", load_testfixture_extensions,0,0 diff --git a/src/test_tclvar.c b/src/test_tclvar.c index 36165bc27..6299960a6 100644 --- a/src/test_tclvar.c +++ b/src/test_tclvar.c @@ -36,11 +36,7 @@ ** according to "fullname" and "value" only. */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include <stdlib.h> #include <string.h> @@ -72,8 +68,8 @@ struct tclvar_cursor { Tcl_Obj *pList1; /* Result of [info vars ?pattern?] */ Tcl_Obj *pList2; /* Result of [array names [lindex $pList1 $i1]] */ - int i1; /* Current item in pList1 */ - int i2; /* Current item (if any) in pList2 */ + Tcl_Size i1; /* Current item in pList1 */ + Tcl_Size i2; /* Current item (if any) in pList2 */ }; /* Methods for the tclvar module */ @@ -150,7 +146,7 @@ static int next2(Tcl_Interp *interp, tclvar_cursor *pCur, Tcl_Obj *pObj){ Tcl_IncrRefCount(pCur->pList2); assert( pCur->i2==0 ); }else{ - int n = 0; + Tcl_Size n = 0; pCur->i2++; Tcl_ListObjLength(0, pCur->pList2, &n); if( pCur->i2>=n ){ @@ -167,7 +163,7 @@ static int next2(Tcl_Interp *interp, tclvar_cursor *pCur, Tcl_Obj *pObj){ static int tclvarNext(sqlite3_vtab_cursor *cur){ Tcl_Obj *pObj; - int n = 0; + Tcl_Size n = 0; int ok = 0; tclvar_cursor *pCur = (tclvar_cursor *)cur; diff --git a/src/test_thread.c b/src/test_thread.c index 126fd9836..7c06d110a 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -16,11 +16,7 @@ */ #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #if SQLITE_THREADSAFE @@ -94,7 +90,7 @@ static int SQLITE_TCLAPI tclScriptEvent(Tcl_Event *evPtr, int flags){ static void postToParent(SqlThread *p, Tcl_Obj *pScript){ EvalEvent *pEvent; char *zMsg; - int nMsg; + Tcl_Size nMsg; zMsg = Tcl_GetStringFromObj(pScript, &nMsg); pEvent = (EvalEvent *)ckalloc(sizeof(EvalEvent)+nMsg+1); @@ -181,8 +177,8 @@ static int SQLITE_TCLAPI sqlthread_spawn( SqlThread *pNew; int rc; - int nVarname; char *zVarname; - int nScript; char *zScript; + Tcl_Size nVarname; char *zVarname; + Tcl_Size nScript; char *zScript; /* Parameters for thread creation */ const int nStack = TCL_THREAD_STACK_DEFAULT; @@ -232,7 +228,7 @@ static int SQLITE_TCLAPI sqlthread_parent( ){ EvalEvent *pEvent; char *zMsg; - int nMsg; + Tcl_Size nMsg; SqlThread *p = (SqlThread *)clientData; assert(objc==3); diff --git a/src/test_vdbecov.c b/src/test_vdbecov.c index a001b1df0..283936aeb 100644 --- a/src/test_vdbecov.c +++ b/src/test_vdbecov.c @@ -15,11 +15,7 @@ #include "sqlite3.h" #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #ifdef SQLITE_VDBE_COVERAGE diff --git a/src/test_vfs.c b/src/test_vfs.c index 312e1a1be..9f84b4f80 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -28,11 +28,7 @@ #include "sqlite3.h" #include "sqliteInt.h" -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" typedef struct Testvfs Testvfs; typedef struct TestvfsShm TestvfsShm; @@ -1150,15 +1146,15 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( return TCL_ERROR; } if( objc==4 ){ - int n; + Tcl_Size n; u8 *a = Tcl_GetByteArrayFromObj(objv[3], &n); int pgsz = pBuffer->pgsz; if( pgsz==0 ) pgsz = 65536; - for(i=0; i*pgsz<n; i++){ + for(i=0; i*pgsz<(int)n; i++){ int nByte = pgsz; tvfsAllocPage(pBuffer, i, pgsz); if( n-i*pgsz<pgsz ){ - nByte = n; + nByte = (int)n; } memcpy(pBuffer->aPage[i], &a[i*pgsz], nByte); } @@ -1203,7 +1199,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( { "xFileControl", TESTVFS_FCNTL_MASK }, }; Tcl_Obj **apElem = 0; - int nElem = 0; + Tcl_Size nElem = 0; int mask = 0; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 2, objv, "LIST"); @@ -1213,7 +1209,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( return TCL_ERROR; } Tcl_ResetResult(interp); - for(i=0; i<nElem; i++){ + for(i=0; i<(int)nElem; i++){ int iMethod; char *zElem = Tcl_GetString(apElem[i]); for(iMethod=0; iMethod<ArraySize(vfsmethod); iMethod++){ @@ -1239,7 +1235,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( */ case CMD_SCRIPT: { if( objc==3 ){ - int nByte; + Tcl_Size nByte; if( p->pScript ){ Tcl_DecrRefCount(p->pScript); p->pScript = 0; @@ -1337,13 +1333,13 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( int j; int iNew = 0; Tcl_Obj **flags = 0; - int nFlags = 0; + Tcl_Size nFlags = 0; if( Tcl_ListObjGetElements(interp, objv[2], &nFlags, &flags) ){ return TCL_ERROR; } - for(j=0; j<nFlags; j++){ + for(j=0; j<(int)nFlags; j++){ int idx = 0; if( Tcl_GetIndexFromObjStruct(interp, flags[j], aFlag, sizeof(aFlag[0]), "flag", 0, &idx) @@ -1491,7 +1487,7 @@ static int SQLITE_TCLAPI testvfs_cmd( if( objc<2 || 0!=(objc%2) ) goto bad_args; for(i=2; i<objc; i += 2){ - int nSwitch; + Tcl_Size nSwitch; char *zSwitch; zSwitch = Tcl_GetStringFromObj(objv[i], &nSwitch); diff --git a/src/test_window.c b/src/test_window.c index 48ab02211..631b20162 100644 --- a/src/test_window.c +++ b/src/test_window.c @@ -16,7 +16,7 @@ #ifdef SQLITE_TEST #include "sqliteInt.h" -#include <tcl.h> +#include "tclsqlite.h" extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); extern const char *sqlite3ErrName(int); diff --git a/src/tokenize.c b/src/tokenize.c index f4d013dee..65d1fbf35 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -437,27 +437,58 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; @@ -622,10 +653,13 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ if( tokenType>=TK_WINDOW ){ assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER ); #else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER + ); #endif /* SQLITE_OMIT_WINDOWFUNC */ if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; @@ -658,7 +692,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else{ + }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; x.n = n; diff --git a/src/treeview.c b/src/treeview.c index 2576532b6..de6716122 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -193,9 +193,11 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); x.printfFlags |= SQLITE_PRINTF_INTERNAL; sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); - if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", - pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + if( pItem->pSTab ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", + pItem->pSTab->zName, pItem->pSTab->nCol, pItem->pSTab, + pItem->colUsed, + pItem->fg.rowidUsed ? "+rowid" : ""); } if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){ sqlite3_str_appendf(&x, " FULL-OUTER-JOIN"); @@ -224,23 +226,30 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ if( pItem->fg.viaCoroutine ) sqlite3_str_appendf(&x, " viaCoroutine"); if( pItem->fg.notCte ) sqlite3_str_appendf(&x, " notCte"); if( pItem->fg.isNestedFrom ) sqlite3_str_appendf(&x, " isNestedFrom"); + if( pItem->fg.fixedSchema ) sqlite3_str_appendf(&x, " fixedSchema"); + if( pItem->fg.hadSchema ) sqlite3_str_appendf(&x, " hadSchema"); + if( pItem->fg.isSubquery ) sqlite3_str_appendf(&x, " isSubquery"); sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, i<pSrc->nSrc-1); n = 0; - if( pItem->pSelect ) n++; + if( pItem->fg.isSubquery ) n++; if( pItem->fg.isTabFunc ) n++; if( pItem->fg.isUsing ) n++; if( pItem->fg.isUsing ){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); } - if( pItem->pSelect ){ - if( pItem->pTab ){ - Table *pTab = pItem->pTab; + if( pItem->fg.isSubquery ){ + assert( n==1 ); + if( pItem->pSTab ){ + Table *pTab = pItem->pSTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); - sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "SUBQUERY"); + sqlite3TreeViewPop(&pView); + sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, 0); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); @@ -282,7 +291,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ n = 1000; }else{ n = 0; - if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ) n++; if( p->pWhere ) n++; if( p->pGroupBy ) n++; if( p->pHaving ) n++; @@ -308,7 +317,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ sqlite3TreeViewPop(&pView); } #endif - if( p->pSrc && p->pSrc->nSrc ){ + if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ){ sqlite3TreeViewPush(&pView, (n--)>0); sqlite3TreeViewLine(pView, "FROM"); sqlite3TreeViewSrcList(pView, p->pSrc); @@ -344,7 +353,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); if( p->pLimit->pRight ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewItem(pView, "OFFSET", 0); sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); sqlite3TreeViewPop(&pView); } @@ -816,7 +825,8 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ case OE_Ignore: zType = "ignore"; break; } assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "RAISE %s", zType); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } #endif @@ -896,9 +906,10 @@ void sqlite3TreeViewBareExprList( sqlite3TreeViewLine(pView, "%s", zLabel); for(i=0; i<pList->nExpr; i++){ int j = pList->a[i].u.x.iOrderByCol; + u8 sortFlags = pList->a[i].fg.sortFlags; char *zName = pList->a[i].zEName; int moreToFollow = i<pList->nExpr - 1; - if( j || zName ){ + if( j || zName || sortFlags ){ sqlite3TreeViewPush(&pView, moreToFollow); moreToFollow = 0; sqlite3TreeViewLine(pView, 0); @@ -919,13 +930,18 @@ void sqlite3TreeViewBareExprList( } } if( j ){ - fprintf(stdout, "iOrderByCol=%d", j); + fprintf(stdout, "iOrderByCol=%d ", j); + } + if( sortFlags & KEYINFO_ORDER_DESC ){ + fprintf(stdout, "DESC "); + }else if( sortFlags & KEYINFO_ORDER_BIGNULL ){ + fprintf(stdout, "NULLS-LAST"); } fprintf(stdout, "\n"); fflush(stdout); } sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow); - if( j || zName ){ + if( j || zName || sortFlags ){ sqlite3TreeViewPop(&pView); } } diff --git a/src/trigger.c b/src/trigger.c index 97ca249be..ff2df82cb 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -152,8 +152,10 @@ void sqlite3BeginTrigger( ** name on pTableName if we are reparsing out of the schema table */ if( db->init.busy && iDb!=1 ){ - sqlite3DbFree(db, pTableName->a[0].zDatabase); - pTableName->a[0].zDatabase = 0; + assert( pTableName->a[0].fg.fixedSchema==0 ); + assert( pTableName->a[0].fg.isSubquery==0 ); + sqlite3DbFree(db, pTableName->a[0].u4.zDatabase); + pTableName->a[0].u4.zDatabase = 0; } /* If the trigger name was unqualified, and the table is a temp table, @@ -631,7 +633,8 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ } assert( pName->nSrc==1 ); - zDb = pName->a[0].zDatabase; + assert( pName->a[0].fg.fixedSchema==0 && pName->a[0].fg.isSubquery==0 ); + zDb = pName->a[0].u4.zDatabase; zName = pName->a[0].zName; assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; i<db->nDb; i++){ @@ -868,7 +871,9 @@ SrcList *sqlite3TriggerStepSrc( Schema *pSchema = pStep->pTrig->pSchema; pSrc->a[0].zName = zName; if( pSchema!=db->aDb[1].pSchema ){ - pSrc->a[0].pSchema = pSchema; + assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); + pSrc->a[0].u4.pSchema = pSchema; + pSrc->a[0].fg.fixedSchema = 1; } if( pStep->pFrom ){ SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); @@ -951,6 +956,72 @@ static ExprList *sqlite3ExpandReturning( return pNew; } +/* If the Expr node is a subquery or an EXISTS operator or an IN operator that +** uses a subquery, and if the subquery is SF_Correlated, then mark the +** expression as EP_VarSelect. +*/ +static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){ + UNUSED_PARAMETER(NotUsed); + if( ExprUseXSelect(pExpr) + && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0 + ){ + testcase( ExprHasProperty(pExpr, EP_VarSelect) ); + ExprSetProperty(pExpr, EP_VarSelect); + } + return WRC_Continue; +} + + +/* +** If the SELECT references the table pWalker->u.pTab, then do two things: +** +** (1) Mark the SELECT as as SF_Correlated. +** (2) Set pWalker->eCode to non-zero so that the caller will know +** that (1) has happened. +*/ +static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ + int i; + SrcList *pSrc; + assert( pSelect!=0 ); + pSrc = pSelect->pSrc; + assert( pSrc!=0 ); + for(i=0; i<pSrc->nSrc; i++){ + if( pSrc->a[i].pSTab==pWalker->u.pTab ){ + testcase( pSelect->selFlags & SF_Correlated ); + pSelect->selFlags |= SF_Correlated; + pWalker->eCode = 1; + break; + } + } + return WRC_Continue; +} + +/* +** Scan the expression list that is the argument to RETURNING looking +** for subqueries that depend on the table which is being modified in the +** statement that is hosting the RETURNING clause (pTab). Mark all such +** subqueries as SF_Correlated. If the subqueries are part of an +** expression, mark the expression as EP_VarSelect. +** +** https://sqlite.org/forum/forumpost/2c83569ce8945d39 +*/ +static void sqlite3ProcessReturningSubqueries( + ExprList *pEList, + Table *pTab +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3ReturningSubqueryCorrelated; + w.u.pTab = pTab; + sqlite3WalkExprList(&w, pEList); + if( w.eCode ){ + w.xExprCallback = sqlite3ReturningSubqueryVarSelect; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkExprList(&w, pEList); + } +} + /* ** Generate code for the RETURNING trigger. Unlike other triggers ** that invoke a subprogram in the bytecode, the code for RETURNING @@ -986,7 +1057,8 @@ static void codeReturningTrigger( sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); sSelect.pSrc = &sFrom; sFrom.nSrc = 1; - sFrom.a[0].pTab = pTab; + sFrom.a[0].pSTab = pTab; + sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ sFrom.a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); if( pParse->nErr==0 ){ @@ -1013,6 +1085,7 @@ static void codeReturningTrigger( int i; int nCol = pNew->nExpr; int reg = pParse->nMem+1; + sqlite3ProcessReturningSubqueries(pNew, pTab); pParse->nMem += nCol+2; pReturning->iRetReg = reg; for(i=0; i<nCol; i++){ diff --git a/src/update.c b/src/update.c index cd7d73f3f..a8e7f7780 100644 --- a/src/update.c +++ b/src/update.c @@ -202,7 +202,7 @@ static void updateFromSelect( Expr *pLimit2 = 0; ExprList *pOrderBy2 = 0; sqlite3 *db = pParse->db; - Table *pTab = pTabList->a[0].pTab; + Table *pTab = pTabList->a[0].pSTab; SrcList *pSrc; Expr *pWhere2; int eDest; @@ -226,8 +226,8 @@ static void updateFromSelect( if( pSrc ){ assert( pSrc->a[0].fg.notCte ); pSrc->a[0].iCursor = -1; - pSrc->a[0].pTab->nTabRef--; - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab->nTabRef--; + pSrc->a[0].pSTab = 0; } if( pPk ){ for(i=0; i<pPk->nKeyCol; i++){ @@ -921,6 +921,9 @@ void sqlite3Update( } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } diff --git a/src/upsert.c b/src/upsert.c index be0d0550d..82295d52a 100644 --- a/src/upsert.c +++ b/src/upsert.c @@ -90,7 +90,8 @@ Upsert *sqlite3UpsertNew( int sqlite3UpsertAnalyzeTarget( Parse *pParse, /* The parsing context */ SrcList *pTabList, /* Table into which we are inserting */ - Upsert *pUpsert /* The ON CONFLICT clauses */ + Upsert *pUpsert, /* The ON CONFLICT clauses */ + Upsert *pAll /* Complete list of all ON CONFLICT clauses */ ){ Table *pTab; /* That table into which we are inserting */ int rc; /* Result code */ @@ -103,7 +104,7 @@ int sqlite3UpsertAnalyzeTarget( int nClause = 0; /* Counter of ON CONFLICT clauses */ assert( pTabList->nSrc==1 ); - assert( pTabList->a[0].pTab!=0 ); + assert( pTabList->a[0].pSTab!=0 ); assert( pUpsert!=0 ); assert( pUpsert->pUpsertTarget!=0 ); @@ -122,7 +123,7 @@ int sqlite3UpsertAnalyzeTarget( if( rc ) return rc; /* Check to see if the conflict target matches the rowid. */ - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; pTarget = pUpsert->pUpsertTarget; iCursor = pTabList->a[0].iCursor; if( HasRowid(pTab) @@ -193,6 +194,14 @@ int sqlite3UpsertAnalyzeTarget( continue; } pUpsert->pUpsertIdx = pIdx; + if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){ + /* Really this should be an error. The isDup ON CONFLICT clause will + ** never fire. But this problem was not discovered until three years + ** after multi-CONFLICT upsert was added, and so we silently ignore + ** the problem to prevent breaking applications that might actually + ** have redundant ON CONFLICT clauses. */ + pUpsert->isDup = 1; + } break; } if( pUpsert->pUpsertIdx==0 ){ @@ -219,9 +228,13 @@ int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ Upsert *pNext; if( NEVER(pUpsert==0) ) return 0; pNext = pUpsert->pNextUpsert; - if( pNext==0 ) return 1; - if( pNext->pUpsertTarget==0 ) return 1; - if( pNext->pUpsertIdx==0 ) return 1; + while( 1 /*exit-by-return*/ ){ + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + if( !pNext->isDup ) return 0; + pNext = pNext->pNextUpsert; + } return 0; } diff --git a/src/util.c b/src/util.c index 207b901ba..0cebb474a 100644 --- a/src/util.c +++ b/src/util.c @@ -68,6 +68,19 @@ int sqlite3IsNaN(double x){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is NaN or +Inf or -Inf. +*/ +int sqlite3IsOverflow(double x){ + int rc; /* The value return */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsOvfl(y); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -312,6 +325,44 @@ void sqlite3DequoteExpr(Expr *p){ } /* +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. +*/ +void sqlite3DequoteNumber(Parse *pParse, Expr *p){ + assert( p!=0 || pParse->db->mallocFailed ); + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); + int iValue; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } + } + }while( *pIn++ ); + if( bHex ) p->op = TK_INTEGER; + + /* tag-20240227-a: If after dequoting, the number is an integer that + ** fits in 32 bits, then it must be converted into EP_IntValue. Other + ** parts of the code expect this. See also tag-20240227-b. */ + if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){ + p->u.iValue = iValue; + p->flags |= EP_IntValue; + } + } +} + +/* ** If the input token p is quoted, try to adjust the token to remove ** the quotes. This is not always possible: ** @@ -627,6 +678,9 @@ do_atof_calc: u64 s2; rr[0] = (double)s; s2 = (u64)rr[0]; +#if defined(_MSC_VER) && _MSC_VER<1700 + if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } +#endif rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); if( e>0 ){ while( e>=100 ){ @@ -961,10 +1015,13 @@ int sqlite3Atoi(const char *z){ ** Decode a floating-point value into an approximate decimal ** representation. ** -** Round the decimal representation to n significant digits if -** n is positive. Or round to -n signficant digits after the -** decimal point if n is negative. No rounding is performed if -** n is zero. +** If iRound<=0 then round to -iRound significant digits to the +** the left of the decimal point, or to a maximum of mxRound total +** significant digits. +** +** If iRound>0 round to min(iRound,mxRound) significant digits total. +** +** mxRound must be positive. ** ** The significant digits of the decimal representation are ** stored in p->z[] which is a often (but not always) a pointer @@ -978,6 +1035,8 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ p->isSpecial = 0; p->z = p->zBuf; + assert( mxRound>0 ); + /* Convert negative numbers to positive. Deal with Infinity, 0.0, and ** NaN. */ if( r<0.0 ){ @@ -1069,7 +1128,7 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ assert( p->n>0 ); assert( p->n<sizeof(p->zBuf) ); p->iDP = p->n + exp; - if( iRound<0 ){ + if( iRound<=0 ){ iRound = p->iDP - iRound; if( iRound==0 && p->zBuf[i+1]>='5' ){ iRound = 1; diff --git a/src/vacuum.c b/src/vacuum.c index c0ae4bc1e..e203f68c6 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -162,6 +162,9 @@ SQLITE_NOINLINE int sqlite3RunVacuum( const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ + u64 iRandom; /* Random value used for zDbVacuum[] */ + char zDbVacuum[42]; /* Name of the ATTACH-ed database used for vacuum */ + if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -202,27 +205,29 @@ SQLITE_NOINLINE int sqlite3RunVacuum( pMain = db->aDb[iDb].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); - /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + /* Attach the temporary database as 'vacuum_XXXXXX'. The synchronous pragma ** can be set to 'off' for this file, as it is not recovered if a crash ** occurs anyway. The integrity of the database is maintained by a ** (possibly synchronous) transaction opened on the main database before ** sqlite3BtreeCopyFile() is called. ** ** An optimization would be to use a non-journaled pager. - ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** (Later:) I tried setting "PRAGMA vacuum_XXXXXX.journal_mode=OFF" but ** that actually made the VACUUM run slower. Very little journalling ** actually occurs when doing a vacuum since the vacuum_db is initially ** empty. Only the journal header is written. Apparently it takes more ** time to parse and run the PRAGMA to turn journalling off than it does ** to write the journal header file. */ + sqlite3_randomness(sizeof(iRandom),&iRandom); + sqlite3_snprintf(sizeof(zDbVacuum), zDbVacuum, "vacuum_%016llx", iRandom); nDb = db->nDb; - rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS %s", zOut, zDbVacuum); db->openFlags = saved_openFlags; if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; - assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); + assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); @@ -299,11 +304,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** the contents to the temporary database. */ rc = execSqlF(db, pzErrMsg, - "SELECT'INSERT INTO vacuum_db.'||quote(name)" + "SELECT'INSERT INTO %s.'||quote(name)" "||' SELECT*FROM\"%w\".'||quote(name)" - "FROM vacuum_db.sqlite_schema " + "FROM %s.sqlite_schema " "WHERE type='table'AND coalesce(rootpage,1)>0", - zDbMain + zDbVacuum, zDbMain, zDbVacuum ); assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 ); db->mDbFlags &= ~DBFLAG_Vacuum; @@ -315,11 +320,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** from the schema table. */ rc = execSqlF(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_schema" + "INSERT INTO %s.sqlite_schema" " SELECT*FROM \"%w\".sqlite_schema" " WHERE type IN('view','trigger')" " OR(type='table'AND rootpage=0)", - zDbMain + zDbVacuum, zDbMain ); if( rc ) goto end_of_vacuum; diff --git a/src/vdbe.c b/src/vdbe.c index 6d45bbbbb..ea69c759c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1129,7 +1129,7 @@ case OP_Return: { /* in1 */ ** ** See also: EndCoroutine */ -case OP_InitCoroutine: { /* jump */ +case OP_InitCoroutine: { /* jump0 */ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2<p->nOp ); assert( pOp->p3>=0 && pOp->p3<p->nOp ); @@ -1152,7 +1152,9 @@ jump_to_p2: ** ** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. -** After the jump, register P1 becomes undefined. +** After the jump, the value register P1 is left with a value +** such that subsequent OP_Yields go back to the this same +** OP_EndCoroutine instruction. ** ** See also: InitCoroutine */ @@ -1164,8 +1166,8 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2<p->nOp ); + pIn1->u.i = (int)(pOp - p->aOp) - 1; pOp = &aOp[pCaller->p2 - 1]; - pIn1->flags = MEM_Undefined; break; } @@ -1182,7 +1184,7 @@ case OP_EndCoroutine: { /* in1 */ ** ** See also: InitCoroutine */ -case OP_Yield: { /* in1, jump */ +case OP_Yield: { /* in1, jump0 */ int pcDest; pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); @@ -1212,7 +1214,7 @@ case OP_HaltIfNull: { /* in3 */ /* no break */ deliberate_fall_through } -/* Opcode: Halt P1 P2 * P4 P5 +/* Opcode: Halt P1 P2 P3 P4 P5 ** ** Exit immediately. All open cursors, etc are closed ** automatically. @@ -1225,18 +1227,22 @@ case OP_HaltIfNull: { /* in3 */ ** then back out all changes that have occurred during this execution of the ** VDBE, but do not rollback the transaction. ** -** If P4 is not null then it is an error message string. +** If P3 is not zero and P4 is NULL, then P3 is a register that holds the +** text of an error message. ** -** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. +** If P3 is zero and P4 is not null then the error message string is held +** in P4. +** +** P5 is a value between 1 and 4, inclusive, then the P4 error message +** string is modified as follows: ** -** 0: (no change) ** 1: NOT NULL constraint failed: P4 ** 2: UNIQUE constraint failed: P4 ** 3: CHECK constraint failed: P4 ** 4: FOREIGN KEY constraint failed: P4 ** -** If P5 is not zero and P4 is NULL, then everything after the ":" is -** omitted. +** If P3 is zero and P5 is not zero and P4 is NULL, then everything after +** the ":" is omitted. ** ** There is an implied "Halt 0 0 0" instruction inserted at the very end of ** every program. So a jump past the last instruction of the program @@ -1249,6 +1255,9 @@ case OP_Halt: { #ifdef SQLITE_DEBUG if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } #endif + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_STATIC + || pOp->p4type==P4_DYNAMIC ); /* A deliberately coded "OP_Halt SQLITE_INTERNAL * * * *" opcode indicates ** something is wrong with the code generator. Raise an assertion in order @@ -1279,7 +1288,12 @@ case OP_Halt: { p->errorAction = (u8)pOp->p2; assert( pOp->p5<=4 ); if( p->rc ){ - if( pOp->p5 ){ + if( pOp->p3>0 && pOp->p4type==P4_NOTUSED ){ + const char *zErr; + assert( pOp->p3<=(p->nMem + 1 - p->nCursor) ); + zErr = sqlite3ValueText(&aMem[pOp->p3], SQLITE_UTF8); + sqlite3VdbeError(p, "%s", zErr); + }else if( pOp->p5 ){ static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", "FOREIGN KEY" }; testcase( pOp->p5==1 ); @@ -1512,19 +1526,15 @@ case OP_Blob: { /* out2 */ break; } -/* Opcode: Variable P1 P2 * P4 * -** Synopsis: r[P2]=parameter(P1,P4) +/* Opcode: Variable P1 P2 * * * +** Synopsis: r[P2]=parameter(P1) ** ** Transfer the values of bound parameter P1 into register P2 -** -** If the parameter is named, then its name appears in P4. -** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); - assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -2045,7 +2055,7 @@ case OP_AddImm: { /* in1 */ ** without data loss, then jump immediately to P2, or if P2==0 ** raise an SQLITE_MISMATCH exception. */ -case OP_MustBeInt: { /* jump, in1 */ +case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); @@ -2086,7 +2096,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) || !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** @@ -2301,7 +2311,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ - if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags1 & MEM_Str)!=0 ){ + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); testcase( pIn1->flags & MEM_IntReal ); @@ -2310,7 +2322,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + if( (flags3 & MEM_Str)!=0 ){ + pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); testcase( pIn3->flags & MEM_IntReal ); @@ -3654,11 +3668,16 @@ case OP_MakeRecord: { switch( len ){ default: zPayload[7] = (u8)(v&0xff); v >>= 8; zPayload[6] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; zPayload[4] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 1: zPayload[0] = (u8)(v&0xff); } zPayload += len; @@ -4317,23 +4336,23 @@ case OP_OpenWrite: if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; } + if( pOp->p5 & OPFLAG_P2ISREG ){ + assert( p2>0 ); + assert( p2<=(u32)(p->nMem+1 - p->nCursor) ); + pIn2 = &aMem[p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); + sqlite3VdbeMemIntegerify(pIn2); + p2 = (int)pIn2->u.i; + /* The p2 value always comes from a prior OP_CreateBtree opcode and + ** that opcode will always set the p2 value to 2 or more or else fail. + ** If there were a failure, the prepared statement would have halted + ** before reaching this instruction. */ + assert( p2>=2 ); + } }else{ wrFlag = 0; - } - if( pOp->p5 & OPFLAG_P2ISREG ){ - assert( p2>0 ); - assert( p2<=(u32)(p->nMem+1 - p->nCursor) ); - assert( pOp->opcode==OP_OpenWrite ); - pIn2 = &aMem[p2]; - assert( memIsValid(pIn2) ); - assert( (pIn2->flags & MEM_Int)!=0 ); - sqlite3VdbeMemIntegerify(pIn2); - p2 = (int)pIn2->u.i; - /* The p2 value always comes from a prior OP_CreateBtree opcode and - ** that opcode will always set the p2 value to 2 or more or else fail. - ** If there were a failure, the prepared statement would have halted - ** before reaching this instruction. */ - assert( p2>=2 ); + assert( (pOp->p5 & OPFLAG_P2ISREG)==0 ); } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; @@ -4577,7 +4596,8 @@ case OP_SequenceTest: { ** is the only cursor opcode that works with a pseudo-table. ** ** P3 is the number of fields in the records that will be stored by -** the pseudo-table. +** the pseudo-table. If P2 is 0 or negative then the pseudo-cursor +** will return NULL for every column. */ case OP_OpenPseudo: { VdbeCursor *pCx; @@ -4720,10 +4740,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group, ncycle */ -case OP_SeekLE: /* jump, in3, group, ncycle */ -case OP_SeekGE: /* jump, in3, group, ncycle */ -case OP_SeekGT: { /* jump, in3, group, ncycle */ +case OP_SeekLT: /* jump0, in3, group, ncycle */ +case OP_SeekLE: /* jump0, in3, group, ncycle */ +case OP_SeekGE: /* jump0, in3, group, ncycle */ +case OP_SeekGT: { /* jump0, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -5288,6 +5308,7 @@ case OP_Found: { /* jump, in3, ncycle */ r.pKeyInfo = pC->pKeyInfo; r.default_rc = 0; #ifdef SQLITE_DEBUG + (void)sqlite3FaultSim(50); /* For use by --counter in TH3 */ for(ii=0; ii<r.nField; ii++){ assert( memIsValid(&r.aMem[ii]) ); assert( (r.aMem[ii].flags & MEM_Zero)==0 || r.aMem[ii].n==0 ); @@ -5390,7 +5411,7 @@ case OP_Found: { /* jump, in3, ncycle */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3, ncycle */ +case OP_SeekRowid: { /* jump0, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6149,7 +6170,7 @@ case OP_NullRow: { ** configured to use Prev, not Next. */ case OP_SeekEnd: /* ncycle */ -case OP_Last: { /* jump, ncycle */ +case OP_Last: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6183,28 +6204,38 @@ case OP_Last: { /* jump, ncycle */ break; } -/* Opcode: IfSmaller P1 P2 P3 * * +/* Opcode: IfSizeBetween P1 P2 P3 P4 * ** -** Estimate the number of rows in the table P1. Jump to P2 if that -** estimate is less than approximately 2**(0.1*P3). +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. +** +** Jump to P2 if X is in between P3 and P4, inclusive. */ -case OP_IfSmaller: { /* jump */ +case OP_IfSizeBetween: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; i64 sz; assert( pOp->p1>=0 && pOp->p1<p->nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640*2 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ sz = sqlite3BtreeRowCountEst(pCrsr); - if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)<pOp->p3 ) res = 1; + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); } + res = sz>=pOp->p3 && sz<=pOp->p4.i; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; break; @@ -6257,7 +6288,7 @@ case OP_Sort: { /* jump ncycle */ ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump, ncycle */ +case OP_Rewind: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -6904,11 +6935,18 @@ case OP_CreateBtree: { /* out2 */ break; } -/* Opcode: SqlExec * * * P4 * +/* Opcode: SqlExec P1 P2 * P4 * ** ** Run the SQL statement or statements specified in the P4 string. -** Disable Auth and Trace callbacks while those statements are running if -** P1 is true. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** */ case OP_SqlExec: { char *zErr; @@ -6916,6 +6954,7 @@ case OP_SqlExec: { sqlite3_xauth xAuth; #endif u8 mTrace; + int savedAnalysisLimit; sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; @@ -6924,18 +6963,23 @@ case OP_SqlExec: { xAuth = db->xAuth; #endif mTrace = db->mTrace; - if( pOp->p1 ){ + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = 0; #endif db->mTrace = 0; } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = xAuth; #endif db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; if( zErr || rc ){ sqlite3VdbeError(p, "%s", zErr); sqlite3_free(zErr); @@ -7087,11 +7131,11 @@ case OP_DropTrigger: { /* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +** register (P1+1) the text of an error message describing any problems. +** If no problems are found, store a NULL in register (P1+1). ** -** The register P3 contains one less than the maximum number of allowed errors. -** At most reg(P3) errors will be reported. +** The register (P1) contains one less than the maximum number of allowed +** errors. At most reg(P1) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** @@ -7111,19 +7155,21 @@ case OP_IntegrityCk: { Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); + assert( pOp->p4type==P4_INTARRAY ); nRoot = pOp->p2; aRoot = pOp->p4.ai; assert( nRoot>0 ); + assert( aRoot!=0 ); assert( aRoot[0]==(Pgno)nRoot ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pnErr = &aMem[pOp->p3]; + assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p1]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1+1]; assert( pOp->p5<db->nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, - (int)pnErr->u.i+1, &nErr, &z); + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], + &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); @@ -7250,7 +7296,9 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** P1 contains the address of the memory cell that contains the first memory ** cell in an array of values used as arguments to the sub-program. P2 ** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address +** exception using the RAISE() function. P2 might be zero, if there is +** no possibility that an IGNORE exception will be raised. +** Register P3 contains the address ** of a memory cell in this (the parent) VM that is used to allocate the ** memory required by the sub-vdbe at runtime. ** @@ -7258,7 +7306,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** ** If P5 is non-zero, then recursive program invocation is enabled. */ -case OP_Program: { /* jump */ +case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ int nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ @@ -7623,18 +7671,29 @@ case OP_AggInverse: case OP_AggStep: { int n; sqlite3_context *pCtx; + u64 nAlloc; assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, n*sizeof(sqlite3_value*) + - (sizeof(pCtx[0]) + sizeof(Mem) - sizeof(sqlite3_value*))); + + /* Allocate space for (a) the context object and (n-1) extra pointers + ** to append to the sqlite3_context.argv[1] array, and (b) a memory + ** cell in which to store the accumulation. Be careful that the memory + ** cell is 8-byte aligned, even on platforms where a pointer is 32-bits. + ** + ** Note: We could avoid this by using a regular memory cell from aMem[] for + ** the accumulator, instead of allocating one here. */ + nAlloc = ROUND8P( sizeof(pCtx[0]) + (n-1)*sizeof(sqlite3_value*) ); + pCtx = sqlite3DbMallocRawNN(db, nAlloc + sizeof(Mem)); if( pCtx==0 ) goto no_mem; - pCtx->pMem = 0; - pCtx->pOut = (Mem*)&(pCtx->argv[n]); + pCtx->pOut = (Mem*)((u8*)pCtx + nAlloc); + assert( EIGHT_BYTE_ALIGNMENT(pCtx->pOut) ); + sqlite3VdbeMemInit(pCtx->pOut, db, MEM_Null); + pCtx->pMem = 0; pCtx->pFunc = pOp->p4.pFunc; pCtx->iOp = (int)(pOp - aOp); pCtx->pVdbe = p; @@ -8807,7 +8866,7 @@ case OP_Filter: { /* jump */ ** error is encountered. */ case OP_Trace: -case OP_Init: { /* jump */ +case OP_Init: { /* jump0 */ int i; #ifndef SQLITE_OMIT_TRACE char *zTrace; @@ -8968,14 +9027,29 @@ case OP_ReleaseReg: { /* Opcode: Noop * * * * * ** -** Do nothing. This instruction is often useful as a jump -** destination. +** Do nothing. Continue downward to the next opcode. */ -/* -** The magic Explain opcode are only inserted when explain==2 (which -** is to say when the EXPLAIN QUERY PLAN syntax is used.) -** This opcode records information from the optimizer. It is the -** the same as a no-op. This opcodesnever appears in a real VM program. +/* Opcode: Explain P1 P2 P3 P4 * +** +** This is the same as OP_Noop during normal query execution. The +** purpose of this opcode is to hold information about the query +** plan for the purpose of EXPLAIN QUERY PLAN output. +** +** The P4 value is human-readable text that describes the query plan +** element. Something like "SCAN t1" or "SEARCH t2 USING INDEX t2x1". +** +** The P1 value is the ID of the current element and P2 is the parent +** element for the case of nested query plan elements. If P2 is zero +** then this element is a top-level element. +** +** For loop elements, P3 is the estimated code of each invocation of this +** element. +** +** As with all opcodes, the meanings of the parameters for OP_Explain +** are subject to change from one release to the next. Applications +** should not attempt to interpret or use any of the information +** contined in the OP_Explain opcode. The information provided by this +** opcode is intended for testing and debugging use only. */ default: { /* This is really OP_Noop, OP_Explain */ assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); diff --git a/src/vdbe.h b/src/vdbe.h index 25bda6be7..f40f68d24 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -32,6 +32,19 @@ typedef struct Vdbe Vdbe; */ typedef struct sqlite3_value Mem; typedef struct SubProgram SubProgram; +typedef struct SubrtnSig SubrtnSig; + +/* +** A signature for a reusable subroutine that materializes the RHS of +** an IN operator. +*/ +struct SubrtnSig { + int selId; /* SELECT-id for the SELECT statement on the RHS */ + char *zAff; /* Affinity of the overall IN expression */ + int iTable; /* Ephemeral table generated by the subroutine */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ +}; /* ** A single instruction of the virtual machine has an opcode @@ -60,6 +73,7 @@ struct VdbeOp { u32 *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ Table *pTab; /* Used when p4type is P4_TABLE */ + SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif @@ -127,6 +141,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ #define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ #define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -296,6 +311,8 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); int sqlite3VdbeHasSubProgram(Vdbe*); +void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); + int sqlite3NotPureFunc(sqlite3_context*); #ifdef SQLITE_ENABLE_BYTECODE_VTAB int sqlite3VdbeBytecodeVtabInit(sqlite3*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 14c6091e0..3182e4070 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2404,7 +2404,6 @@ int sqlite3_stmt_scanstatus_v2( } if( flags & SQLITE_SCANSTAT_COMPLEX ){ idx = iScan; - pScan = &p->aScan[idx]; }else{ /* If the COMPLEX flag is clear, then this function must ignore any ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ @@ -2417,6 +2416,8 @@ int sqlite3_stmt_scanstatus_v2( } } if( idx>=p->nScan ) return 1; + assert( pScan==0 || pScan==&p->aScan[idx] ); + pScan = &p->aScan[idx]; switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 420365e93..f1e0cccdc 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -939,6 +939,15 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } + + /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes + ** might */ + assert( pOp->p2>0 + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 ); + + /* Jumps never go off the end of the bytecode array */ + assert( pOp->p2<p->nOp + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 ); break; } } @@ -1404,6 +1413,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4); break; } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = (SubrtnSig*)p4; + sqlite3DbFree(db, pSig->zAff); + sqlite3DbFree(db, pSig); + break; + } } } @@ -1983,6 +1998,11 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = pOp->p4.pTab->zName; break; } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = pOp->p4.pSubrtnSig; + sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff); + break; + } default: { zP4 = pOp->p4.z; } @@ -3346,9 +3366,9 @@ int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFk(p, 0); } - + /* If the auto-commit flag is set and this is the only active writer ** VM, then we do either a commit or rollback of the current transaction. ** @@ -4060,6 +4080,23 @@ static void serialGet( pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } } +static int serialGet7( + const unsigned char *buf, /* Buffer to deserialize from */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + if( IsNaN(x) ){ + pMem->flags = MEM_Null; + return 1; + } + pMem->flags = MEM_Real; + return 0; +} void sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ @@ -4499,17 +4536,15 @@ int sqlite3IntFloatCompare(i64 i, double r){ return (x<r) ? -1 : (x>r); }else{ i64 y; - double s; if( r<-9223372036854775808.0 ) return +1; if( r>=9223372036854775808.0 ) return -1; y = (i64)r; if( i<y ) return -1; if( i>y ) return +1; - s = (double)i; - testcase( doubleLt(s,r) ); - testcase( doubleLt(r,s) ); - testcase( doubleEq(r,s) ); - return (s<r) ? -1 : (s>r); + testcase( doubleLt(((double)i),r) ); + testcase( doubleLt(r,((double)i)) ); + testcase( doubleEq(r,((double)i)) ); + return (((double)i)<r) ? -1 : (((double)i)>r); } } @@ -4739,7 +4774,7 @@ int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + serialGet7(&aKey1[d1], &mem1); rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); @@ -4764,14 +4799,18 @@ int sqlite3VdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else{ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - if( mem1.u.r<pRhs->u.r ){ + if( serialGet7(&aKey1[d1], &mem1) ){ + rc = -1; /* mem1 is a NaN */ + }else if( mem1.u.r<pRhs->u.r ){ rc = -1; }else if( mem1.u.r>pRhs->u.r ){ rc = +1; + }else{ + assert( rc==0 ); } }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } @@ -4841,7 +4880,14 @@ int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0 && serial_type!=10); + if( serial_type==0 + || serial_type==10 + || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0) + ){ + assert( rc==0 ); + }else{ + rc = 1; + } } if( rc!=0 ){ @@ -5301,7 +5347,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ @@ -5321,7 +5368,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ */ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( iVar>=32 ){ v->expmask |= 0x80000000; }else{ diff --git a/src/vdbemem.c b/src/vdbemem.c index d52716468..8e2aa4a6c 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -943,6 +943,13 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ } } +/* +** Set the iIdx'th entry of array aMem[] to contain integer value val. +*/ +void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){ + sqlite3VdbeMemSetInt64(&aMem[iIdx], val); +} + /* A no-op destructor */ void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } @@ -1631,14 +1638,20 @@ static int valueFromExpr( } /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + ** case when the value is -9223372036854775808. Except - do not do this + ** for hexadecimal literals. */ + if( op==TK_UMINUS ){ + Expr *pLeft = pExpr->pLeft; + if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){ + if( ExprHasProperty(pLeft, EP_IntValue) + || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X' + ){ + pExpr = pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + } } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ @@ -1647,12 +1660,26 @@ static int valueFromExpr( if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + i64 iVal; + if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){ + sqlite3VdbeMemSetInt64(pVal, iVal*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + if( affinity==SQLITE_AFF_BLOB ){ + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + pVal->flags = MEM_Real; + }else if( op==TK_INTEGER ){ + /* This case is required by -9223372036854775808 and other strings + ** that look like integers but cannot be handled by the + ** sqlite3DecOrHexToI64() call above. */ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + } }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } @@ -1922,17 +1949,17 @@ int sqlite3Stat4Column( sqlite3_value **ppVal /* OUT: Extracted value */ ){ u32 t = 0; /* a column type code */ - int nHdr; /* Size of the header in the record */ - int iHdr; /* Next unread header byte */ - int iField; /* Next unread data byte */ - int szField = 0; /* Size of the current data field */ + u32 nHdr; /* Size of the header in the record */ + u32 iHdr; /* Next unread header byte */ + i64 iField; /* Next unread data byte */ + u32 szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ assert( iCol>0 ); iHdr = getVarint32(a, nHdr); - if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; iField = nHdr; for(i=0; i<=iCol; i++){ iHdr += getVarint32(&a[iHdr], t); diff --git a/src/vdbesort.c b/src/vdbesort.c index 008369030..239c0a0f3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -556,13 +556,14 @@ static int vdbePmaReadBlob( while( nRem>0 ){ int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ - u8 *aNext; /* Pointer to buffer to copy data from */ + u8 *aNext = 0; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); + assert( aNext!=0 ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); nRem -= nCopy; } diff --git a/src/vdbevtab.c b/src/vdbevtab.c index b295dff7b..1c9909a1a 100644 --- a/src/vdbevtab.c +++ b/src/vdbevtab.c @@ -286,10 +286,10 @@ static int bytecodevtabColumn( #ifdef SQLITE_ENABLE_STMT_SCANSTATUS case 9: /* nexec */ - sqlite3_result_int(ctx, pOp->nExec); + sqlite3_result_int64(ctx, pOp->nExec); break; case 10: /* ncycle */ - sqlite3_result_int(ctx, pOp->nCycle); + sqlite3_result_int64(ctx, pOp->nCycle); break; #else case 9: /* nexec */ diff --git a/src/vtab.c b/src/vtab.c index f83921678..1036eed44 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -611,6 +611,8 @@ static int vtabCallConstructor( db->pVtabCtx = &sCtx; pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + assert( pTab!=0 ); + assert( pTab->nTabRef>1 || rc!=SQLITE_OK ); sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); @@ -633,7 +635,7 @@ static int vtabCallConstructor( pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + *pzErr = sqlite3MPrintf(db, zFormat, zModuleName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -811,12 +813,30 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; Parse sParse; int initBusy; + int i; + const unsigned char *z; + static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif + + /* Verify that the first two keywords in the CREATE TABLE statement + ** really are "CREATE" and "TABLE". If this is not the case, then + ** sqlite3_declare_vtab() is being misused. + */ + z = (const unsigned char*)zCreateTable; + for(i=0; aKeyword[i]; i++){ + int tokenType = 0; + do{ z += sqlite3GetToken(z, &tokenType); }while( tokenType==TK_SPACE ); + if( tokenType!=aKeyword[i] ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error"); + return SQLITE_ERROR; + } + } + sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ @@ -824,6 +844,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( IsVirtual(pTab) ); @@ -837,11 +858,10 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ initBusy = db->init.busy; db->init.busy = 0; sParse.nQueryLoop = 1; - if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) - && ALWAYS(sParse.pNewTable!=0) - && ALWAYS(!db->mallocFailed) - && IsOrdinaryTable(sParse.pNewTable) - ){ + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){ + assert( sParse.pNewTable!=0 ); + assert( !db->mallocFailed ); + assert( IsOrdinaryTable(sParse.pNewTable) ); assert( sParse.zErrMsg==0 ); if( !pTab->aCol ){ Table *pNew = sParse.pNewTable; @@ -3176,9 +3176,13 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); walDisableBlocking(pWal); if( rc ){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT if( rc==SQLITE_BUSY_TIMEOUT ){ *pCnt |= WAL_RETRY_BLOCKED_MASK; } +#else + assert( rc!=SQLITE_BUSY_TIMEOUT ); +#endif assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT ); return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; } diff --git a/src/walker.c b/src/walker.c index 0fe4a1d37..c8735e39b 100644 --- a/src/walker.c +++ b/src/walker.c @@ -171,7 +171,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ pSrc = p->pSrc; if( ALWAYS(pSrc) ){ for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + if( pItem->fg.isSubquery + && sqlite3WalkSelect(pWalker, pItem->u4.pSubq->pSelect) + ){ return WRC_Abort; } if( pItem->fg.isTabFunc diff --git a/src/where.c b/src/where.c index 4ff2815ba..2b835009d 100644 --- a/src/where.c +++ b/src/where.c @@ -303,6 +303,42 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ } /* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + assert( pTerm->eOperator & WO_IN ); + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; +} + +/* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). ** Return NULL if there are no more matching WhereTerms. @@ -352,16 +388,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; + }else{ + CollSeq *pColl; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - assert(pX->pLeft); - pColl = sqlite3ExprCompareCollSeq(pParse, pX); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } @@ -600,7 +644,7 @@ static int isDistinctRedundant( ** clause is redundant. */ if( pTabList->nSrc!=1 ) return 0; iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; /* If any of the expressions is an IPK column on table iBase, then return ** true. Note: The (p->iTable==iBase) part of this test may be false if the @@ -675,6 +719,12 @@ static void translateColumnToCopy( VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); if( pParse->db->mallocFailed ) return; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("CHECKING for column-to-copy on cursor %d for %d..%d\n", + iTabCur, iStart, iEnd); + } +#endif for(; iStart<iEnd; iStart++, pOp++){ if( pOp->p1!=iTabCur ) continue; if( pOp->opcode==OP_Column ){ @@ -713,9 +763,13 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName); for(i=0; i<p->nConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", @@ -733,9 +787,13 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName); for(i=0; i<p->nConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -749,8 +807,8 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define whereTraceIndexInfoInputs(A) -#define whereTraceIndexInfoOutputs(A) +#define whereTraceIndexInfoInputs(A,B) +#define whereTraceIndexInfoOutputs(A,B) #endif /* @@ -788,6 +846,40 @@ static int constraintCompatibleWithOuterJoin( return 1; } +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return true if column iCol of table pTab seem like it might be a +** good column to use as part of a query-time index. +** +** Current algorithm (subject to improvement!): +** +** 1. If iCol is already the left-most column of some other index, +** then return false. +** +** 2. If iCol is part of an existing index that has an aiRowLogEst of +** more than 20, then return false. +** +** 3. If no disqualifying conditions above are found, return true. +*/ +static SQLITE_NOINLINE int columnIsGoodIndexCandidate( + const Table *pTab, + int iCol +){ + const Index *pIdx; + for(pIdx = pTab->pIndex; pIdx!=0; pIdx=pIdx->pNext){ + int j; + for(j=0; j<pIdx->nKeyCol; j++){ + if( pIdx->aiColumn[j]==iCol ){ + if( j==0 ) return 0; + if( pIdx->hasStat1 && pIdx->aiRowLogEst[j+1]>20 ) return 0; + break; + } + } + } + return 1; +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX @@ -802,6 +894,8 @@ static int termCanDriveIndex( const Bitmask notReady /* Tables in outer loops of the join */ ){ char aff; + int leftCol; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); @@ -812,11 +906,12 @@ static int termCanDriveIndex( } if( (pTerm->prereqRight & notReady)!=0 ) return 0; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); - if( pTerm->u.x.leftColumn<0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity; + leftCol = pTerm->u.x.leftColumn; + if( leftCol<0 ) return 0; + aff = pSrc->pSTab->aCol[leftCol].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; testcase( pTerm->pExpr->op==TK_IS ); - return 1; + return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol); } #endif @@ -924,7 +1019,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( nKeyCol = 0; pTabList = pWC->pWInfo->pTabList; pSrc = &pTabList->a[pLevel->iFrom]; - pTable = pSrc->pTab; + pTable = pSrc->pSTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; @@ -934,7 +1029,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -976,7 +1071,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** if they go out of sync. */ if( IsView(pTable) ){ - extraCols = ALLBITS; + extraCols = ALLBITS & ~idxCols; }else{ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); } @@ -1066,12 +1161,17 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Fill the automatic index with content */ assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); if( pSrc->fg.viaCoroutine ){ - int regYield = pSrc->regReturn; + int regYield; + Subquery *pSubq; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + assert( pSubq!=0 ); + regYield = pSubq->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pSrc->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } @@ -1093,11 +1193,12 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pSrc->fg.viaCoroutine ){ + assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 ); sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pSrc->regResult, pLevel->iIdxCur); + pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; }else{ @@ -1188,7 +1289,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( iSrc = pLevel->iFrom; pItem = &pTabList->a[iSrc]; assert( pItem!=0 ); - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); sz = sqlite3LogEstToInt(pTab->nRowLogEst); if( sz<10000 ){ @@ -1203,7 +1304,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){ Expr *pExpr = pTerm->pExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -1219,7 +1320,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jj<n; jj++){ - assert( pIdx->pTable==pItem->pTab ); + assert( pIdx->pTable==pItem->pSTab ); sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); @@ -1258,6 +1359,20 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( #ifndef SQLITE_OMIT_VIRTUALTABLE /* +** Return term iTerm of the WhereClause passed as the first argument. Terms +** are numbered from 0 upwards, starting with the terms in pWC->a[], then +** those in pWC->pOuter->a[] (if any), and so on. +*/ +static WhereTerm *termFromWhereClause(WhereClause *pWC, int iTerm){ + WhereClause *p; + for(p=pWC; p; p=p->pOuter){ + if( iTerm<p->nTerm ) return &p->a[iTerm]; + iTerm -= p->nTerm; + } + return 0; +} + +/* ** Allocate and populate an sqlite3_index_info structure. It is the ** responsibility of the caller to eventually release the structure ** by passing the pointer returned by this function to freeIndexInfo(). @@ -1283,9 +1398,10 @@ static sqlite3_index_info *allocateIndexInfo( const Table *pTab; int eDistinct = 0; ExprList *pOrderBy = pWInfo->pOrderBy; + WhereClause *p; assert( pSrc!=0 ); - pTab = pSrc->pTab; + pTab = pSrc->pSTab; assert( pTab!=0 ); assert( IsVirtual(pTab) ); @@ -1293,28 +1409,30 @@ static sqlite3_index_info *allocateIndexInfo( ** Mark each term with the TERM_OK flag. Set nTerm to the number of ** terms found. */ - for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ - pTerm->wtFlags &= ~TERM_OK; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; + for(p=pWC, nTerm=0; p; p=p->pOuter){ + for(i=0, pTerm=p->a; i<p->nTerm; i++, pTerm++){ + pTerm->wtFlags &= ~TERM_OK; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; - assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); - assert( pTerm->u.x.leftColumn>=XN_ROWID ); - assert( pTerm->u.x.leftColumn<pTab->nCol ); - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 - && !constraintCompatibleWithOuterJoin(pTerm,pSrc) - ){ - continue; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + assert( pTerm->u.x.leftColumn>=XN_ROWID ); + assert( pTerm->u.x.leftColumn<pTab->nCol ); + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; + } + nTerm++; + pTerm->wtFlags |= TERM_OK; } - nTerm++; - pTerm->wtFlags |= TERM_OK; } /* If the ORDER BY clause contains only columns in the current @@ -1329,7 +1447,7 @@ static sqlite3_index_info *allocateIndexInfo( Expr *pE2; /* Skip over constant terms in the ORDER BY clause */ - if( sqlite3ExprIsConstant(pExpr) ){ + if( sqlite3ExprIsConstant(0, pExpr) ){ continue; } @@ -1364,7 +1482,7 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n ){ nOrderBy = n; - if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ eDistinct = 1; @@ -1389,59 +1507,75 @@ static sqlite3_index_info *allocateIndexInfo( pIdxInfo->aConstraint = pIdxCons; pIdxInfo->aOrderBy = pIdxOrderBy; pIdxInfo->aConstraintUsage = pUsage; + pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + if( HasRowid(pTab)==0 ){ + /* Ensure that all bits associated with PK columns are set. This is to + ** ensure they are available for cases like RIGHT joins or OR loops. */ + Index *pPk = sqlite3PrimaryKeyIndex((Table*)pTab); + assert( pPk!=0 ); + for(i=0; i<pPk->nKeyCol; i++){ + int iCol = pPk->aiColumn[i]; + assert( iCol>=0 ); + if( iCol>=BMS-1 ) iCol = BMS-1; + pIdxInfo->colUsed |= MASKBIT(iCol); + } + } pHidden->pWC = pWC; pHidden->pParse = pParse; pHidden->eDistinct = eDistinct; pHidden->mIn = 0; - for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ - u16 op; - if( (pTerm->wtFlags & TERM_OK)==0 ) continue; - pIdxCons[j].iColumn = pTerm->u.x.leftColumn; - pIdxCons[j].iTermOffset = i; - op = pTerm->eOperator & WO_ALL; - if( op==WO_IN ){ - if( (pTerm->wtFlags & TERM_SLICE)==0 ){ - pHidden->mIn |= SMASKBIT32(j); + for(p=pWC, i=j=0; p; p=p->pOuter){ + int nLast = i+p->nTerm;; + for(pTerm=p->a; i<nLast; i++, pTerm++){ + u16 op; + if( (pTerm->wtFlags & TERM_OK)==0 ) continue; + pIdxCons[j].iColumn = pTerm->u.x.leftColumn; + pIdxCons[j].iTermOffset = i; + op = pTerm->eOperator & WO_ALL; + if( op==WO_IN ){ + if( (pTerm->wtFlags & TERM_SLICE)==0 ){ + pHidden->mIn |= SMASKBIT32(j); + } + op = WO_EQ; } - op = WO_EQ; - } - if( op==WO_AUX ){ - pIdxCons[j].op = pTerm->eMatchOp; - }else if( op & (WO_ISNULL|WO_IS) ){ - if( op==WO_ISNULL ){ - pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; + if( op==WO_AUX ){ + pIdxCons[j].op = pTerm->eMatchOp; + }else if( op & (WO_ISNULL|WO_IS) ){ + if( op==WO_ISNULL ){ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; + }else{ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; + } }else{ - pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; - } - }else{ - pIdxCons[j].op = (u8)op; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); - - if( op & (WO_LT|WO_LE|WO_GT|WO_GE) - && sqlite3ExprIsVector(pTerm->pExpr->pRight) - ){ - testcase( j!=i ); - if( j<16 ) mNoOmit |= (1 << j); - if( op==WO_LT ) pIdxCons[j].op = WO_LE; - if( op==WO_GT ) pIdxCons[j].op = WO_GE; + pIdxCons[j].op = (u8)op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); + + if( op & (WO_LT|WO_LE|WO_GT|WO_GE) + && sqlite3ExprIsVector(pTerm->pExpr->pRight) + ){ + testcase( j!=i ); + if( j<16 ) mNoOmit |= (1 << j); + if( op==WO_LT ) pIdxCons[j].op = WO_LE; + if( op==WO_GT ) pIdxCons[j].op = WO_GE; + } } - } - j++; + j++; + } } assert( j==nTerm ); pIdxInfo->nConstraint = j; for(i=j=0; i<nOrderBy; i++){ Expr *pExpr = pOrderBy->a[i].pExpr; - if( sqlite3ExprIsConstant(pExpr) ) continue; + if( sqlite3ExprIsConstant(0, pExpr) ) continue; assert( pExpr->op==TK_COLUMN || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN && pExpr->iColumn==pExpr->pLeft->iColumn) ); @@ -1456,6 +1590,17 @@ static sqlite3_index_info *allocateIndexInfo( } /* +** Free and zero the sqlite3_index_info.idxStr value if needed. +*/ +static void freeIdxStr(sqlite3_index_info *pIdxInfo){ + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->needToFreeIdxStr = 0; + } +} + +/* ** Free an sqlite3_index_info structure allocated by allocateIndexInfo() ** and possibly modified by xBestIndex methods. */ @@ -1470,6 +1615,7 @@ static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */ pHidden->aRhs[i] = 0; } + freeIdxStr(pIdxInfo); sqlite3DbFree(db, pIdxInfo); } @@ -1493,11 +1639,11 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; - whereTraceIndexInfoInputs(p); + whereTraceIndexInfoInputs(p, pTab); pParse->db->nSchemaLock++; rc = pVtab->pModule->xBestIndex(pVtab, p); pParse->db->nSchemaLock--; - whereTraceIndexInfoOutputs(p); + whereTraceIndexInfoOutputs(p, pTab); if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ @@ -2263,7 +2409,7 @@ void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ WhereInfo *pWInfo = pWC->pWInfo; int nb = 1+(pWInfo->pTabList->nSrc+3)/4; SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; + Table *pTab = pItem->pSTab; Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); @@ -2821,7 +2967,7 @@ static void whereLoopOutputAdjust( Expr *pRight = pTerm->pExpr->pRight; int k = 0; testcase( pTerm->pExpr->op==TK_IS ); - if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ k = 20; @@ -2975,7 +3121,9 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered || pProbe->bLowQual ){ if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); + if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ + opMask &= ~(WO_EQ|WO_IN|WO_IS); + } } assert( pNew->u.btree.nEq<pProbe->nColumn ); @@ -3116,7 +3264,7 @@ static int whereLoopAddBtreeIndex( || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ if( iCol==XN_ROWID || pProbe->uniqNotNull - || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) + || (pProbe->nKeyCol==1 && pProbe->onError && (eOp & WO_EQ)) ){ pNew->wsFlags |= WHERE_ONEROW; }else{ @@ -3242,11 +3390,14 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ - assert( pSrc->pTab->szTabRow>0 ); + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ + assert( pSrc->pSTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior ** pages are small. Thus szIdxRow gives a good estimate of seek cost. @@ -3254,9 +3405,17 @@ static int whereLoopAddBtreeIndex( ** under-estimate the scanning cost. */ rCostIdx = pNew->nOut + 16; }else{ - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow; } - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } @@ -3362,7 +3521,9 @@ static int indexMightHelpWithOrderBy( for(ii=0; ii<pOB->nExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( NEVER(pExpr==0) ) continue; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) + && pExpr->iTable==iCursor + ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jj<pIndex->nKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; @@ -3619,7 +3780,7 @@ static void wherePartIdxExpr( u8 aff; if( pLeft->op!=TK_COLUMN ) return; - if( !sqlite3ExprIsConstant(pRight) ) return; + if( !sqlite3ExprIsConstant(0, pRight) ) return; if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; if( pLeft->iColumn<0 ) return; aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; @@ -3709,9 +3870,9 @@ static int whereLoopAddBtree( pWInfo = pBuilder->pWInfo; pTabList = pWInfo->pTabList; pSrc = pTabList->a + pNew->iTab; - pTab = pSrc->pTab; + pTab = pSrc->pSTab; pWC = pBuilder->pWC; - assert( !IsVirtual(pSrc->pTab) ); + assert( !IsVirtual(pSrc->pSTab) ); if( pSrc->fg.isIndexedBy ){ assert( pSrc->fg.isCte==0 ); @@ -3736,7 +3897,7 @@ static int whereLoopAddBtree( sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; - pFirst = pSrc->pTab->pIndex; + pFirst = pSrc->pSTab->pIndex; if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ @@ -3826,6 +3987,7 @@ static int whereLoopAddBtree( pNew->prereq = mPrereq; pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; + pNew->u.btree.pOrderBy = 0; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ @@ -3855,6 +4017,10 @@ static int whereLoopAddBtree( #endif ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); + if( pSrc->fg.isSubquery ){ + if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -3892,7 +4058,9 @@ static int whereLoopAddBtree( " according to whereIsCoveringIndex()\n", pProbe->zName)); } } - }else if( m==0 ){ + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ WHERETRACE(0x200, ("-> %s a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); @@ -3968,7 +4136,7 @@ static int whereLoopAddBtree( ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the ** plan */ - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; } #ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); @@ -3991,6 +4159,21 @@ static int isLimitTerm(WhereTerm *pTerm){ } /* +** Return true if the first nCons constraints in the pUsage array are +** marked as in-use (have argvIndex>0). False otherwise. +*/ +static int allConstraintsUsed( + struct sqlite3_index_constraint_usage *aUsage, + int nCons +){ + int ii; + for(ii=0; ii<nCons; ii++){ + if( aUsage[ii].argvIndex<=0 ) return 0; + } + return 1; +} + +/* ** Argument pIdxInfo is already populated with all constraints that may ** be used by the virtual table identified by pBuilder->pNew->iTab. This ** function marks a subset of those constraints usable, invokes the @@ -4040,7 +4223,7 @@ static int whereLoopAddVirtualOne( ** arguments mUsable and mExclude. */ pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; i<nConstraint; i++, pIdxCons++){ - WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset]; + WhereTerm *pTerm = termFromWhereClause(pWC, pIdxCons->iTermOffset); pIdxCons->usable = 0; if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight && (pTerm->eOperator & mExclude)==0 @@ -4059,11 +4242,10 @@ static int whereLoopAddVirtualOne( pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; pIdxInfo->estimatedRows = 25; pIdxInfo->idxFlags = 0; - pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; pHidden->mHandleIn = 0; /* Invoke the virtual table xBestIndex() method */ - rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); + rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo); if( rc ){ if( rc==SQLITE_CONSTRAINT ){ /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means @@ -4071,6 +4253,7 @@ static int whereLoopAddVirtualOne( ** Make no entries in the loop table. */ WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); + freeIdxStr(pIdxInfo); return SQLITE_OK; } return rc; @@ -4088,18 +4271,17 @@ static int whereLoopAddVirtualOne( int j = pIdxCons->iTermOffset; if( iTerm>=nConstraint || j<0 - || j>=pWC->nTerm + || (pTerm = termFromWhereClause(pWC, j))==0 || pNew->aLTerm[iTerm]!=0 || pIdxCons->usable==0 ){ - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); - testcase( pIdxInfo->needToFreeIdxStr ); + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); + freeIdxStr(pIdxInfo); return SQLITE_ERROR; } testcase( iTerm==nConstraint-1 ); testcase( j==0 ); testcase( j==pWC->nTerm-1 ); - pTerm = &pWC->a[j]; pNew->prereq |= pTerm->prereqRight; assert( iTerm<pNew->nLSlot ); pNew->aLTerm[iTerm] = pTerm; @@ -4130,18 +4312,21 @@ static int whereLoopAddVirtualOne( *pbIn = 1; assert( (mExclude & WO_IN)==0 ); } + /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET + ** terms. And if there are any, they should follow all other terms. */ assert( pbRetryLimit || !isLimitTerm(pTerm) ); - if( isLimitTerm(pTerm) && *pbIn ){ + assert( !isLimitTerm(pTerm) || i>=nConstraint-2 ); + assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) ); + + if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){ /* If there is an IN(...) term handled as an == (separate call to ** xFilter for each value on the RHS of the IN) and a LIMIT or - ** OFFSET term handled as well, the plan is unusable. Set output - ** variable *pbRetryLimit to true to tell the caller to retry with - ** LIMIT and OFFSET disabled. */ - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - pIdxInfo->idxStr = 0; - pIdxInfo->needToFreeIdxStr = 0; - } + ** OFFSET term handled as well, the plan is unusable. Similarly, + ** if there is a LIMIT/OFFSET and there are other unused terms, + ** the plan cannot be used. In these cases set variable *pbRetryLimit + ** to true to tell the caller to retry with LIMIT and OFFSET + ** disabled. */ + freeIdxStr(pIdxInfo); *pbRetryLimit = 1; return SQLITE_OK; } @@ -4153,8 +4338,8 @@ static int whereLoopAddVirtualOne( if( pNew->aLTerm[i]==0 ){ /* The non-zero argvIdx values must be contiguous. Raise an ** error if they are not */ - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); - testcase( pIdxInfo->needToFreeIdxStr ); + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); + freeIdxStr(pIdxInfo); return SQLITE_ERROR; } } @@ -4165,6 +4350,7 @@ static int whereLoopAddVirtualOne( pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? pIdxInfo->nOrderBy : 0); + pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0; pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); @@ -4209,7 +4395,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){ if( iCons>=0 && iCons<pIdxInfo->nConstraint ){ CollSeq *pC = 0; int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; - Expr *pX = pHidden->pWC->a[iTerm].pExpr; + Expr *pX = termFromWhereClause(pHidden->pWC, iTerm)->pExpr; if( pX->pLeft ){ pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); } @@ -4255,7 +4441,9 @@ int sqlite3_vtab_rhs_value( rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */ }else{ if( pH->aRhs[iCons]==0 ){ - WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset]; + WhereTerm *pTerm = termFromWhereClause( + pH->pWC, pIdxInfo->aConstraint[iCons].iTermOffset + ); rc = sqlite3ValueFromExpr( pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db), SQLITE_AFF_BLOB, &pH->aRhs[iCons] @@ -4353,7 +4541,7 @@ static int whereLoopAddVirtual( pWC = pBuilder->pWC; pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; - assert( IsVirtual(pSrc->pTab) ); + assert( IsVirtual(pSrc->pSTab) ); p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; @@ -4367,7 +4555,7 @@ static int whereLoopAddVirtual( } /* First call xBestIndex() with all constraints usable. */ - WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); + WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName)); WHERETRACE(0x800, (" VirtualOne: all usable\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry @@ -4411,9 +4599,8 @@ static int whereLoopAddVirtual( Bitmask mNext = ALLBITS; assert( mNext>0 ); for(i=0; i<nConstraint; i++){ - Bitmask mThis = ( - pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq - ); + int iTerm = p->aConstraint[i].iTermOffset; + Bitmask mThis = termFromWhereClause(pWC, iTerm)->prereqRight & ~mPrereq; if( mThis>mPrev && mThis<mNext ) mNext = mThis; } mPrev = mNext; @@ -4449,9 +4636,8 @@ static int whereLoopAddVirtual( } } - if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); freeIndexInfo(pParse->db, p); - WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc)); + WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc)); return rc; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -4523,7 +4709,7 @@ static int whereLoopAddOr( } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); }else #endif @@ -4637,7 +4823,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ SrcItem *p; for(p=&pItem[1]; p<pEnd; p++){ if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){ @@ -4669,6 +4855,97 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ return rc; } +/* Implementation of the order-by-subquery optimization: +** +** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really +** a subquery or CTE that has an ORDER BY clause. See if any of the terms +** in the subquery ORDER BY clause will satisfy pOrderBy from the outer +** query. Mark off all satisfied terms (by setting bits in *pOBSat) and +** return TRUE if they do. If not, return false. +** +** Example: +** +** CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b)); +** CREATE TABLE t2(x,y); +** WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y) +** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b; +** +** The CTE named "t3" comes out in the natural order of "p", so the first +** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3" +** and sorting only needs to occur on the second term "b". +** +** Limitations: +** +** (1) The optimization is not applied if the outer ORDER BY contains +** a COLLATE clause. The optimization might be applied if the +** outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as +** long as the subquery ORDER BY does the same. But if the +** outer ORDER BY uses COLLATE, even a redundant COLLATE, the +** optimization is bypassed. +** +** (2) The subquery ORDER BY terms must exactly match subquery result +** columns, including any COLLATE annotations. This routine relies +** on iOrderByCol to do matching between order by terms and result +** columns, and iOrderByCol will not be set if the result column +** and ORDER BY collations differ. +** +** (3) The subquery and outer ORDER BY can be in opposite directions as +** long as the subquery is materialized. If the subquery is +** implemented as a co-routine, the sort orders must be in the same +** direction because there is no way to run a co-routine backwards. +*/ +static SQLITE_NOINLINE int wherePathMatchSubqueryOB( + WhereInfo *pWInfo, /* The WHERE clause */ + WhereLoop *pLoop, /* The nested loop term that is a subquery */ + int iLoop, /* Which level of the nested loop. 0==outermost */ + int iCur, /* Cursor used by the this loop */ + ExprList *pOrderBy, /* The ORDER BY clause on the whole query */ + Bitmask *pRevMask, /* When loops need to go in reverse order */ + Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */ +){ + int iOB; /* Index into pOrderBy->a[] */ + int jSub; /* Index into pSubOB->a[] */ + u8 rev = 0; /* True if iOB and jSub sort in opposite directions */ + u8 revIdx = 0; /* Sort direction for jSub */ + Expr *pOBExpr; /* Current term of outer ORDER BY */ + ExprList *pSubOB; /* Complete ORDER BY on the subquery */ + + pSubOB = pLoop->u.btree.pOrderBy; + assert( pSubOB!=0 ); + for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){} + for(jSub=0; jSub<pSubOB->nExpr && iOB<pOrderBy->nExpr; jSub++, iOB++){ + if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break; + pOBExpr = pOrderBy->a[iOB].pExpr; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; + if( pOBExpr->iTable!=iCur ) break; + if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break; + if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */ + u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */ + if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){ + break; + } + revIdx = sfSub & KEYINFO_ORDER_DESC; + if( jSub>0 ){ + if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){ + break; + } + }else{ + rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC); + if( rev ){ + if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ + /* Cannot run a co-routine in reverse order */ + break; + } + *pRevMask |= MASKBIT(iLoop); + } + } + } + *pOBSat |= MASKBIT(iOB); + } + return jSub>0; +} + /* ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th ** parameters) to see if it outputs rows in the requested ORDER BY @@ -4814,9 +5091,18 @@ static i8 wherePathSatisfiesOrderBy( if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ if( pLoop->wsFlags & WHERE_IPK ){ + if( pLoop->u.btree.pOrderBy + && OptimizationEnabled(db, SQLITE_OrderBySubq) + && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur, + pOrderBy,pRevMask, &obSat) + ){ + nColumn = 0; + isOrderDistinct = 0; + }else{ + nColumn = 1; + } pIndex = 0; nKeyCol = 0; - nColumn = 1; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ @@ -4826,7 +5112,7 @@ static i8 wherePathSatisfiesOrderBy( assert( pIndex->aiColumn[nColumn-1]==XN_ROWID || !HasRowid(pIndex->pTable)); /* All relevant terms of the index must also be non-NULL in order - ** for isOrderDistinct to be true. So the isOrderDistint value + ** for isOrderDistinct to be true. So the isOrderDistinct value ** computed here might be a false positive. Corrections will be ** made at tag-20210426-1 below */ isOrderDistinct = IsUniqueIndex(pIndex) @@ -4911,7 +5197,7 @@ static i8 wherePathSatisfiesOrderBy( } /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and mark that ORDER BY term off + ** of the index and mark that ORDER BY term having been satisfied. */ isMatch = 0; for(i=0; bOnce && i<nOrderBy; i++){ @@ -4993,7 +5279,7 @@ static i8 wherePathSatisfiesOrderBy( if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } @@ -5119,6 +5405,83 @@ static LogEst whereSortingCost( } /* +** Compute the maximum number of paths in the solver algorithm, for +** queries that have three or more terms in the FROM clause. Queries with +** two or fewer FROM clause terms are handled by the caller. +** +** Query planning is NP-hard. We must limit the number of paths at +** each step of the solver search algorithm to avoid exponential behavior. +** +** The value returned is a tuning parameter. Currently the value is: +** +** 18 for star queries +** 12 otherwise +** +** For the purposes of SQLite, a star-query is defined as a query +** with a large central table that is joined against four or more +** smaller tables. The central table is called the "fact" table. +** The smaller tables that get joined are "dimension tables". +** +** SIDE EFFECT: (and really the whole point of this subroutine) +** +** If pWInfo describes a star-query, then the cost on WhereLoops for the +** fact table is reduced. This heuristic helps keep fact tables in +** outer loops. Without this heuristic, paths with fact tables in outer +** loops tend to get pruned by the mxChoice limit on the number of paths, +** resulting in poor query plans. The total amount of heuristic cost +** adjustment is stored in pWInfo->nOutStarDelta and the cost adjustment +** for each WhereLoop is stored in its rStarDelta field. +*/ +static int computeMxChoice(WhereInfo *pWInfo, LogEst nRowEst){ + int nLoop = pWInfo->nLevel; /* Number of terms in the join */ + if( nRowEst==0 && nLoop>=5 ){ + /* Check to see if we are dealing with a star schema and if so, reduce + ** the cost of fact tables relative to dimension tables, as a heuristic + ** to help keep the fact tables in outer loops. + */ + int iLoop; /* Counter over join terms */ + Bitmask m; /* Bitmask for current loop */ + assert( pWInfo->nOutStarDelta==0 ); + for(iLoop=0, m=1; iLoop<nLoop; iLoop++, m<<=1){ + WhereLoop *pWLoop; /* For looping over WhereLoops */ + int nDep = 0; /* Number of dimension tables */ + LogEst rDelta; /* Heuristic cost adjustment */ + Bitmask mSeen = 0; /* Mask of dimension tables */ + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( (pWLoop->prereq & m)!=0 && (pWLoop->maskSelf & mSeen)==0 ){ + nDep++; + mSeen |= pWLoop->maskSelf; + } + } + if( nDep<=3 ) continue; + rDelta = 15*(nDep-3); +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + SrcItem *pItem = pWInfo->pTabList->a + iLoop; + sqlite3DebugPrintf("Fact-table %s: %d dimensions, cost reduced %d\n", + pItem->zAlias ? pItem->zAlias : pItem->pSTab->zName, + nDep, rDelta); + } +#endif + if( pWInfo->nOutStarDelta==0 ){ + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + pWLoop->rStarDelta = 0; + } + } + pWInfo->nOutStarDelta += rDelta; + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( pWLoop->maskSelf==m ){ + pWLoop->rRun -= rDelta; + pWLoop->nOut -= rDelta; + pWLoop->rStarDelta = rDelta; + } + } + } + } + return pWInfo->nOutStarDelta>0 ? 18 : 12; +} + +/* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop ** once. This path is then loaded into the pWInfo->a[].pWLoop fields. @@ -5153,13 +5516,25 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pParse = pWInfo->pParse; nLoop = pWInfo->nLevel; - /* TUNING: For simple queries, only the best path is tracked. - ** For 2-way joins, the 5 best paths are followed. - ** For joins of 3 or more tables, track the 10 best paths */ - mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); - assert( nLoop<=pWInfo->pTabList->nSrc ); WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", nRowEst, pParse->nQueryLoop)); + /* TUNING: mxChoice is the maximum number of possible paths to preserve + ** at each step. Based on the number of loops in the FROM clause: + ** + ** nLoop mxChoice + ** ----- -------- + ** 1 1 // the most common case + ** 2 5 + ** 3+ 12 or 18 // see computeMxChoice() + */ + if( nLoop<=1 ){ + mxChoice = 1; + }else if( nLoop==2 ){ + mxChoice = 5; + }else{ + mxChoice = computeMxChoice(pWInfo, nRowEst); + } + assert( nLoop<=pWInfo->pTabList->nSrc ); /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this ** case the purpose of this call is to estimate the number of rows returned @@ -5242,7 +5617,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = pWLoop->rRun + pFrom->nRow; + if( pWLoop->rSetup ){ + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup, rUnsorted); + } rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; @@ -5287,6 +5665,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range ** of legal values for isOrdered, -1..64. */ + testcase( nTo==0 ); for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){ if( pTo->maskLoop==maskNew && ((pTo->isOrdered^isOrdered)&0x80)==0 @@ -5403,16 +5782,28 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #ifdef WHERETRACE_ENABLED /* >=2 */ if( sqlite3WhereTrace & 0x02 ){ + LogEst rMin, rFloor = 0; + int nDone = 0; sqlite3DebugPrintf("---- after round %d ----\n", iLoop); - for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){ - sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c", - wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); - if( pTo->isOrdered>0 ){ - sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); - }else{ - sqlite3DebugPrintf("\n"); + while( nDone<nTo ){ + rMin = 0x7fff; + for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){ + if( pTo->rCost>rFloor && pTo->rCost<rMin ) rMin = pTo->rCost; } + for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){ + if( pTo->rCost==rMin ){ + sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } + nDone++; + } + } + rFloor = rMin; } } #endif @@ -5462,10 +5853,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } - if( pWInfo->pSelect->pOrderBy - && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ - pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; - } + /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */ + assert( pWInfo->pSelect->pOrderBy==0 + || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr ); }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -5508,8 +5898,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - - pWInfo->nRowOut = pFrom->nRow; + pWInfo->nRowOut = pFrom->nRow + pWInfo->nOutStarDelta; /* Free temporary memory and return success */ sqlite3StackFreeNN(pParse->db, pSpace); @@ -5517,6 +5906,83 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } /* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; i<pWInfo->nLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + +/* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts ** to plan those simple cases using much less ceremony than the @@ -5543,7 +6009,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; - pTab = pItem->pTab; + pTab = pItem->pSTab; if( IsVirtual(pTab) ) return 0; if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ testcase( pItem->fg.isIndexedBy ); @@ -5684,6 +6150,10 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** the right-most table of a subquery that was flattened into the ** main query and that subquery was the right-hand operand of an ** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. ** ** For example, given: ** @@ -5802,9 +6272,9 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( WhereLoop *pLoop = pWInfo->a[i].pWLoop; const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; - Table *pTab = pItem->pTab; + Table *pTab = pItem->pSTab; if( (pTab->tabFlags & TF_HasStat1)==0 ) break; - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; if( i>=1 && (pLoop->wsFlags & reqFlags)==reqFlags /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ @@ -5822,10 +6292,63 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } nSearch += pLoop->nOut; + if( pWInfo->nOutStarDelta ) nSearch += pLoop->rStarDelta; } } /* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + +/* ** The index pIdx is used by a query and contains one or more expressions. ** In other words pIdx is an index on an expression. iIdxCur is the cursor ** number for the index and iDataCur is the cursor number for the corresponding @@ -5850,33 +6373,19 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( for(i=0; i<pIdx->nColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } - if( sqlite3ExprIsConstant(pExpr) ) continue; - if( pExpr->op==TK_FUNCTION ){ + if( sqlite3ExprIsConstant(0,pExpr) ) continue; + if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){ /* Functions that might set a subtype should not be replaced by the ** value taken from an expression index since the index omits the ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ - int n; - FuncDef *pDef; - sqlite3 *db = pParse->db; - assert( ExprUseXList(pExpr) ); - n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; - pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); - if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ - continue; - } + continue; } p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; @@ -5891,7 +6400,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } @@ -5920,8 +6429,8 @@ static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ SrcItem *pItem = &pWInfo->pTabList->a[ii]; if( !pItem->fg.isCte || pItem->u2.pCteUse->eM10d!=M10d_Yes - || NEVER(pItem->pSelect==0) - || pItem->pSelect->pOrderBy==0 + || NEVER(pItem->fg.isSubquery==0) + || pItem->u4.pSubq->pSelect->pOrderBy==0 ){ pWInfo->revMask |= MASKBIT(ii); } @@ -6056,7 +6565,11 @@ WhereInfo *sqlite3WhereBegin( /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); - if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; + if( pOrderBy && pOrderBy->nExpr>=BMS ){ + pOrderBy = 0; + wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ + } /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask @@ -6138,7 +6651,11 @@ WhereInfo *sqlite3WhereBegin( ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } - ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + if( ALWAYS(pWInfo->pSelect) + && (pWInfo->pSelect->selFlags & SF_MultiValue)==0 + ){ + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + } }else{ /* Assign a bit from the bitmask to every term in the FROM clause. ** @@ -6291,7 +6808,8 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ - wherePathSolver(pWInfo, pWInfo->nRowOut+1); + whereInterstageHeuristic(pWInfo); + wherePathSolver(pWInfo, pWInfo->nRowOut<0 ? 1 : pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } @@ -6351,10 +6869,10 @@ WhereInfo *sqlite3WhereBegin( ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. */ notReady = ~(Bitmask)0; - if( pWInfo->nLevel>=2 - && pResultSet!=0 /* these two combine to guarantee */ - && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ ){ notReady = whereOmitNoopJoin(pWInfo, notReady); nTabList = pWInfo->nLevel; @@ -6402,15 +6920,15 @@ WhereInfo *sqlite3WhereBegin( if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ int wsFlags = pWInfo->a[0].pWLoop->wsFlags; int bOnerow = (wsFlags & WHERE_ONEROW)!=0; - assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); + assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) ); if( bOnerow || ( 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) - && !IsVirtual(pTabList->a[0].pTab) + && !IsVirtual(pTabList->a[0].pSTab) && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) && OptimizationEnabled(db, SQLITE_OnePass) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; - if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){ if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ bFordelete = OPFLAG_FORDELETE; } @@ -6428,7 +6946,7 @@ WhereInfo *sqlite3WhereBegin( SrcItem *pTabItem; pTabItem = &pTabList->a[pLevel->iFrom]; - pTab = pTabItem->pTab; + pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ @@ -6499,7 +7017,7 @@ WhereInfo *sqlite3WhereBegin( iIndexCur = pLevel->iTabCur; op = 0; }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ - Index *pJ = pTabItem->pTab->pIndex; + Index *pJ = pTabItem->pSTab->pIndex; iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ @@ -6566,7 +7084,7 @@ WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); pRJ->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); - assert( pTab==pTabItem->pTab ); + assert( pTab==pTabItem->pSTab ); if( HasRowid(pTab) ){ KeyInfo *pInfo; sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); @@ -6605,13 +7123,18 @@ WhereInfo *sqlite3WhereBegin( wsFlags = pLevel->pWLoop->wsFlags; pSrc = &pTabList->a[pLevel->iFrom]; if( pSrc->fg.isMaterialized ){ - if( pSrc->fg.isCorrelated ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); + Subquery *pSubq; + int iOnce = 0; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + if( pSrc->fg.isCorrelated==0 ){ + iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); }else{ - int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); - sqlite3VdbeJumpHere(v, iOnce); + iOnce = 0; } + sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); + VdbeComment((v, "materialize %!S", pSrc)); + if( iOnce ) sqlite3VdbeJumpHere(v, iOnce); } assert( pTabList == pWInfo->pTabList ); if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ @@ -6674,26 +7197,6 @@ whereBeginError: } #endif -#ifdef SQLITE_DEBUG -/* -** Return true if cursor iCur is opened by instruction k of the -** bytecode. Used inside of assert() only. -*/ -static int cursorIsOpen(Vdbe *v, int iCur, int k){ - while( k>=0 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); - if( pOp->p1!=iCur ) continue; - if( pOp->opcode==OP_Close ) return 0; - if( pOp->opcode==OP_OpenRead ) return 1; - if( pOp->opcode==OP_OpenWrite ) return 1; - if( pOp->opcode==OP_OpenDup ) return 1; - if( pOp->opcode==OP_OpenAutoindex ) return 1; - if( pOp->opcode==OP_OpenEphemeral ) return 1; - } - return 0; -} -#endif /* SQLITE_DEBUG */ - /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -6840,7 +7343,16 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); if( (ws & WHERE_IDX_ONLY)==0 ){ - assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + SrcItem *pSrc = &pTabList->a[pLevel->iFrom]; + assert( pLevel->iTabCur==pSrc->iCursor ); + if( pSrc->fg.viaCoroutine ){ + int m, n; + assert( pSrc->fg.isSubquery ); + n = pSrc->u4.pSubq->regResult; + assert( pSrc->pSTab!=0 ); + m = pSrc->pSTab->nCol; + sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); + } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } if( (ws & WHERE_INDEXED) @@ -6862,7 +7374,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeJumpHere(v, addr); } VdbeModuleComment((v, "End WHERE-loop%d: %s", i, - pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); + pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName)); } assert( pWInfo->nLevel<=pTabList->nSrc ); @@ -6871,7 +7383,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeOp *pOp, *pLastOp; Index *pIdx = 0; SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; assert( pTab!=0 ); pLoop = pLevel->pWLoop; @@ -6890,8 +7402,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); + assert( pTabItem->fg.isSubquery ); + assert( pTabItem->u4.pSubq->regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult, 0); + pTabItem->u4.pSubq->regResult, 0); continue; } @@ -6984,16 +7498,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** reference. Verify that this is harmless - that the ** table being referenced really is open. */ -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - || pOp->opcode==OP_Offset - ); -#else - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - ); -#endif + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + } } }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; diff --git a/src/whereInt.h b/src/whereInt.h index f3cc5776a..8247528a9 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -143,11 +143,13 @@ struct WhereLoop { u16 nTop; /* Size of TOP vector */ u16 nDistinctCol; /* Index columns used to sort for DISTINCT */ Index *pIndex; /* Index used, or NULL */ + ExprList *pOrderBy; /* ORDER BY clause if this is really a subquery */ } btree; struct { /* Information for virtual tables */ int idxNum; /* Index number */ u32 needFree : 1; /* True if sqlite3_free(idxStr) is needed */ u32 bOmitOffset : 1; /* True to let virtual table handle offset */ + u32 bIdxNumHex : 1; /* Show idxNum as hex in EXPLAIN QUERY PLAN */ i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ @@ -160,6 +162,8 @@ struct WhereLoop { /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + LogEst rStarDelta; /* Cost delta due to star-schema heuristic. Not + ** initialized unless pWInfo->nOutStarDelta>0 */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ @@ -482,6 +486,7 @@ struct WhereInfo { unsigned untestedTerms :1; /* Not all WHERE terms resolved by outer loop */ unsigned bOrderedInnerLoop:1;/* True if only the inner-most loop is ordered */ unsigned sorted :1; /* True if really sorted (not just grouped) */ + LogEst nOutStarDelta; /* Artifical nOut reduction for star-query */ LogEst nRowOut; /* Estimated number of output rows */ int iTop; /* The very beginning of the WHERE loop */ int iEndWhere; /* End of the WHERE clause itself */ @@ -633,7 +638,8 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ - /* 0x02000000 -- available for reuse */ +#define WHERE_COROUTINE 0x02000000 /* Implemented by co-routine. + ** NB: False-negatives are possible */ #define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ diff --git a/src/wherecode.c b/src/wherecode.c index 47ce36ce3..0951e5e20 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -157,7 +157,7 @@ int sqlite3WhereExplainOneScan( assert( pLoop->u.btree.pIndex!=0 ); pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( !HasRowid(pItem->pSTab) && IsPrimaryKeyIndex(pIdx) ){ if( isSearch ){ zFmt = "PRIMARY KEY"; } @@ -200,7 +200,9 @@ int sqlite3WhereExplainOneScan( } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3_str_appendf(&str, " VIRTUAL TABLE INDEX %d:%s", + sqlite3_str_appendall(&str, " VIRTUAL TABLE INDEX "); + sqlite3_str_appendf(&str, + pLoop->u.vtab.bIdxNumHex ? "0x%x:%s" : "%d:%s", pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif @@ -218,7 +220,8 @@ int sqlite3WhereExplainOneScan( zMsg = sqlite3StrAccumFinish(&str); sqlite3ExplainBreakpoint("",zMsg); ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), - pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + pParse->addrExplain, pLoop->rRun, + zMsg, P4_DYNAMIC); } return ret; } @@ -253,7 +256,7 @@ int sqlite3WhereExplainBloomFilter( sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem); pLoop = pLevel->pWLoop; if( pLoop->wsFlags & WHERE_IPK ){ - const Table *pTab = pItem->pTab; + const Table *pTab = pItem->pSTab; if( pTab->iPKey>=0 ){ sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName); }else{ @@ -316,7 +319,9 @@ void sqlite3WhereAddScanStatus( sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); } }else{ - int addr = pSrclist->a[pLvl->iFrom].addrFillSub; + int addr; + assert( pSrclist->a[pLvl->iFrom].fg.isSubquery ); + addr = pSrclist->a[pLvl->iFrom].u4.pSubq->addrFillSub; VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); @@ -460,6 +465,39 @@ static void updateRangeAffinityStr( } } +/* +** The pOrderBy->a[].u.x.iOrderByCol values might be incorrect because +** columns might have been rearranged in the result set. This routine +** fixes them up. +** +** pEList is the new result set. The pEList->a[].u.x.iOrderByCol values +** contain the *old* locations of each expression. This is a temporary +** use of u.x.iOrderByCol, not its intended use. The caller must reset +** u.x.iOrderByCol back to zero for all entries in pEList before the +** caller returns. +** +** This routine changes pOrderBy->a[].u.x.iOrderByCol values from +** pEList->a[N].u.x.iOrderByCol into N+1. (The "+1" is because of the 1-based +** indexing used by iOrderByCol.) Or if no match, iOrderByCol is set to zero. +*/ +static void adjustOrderByCol(ExprList *pOrderBy, ExprList *pEList){ + int i, j; + if( pOrderBy==0 ) return; + for(i=0; i<pOrderBy->nExpr; i++){ + int t = pOrderBy->a[i].u.x.iOrderByCol; + if( t==0 ) continue; + for(j=0; j<pEList->nExpr; j++){ + if( pEList->a[j].u.x.iOrderByCol==t ){ + pOrderBy->a[i].u.x.iOrderByCol = j+1; + break; + } + } + if( j>=pEList->nExpr ){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + } +} + /* ** pX is an expression of the form: (vector) IN (SELECT ...) @@ -522,7 +560,8 @@ static Expr *removeUnindexableInClauseTerms( iField = pLoop->aLTerm[i]->u.x.iField - 1; if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); - pOrigRhs->a[iField].pExpr = 0; + pOrigRhs->a[iField].pExpr = 0; + if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1; if( pOrigLhs ){ assert( pOrigLhs->a[iField].pExpr!=0 ); pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr); @@ -545,18 +584,16 @@ static Expr *removeUnindexableInClauseTerms( sqlite3ExprDelete(db, pNew->pLeft); pNew->pLeft = p; } - if( pSelect->pOrderBy ){ - /* If the SELECT statement has an ORDER BY clause, zero the - ** iOrderByCol variables. These are set to non-zero when an - ** ORDER BY term exactly matches one of the terms of the - ** result-set. Since the result-set of the SELECT statement may - ** have been modified or reordered, these variables are no longer - ** set correctly. Since setting them is just an optimization, - ** it's easiest just to zero them here. */ - ExprList *pOrderBy = pSelect->pOrderBy; - for(i=0; i<pOrderBy->nExpr; i++){ - pOrderBy->a[i].u.x.iOrderByCol = 0; - } + + /* If either the ORDER BY clause or the GROUP BY clause contains + ** references to result-set columns, those references might now be + ** obsolete. So fix them up. + */ + assert( pRhs!=0 || db->mallocFailed ); + if( pRhs ){ + adjustOrderByCol(pSelect->pOrderBy, pRhs); + adjustOrderByCol(pSelect->pGroupBy, pRhs); + for(i=0; i<pRhs->nExpr; i++) pRhs->a[i].u.x.iOrderByCol = 0; } #if 0 @@ -571,6 +608,147 @@ static Expr *removeUnindexableInClauseTerms( } +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for a single X IN (....) term of the WHERE clause. +** +** This is a special-case of codeEqualityTerm() that works for IN operators +** only. It is broken out into a subroutine because this case is +** uncommon and by splitting it off into a subroutine, the common case +** runs faster. +** +** The current value for the constraint is left in register iTarget. +** This routine sets up a loop that will iterate over all values of X. +*/ +static SQLITE_NOINLINE void codeINTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + int eType = IN_INDEX_NOOP; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + Vdbe *v = pParse->pVdbe; + int i; + int nEq = 0; + int *aiMap = 0; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + + for(i=0; i<iEq; i++){ + if( pLoop->aLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ + disableTerm(pLevel, pTerm); + return; + } + } + for(i=iEq;i<pLoop->nLTerm; i++){ + assert( pLoop->aLTerm[i]!=0 ); + if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; + } + + iTab = 0; + if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); + }else{ + Expr *pExpr = pTerm->pExpr; + if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ + sqlite3 *db = pParse->db; + pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); + pExpr->iTable = iTab; + } + sqlite3ExprDelete(db, pX); + }else{ + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); + } + pX = pExpr; + } + + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + } + if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){ + pLoop->wsFlags |= WHERE_IN_EARLYOUT; + } + + i = pLevel->u.in.nIn; + pLevel->u.in.nIn += nEq; + pLevel->u.in.aInLoop = + sqlite3WhereRealloc(pTerm->pWC->pWInfo, + pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + int iMap = 0; /* Index in aiMap[] */ + pIn += i; + for(i=iEq;i<pLoop->nLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iOut = iTarget + i - iEq; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); + }else{ + int iCol = aiMap ? aiMap[iMap++] : 0; + pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); + } + sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); + if( i==iEq ){ + pIn->iCur = iTab; + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; + if( iEq>0 ){ + pIn->iBase = iTarget - i; + pIn->nPrefix = i; + }else{ + pIn->nPrefix = 0; + } + }else{ + pIn->eEndLoopOp = OP_Noop; + } + pIn++; + } + } + testcase( iEq>0 + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 + && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ); + if( iEq>0 + && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0 + ){ + sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq); + } + }else{ + pLevel->u.in.nIn = 0; + } + sqlite3DbFree(pParse->db, aiMap); +} +#endif + + /* ** Generate code for a single equality term of the WHERE clause. An equality ** term can be either X=expr or X IN (...). pTerm is the term to be @@ -595,7 +773,6 @@ static int codeEqualityTerm( int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; int iReg; /* Register holding results */ assert( pLevel->pWLoop->aLTerm[iEq]==pTerm ); @@ -604,125 +781,12 @@ static int codeEqualityTerm( iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); }else if( pX->op==TK_ISNULL ){ iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Null, 0, iReg); #ifndef SQLITE_OMIT_SUBQUERY }else{ - int eType = IN_INDEX_NOOP; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; - int i; - int nEq = 0; - int *aiMap = 0; - - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; - } assert( pX->op==TK_IN ); iReg = iTarget; - - for(i=0; i<iEq; i++){ - if( pLoop->aLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ - disableTerm(pLevel, pTerm); - return iTarget; - } - } - for(i=iEq;i<pLoop->nLTerm; i++){ - assert( pLoop->aLTerm[i]!=0 ); - if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; - } - - iTab = 0; - if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); - }else{ - Expr *pExpr = pTerm->pExpr; - if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ - sqlite3 *db = pParse->db; - pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); - if( !db->mallocFailed ){ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); - pExpr->iTable = iTab; - } - sqlite3ExprDelete(db, pX); - }else{ - int n = sqlite3ExprVectorSize(pX->pLeft); - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); - } - pX = pExpr; - } - - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; - } - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); - } - if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){ - pLoop->wsFlags |= WHERE_IN_EARLYOUT; - } - - i = pLevel->u.in.nIn; - pLevel->u.in.nIn += nEq; - pLevel->u.in.aInLoop = - sqlite3WhereRealloc(pTerm->pWC->pWInfo, - pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - int iMap = 0; /* Index in aiMap[] */ - pIn += i; - for(i=iEq;i<pLoop->nLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iOut = iReg + i - iEq; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); - }else{ - int iCol = aiMap ? aiMap[iMap++] : 0; - pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); - } - sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); - if( i==iEq ){ - pIn->iCur = iTab; - pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; - if( iEq>0 ){ - pIn->iBase = iReg - i; - pIn->nPrefix = i; - }else{ - pIn->nPrefix = 0; - } - }else{ - pIn->eEndLoopOp = OP_Noop; - } - pIn++; - } - } - testcase( iEq>0 - && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 - && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ); - if( iEq>0 - && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0 - ){ - sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq); - } - }else{ - pLevel->u.in.nIn = 0; - } - sqlite3DbFree(pParse->db, aiMap); + codeINTerm(pParse, pTerm, pLevel, iEq, bRev, iTarget); #endif } @@ -1338,6 +1402,27 @@ static SQLITE_NOINLINE void filterPullDown( } /* +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one +** row for each key generated for the index. +*/ +static int whereLoopIsOneRow(WhereLoop *pLoop){ + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + ){ + int ii; + for(ii=0; ii<pLoop->u.btree.nEq; ii++){ + if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){ + return 0; + } + } + return 1; + } + return 0; +} + +/* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. */ @@ -1373,7 +1458,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( iCur = pTabItem->iCursor; pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + VdbeModuleComment((v, "Begin WHERE-loop%d: %s", + iLevel, pTabItem->pSTab->zName)); #if WHERETRACE_ENABLED /* 0x4001 */ if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", @@ -1415,7 +1501,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + VdbeComment((v, "init LEFT JOIN match flag")); } /* Compute a safe address to jump to if we discover that the table for @@ -1428,11 +1514,15 @@ Bitmask sqlite3WhereCodeOneLoopStart( /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + int regYield; + Subquery *pSubq; + assert( pTabItem->fg.isSubquery && pTabItem->u4.pSubq!=0 ); + pSubq = pTabItem->u4.pSubq; + regYield = pSubq->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pSTab->zName)); pLevel->op = OP_Goto; }else @@ -2084,7 +2174,9 @@ Bitmask sqlite3WhereCodeOneLoopStart( } /* Record the instruction used to terminate the loop. */ - if( pLoop->wsFlags & WHERE_ONEROW ){ + if( (pLoop->wsFlags & WHERE_ONEROW) + || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) + ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -2159,7 +2251,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); @@ -2474,6 +2566,12 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ iLoop = (pIdx ? 1 : 2); do{ @@ -2612,7 +2710,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** least once. This is accomplished by storing the PK for the row in ** both the iMatch index and the regBloom Bloom filter. */ - pTab = pWInfo->pTabList->a[pLevel->iFrom].pTab; + pTab = pWInfo->pTabList->a[pLevel->iFrom].pSTab; if( HasRowid(pTab) ){ r = sqlite3GetTempRange(pParse, 2); sqlite3ExprCodeGetColumnOfTable(v, pTab, pLevel->iTabCur, -1, r+1); @@ -2719,12 +2817,25 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( Bitmask mAll = 0; int k; - ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pTab->zName)); + ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pSTab->zName)); sqlite3VdbeNoJumpsOutsideSubrtn(v, pRJ->addrSubrtn, pRJ->endSubrtn, pRJ->regReturn); for(k=0; k<iLevel; k++){ int iIdxCur; + SrcItem *pRight; + assert( pWInfo->a[k].pWLoop->iTab == pWInfo->a[k].iFrom ); + pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; mAll |= pWInfo->a[k].pWLoop->maskSelf; + if( pRight->fg.viaCoroutine ){ + Subquery *pSubq; + assert( pRight->fg.isSubquery && pRight->u4.pSubq!=0 ); + pSubq = pRight->u4.pSubq; + assert( pSubq->pSelect!=0 && pSubq->pSelect->pEList!=0 ); + sqlite3VdbeAddOp3( + v, OP_Null, 0, pSubq->regResult, + pSubq->regResult + pSubq->pSelect->pEList->nExpr-1 + ); + } sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); iIdxCur = pWInfo->a[k].iIdxCur; if( iIdxCur ){ @@ -2760,7 +2871,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( int nPk; int jmp; int addrCont = sqlite3WhereContinueLabel(pSubWInfo); - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; if( HasRowid(pTab) ){ sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r); nPk = 1; diff --git a/src/whereexpr.c b/src/whereexpr.c index daf3d5d95..7ea2956a7 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -101,7 +101,12 @@ static int allowedOp(int op){ assert( TK_LT>TK_EQ && TK_LT<TK_GE ); assert( TK_LE>TK_EQ && TK_LE<TK_GE ); assert( TK_GE==TK_EQ+4 ); - return op==TK_IN || (op>=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; + assert( TK_IN<TK_EQ ); + assert( TK_IS<TK_EQ ); + assert( TK_ISNULL<TK_EQ ); + if( op>TK_GE ) return 0; + if( op>=TK_EQ ) return 1; + return op==TK_IN || op==TK_ISNULL || op==TK_IS; } /* @@ -134,15 +139,16 @@ static u16 exprCommute(Parse *pParse, Expr *pExpr){ static u16 operatorMask(int op){ u16 c; assert( allowedOp(op) ); - if( op==TK_IN ){ + if( op>=TK_EQ ){ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + }else if( op==TK_IN ){ c = WO_IN; }else if( op==TK_ISNULL ){ c = WO_ISNULL; - }else if( op==TK_IS ){ - c = WO_IS; }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); + assert( op==TK_IS ); + c = WO_IS; } assert( op!=TK_ISNULL || c==WO_ISNULL ); assert( op!=TK_IN || c==WO_IN ); @@ -214,11 +220,20 @@ static int isLikeOrGlob( } if( z ){ - /* Count the number of prefix characters prior to the first wildcard */ + /* Count the number of prefix characters prior to the first wildcard. + ** If the underlying database has a UTF16LE encoding, then only consider + ** ASCII characters. Note that the encoding of z[] is UTF8 - we are + ** dealing with only UTF8 here in this code, but the database engine + ** itself might be processing content using a different encoding. */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; - if( c==wc[3] && z[cnt]!=0 ) cnt++; + if( c==wc[3] && z[cnt]!=0 ){ + cnt++; + }else if( c>=0x80 && ENC(db)==SQLITE_UTF16LE ){ + cnt--; + break; + } } /* The optimization is possible only if (1) the pattern does not begin @@ -233,7 +248,7 @@ static int isLikeOrGlob( Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ - *pisComplete = c==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE; /* Get the pattern prefix. Remove all escapes from the prefix. */ pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); @@ -429,6 +444,13 @@ static int isAuxiliaryVtabOperator( } } } + }else if( pExpr->op>=TK_EQ ){ + /* Comparison operators are a common case. Save a few comparisons for + ** that common case by terminating early. */ + assert( TK_NE < TK_EQ ); + assert( TK_ISNOT < TK_EQ ); + assert( TK_NOTNULL < TK_EQ ); + return 0; }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){ int res = 0; Expr *pLeft = pExpr->pLeft; @@ -945,7 +967,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ if( ALWAYS(pSrc!=0) ){ int i; for(i=0; i<pSrc->nSrc; i++){ - mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + if( pSrc->a[i].fg.isSubquery ){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].u4.pSubq->pSelect); + } if( pSrc->a[i].fg.isUsing==0 ){ mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn); } @@ -983,13 +1007,13 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( int iCur; do{ iCur = pFrom->a[j].iCursor; - for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + for(pIdx=pFrom->a[j].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->aColExpr==0 ) continue; for(i=0; i<pIdx->nKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 - && pExpr->op!=TK_STRING + && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr) ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR; @@ -1027,7 +1051,7 @@ static int exprMightBeIndexed( for(i=0; i<pFrom->nSrc; i++){ Index *pIdx; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + for(pIdx=pFrom->a[i].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->aColExpr ){ return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i); } @@ -1570,7 +1594,7 @@ static void whereAddLimitExpr( Expr *pNew; int iVal = 0; - if( sqlite3ExprIsInteger(pExpr, &iVal) && iVal>=0 ){ + if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); if( pVal==0 ) return; ExprSetProperty(pVal, EP_IntValue); @@ -1615,7 +1639,7 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ if( p->pGroupBy==0 && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ - && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ + && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pSTab)) /* 3 */ ){ ExprList *pOrderBy = p->pOrderBy; int iCsr = p->pSrc->a[0].iCursor; @@ -1638,6 +1662,7 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ continue; } if( pWC->a[ii].leftCursor!=iCsr ) return; + if( pWC->a[ii].prereqRight!=0 ) return; } /* Check condition (5). Return early if it is not met. */ @@ -1652,12 +1677,14 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ /* All conditions are met. Add the terms to the where-clause object. */ assert( p->pLimit->op==TK_LIMIT ); - whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, - iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); - if( p->iOffset>0 ){ + if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){ whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight, iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET); } + if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, + iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); + } } } @@ -1833,7 +1860,7 @@ void sqlite3WhereTabFuncArgs( Expr *pColRef; Expr *pTerm; if( pItem->fg.isTabFunc==0 ) return; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); pArgs = pItem->u1.pFuncArg; if( pArgs==0 ) return; diff --git a/src/window.c b/src/window.c index 62df349fb..d4083beeb 100644 --- a/src/window.c +++ b/src/window.c @@ -909,7 +909,7 @@ static ExprList *exprListAppendList( int iDummy; Expr *pSub; pSub = sqlite3ExprSkipCollateAndLikely(pDup); - if( sqlite3ExprIsInteger(pSub, &iDummy) ){ + if( sqlite3ExprIsInteger(pSub, &iDummy, 0) ){ pSub->op = TK_NULL; pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); pSub->u.zToken = 0; @@ -1077,9 +1077,10 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside ** of sqlite3DbMallocRawNN() called from ** sqlite3SrcListAppend() */ - if( p->pSrc ){ + if( p->pSrc==0 ){ + sqlite3SelectDelete(db, pSub); + }else if( sqlite3SrcItemAttachSubquery(pParse, &p->pSrc->a[0], pSub, 0) ){ Table *pTab2; - p->pSrc->a[0].pSelect = pSub; p->pSrc->a[0].fg.isCorrelated = 1; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded|SF_OrderByReqd; @@ -1093,7 +1094,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ }else{ memcpy(pTab, pTab2, sizeof(Table)); pTab->tabFlags |= TF_Ephemeral; - p->pSrc->a[0].pTab = pTab; + p->pSrc->a[0].pSTab = pTab; pTab = pTab2; memset(&w, 0, sizeof(w)); w.xExprCallback = sqlite3WindowExtraAggFuncDepth; @@ -1101,8 +1102,6 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ w.xSelectCallback2 = sqlite3WalkerDepthDecrease; sqlite3WalkSelect(&w, pSub); } - }else{ - sqlite3SelectDelete(db, pSub); } if( db->mallocFailed ) rc = SQLITE_NOMEM; @@ -1164,7 +1163,7 @@ void sqlite3WindowListDelete(sqlite3 *db, Window *p){ ** variable values in the expression tree. */ static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ - if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( 0==sqlite3ExprIsConstant(0,pExpr) ){ if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); sqlite3ExprDelete(pParse->db, pExpr); pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); @@ -1389,10 +1388,15 @@ int sqlite3WindowCompare( ** and initialize registers and cursors used by sqlite3WindowCodeStep(). */ void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ - int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr; - Window *pMWin = pSelect->pWin; Window *pWin; - Vdbe *v = sqlite3GetVdbe(pParse); + int nEphExpr; + Window *pMWin; + Vdbe *v; + + assert( pSelect->pSrc->a[0].fg.isSubquery ); + nEphExpr = pSelect->pSrc->a[0].u4.pSubq->pSelect->pEList->nExpr; + pMWin = pSelect->pWin; + v = sqlite3GetVdbe(pParse); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr); sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); @@ -2789,7 +2793,7 @@ void sqlite3WindowCodeStep( Vdbe *v = sqlite3GetVdbe(pParse); int csrWrite; /* Cursor used to write to eph. table */ int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ - int nInput = p->pSrc->a[0].pTab->nCol; /* Number of cols returned by sub */ + int nInput = p->pSrc->a[0].pSTab->nCol; /* Number of cols returned by sub */ int iInput; /* To iterate through sub cols */ int addrNe; /* Address of OP_Ne */ int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ diff --git a/test/alter2.test b/test/alter2.test index aae0061ad..20b75b59e 100644 --- a/test/alter2.test +++ b/test/alter2.test @@ -371,7 +371,7 @@ do_test alter2-7.5 { execsql { SELECT a, typeof(a), b, typeof(b), c, typeof(c) FROM t1 LIMIT 1; } -} {1 integer -123 integer 5 text} +} {1 integer -123.0 real 5 text} #----------------------------------------------------------------------- # Test that UPDATE trigger tables work with default values, and that when @@ -397,11 +397,11 @@ do_test alter2-8.2 { UPDATE t1 SET c = 10 WHERE a = 1; SELECT a, typeof(a), b, typeof(b), c, typeof(c) FROM t1 LIMIT 1; } -} {1 integer -123 integer 10 text} +} {1 integer -123.0 real 10 text} ifcapable trigger { do_test alter2-8.3 { set ::val - } {-123 integer 5 text -123 integer 10 text} + } {-123.0 real 5 text -123.0 real 10 text} } #----------------------------------------------------------------------- @@ -425,7 +425,7 @@ ifcapable trigger { DELETE FROM t1 WHERE a = 2; } set ::val - } {-123 integer 5 text} + } {-123.0 real 5 text} } #----------------------------------------------------------------------- diff --git a/test/altertab2.test b/test/altertab2.test index def9e5668..576dc4967 100644 --- a/test/altertab2.test +++ b/test/altertab2.test @@ -360,4 +360,26 @@ do_catchsql_test 8.6 { SELECT sql FROM sqlite_master WHERE name='i0'; } {1 {error in index i0: second argument to likelihood() must be a constant between 0.0 and 1.0}} + +reset_db + +do_execsql_test 9.0 { + CREATE TABLE t1(a,b,c,d); + CREATE TABLE t2(a,b,c,d,x); + + CREATE TRIGGER AFTER INSERT ON t2 BEGIN + + SELECT group_conct( + 123 ORDER BY ( + SELECT 1 FROM ( VALUES(a, 'b'), ('c') ) + )) + FROM t1; + + END; +} + +do_catchsql_test 9.1 { + ALTER TABLE t2 RENAME TO newname; +} {1 {error in trigger AFTER: all VALUES must have the same number of terms}} + finish_test diff --git a/test/altertab3.test b/test/altertab3.test index 5fd17f3a2..5f5c11b0b 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -736,4 +736,54 @@ do_execsql_test 29.7 { END} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 30.0 { + CREATE TABLE t1(a, b); + CREATE VIEW v1 AS + SELECT ( VALUES(a), (b) ) FROM ( + SELECT a, b FROM t1 + ) + ; +} + +do_execsql_test 30.1 { + SELECT * FROM v1 +} + +do_execsql_test 30.1 { + ALTER TABLE t1 RENAME TO t2; +} +do_execsql_test 30.2 { + SELECT sql FROM sqlite_schema WHERE type='view' +} { + {CREATE VIEW v1 AS + SELECT ( VALUES(a), (b) ) FROM ( + SELECT a, b FROM "t2" + )} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 31.0 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, tt INTEGER, rr REAL); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000 + ) + INSERT INTO t1 SELECT NULL, i, 5.0 FROM s; +} + +do_test 31.1 { + set pg [db one {PRAGMA page_count}] + execsql { + ALTER TABLE t1 DROP COLUMN tt; + } + set pg2 [db one {PRAGMA page_count}] + expr $pg==$pg2 +} {1} + +do_execsql_test 31.2 { + SELECT rr FROM t1 LIMIT 1 +} {5.0} + finish_test diff --git a/test/autoindex1.test b/test/autoindex1.test index 08bd9f74e..b294a2721 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -185,7 +185,8 @@ do_eqp_test autoindex1-500.1 { QUERY PLAN |--SEARCH t501 USING INTEGER PRIMARY KEY (rowid=?) `--LIST SUBQUERY xxxxxx - `--SCAN t502 + |--SCAN t502 + `--CREATE BLOOM FILTER } do_eqp_test autoindex1-501 { SELECT b FROM t501 diff --git a/test/autoindex3.test b/test/autoindex3.test index 3da7a7058..aa6aa0012 100644 --- a/test/autoindex3.test +++ b/test/autoindex3.test @@ -90,5 +90,40 @@ do_eqp_test 220 { `--SEARCH u USING AUTOMATIC COVERING INDEX (b=?) } +# 2024-05-27 +# ticket https://sqlite.org/src/tktview/8ff324e120 +# forum post https://sqlite.org/forum/forumpost/b21c2101a559be0a +# +# If an index with STAT1 data indicates that a column is not very +# selective, then do not attempt to create an automatic index on +# that column. +# +reset_db +do_execsql_test 300 { + CREATE TABLE t1(id INTEGER PRIMARY KEY); + CREATE TABLE t2(cid INT, pid INT, rx INT, PRIMARY KEY(cid, pid, rx)); + CREATE INDEX x1 ON t2(pid, rx); + ANALYZE sqlite_schema; + REPLACE INTO sqlite_stat1(tbl, idx, stat) VALUES + ('t2', 'x1', '500000 250 250'), + ('t2','sqlite_autoindex_t2_1','500000 1 1 1'); + ANALYZE sqlite_schema; +} +do_eqp_test 310 { + WITH RECURSIVE children(id) AS ( + SELECT cid FROM t2 WHERE pid = ?1 AND rx = ?2 + UNION + SELECT cid FROM t2 JOIN children ON t2.pid = children.id AND rx = ?2 + ) SELECT count(id) FROM children; +} { + QUERY PLAN + |--CO-ROUTINE children + | |--SETUP + | | `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?) + | `--RECURSIVE STEP + | |--SCAN children + | `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?) + `--SCAN children +} finish_test diff --git a/test/avfs.test b/test/avfs.test index 2ebd608ba..ffd6b309f 100644 --- a/test/avfs.test +++ b/test/avfs.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # This file implements tests for the appendvfs extension. # diff --git a/test/backcompat.test b/test/backcompat.test index 87ffc4b3e..d477d4466 100644 --- a/test/backcompat.test +++ b/test/backcompat.test @@ -112,7 +112,7 @@ proc read_file {zFile} { set zData {} if {[file exists $zFile]} { set fd [open $zFile] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary if {[file size $zFile]<=$::sqlite_pending_byte || $zFile != "test.db"} { set zData [read $fd] @@ -129,7 +129,7 @@ proc read_file {zFile} { } proc write_file {zFile zData} { set fd [open $zFile w] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary puts -nonewline $fd $zData close $fd } diff --git a/test/bestindex8.test b/test/bestindex8.test index e95c3c6dc..3ed7f6703 100644 --- a/test/bestindex8.test +++ b/test/bestindex8.test @@ -158,7 +158,7 @@ do_test 2.2 { set ::lFilterArgs [list] execsql { SELECT * FROM vt1 LIMIT 5 OFFSET 50 } set ::lFilterArgs -} {{5 50}} +} {{50 5}} do_test 2.3 { set ::lFilterArgs [list] diff --git a/test/bestindexC.test b/test/bestindexC.test new file mode 100644 index 000000000..48f3a2765 --- /dev/null +++ b/test/bestindexC.test @@ -0,0 +1,352 @@ +# 2024-04-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bestindexC + +ifcapable !vtab { + finish_test + return +} + +register_tcl_module db + +proc vtab_command {lVal method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a)" + } + + xBestIndex { + set hdl [lindex $args 0] + set clist [$hdl constraints] + set orderby [$hdl orderby] + + set idxstr [list] + set res [list] + + set idx 0 + foreach c $clist { + array set a $c + if {$a(usable)==0} continue + if {$a(op)=="limit" && ![info exists ::do_not_use_limit]} { + lappend idxstr limit + lappend res omit $idx + } + if {$a(op)=="offset" && ![info exists ::do_not_use_offset]} { + lappend idxstr offset + lappend res omit $idx + } + incr idx + } + + return "cost 1000000 rows 1000000 idxnum 0 idxstr {$idxstr} $res" + } + + xFilter { + set idxstr [lindex $args 1] + set LIMIT "" + foreach a $idxstr b [lindex $args 2] { + set x($a) $b + } + + if {![info exists x(limit)]} { set x(limit) -1 } + if {![info exists x(offset)]} { set x(offset) -1 } + set LIMIT " LIMIT $x(limit) OFFSET $x(offset)" + + set idx 1 + foreach v $lVal { + lappend lRow "($idx, '$v')" + incr idx + } + + return [list sql " + SELECT * FROM ( VALUES [join $lRow ,]) $LIMIT + "] + } + } + + return {} +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command "a b c d e f"); + CREATE VIRTUAL TABLE x2 USING tcl(vtab_command "A B C D E F a b"); +} {} + +do_execsql_test 1.1 { + CREATE TEMP TABLE t_unionall AS + SELECT * FROM x1 UNION ALL SELECT * FROM x2; + + CREATE TEMP TABLE t_intersect AS + SELECT * FROM x1 INTERSECT SELECT * FROM x2; + + CREATE TEMP TABLE t_union AS + SELECT * FROM x1 UNION SELECT * FROM x2; + + CREATE TEMP TABLE t_except AS + SELECT * FROM x1 EXCEPT SELECT * FROM x2; +} + +foreach {tn limit} { + 1 "LIMIT 8" + 2 "LIMIT 4" + 3 "LIMIT 4 OFFSET 2" + 4 "LIMIT 8 OFFSET 4" +} { + + foreach {op tbl} { + "UNION ALL" t_unionall + "UNION" t_union + "INTERSECT" t_intersect + "EXCEPT" t_except + } { + + set expect [execsql "SELECT * FROM $tbl $limit"] + do_execsql_test 1.2.$tbl.$tn "SELECT * FROM ( + SELECT * FROM x1 $op SELECT * FROM x2 + ) $limit" $expect + + } + +} + +#------------------------------------------------------------------------- +reset_db +register_tcl_module db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command "a b c d e f"); + CREATE VIRTUAL TABLE x2 USING tcl(vtab_command "a b e f"); +} {} + +do_execsql_test 2.1 { + SELECT * FROM x1 + EXCEPT + SELECT * FROM x2 + LIMIT 3 +} {c d} + +#------------------------------------------------------------------------- +reset_db +register_tcl_module db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "1 2 3 4 5 6 7 8 9 10"); +} {} + +do_execsql_test 3.1 { + SELECT * FROM y1 WHERE a = COALESCE('8', a) LIMIT 3 +} {8} + +do_execsql_test 3.2 { + SELECT * FROM y1 WHERE a = '2' LIMIT 3 +} {2} + +load_static_extension db series +do_execsql_test 3.3 { + SELECT * FROM generate_series(1, 5) WHERE value = (value & 14) LIMIT 3 +} {2 4} + +do_execsql_test 3.4 { + SELECT value FROM generate_series(1,10) WHERE value>2 LIMIT 4 OFFSET 1; +} {4 5 6 7} + +set ::do_not_use_limit 1 +do_execsql_test 3.5 { + SELECT * FROM y1 LIMIT 5 OFFSET 3 +} {4 5 6 7 8} +unset ::do_not_use_limit +set ::do_not_use_offset 1 +do_execsql_test 3.6 { + SELECT * FROM y1 LIMIT 5 OFFSET 3 +} {4 5 6 7 8} +unset ::do_not_use_offset + +#------------------------------------------------------------------------- +reset_db +proc vtab_command {lVal method args} { + switch -- $method { + xConnect { error "not happy!" } + } + + return {} +} + +register_tcl_module db +do_catchsql_test 4.0 { + CREATE VIRTUAL TABLE y1 USING tcl(vtab_command 1); +} {1 {not happy!}} +do_test 4.1 { + sqlite3_errcode db +} SQLITE_ERROR + +proc vtab_command {lVal method args} { + switch -- $method { + xConnect { + return $lVal + } + } + return {} +} + +do_catchsql_test 4.2 { + CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "PRAGMA page_size=1024"); +} {1 {declare_vtab: syntax error}} +do_catchsql_test 4.3 { + CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "CREATE TABLE x1("); +} {1 {declare_vtab: incomplete input}} +do_catchsql_test 4.4 { + CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "CREATE TABLE x1(insert)"); +} {1 {declare_vtab: near "insert": syntax error}} + +#------------------------------------------------------------------------- +reset_db +register_tcl_module db + +proc quote {str} { + return "'[string map {' ''} $str]'" +} + +proc vtab_command {lVal method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c, d)" + } + + xBestIndex { + set hdl [lindex $args 0] + set clist [$hdl constraints] + + set res [list] + set idx 0 + set idxnum 0 + + set cols(0) a + set cols(1) b + set cols(2) c + + set lCons [list] + + foreach c $clist { + array set a $c + if {$a(usable)==0} continue + + if {($a(op)=="eq" || $a(op)=="is") && [info exists cols($a(column))]} { + lappend res omit $idx + set coll [$hdl collation $idx] + lappend lCons "$cols($a(column)) = %[llength $lCons]% COLLATE $coll" + + set idxnum [expr {$idx + (1 << $a(column))}] + catch { unset cols($a(column)) } + } + + incr idx + } + + if {[llength [array names cols]]>0} { + set missing [list] + for {set i 0} {$i < 3} {incr i} { + catch { lappend missing $cols($i) } + } + set msg "missing required constraints: [join $missing ,]" + return [list constraint $msg] + } + + set idxstr [join $lCons " AND "] + return "cost 1000 rows 1000 idxnum $idxnum $res idxstr {$idxstr}" + } + + xFilter { + foreach {idxnum idxstr lArg} $args {} + set i 0 + set where $idxstr + foreach a $lArg { + set where [string map [list %$i% [quote $a]] $where] + incr i + } + # puts $where + return [list sql "SELECT rowid, * FROM $lVal WHERE $where"] + } + } + + return {} +} + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command t1); + CREATE TABLE t1(a, b, c, d); +} + +foreach {tn where ok} { + 0 "WHERE a=? AND b=? AND c=? AND c=?" 1 + 1 "WHERE a=? AND b=? AND c=?" 1 + 2 "WHERE a=? AND b=? AND (c=? OR c=?)" 1 + 3 "WHERE a=? AND b=? AND (c=? OR c=? OR c=?)" 1 + 4 "WHERE a=? AND b=? AND (c IS ? OR c IS ?)" 1 + 5 "WHERE a=? AND ((b=? AND c=?) OR (c=? AND b=?))" 1 + 6 "WHERE a=? AND ((b=? AND c=?) OR (c=?))" 0 +} { + do_test 5.2.$tn { + catch { execsql "SELECT * FROM x1 $::where" } msg +# if {$tn==0 || $tn==2 || $tn==3} { puts "MSG: $msg" } + } [expr !$ok] +} + +do_execsql_test 5.3 { + SELECT * FROM x1 WHERE (a, b, c) = (?, ?, ?); +} + +do_execsql_test 5.4 { + INSERT INTO t1(rowid, a, b, c, d) VALUES(1, 'x', 'y', 'z', 'one'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(2, 'X', 'Y', 'Z', 'two'); + SELECT * FROM x1 WHERE (a, b, c) = ('X', 'Y', 'Z'); +} {X Y Z two} +do_execsql_test 5.5 { + SELECT * FROM x1 WHERE a='x' AND b='y' AND c='z'; +} {x y z one} +do_execsql_test 5.6 { + SELECT * FROM x1 + WHERE a='x' COLLATE nocase AND b='y' COLLATE nocase AND c='z'COLLATE nocase; +} {x y z one X Y Z two} + +do_execsql_test 5.7 { + DELETE FROM t1; + + INSERT INTO t1(rowid, a, b, c, d) VALUES(0, 'x', 'y', 'z', 'zero'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(1, 'x', 'y', 'Z', 'one'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(2, 'x', 'Y', 'z', 'two'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(3, 'x', 'Y', 'Z', 'three'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(4, 'X', 'y', 'z', 'four'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(5, 'X', 'y', 'Z', 'five'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(6, 'X', 'Y', 'z', 'six'); + INSERT INTO t1(rowid, a, b, c, d) VALUES(7, 'X', 'Y', 'z', 'seven'); +} + +do_execsql_test 5.8 { + SELECT d FROM x1 + WHERE a='x' AND ((b='y' AND c='z') OR (b='Y' AND c='z' COLLATE nocase)) +} { + zero two three +} + +do_execsql_test 5.9 { + SELECT d FROM x1 + WHERE a='x' COLLATE nocase + AND ((b='y' AND c='z') OR (b='Y' AND c='z' COLLATE nocase)) +} { + zero four two + three six seven +} + +finish_test diff --git a/test/bestindexD.test b/test/bestindexD.test new file mode 100644 index 000000000..b06d6b427 --- /dev/null +++ b/test/bestindexD.test @@ -0,0 +1,93 @@ +# 2024-08-03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bestindexD + +ifcapable !vtab { + finish_test + return +} + +register_tcl_module db + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID" + } + + xBestIndex { + set hdl [lindex $args 0] + set ::colUsed [$hdl mask] + + set cost 1000000 + set used "" + + set cons 0 + foreach c [$hdl constraints] { + set cost [expr $cost/10] + append used " use $cons" + incr cons + } + + return "cost $cost rows $cost $used" + } + } + + return {} +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + + CREATE TABLE t2(a, b); +} {} + +# This proc assumes that there is only one use of a virtual table - x1 - +# in SQL statement $sql. It tests that the colUsed value passed to the +# xBestIndex method matches the actual columns used, which is ascertained +# by searching the compiled VM code for VColumn instructions. +# +proc do_colsused_test {tn sql} { + set ::colUsed "" + execsql $sql + set got $::colUsed + + set expect 0 + db eval "EXPLAIN $sql" x { + if {$x(opcode)=="VColumn"} { + set expect [expr $expect | (1<<$x(p2))] + } + } + + uplevel [list do_test $tn.($expect/$got) [list expr ($expect & $got)] $expect] +} + +do_colsused_test 1.1 { SELECT a FROM x1 } +do_colsused_test 1.2 { SELECT a,c FROM x1 } +do_colsused_test 1.3 { SELECT b FROM x1 } +do_colsused_test 1.4 { SELECT b FROM x1 WHERE c=? } + +do_colsused_test 1.5 { + select 1 from t2 full join x1; +} + +do_colsused_test 1.6 { + select 1 from x1 WHERE (b=? AND c=?) OR (b=? AND c=?) +} + +finish_test + + diff --git a/test/busy.test b/test/busy.test index be0515b01..896c7fa65 100644 --- a/test/busy.test +++ b/test/busy.test @@ -106,7 +106,7 @@ do_test 3.4 { proc busy_handler {n} { return 1 } do_test 3.5 { catchsql { PRAGMA optimize } -} {0 {}} +} {1 {database is locked}} do_test 3.6 { execsql { COMMIT } db2 diff --git a/test/capi3.test b/test/capi3.test index 40d87ac8e..e65f90e3a 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -181,7 +181,7 @@ do_test capi3-3.4 { do_test capi3-3.5 { list [sqlite3_system_errno $db2] [sqlite3_close $db2] } [list $::capi3_errno SQLITE_OK] -if {[clang_sanitize_address]==0} { +if {[clang_sanitize_address]==0 && 0} { do_test capi3-3.6.1-misuse { sqlite3_close $db2 } {SQLITE_MISUSE} diff --git a/test/cast.test b/test/cast.test index ebfea5aa0..7f48ed80b 100644 --- a/test/cast.test +++ b/test/cast.test @@ -234,6 +234,7 @@ do_test cast-3.1 { do_test cast-3.2 { execsql {SELECT CAST(9223372036854774800 AS numeric)} } 9223372036854774800 +breakpoint do_realnum_test cast-3.3 { execsql {SELECT CAST(9223372036854774800 AS real)} } 9.22337203685477e+18 diff --git a/test/cksumvfs.test b/test/cksumvfs.test new file mode 100644 index 000000000..8c7bcf551 --- /dev/null +++ b/test/cksumvfs.test @@ -0,0 +1,33 @@ +# 2024 March 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cksumvfs + +sqlite3_register_cksumvfs +db close +sqlite3 db test.db +file_control_reservebytes db 8 + +set text [db one "SELECT hex(randomblob(5000))"] + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, $text); +} + +do_execsql_test 1.1 { + SELECT * FROM t1; +} [list 1 $text] + +finish_test diff --git a/test/colname.test b/test/colname.test index 5fa0b601f..24fb51acf 100644 --- a/test/colname.test +++ b/test/colname.test @@ -424,7 +424,7 @@ do_catchsql_test colname-9.400 { # do_catchsql_test colname-9.410 { CREATE TABLE t5 AS SELECT RAISE(abort,a); -} {1 {RAISE() may only be used within a trigger-program}} +} {1 {no such column: a}} # Make sure the quotation marks get removed from the column names # when constructing a new table from an aggregate SELECT. diff --git a/test/corrupt.test b/test/corrupt.test index 62ead4d9a..ff61cc0bb 100644 --- a/test/corrupt.test +++ b/test/corrupt.test @@ -290,7 +290,7 @@ ifcapable oversize_cell_check { # detecting corruption was not possible at that point, and an assert() failed. # set fd [open test.db r+] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd [expr 1024+8] puts -nonewline $fd "\x03\x14" close $fd diff --git a/test/corrupt2.test b/test/corrupt2.test index 2e36cbd30..fc24cb376 100644 --- a/test/corrupt2.test +++ b/test/corrupt2.test @@ -69,7 +69,7 @@ do_test corrupt2-1.3 { forcedelete corrupt.db-journal forcecopy test.db corrupt.db set f [open corrupt.db RDWR] - fconfigure $f -encoding binary + fconfigure $f -translation binary seek $f 16 start puts -nonewline $f "\x00\xFF" close $f @@ -89,7 +89,7 @@ do_test corrupt2-1.4 { forcedelete corrupt.db-journal forcecopy test.db corrupt.db set f [open corrupt.db RDWR] - fconfigure $f -encoding binary + fconfigure $f -translation binary seek $f 101 start puts -nonewline $f "\xFF\xFF" close $f @@ -109,7 +109,7 @@ do_test corrupt2-1.5 { forcedelete corrupt.db-journal forcecopy test.db corrupt.db set f [open corrupt.db RDWR] - fconfigure $f -encoding binary + fconfigure $f -translation binary seek $f 101 start puts -nonewline $f "\x00\xC8" seek $f 200 start @@ -179,7 +179,7 @@ do_test corrupt2-3.1 { # of the DROP TABLE operation below. # set fd [open corrupt.db r+] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary seek $fd [expr 1024*3 + 12] set zCelloffset [read $fd 2] binary scan $zCelloffset S iCelloffset @@ -227,7 +227,7 @@ do_test corrupt2-5.1 { # This block links a page from table t2 into the t1 table structure. # set fd [open corrupt.db r+] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary seek $fd [expr 1024 + 12] set zCelloffset [read $fd 2] binary scan $zCelloffset S iCelloffset @@ -392,7 +392,7 @@ corruption_test -sqlprep $sqlprep -corrupt { # 0x0D (interpreted by SQLite as "leaf page of a table B-Tree"). # set fd [open corrupt.db r+] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd [expr 1024*2 + 8] set zRightChild [read $fd 4] binary scan $zRightChild I iRightChild @@ -410,7 +410,7 @@ corruption_test -sqlprep $sqlprep -corrupt { # The corruption is detected as part of an OP_Prev opcode. # set fd [open corrupt.db r+] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd [expr 1024*2 + 12] set zCellOffset [read $fd 2] binary scan $zCellOffset S iCellOffset @@ -431,7 +431,7 @@ corruption_test -sqlprep $sqlprep -corrupt { # 0x0A (interpreted by SQLite as "leaf page of an index B-Tree"). # set fd [open corrupt.db r+] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd [expr 1024*1 + 8] set zRightChild [read $fd 4] binary scan $zRightChild I iRightChild @@ -459,7 +459,7 @@ corruption_test -sqlprep { CREATE TABLE x6(a, b, c); CREATE TABLE xD(a, b, c); CREATE TABLE xJ(a, b, c); } -corrupt { set fd [open corrupt.db r+] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd 108 set zRightChild [read $fd 4] binary scan $zRightChild I iRightChild diff --git a/test/corrupt4.test b/test/corrupt4.test index 5b0965a83..544ac3305 100644 --- a/test/corrupt4.test +++ b/test/corrupt4.test @@ -122,7 +122,7 @@ proc put4byte {fd offset val} { # the second rightmost child page number of page 1 to 1. # set fd [open test.db r+] -fconfigure $fd -encoding binary -translation binary +fconfigure $fd -translation binary set nChild [get2byte $fd 103] set offChild [get2byte $fd [expr 100+12+($nChild-2)*2]] set pgnoChild [get4byte $fd $offChild] diff --git a/test/corruptC.test b/test/corruptC.test index f5733a818..bf324c120 100644 --- a/test/corruptC.test +++ b/test/corruptC.test @@ -98,7 +98,7 @@ do_test corruptC-2.1 { sqlite3 db test.db catchsql {PRAGMA integrity_check} } {0 {{*** in database main *** -Tree 3 page 3: free space corruption}}} +Tree 3 page 3: free space corruption} {wrong # of entries in index t1i1}}} # test that a corrupt content offset size is handled (seed 5649) # diff --git a/test/corruptD.test b/test/corruptD.test index c35388adf..381e52c80 100644 --- a/test/corruptD.test +++ b/test/corruptD.test @@ -113,7 +113,7 @@ do_test corruptD-1.1.1 { hexio_write test.db [expr 1024+1] FFFF catchsql { PRAGMA quick_check } } {0 {{*** in database main *** -Tree 2 page 2: free space corruption}}} +Tree 2 page 2: free space corruption} {wrong # of entries in index i1}}} do_test corruptD-1.1.2 { incr_change_counter hexio_write test.db [expr 1024+1] [hexio_render_int32 1021] diff --git a/test/corruptK.test b/test/corruptK.test index 1569afe4a..0bdb3c45b 100644 --- a/test/corruptK.test +++ b/test/corruptK.test @@ -60,7 +60,7 @@ do_test 1.2 { sqlite3 db test.db set fd [db incrblob t1 x 3] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd 30 puts -nonewline $fd "\x18" close $fd @@ -102,7 +102,7 @@ do_test 2.2 { sqlite3 db test.db set fd [db incrblob t1 x 5] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd 22 puts -nonewline $fd "\x5d" diff --git a/test/corruptL.test b/test/corruptL.test index cf3876441..52adf6fd7 100644 --- a/test/corruptL.test +++ b/test/corruptL.test @@ -1505,4 +1505,89 @@ do_catchsql_test 19.4 { PRAGMA integrity_check; } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 18.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 20480 pagesize 4096 filename crash-a4150b729051e4.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 64: 00 00 00 00 00 00 00 00 00 00 ff f0 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 04 0e e5 00 0f c2 0f 75 ...............u +| 112: 0f 19 0e e5 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 3808: 00 00 00 00 00 32 04 06 17 17 11 01 4b 69 6e 64 .....2......Kind +| 3824: 65 78 74 31 61 62 63 74 31 05 43 52 45 41 54 45 ext1abct1.CREATE +| 3840: 20 49 4e 44 45 58 20 74 31 61 62 63 20 4f 4e 20 INDEX t1abc ON +| 3856: 74 31 28 61 2c 62 2c 63 29 5a 03 06 17 25 25 01 t1(a,b,c)Z...%%. +| 3872: 79 74 61 62 6c 65 73 71 6c 69 74 65 5f 73 74 61 ytablesqlite_sta +| 3888: 74 34 73 71 6c 69 74 65 5f 73 74 61 74 34 04 43 t4sqlite_stat4.C +| 3904: 52 45 41 54 45 20 54 41 42 4c 45 20 73 71 6c 69 REATE TABLE sqli +| 3920: 74 65 5f 73 74 61 74 34 28 74 62 6c 2c 69 64 78 te_stat4(tbl,idx +| 3936: 2c 6e 65 71 2c 6e 6c 74 2c 6e 64 6c 74 2c 73 61 ,neq,nlt,ndlt,sa +| 3952: 6d 70 6c 65 29 4b 02 06 17 25 25 01 5b 74 61 62 mple)K...%%.[tab +| 3968: 6c 65 73 71 6c 69 74 65 5f 73 74 61 74 31 73 71 lesqlite_stat1sq +| 3984: 6c 69 74 65 5f 73 74 61 74 31 03 43 52 45 41 54 lite_stat1.CREAT +| 4000: 45 20 54 41 42 4c 45 20 73 71 6c 69 74 65 5f 73 E TABLE sqlite_s +| 4016: 74 61 74 31 28 74 62 6c 2c 69 64 78 2c 73 74 61 tat1(tbl,idx,sta +| 4032: 74 29 3c 01 06 17 11 11 01 65 74 61 62 6c 65 74 t)<......etablet +| 4048: 31 74 31 02 43 52 45 41 54 45 20 54 41 42 4c 45 1t1.CREATE TABLE +| 4064: 20 74 31 28 61 20 54 45 58 54 2c 20 62 20 49 4e t1(a TEXT, b IN +| 4080: 54 2c 20 63 20 49 4e 54 2c 20 64 20 49 4e 54 29 T, c INT, d INT) +| page 2 offset 4096 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4000: 0b 07 05 13 01 01 01 62 63 64 64 06 0b 0c 06 05 .......bcdd..... +| 4016: 13 02 01 01 64 65 66 01 59 09 0a 0c 05 05 13 03 ....def.Y....... +| 4032: 01 01 64 65 66 02 6f 08 09 0c 04 05 13 02 01 01 ..def.o......... +| 4048: 61 62 63 01 59 07 08 0c 03 05 13 02 01 01 87 62 abc.Y..........b +| 4064: 63 00 ea 06 07 0c 02 05 13 02 01 01 61 62 63 00 c...........abc. +| 4080: ea 06 06 0b 01 05 13 01 01 01 61 62 63 7b 04 04 ..........abc... +| page 3 offset 8192 +| 0: 0d 00 00 00 01 0f e0 00 0f e1 00 00 00 00 00 00 ................ +| 4064: 00 1d 01 04 11 17 31 74 31 74 31 61 62 63 31 30 ......1t1t1abc10 +| 4080: 30 30 30 20 35 30 30 30 20 32 30 30 30 20 31 30 000 5000 2000 10 +| page 4 offset 12288 +| 0: 0d 00 00 00 07 0e ac 00 0f d1 0f a0 0f 6f 0f 3e .............o.> +| 16: 0f 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3744: 00 00 00 00 00 00 00 00 00 00 00 00 2f 07 07 11 ............/... +| 3760: 17 1b 1b 1b 24 74 31 74 31 61 62 63 32 20 31 20 ....$t1t1abc2 1 +| 3776: 31 20 31 35 20 36 20 36 20 36 32 20 35 20 36 20 1 15 6 6 62 5 6 +| 3792: 36 05 13 02 01 01 64 65 66 02 37 08 05 2f 06 07 6.....def.7../.. +| 3808: 11 17 1b 1b 1b 24 74 41 74 31 61 62 63 32 20 31 .....$tAt1abc2 1 +| 3824: 20 31 20 31 35 20 35 20 55 20 35 32 20 34 20 35 1 15 5 U 52 4 5 +| 3840: 20 35 05 13 02 01 01 64 65 66 01 59 09 06 2e 05 5.....def.Y.... +| 3856: 07 11 17 1b 1b 1b 22 74 31 74 31 61 62 63 31 20 .......t1t1abc1 +| 3872: 31 20 31 20 31 34 20 34 20 34 20 34 31 20 33 20 1 1 14 4 4 41 3 +| 3888: 34 20 34 08 b3 cd f0 f1 62 63 64 64 06 07 2f 05 4 4.....bcdd../. +| 3904: 07 11 17 1b 1b 1b 24 74 37 74 31 61 62 63 34 20 ......$t7t1abc4 +| 3920: 31 20 31 20 31 30 20 33 20 33 20 33 30 20 32 20 1 1 10 3 3 30 2 +| 3936: 33 20 33 05 13 02 01 01 61 62 63 01 59 07 04 2f 3 3.....abc.Y../ +| 3952: 03 07 11 17 1b 1b 1b 24 74 31 74 31 61 62 63 34 .......$t1t1abc4 +| 3968: 20 32 20 31 20 31 30 20 31 20 32 20 32 30 20 31 2 1 10 1 2 20 1 +| 3984: 20 32 20 32 05 13 02 01 01 61 62 63 00 ea 06 03 2 2.....abc.... +| 4000: 2f 02 07 11 17 1b 1b 1b 24 74 31 74 31 61 62 63 /.......$t1t1abc +| 4016: 34 20 32 20 31 20 31 30 20 31 20 31 20 31 30 20 4 2 1 10 1 1 10 +| 4032: 31 20 31 20 31 05 13 02 01 01 61 62 63 00 ea 05 1 1 1.....abc... +| 4048: 02 2d 01 07 11 17 1b 1b 1b 20 74 31 74 31 61 62 .-....... t1t1ab +| 4064: 63 34 20 31 20 31 20 31 30 20 30 20 30 1f 30 30 c4 1 1 10 0 0.00 +| 4080: 20 30 20 30 20 30 05 13 01 01 09 61 62 63 7b 04 0 0 0.....abc.. +| page 5 offset 16384 +| 0: 0a 00 00 00 07 0f a8 00 0f f5 00 00 00 00 00 00 ................ +| 4000: 00 00 00 00 00 00 00 00 0c 05 13 02 01 01 64 65 ..............de +| 4016: 66 02 37 08 05 0c 05 13 02 01 01 64 65 66 01 59 f.7........def.Y +| 4032: 09 06 0b 05 12 01 01 01 62 63 64 64 06 07 0c 05 ........bcdd.... +| 4048: 13 02 01 01 61 62 63 01 59 07 01 2c 05 13 02 01 ....abc.Y..,.... +| 4064: 01 61 62 63 00 ea 06 03 0c 05 13 02 01 01 61 62 .abc..........ab +| 4080: 63 00 ea 05 00 00 00 00 00 00 00 00 00 00 00 00 c............... +| end crash-a4150b729051e4.db +}]} {} + +do_catchsql_test 18.1 { + SELECT a FROM t1 WHERE b GLOB b AND b GLOB '0^x]␅6␚xz]'; +} {1 {database disk image is malformed}} + finish_test diff --git a/test/cost.test b/test/cost.test index 5684177a1..6106caba8 100644 --- a/test/cost.test +++ b/test/cost.test @@ -103,7 +103,7 @@ do_eqp_test 5.2 { } { QUERY PLAN |--SCAN t2 USING INDEX t2i1 - `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY } do_eqp_test 5.3 { diff --git a/test/crash.test b/test/crash.test index c1901daec..f63163659 100644 --- a/test/crash.test +++ b/test/crash.test @@ -399,7 +399,7 @@ do_test crash-7.1 { # Change the checksum value for the master journal name. set f [open test.db-journal a] - fconfigure $f -encoding binary + fconfigure $f -translation binary seek $f [expr [file size test.db-journal] - 12] puts -nonewline $f "\00\00\00\00" close $f diff --git a/test/date.test b/test/date.test index 19cecc2db..d22b652b4 100644 --- a/test/date.test +++ b/test/date.test @@ -209,8 +209,8 @@ datetest 3.16 "strftime('[repeat 200 %Y]','2003-10-31')" [repeat 200 2003] datetest 3.17 "strftime('[repeat 200 abc%m123]','2003-10-31')" \ [repeat 200 abc10123] -foreach c {a b c g h i n o q r t v x y z - A B C D E G K L N O Q V Z +foreach c {a b c h i n o q r t v x y z + A B C D E K L N O Q Z 0 1 2 3 4 5 6 6 7 9 _} { datetest 3.18.$c "strftime('%$c','2003-10-31')" NULL } @@ -262,7 +262,7 @@ datetest 5.15 {datetime('1994-04-16 14:00:00 +05:00 Z')} NULL # localtime->utc and utc->localtime conversions. # # Use SQLITE_TESTCTRL_LOCALTIME_FAULT=2 to set an alternative localtime_r() -# implementation that is not locale-dependent. This testing localtime_r() +# implementation that is not locale-dependent. The testing localtime_r() # operates as follows: # # (1) Localtime is 30 minutes earlier than (west of) UTC on @@ -321,6 +321,38 @@ utc_to_local 6.22 {1800-10-29 12:30:00} {1800-10-29 12:00:00} local_to_utc 6.23 {3000-10-30 12:00:00} {3000-10-30 11:30:00} utc_to_local 6.24 {3000-10-30 11:30:00} {3000-10-30 12:00:00} +# If the time is specified to be ZULU, or if it has an explicit +# timezone extension, then the time will already be UTC and subsequent +# 'utc' modifiers are no-ops. +# +do_execsql_test date-6.25 { + SELECT datetime('2000-10-29 12:00Z','utc','utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.26 { + SELECT datetime('2000-10-29 12:00:00+05:00'); +} {{2000-10-29 07:00:00}} +do_execsql_test date-6.27 { + SELECT datetime('2000-10-29 12:00:00+05:00', 'utc'); +} {{2000-10-29 07:00:00}} + +# Multiple back-and-forth UTC to LOCAL to UTC... +do_execsql_test date-6.28 { + SELECT datetime('2000-10-29 12:00:00Z', 'localtime'); +} {{2000-10-29 12:30:00}} +do_execsql_test date-6.29 { + SELECT datetime('2000-10-29 12:00:00Z', 'utc', 'localtime'); +} {{2000-10-29 12:30:00}} +do_execsql_test date-6.30 { + SELECT datetime('2000-10-29 12:00:00Z', 'utc', 'localtime', 'utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.31 { + SELECT datetime('2000-10-29 12:00:00Z', 'utc','localtime','utc','localtime'); +} {{2000-10-29 12:30:00}} +do_execsql_test date-6.32 { + SELECT datetime('2000-10-29 12:00:00Z', 'localtime','localtime'); +} {{2000-10-29 12:30:00}} + + # Restore the use of the OS localtime_r() before going on... sqlite3_test_control SQLITE_TESTCTRL_LOCALTIME_FAULT 0 @@ -573,4 +605,51 @@ datetest 18.2 {unixepoch('1970-01-01T00:00:00.1', 'subsec')} {0.1} datetest 18.3 {unixepoch('1970-01-01T00:00:00.2', 'subsecond')} {0.2} datetest 18.4 {julianday('-4713-11-24 13:40:48.864', 'subsec')} {0.07001} datetest 18.5 {typeof(unixepoch('now', 'subsecond'))} {real} + +# 2024-03-03 the 'ceiling' and 'floor' operators. +# +datetest 19.1 {date('2000-01-31','floor')} {2000-01-31} +datetest 19.2a {date('2000-02-31','floor')} {2000-02-29} +datetest 19.2b {date('1999-02-31','floor')} {1999-02-28} +datetest 19.2c {date('1900-02-31','floor')} {1900-02-28} +datetest 19.3 {date('2000-03-31','floor')} {2000-03-31} +datetest 19.4 {date('2000-04-31','floor')} {2000-04-30} +datetest 19.5 {date('2000-05-31','floor')} {2000-05-31} +datetest 19.6 {date('2000-06-31','floor')} {2000-06-30} +datetest 19.7 {date('2000-07-31','floor')} {2000-07-31} +datetest 19.8 {date('2000-08-31','floor')} {2000-08-31} +datetest 19.9 {date('2000-09-31','floor')} {2000-09-30} +datetest 19.10 {date('2000-10-31','floor')} {2000-10-31} +datetest 19.11 {date('2000-11-31','floor')} {2000-11-30} +datetest 19.12 {date('2000-12-31','floor')} {2000-12-31} +datetest 19.21 {date('2000-01-31','ceiling')} {2000-01-31} +datetest 19.22a {date('2000-02-31','ceiling')} {2000-03-02} +datetest 19.22b {date('1999-02-31','ceiling')} {1999-03-03} +datetest 19.22c {date('1900-02-31','ceiling')} {1900-03-03} +datetest 19.23 {date('2000-03-31','ceiling')} {2000-03-31} +datetest 19.24 {date('2000-04-31','ceiling')} {2000-05-01} +datetest 19.25 {date('2000-05-31','ceiling')} {2000-05-31} +datetest 19.26 {date('2000-06-31','ceiling')} {2000-07-01} +datetest 19.27 {date('2000-07-31','ceiling')} {2000-07-31} +datetest 19.28 {date('2000-08-31','ceiling')} {2000-08-31} +datetest 19.29 {date('2000-09-31','ceiling')} {2000-10-01} +datetest 19.30 {date('2000-10-31','ceiling')} {2000-10-31} +datetest 19.31 {date('2000-11-31','ceiling')} {2000-12-01} +datetest 19.32 {date('2000-12-31','ceiling')} {2000-12-31} +datetest 19.40 {date('2024-01-31','+1 month','ceiling')} {2024-03-02} +datetest 19.41 {date('2024-01-31','+1 month','floor')} {2024-02-29} +datetest 19.42 {date('2023-01-31','+1 month','ceiling')} {2023-03-03} +datetest 19.43 {date('2023-01-31','+1 month','floor')} {2023-02-28} +datetest 19.44 {date('2024-02-29','+1 year','ceiling')} {2025-03-01} +datetest 19.45 {date('2024-02-29','+1 year','floor')} {2025-02-28} +datetest 19.46 {date('2024-02-29','-110 years','ceiling')} {1914-03-01} +datetest 19.47 {date('2024-02-29','-110 years','floor')} {1914-02-28} +datetest 19.48 {date('2024-02-29','-0110-00-00','floor')} {1914-02-28} +datetest 19.49 {date('2024-02-29','-0110-00-00','ceiling')} {1914-03-01} +datetest 19.50 {date('2000-08-31','+0023-06-00','floor')} {2024-02-29} +datetest 19.51 {date('2000-08-31','+0022-06-00','floor')} {2023-02-28} +datetest 19.52 {date('2000-08-31','+0023-06-00','ceiling')} {2024-03-02} +datetest 19.53 {date('2000-08-31','+0022-06-00','ceiling')} {2023-03-03} + + finish_test diff --git a/test/date4.test b/test/date4.test index 0d820a0a4..56a9090b1 100644 --- a/test/date4.test +++ b/test/date4.test @@ -24,12 +24,12 @@ ifcapable {!datetime} { } if {$tcl_platform(os)=="Linux"} { - set FMT {%d,%e,%F,%H,%k,%I,%l,%j,%m,%M,%u,%w,%W,%Y,%%,%P,%p} + set FMT {%d,%e,%F,%H,%k,%I,%l,%j,%m,%M,%u,%w,%W,%Y,%%,%P,%p,%U,%V,%G,%g} } else { set FMT {%d,%e,%F,%H,%I,%j,%p,%R,%u,%w,%W,%%} } -for {set i 0} {$i<=24854} {incr i} { - set TS [expr {$i*86401}] +for {set i 0} {$i<=24858} {incr i} { + set TS [expr {$i*86390}] do_execsql_test date4-$i { SELECT strftime($::FMT,$::TS,'unixepoch'); } [list [strftime $FMT $TS]] diff --git a/test/date5.test b/test/date5.test new file mode 100644 index 000000000..688f84d0f --- /dev/null +++ b/test/date5.test @@ -0,0 +1,86 @@ +# 2024-08-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# https://sqlite.org/forum/forumpost/eaa0a09786c6368b +# +# Apparently SQLite has been miscomputing leap-year dates before +# the year 0400. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Skip this whole file if date and time functions are omitted +# at compile-time +# +ifcapable {!datetime} { + finish_test + return +} + +# Data sources: +# 1-10 https://ssd.jpl.nasa.gov/tools/jdc/#/cd +# 11 Jean Meeus, Astronomical Algorithms, ISBN 0-943396-61-1, p.59 +# 12 https://en.wikipedia.org/wiki/Julian_day +# +# ID YEAR MONTH DAY JD +set date5data { + 1 2024 2 29 2460369.5 + 2 2024 3 1 2460370.5 + 3 2023 2 28 2460003.5 + 4 2023 3 1 2460004.5 + 5 2000 2 29 2451603.5 + 6 2000 3 1 2451604.5 + 7 1900 2 28 2415078.5 + 8 1900 3 1 2415079.5 + 9 1712 2 29 2346413.5 + 10 1712 3 1 2346414.5 + 11 1977 4 26 2443259.5 + 12 2013 1 1 2456293.5 +} + +foreach {id y m d jd} $date5data { + set date [format %04d-%02d-%02d $y $m $d] + do_execsql_test date5-jd$jd { + SELECT date($::jd); + } $date + do_execsql_test date5-cal/$date { + SELECT julianday($::date); + } $jd + for {set i 1} {$y+400*$i<=9999} {incr i} { + set y2 [expr {$y+400*$i}] + set date2 [format %04d-%02d-%02d $y2 $m $d] + set jd2 [expr {$jd+146097*$i}] + do_execsql_test date5-jd$jd2 { + SELECT date($::jd2); + } $date2 + do_execsql_test date5-cal/$date2 { + SELECT julianday($::date2); + } $jd2 + } + for {set i 1} {$y-400*$i>=-4712} {incr i} { + set y2 [expr {$y-400*$i}] + if {$y2<0} { + set date2 [format -%04d-%02d-%02d [expr {-$y2}] $m $d] + } else { + set date2 [format %04d-%02d-%02d $y2 $m $d] + } + set jd2 [expr {$jd-146097*$i}] + do_execsql_test date5-jd$jd2 { + SELECT date($::jd2); + } $date2 + do_execsql_test date5-cal/$date2 { + SELECT julianday($::date2); + } $jd2 + } +} + +finish_test diff --git a/test/default.test b/test/default.test index 06a180c1d..192b3d2ff 100644 --- a/test/default.test +++ b/test/default.test @@ -135,6 +135,10 @@ reset_db do_catchsql_test default-5.1 { CREATE TABLE t1 (a,b DEFAULT(random() NOTNULL IN (RAISE(IGNORE),2,3))); INSERT INTO t1(a) VALUES(1); -} {1 {RAISE() may only be used within a trigger-program}} +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-5.2 { + CREATE TABLE Table0 (Col0 DEFAULT (RAISE(IGNORE) ) ) ; + INSERT INTO Table0 DEFAULT VALUES ; +} {1 {default value of column [Col0] is not constant}} finish_test diff --git a/test/distinct2.test b/test/distinct2.test index 958d9634f..980b0b1e3 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -313,4 +313,71 @@ do_execsql_test 4020 { SELECT b FROM t1 UNION SELECT 1; } {1 { }} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5010 { + CREATE TABLE cnt(a); + WITH RECURSIVE cnt2(x) AS ( + VALUES(1) UNION ALL SELECT x+1 FROM cnt2 WHERE x<50 + ) + INSERT INTO cnt SELECT x FROM cnt2; +} + +do_execsql_test 5020 { + SELECT DISTINCT abs(random())%5 AS r FROM cnt ORDER BY r; +} {0 1 2 3 4} + +do_execsql_test 5030 { + SELECT abs(random())%5 AS r FROM cnt GROUP BY 1 ORDER BY 1; +} {0 1 2 3 4} + +do_execsql_test 5040 { + SELECT a FROM cnt WHERE a>45 GROUP BY 1; +} {46 47 48 49 50} + + +# 2024-06-03 dbsqlfuzz 8a44f675401a8b1f68a43bf813c4f4f72ad8f0ea +# Use of uninitialized bytecode register due to the call-function-once +# optimization at check-in 663f5dd32d9db832 +# +db null NULL +do_execsql_test 5050 { + CREATE TABLE t0(a TEXT); INSERT INTO t0 VALUES('abcd'); + CREATE TABLE t1(b TEXT); + CREATE TABLE t2(c TEXT); + CREATE TABLE t3(d TEXT); INSERT INTO t3 VALUES('wxyz'); + CREATE VIEW v4(e) AS SELECT (SELECT t2.c FROM t0, t1 GROUP BY 1) FROM t2; + SELECT v4.e FROM t3 LEFT JOIN v4 ON true GROUP BY 1; +} NULL +do_execsql_test 5060 { + DROP VIEW v4; + CREATE VIEW v4(e) AS SELECT (SELECT t2.c COLLATE nocase FROM t0, t1 GROUP BY 1) FROM t2; + SELECT v4.e FROM t3 LEFT JOIN v4 ON true GROUP BY 1; +} NULL + +do_execsql_test 5070 { + DROP VIEW v4; + CREATE VIEW v4(e) AS SELECT (SELECT unlikely(t2.c COLLATE nocase) FROM t0, t1 GROUP BY 1) FROM t2; + SELECT v4.e FROM t3 LEFT JOIN v4 ON true GROUP BY 1; +} NULL + +# 2024-06-28 dbsqlfuzz 46343912848a603e32c6072cae792eb056bac897 +# Do not call sqlite3ExprToRegister() on an expression that is already +# a register. +# +do_execsql_test 5080 { + CREATE TABLE dual(dummy TEXT); + INSERT INTO dual VALUES('X'); + SELECT 11 = ( + SELECT b + FROM ( + SELECT a AS b + FROM dual + LEFT JOIN (SELECT 22 AS a FROM dual) + ) + GROUP BY b, b + ); +} 0 + finish_test diff --git a/test/distinctagg.test b/test/distinctagg.test index 199ca0666..9eedd35bd 100644 --- a/test/distinctagg.test +++ b/test/distinctagg.test @@ -95,7 +95,7 @@ foreach {tn use_eph sql res} { 7 0 "SELECT count(DISTINCT a) FROM t2, t1" 5 8 1 "SELECT count(DISTINCT a+b) FROM t1, t2, t2, t2" 6 9 0 "SELECT count(DISTINCT c) FROM t1 WHERE c=2" 1 - 10 1 "SELECT count(DISTINCT t1.rowid) FROM t1, t2" 10 + 10 0 "SELECT count(DISTINCT t1.rowid) FROM t1, t2" 10 } { do_test 3.$tn.1 { set prg [db eval "EXPLAIN $sql"] @@ -148,6 +148,10 @@ do_execsql_test 3.0 { CREATE TABLE t3(x, y, z); INSERT INTO t3 VALUES(1,1,1); INSERT INTO t3 VALUES(2,2,2); + + CREATE TABLE t4(a); + CREATE INDEX t4a ON t4(a); + INSERT INTO t4 VALUES(1), (2), (2), (3), (1); } foreach {tn use_eph sql res} { @@ -158,6 +162,9 @@ foreach {tn use_eph sql res} { 4 0 "SELECT count(DISTINCT f) FROM t2 GROUP BY d, e" {1 2 2 3} 5 1 "SELECT count(DISTINCT f) FROM t2 GROUP BY d" {2 3} 6 0 "SELECT count(DISTINCT f) FROM t2 WHERE d IS 1 GROUP BY e" {1 2 2} + + 7 0 "SELECT count(DISTINCT a) FROM t1" {4} + 8 0 "SELECT count(DISTINCT a) FROM t4" {3} } { do_test 4.$tn.1 { set prg [db eval "EXPLAIN $sql"] diff --git a/test/e_reindex.test b/test/e_reindex.test index 00291b76a..50d2e7d51 100644 --- a/test/e_reindex.test +++ b/test/e_reindex.test @@ -73,12 +73,12 @@ sqlite3 db test.db do_execsql_test e_reindex-1.3 { PRAGMA integrity_check; } [list \ + {wrong # of entries in index i2} \ + {wrong # of entries in index i1} \ {row 3 missing from index i2} \ {row 3 missing from index i1} \ {row 4 missing from index i2} \ - {row 4 missing from index i1} \ - {wrong # of entries in index i2} \ - {wrong # of entries in index i1} + {row 4 missing from index i1} ] do_execsql_test e_reindex-1.4 { diff --git a/test/eqp.test b/test/eqp.test index 61fd617d8..5d2659be7 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -288,6 +288,7 @@ det 3.2.1 { |--SCAN (subquery-xxxxxx) `--USE TEMP B-TREE FOR ORDER BY } + det 3.2.2 { SELECT * FROM (SELECT * FROM t1 ORDER BY x LIMIT 10) AS x1, @@ -305,13 +306,24 @@ det 3.2.2 { `--USE TEMP B-TREE FOR ORDER BY } +det 3.2.3 { + SELECT * FROM (SELECT * FROM t1 ORDER BY x LIMIT 10) ORDER BY x LIMIT 5 +} { + QUERY PLAN + |--CO-ROUTINE (subquery-xxxxxx) + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + `--SCAN (subquery-xxxxxx) +} + det 3.3.1 { SELECT * FROM t1 WHERE y IN (SELECT y FROM t2) } { QUERY PLAN |--SCAN t1 `--LIST SUBQUERY xxxxxx - `--SCAN t2 + |--SCAN t2 + `--CREATE BLOOM FILTER } det 3.3.2 { SELECT * FROM t1 WHERE y IN (SELECT y FROM t2 WHERE t1.x!=t2.x) @@ -413,7 +425,7 @@ do_eqp_test 4.2.3 { | `--USE TEMP B-TREE FOR ORDER BY `--RIGHT |--SCAN t2 USING INDEX t2i1 - `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY } do_eqp_test 4.2.4 { SELECT * FROM t1 INTERSECT SELECT * FROM t2 ORDER BY 1 @@ -425,7 +437,7 @@ do_eqp_test 4.2.4 { | `--USE TEMP B-TREE FOR ORDER BY `--RIGHT |--SCAN t2 USING INDEX t2i1 - `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY } do_eqp_test 4.2.5 { SELECT * FROM t1 EXCEPT SELECT * FROM t2 ORDER BY 1 @@ -437,7 +449,7 @@ do_eqp_test 4.2.5 { | `--USE TEMP B-TREE FOR ORDER BY `--RIGHT |--SCAN t2 USING INDEX t2i1 - `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY } do_eqp_test 4.3.1 { @@ -806,6 +818,7 @@ do_execsql_test 9.0 { CREATE INDEX event_i1 ON event(mtime); CREATE TABLE private(rid INTEGER PRIMARY KEY); } +optimization_control db order-by-subquery off do_eqp_test 9.1 { WITH thread(age,duration,cnt,root,last) AS ( SELECT @@ -846,5 +859,46 @@ do_eqp_test 9.1 { |--SEARCH event USING INTEGER PRIMARY KEY (rowid=?) `--USE TEMP B-TREE FOR ORDER BY } +optimization_control db all on +db cache flush +do_eqp_test 9.2 { + WITH thread(age,duration,cnt,root,last) AS ( + SELECT + julianday('now') - max(fmtime) AS age, + max(fmtime) - min(fmtime) AS duration, + sum(fprev IS NULL) AS msg_count, + froot, + (SELECT fpid FROM forumpost + WHERE froot=x.froot + AND fpid NOT IN private + ORDER BY fmtime DESC LIMIT 1) + FROM forumpost AS x + WHERE fpid NOT IN private --- Ensure this table mentioned in EQP output! + GROUP BY froot + ORDER BY 1 LIMIT 26 OFFSET 5 + ) + SELECT + thread.age, + thread.duration, + thread.cnt, + blob.uuid, + substr(event.comment,instr(event.comment,':')+1) + FROM thread, blob, event + WHERE blob.rid=thread.last + AND event.objid=thread.last + ORDER BY 1; +} { + QUERY PLAN + |--CO-ROUTINE thread + | |--SCAN x USING INDEX forumthread + | |--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR + | |--CORRELATED SCALAR SUBQUERY xxxxxx + | | |--SEARCH forumpost USING COVERING INDEX forumthread (froot=?) + | | `--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN thread + |--SEARCH blob USING INTEGER PRIMARY KEY (rowid=?) + `--SEARCH event USING INTEGER PRIMARY KEY (rowid=?) +} finish_test diff --git a/test/eqp2.test b/test/eqp2.test new file mode 100644 index 000000000..3c634fc28 --- /dev/null +++ b/test/eqp2.test @@ -0,0 +1,49 @@ +# 2024 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix eqp2 + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, d); + CREATE INDEX i1 ON t1(a, b, c); +} + +do_eqp_test 1.1 { + SELECT * FROM t1 ORDER BY a, b, c +} { + QUERY PLAN + `--SCAN t1 USING INDEX i1 +} + + +do_eqp_test 1.2 { + SELECT * FROM t1 ORDER BY a, b, +c +} { + QUERY PLAN + |--SCAN t1 USING INDEX i1 + `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY +} + +do_eqp_test 1.3 { + SELECT * FROM t1 ORDER BY a, +b, +c +} { + QUERY PLAN + |--SCAN t1 USING INDEX i1 + `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY +} + +finish_test + + diff --git a/test/exclusive2.test b/test/exclusive2.test index 92fcd76ab..0a5d2b6eb 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -41,7 +41,7 @@ sqlite3_soft_heap_limit 0 proc pagerChangeCounter {filename new {fd ""}} { if {$fd==""} { set fd [open $filename RDWR] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set needClose 1 } else { set needClose 0 @@ -70,7 +70,7 @@ proc pagerChangeCounter {filename new {fd ""}} { proc readPagerChangeCounter {filename} { set fd [open $filename RDONLY] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary seek $fd 24 foreach {a b c d} [list 0 0 0 0] {} diff --git a/test/exprfault2.test b/test/exprfault2.test new file mode 100644 index 000000000..acbead59f --- /dev/null +++ b/test/exprfault2.test @@ -0,0 +1,35 @@ +# 2024-05-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix exprfault2 + +do_execsql_test 1.0 { + CREATE TABLE t1(a,b,c,d,f,PRIMARY KEY(b,b)); + CREATE TABLE t2(x INT PRIMARY KEY, y, z); + CREATE TABLE t3(a,b,c,d,e,PRIMARY KEY(a,b))WITHOUT ROWID; +} +faultsim_save_and_close + + +do_faultsim_test 1 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE t3 SET (d,d,d,d, a )=(SELECT EXISTS(SELECT 1 NOT IN(SELECT EXISTS(SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT (SELECT max( 1 IN(SELECT x NOT IN(SELECT 1 NOT IN(SELECT EXISTS(SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT (SELECT max( (SELECT x NOT IN(SELECT 1 NOT IN(SELECT EXISTS(SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT (SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT (SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) a)| (SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(ORDER BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) z)|9 AS blob) IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))EXCEPT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) z) ORDER BY 1) IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT (SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 x ORDER BY 1)))EXCEPT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) a)| (SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) z)|9 AS blob) IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))EXCEPT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) z) ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 5 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(ORDER BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) z)| 1 AS blob) IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) z)| (SELECT 1 IN(SELECT max( 1 IN(SELECT c ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT (SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) e)|9 AS blob) FROM t2 WHERE a<x), e= BY 1) FROM t2 UNION SELECT 1 ORDER BY 1) ORDER BY 1)) a) FILTER (GROUP BY 1 HAVING b<= OVER(ORDER BY (SELECT max(x INPARTITION BY sum((SELECT y FROM t2 UNION SELECT x IN(SELECT 1 ORDER BY 1) ORDER 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 WHERE xBY 1) ORDER BY 1)) FROM77; + } +} -test { + faultsim_test_result {1 {near ")": syntax error}} +} + +finish_test diff --git a/test/fts3fault3.test b/test/fts3fault3.test index 6e1c0cd67..ae204718b 100644 --- a/test/fts3fault3.test +++ b/test/fts3fault3.test @@ -50,5 +50,33 @@ do_faultsim_test 1 -faults oom* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + BEGIN; + CREATE VIRTUAL TABLE t1 USING fts3(a); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50 + ) + INSERT INTO t1 SELECT 'abc def ghi jkl mno pqr' FROM s; + COMMIT; +} + +faultsim_save_and_close +do_faultsim_test 2 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + CREATE TABLE x1(a PRIMARY KEY); + } +} -body { + execsql { + PRAGMA integrity_check; + } +} -test { + faultsim_test_result {0 ok} $::TMPDBERROR +} + finish_test diff --git a/test/fts3snippet.test b/test/fts3snippet.test index ae022b68a..ad1fbb3be 100644 --- a/test/fts3snippet.test +++ b/test/fts3snippet.test @@ -561,7 +561,6 @@ do_test 4.3 { }] } {64} - #------------------------------------------------------------------------- # Request a snippet from a query with more than 64 phrases. # @@ -588,5 +587,9 @@ do_execsql_test 5.1 { {[a70] [a71] [a72]} } +do_execsql_test 5.2 { + SELECT snippet(t5, '[', ']', -1, 0) FROM t5 WHERE t5 MATCH 'a5' +} {{a4 [a5] a6}} + set sqlite_fts3_enable_parentheses 0 finish_test diff --git a/test/fts3snippet2.test b/test/fts3snippet2.test index 607b01e0f..f55a5f203 100644 --- a/test/fts3snippet2.test +++ b/test/fts3snippet2.test @@ -55,5 +55,16 @@ do_execsql_test 2.2 { '(def AND (one NEAR abc)) OR one' } {<b>one</b>} +#------------------------------------------------------------------------- + +do_execsql_test 3.0 { +CREATE VIRTUAL TABLE f USING fts3(a,b); +INSERT INTO f VALUES (101,x'056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f5010201010101014101017373737373737373737373737373737373737373737373737373737330737373737373737373737373737365056505650d051e05650565056528056505650d05650505656505650565650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565286505c705650565050565059494949494949494949494949494949494949494949494949494949494949494949494650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650505650565056505650565056505650565056500000000000000000000000000000000000000000000000000000000000000000565056505656505650565056505650d056500000000000000000000000000000000000000000000000000000000000000000100000000000000ed0000000000ffffffffffffffffffffff0007ffffff0001c5c50001c5c50001c5c50001c5c50001c5c50001c5c50001e5c50001c5c50001c5c50001c5c50001c5c50001c5c5000100000014720000000000000016dac5c50001c5c50001c5c50001c5c50001c5c50001c5c50d0505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800465056505650565056505651e650565056505650565056505650d05650505656505650565050565650584058005650565056505650565056505650565056522650565056505650d051e056505650561286505c70565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565226505737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373733a73737373737373737373737373737373737373737373737373737373737373737373737373737373737c7373737365056505650d051e05650565056528650565056505650505650565056505650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505'); +} + +do_execsql_test 3.1 { + SELECT length(snippet(f)) FROM f WHERE b MATCH x'0565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565056505650565050565056505640565056505650565056505651e05650565056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f50102010101010141010141010001017bf15905000000000017'; +} {192} + set sqlite_fts3_enable_parentheses 0 finish_test diff --git a/test/fts4intck1.test b/test/fts4intck1.test index abdc46bf5..6596b2f99 100644 --- a/test/fts4intck1.test +++ b/test/fts4intck1.test @@ -54,5 +54,22 @@ do_execsql_test 2.3 { PRAGMA integrity_check(t2); } {{malformed inverted index for FTS4 table main.t2}} +#------------------------------------------------------------------------- +# Test that integrity-check works on a read-only database. +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts4(a, b); + INSERT INTO x1 VALUES('one', 'two'); + INSERT INTO x1 VALUES('three', 'four'); +} +db close +sqlite3 db test.db -readonly 1 + +do_execsql_test 3.1 { + PRAGMA integrity_check; +} {ok} + + finish_test diff --git a/test/func.test b/test/func.test index 883950a0c..a3ecd4e30 100644 --- a/test/func.test +++ b/test/func.test @@ -786,6 +786,11 @@ do_test func-16.1 { } } {X'616263' NULL} +# Test the quote function for +Inf and -Inf +do_execsql_test func-16.2 { + SELECT quote(4.2e+859), quote(-7.8e+904); +} {9.0e+999 -9.0e+999} + # Correctly handle function error messages that include %. Ticket #1354 # do_test func-17.1 { @@ -1042,6 +1047,9 @@ do_test func-21.8 { SELECT replace('aaaaaaa', 'a', '0123456789'); } } {0123456789012345678901234567890123456789012345678901234567890123456789} +do_execsql_test func-21.9 { + SELECT typeof(replace(1,'',0)); +} {text} ifcapable tclvar { do_test func-21.9 { @@ -1553,4 +1561,25 @@ do_execsql_test func-38.100 { WITH t1(x) AS (VALUES(-9e+999)) SELECT sum(x), avg(x), total(x) FROM t1; } {Inf Inf Inf -Inf -Inf -Inf} +# 2024-03-21 https://sqlite.org/forum/forumpost/23b8688ef4 +# Another problem with Kahan-Babushka-Neumaier summation and +# infinities. +# +do_execsql_test func-39.101 { + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1) + SELECT sum(1.7976931348623157e308), + avg(1.7976931348623157e308), + total(1.7976931348623157e308) + FROM c; +} {1.79769313486232e+308 1.79769313486232e+308 1.79769313486232e+308} +for {set i 2} {$i<10} {incr i} { + do_execsql_test func-39.[expr {10*$i+100}] { + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<$i) + SELECT sum(1.7976931348623157e308), + avg(1.7976931348623157e308), + total(1.7976931348623157e308) + FROM c; + } {Inf Inf Inf} +} + finish_test diff --git a/test/func2.test b/test/func2.test index 08ad85750..a7c7ec3fd 100644 --- a/test/func2.test +++ b/test/func2.test @@ -508,4 +508,27 @@ do_test func2-3.9.2 { bin_to_hex [lindex $blob 0] } "12" +#------------------------------------------------------------------------- +# At one point this was extremely slow to compile. +# +do_test func2-3.10 { + set tm [time { + execsql { + SELECT '' IN (zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB( + zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(1) + ))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) + } + }] + + set tm [lindex $tm 0] + expr $tm<2000000 +} {1} + finish_test diff --git a/test/func4.test b/test/func4.test index 64c7a11ed..56cc9063a 100644 --- a/test/func4.test +++ b/test/func4.test @@ -1,4 +1,4 @@ -# 2013 March 10 +# 2023-03-10 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -9,7 +9,10 @@ # #*********************************************************************** # This file implements regression tests for SQLite library. The focus of -# this file is testing the tointeger() and toreal() functions. +# this file is testing the tointeger() and toreal() functions that are +# part of the "totype.c" extension. This file does not test the core +# SQLite library. Failures of tests in this file are related to the +# ext/misc/totype.c extension. # # Several of the toreal() tests are disabled on platforms where floating # point precision is not high enough to represent their constant integer @@ -23,6 +26,20 @@ load_static_extension db totype set highPrecision(1) [expr \ {[db eval {SELECT tointeger(9223372036854775807 + 1);}] eq {{}}}] +set highPrecision(2) [expr \ + {[db eval {SELECT toreal(-9223372036854775808 + 1);}] eq {{}}}] + +# highPrecision(3) is only known to be false on i586 with gcc-13 and -O2. +# It is true on the exact same platform with -O0. Both results seem +# reasonable, so we'll just very the expectation accordingly. +# +set highPrecision(3) [expr \ + {[db eval {SELECT toreal(9007199254740992 + 1);}] eq {{}}}] + +if {!$highPrecision(1) || !$highPrecision(2) || !$highPrecision(3)} { + puts "NOTICE: use_long_double: [use_long_double] \ + highPrecision: $highPrecision(1) $highPrecision(2) $highPrecision(3)" +} do_execsql_test func4-1.1 { SELECT tointeger(NULL); @@ -195,8 +212,6 @@ do_execsql_test func4-1.55 { } {{}} ifcapable floatingpoint { - set highPrecision(2) [expr \ - {[db eval {SELECT toreal(-9223372036854775808 + 1);}] eq {{}}}] do_execsql_test func4-2.1 { SELECT toreal(NULL); @@ -341,10 +356,14 @@ ifcapable floatingpoint { do_execsql_test func4-2.45 { SELECT toreal(9007199254740992); } {9007199254740992.0} - if {$highPrecision(2)} { + if {$highPrecision(3)} { do_execsql_test func4-2.46 { SELECT toreal(9007199254740992 + 1); } {{}} + } else { + do_execsql_test func4-2.46 { + SELECT toreal(9007199254740992 + 1); + } {9007199254740992.0} } do_execsql_test func4-2.47 { SELECT toreal(9007199254740992 + 2); @@ -626,10 +645,14 @@ ifcapable floatingpoint { do_execsql_test func4-5.22 { SELECT tointeger(toreal(9007199254740992)); } {9007199254740992} - if {$highPrecision(2)} { + if {$highPrecision(3)} { do_execsql_test func4-5.23 { SELECT tointeger(toreal(9007199254740992 + 1)); } {{}} + } else { + do_execsql_test func4-5.23 { + SELECT tointeger(toreal(9007199254740992 + 1)); + } {9007199254740992} } do_execsql_test func4-5.24 { SELECT tointeger(toreal(9007199254740992 + 2)); diff --git a/test/func6.test b/test/func6.test index fe90a755d..acca490f3 100644 --- a/test/func6.test +++ b/test/func6.test @@ -43,7 +43,7 @@ do_execsql_test func6-100 { # string. proc loadhex {file} { set fd [open $file] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set data [read $fd] close $fd binary encode hex $data diff --git a/test/func7.test b/test/func7.test index bb4f80b32..6026b557f 100644 --- a/test/func7.test +++ b/test/func7.test @@ -100,13 +100,13 @@ do_execsql_test func7-pg-300 { SELECT acos(1); } {0.0} do_execsql_test func7-pg-301 { - SELECT degrees(acos(0.5)); + SELECT format('%f',degrees(acos(0.5))); } {60.0} do_execsql_test func7-pg-310 { SELECT round( asin(1), 7); } {1.5707963} do_execsql_test func7-pg-311 { - SELECT degrees( asin(0.5) ); + SELECT format('%f',degrees( asin(0.5) )); } {30.0} do_execsql_test func7-pg-320 { SELECT round( atan(1), 7); @@ -139,7 +139,7 @@ do_execsql_test func7-pg-420 { SELECT round( tan(1), 7); } {1.5574077} do_execsql_test func7-pg-421 { - SELECT tan( radians(45) ); + SELECT round(tan( radians(45) ),10); } {1.0} do_execsql_test func7-pg-500 { SELECT round( sinh(1), 7); diff --git a/test/fuzz3.test b/test/fuzz3.test index bf778ed2b..8ac1f4d58 100644 --- a/test/fuzz3.test +++ b/test/fuzz3.test @@ -83,7 +83,7 @@ proc modify_database {iMod} { set offset [expr {$iMod>>8}] set fd [open test.db r+] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary seek $fd $offset set old_blob [read $fd 1] seek $fd $offset diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index dd4912011..eaeb54d87 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -161,8 +161,8 @@ static struct GlobalVars { /* ** Include the external vt02.c and randomjson.c modules. */ -extern int sqlite3_vt02_init(sqlite3*,char***,void*); -extern int sqlite3_randomjson_init(sqlite3*,char***,void*); +extern int sqlite3_vt02_init(sqlite3*,char**,const sqlite3_api_routines*); +extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*); /* @@ -979,7 +979,8 @@ extern int fuzz_invariant( int iRow, /* The row number for pStmt */ int nRow, /* Total number of output rows */ int *pbCorrupt, /* IN/OUT: Flag indicating a corrupt database file */ - int eVerbosity /* How much debugging output */ + int eVerbosity, /* How much debugging output */ + unsigned int dbOpt /* Default optimization flags */ ); /* Implementation of sqlite_dbdata and sqlite_dbptr */ @@ -1027,11 +1028,40 @@ static int recoverDatabase(sqlite3 *db){ } return rc; } +/* +** Special parameter binding, for testing and debugging purposes. +** +** $int_NNN -> integer value NNN +** $text_TTTT -> floating point value TTT with destructor +*/ +static void bindDebugParameters(sqlite3_stmt *pStmt){ + int nVar = sqlite3_bind_parameter_count(pStmt); + int i; + for(i=0; i<nVar; i++){ + const char *zVar = sqlite3_bind_parameter_name(pStmt, i+1); + if( zVar==0 ) continue; + if( strncmp(zVar, "$int_", 5)==0 ){ + sqlite3_bind_int(pStmt, i+1, atoi(&zVar[5])); + }else + if( strncmp(zVar, "$text_", 6)==0 ){ + char *zBuf = sqlite3_malloc64( strlen(zVar)-5 ); + if( zBuf ){ + memcpy(zBuf, &zVar[6], strlen(zVar)-5); + sqlite3_bind_text64(pStmt, i+1, zBuf, -1, sqlite3_free, SQLITE_UTF8); + } + } + } +} /* ** Run the SQL text */ -static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ +static int runDbSql( + sqlite3 *db, /* Run SQL on this database connection */ + const char *zSql, /* The SQL to be run */ + unsigned int *pBtsFlags, + unsigned int dbOpt /* Default optimization flags */ +){ int rc; sqlite3_stmt *pStmt; int bCorrupt = 0; @@ -1045,6 +1075,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc==SQLITE_OK ){ int nRow = 0; + bindDebugParameters(pStmt); while( (rc = sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; if( eVerbosity>=4 ){ @@ -1107,7 +1138,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ iRow++; for(iCnt=0; iCnt<99999; iCnt++){ rc = fuzz_invariant(db, pStmt, iCnt, iRow, nRow, - &bCorrupt, eVerbosity); + &bCorrupt, eVerbosity, dbOpt); if( rc==SQLITE_DONE ) break; if( rc!=SQLITE_ERROR ) g.nInvariant++; if( eVerbosity>0 ){ @@ -1330,7 +1361,7 @@ int runCombinedDbSqlInput( char cSaved = zSql[i+1]; zSql[i+1] = 0; if( sqlite3_complete(zSql+j) ){ - rc = runDbSql(cx.db, zSql+j, &btsFlags); + rc = runDbSql(cx.db, zSql+j, &btsFlags, dbOpt); j = i+1; } zSql[i+1] = cSaved; @@ -1340,7 +1371,7 @@ int runCombinedDbSqlInput( } } if( j<i ){ - runDbSql(cx.db, zSql+j, &btsFlags); + runDbSql(cx.db, zSql+j, &btsFlags, dbOpt); } } testrun_finished: diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 5d473f19f..36121963f 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -30,7 +30,38 @@ /* Forward references */ static char *fuzz_invariant_sql(sqlite3_stmt*, int); static int sameValue(sqlite3_stmt*,int,sqlite3_stmt*,int,sqlite3_stmt*); -static void reportInvariantFailed(sqlite3_stmt*,sqlite3_stmt*,int); +static void reportInvariantFailed( + sqlite3_stmt *pOrig, /* The original query */ + sqlite3_stmt *pTest, /* The alternative test query with a missing row */ + int iRow, /* Row number in pOrig */ + unsigned int dbOpt, /* Optimization flags on pOrig */ + int noOpt /* True if opt flags inverted for pTest */ +); + +/* +** Special parameter binding, for testing and debugging purposes. +** +** $int_NNN -> integer value NNN +** $text_TTTT -> floating point value TTT with destructor +*/ +static void bindDebugParameters(sqlite3_stmt *pStmt){ + int nVar = sqlite3_bind_parameter_count(pStmt); + int i; + for(i=0; i<nVar; i++){ + const char *zVar = sqlite3_bind_parameter_name(pStmt, i+1); + if( zVar==0 ) continue; + if( strncmp(zVar, "$int_", 5)==0 ){ + sqlite3_bind_int(pStmt, i+1, atoi(&zVar[5])); + }else + if( strncmp(zVar, "$text_", 6)==0 ){ + char *zBuf = sqlite3_malloc64( strlen(zVar)-5 ); + if( zBuf ){ + memcpy(zBuf, &zVar[6], strlen(zVar)-5); + sqlite3_bind_text64(pStmt, i+1, zBuf, -1, sqlite3_free, SQLITE_UTF8); + } + } + } +} /* ** Do an invariant check on pStmt. iCnt determines which invariant check to @@ -68,7 +99,8 @@ int fuzz_invariant( int iRow, /* Current row number */ int nRow, /* Number of output rows from pStmt */ int *pbCorrupt, /* IN/OUT: Flag indicating a corrupt database file */ - int eVerbosity /* How much debugging output */ + int eVerbosity, /* How much debugging output */ + unsigned int dbOpt /* Default optimization flags */ ){ char *zTest; sqlite3_stmt *pTestStmt = 0; @@ -76,13 +108,20 @@ int fuzz_invariant( int i; int nCol; int nParam; + int noOpt = (iCnt%3)==0; if( *pbCorrupt ) return SQLITE_DONE; nParam = sqlite3_bind_parameter_count(pStmt); if( nParam>100 ) return SQLITE_DONE; zTest = fuzz_invariant_sql(pStmt, iCnt); if( zTest==0 ) return SQLITE_DONE; + if( noOpt ){ + sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, ~dbOpt); + } rc = sqlite3_prepare_v2(db, zTest, -1, &pTestStmt, 0); + if( noOpt ){ + sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, dbOpt); + } if( rc ){ if( eVerbosity ){ printf("invariant compile failed: %s\n%s\n", @@ -93,6 +132,7 @@ int fuzz_invariant( return rc; } sqlite3_free(zTest); + bindDebugParameters(pTestStmt); nCol = sqlite3_column_count(pStmt); for(i=0; i<nCol; i++){ rc = sqlite3_bind_value(pTestStmt,i+1+nParam,sqlite3_column_value(pStmt,i)); @@ -157,6 +197,7 @@ int fuzz_invariant( printf("invariant-validity-check #2:\n%s\n", zSql); sqlite3_free(zSql); } + bindDebugParameters(pCk); while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){ for(i=0; i<nCol; i++){ if( !sameValue(pStmt, i, pTestStmt, i, 0) ) break; @@ -185,6 +226,7 @@ int fuzz_invariant( } sqlite3_reset(pTestStmt); + bindDebugParameters(pCk); while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ for(i=0; i<nCol; i++){ if( !sameValue(pStmt, i, pTestStmt, i, pCk) ) break; @@ -212,7 +254,7 @@ int fuzz_invariant( } sqlite3_finalize(pCk); if( rc==SQLITE_DONE ){ - reportInvariantFailed(pStmt, pTestStmt, iRow); + reportInvariantFailed(pStmt, pTestStmt, iRow, dbOpt, noOpt); return SQLITE_INTERNAL; }else if( eVerbosity>0 ){ printf("invariant-error ignored due to the use of virtual tables\n"); @@ -223,7 +265,6 @@ not_a_fault: return SQLITE_OK; } - /* ** Generate SQL used to test a statement invariant. ** @@ -285,6 +326,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ sqlite3_finalize(pBase); pBase = pStmt; } + bindDebugParameters(pBase); for(i=0; i<sqlite3_column_count(pStmt); i++){ const char *zColName = sqlite3_column_name(pBase,i); const char *zSuffix = zColName ? strrchr(zColName, ':') : 0; @@ -489,13 +531,17 @@ static void printRow(sqlite3_stmt *pStmt, int iRow){ static void reportInvariantFailed( sqlite3_stmt *pOrig, /* The original query */ sqlite3_stmt *pTest, /* The alternative test query with a missing row */ - int iRow /* Row number in pOrig */ + int iRow, /* Row number in pOrig */ + unsigned int dbOpt, /* Optimization flags on pOrig */ + int noOpt /* True if opt flags inverted for pTest */ ){ int iTestRow = 0; printf("Invariant check failed on row %d.\n", iRow); - printf("Original query --------------------------------------------------\n"); + printf("Original query (opt-flags: 0x%08x) --------------------------\n", + dbOpt); printf("%s\n", sqlite3_expanded_sql(pOrig)); - printf("Alternative query -----------------------------------------------\n"); + printf("Alternative query (opt-flags: 0x%08x) -----------------------\n", + noOpt ? ~dbOpt : dbOpt); printf("%s\n", sqlite3_expanded_sql(pTest)); printf("Result row that is missing from the alternative -----------------\n"); printRow(pOrig, iRow); diff --git a/test/icu.test b/test/icu.test index 644cbb1f0..c1b5653d4 100644 --- a/test/icu.test +++ b/test/icu.test @@ -149,7 +149,7 @@ ifcapable icu { # 2020-03-19 # The ESCAPE clause on LIKE takes precedence over wildcards # -do_execsql_test idu-6.0 { +do_execsql_test icu-6.0 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(id INTEGER PRIMARY KEY, x TEXT); INSERT INTO t1 VALUES @@ -164,4 +164,20 @@ do_execsql_test icu-6.1 { SELECT id FROM t1 WHERE x LIKE 'abc__' ESCAPE '_'; } {2} +# 2024-04-02 +# Optional 3rd argument to icu_load_collation() that specifies +# the "strength" of comparison. +# +reset_db +do_catchsql_test icu-7.1 { + SELECT icu_load_collation('en_US','error','xyzzy'); +} {1 {unknown collation strength "xyzzy" - should be one of: PRIMARY SECONDARY TERTIARY DEFAULT QUARTERNARY IDENTICAL}} +do_execsql_test icu-7.2 { + SELECT icu_load_collation('en_US','prim','PRIMARY'), + icu_load_collation('en_US','dflt','DEFAULT'); +} {{} {}} +do_execsql_test icu-7.3 { + SELECT char(0x100)=='a', char(0x100)=='a' COLLATE dflt, char(0x100)=='a' COLLATE prim; +} {0 0 1} + finish_test diff --git a/test/in4.test b/test/in4.test index a3fe22e78..71993e700 100644 --- a/test/in4.test +++ b/test/in4.test @@ -458,14 +458,14 @@ do_execsql_test 11.0 { do_execsql_test 11.1 { SELECT * FROM t1 WHERE b IN (345, (SELECT 1 FROM t1 - WHERE b IN (345 NOT GLOB 510) + WHERE b IN (coalesce(1,random())) AND c GLOB 'abc*xyz')) AND c BETWEEN 'abc' AND 'xyz'; } {xyz 1 abcdefxyz 99} do_execsql_test 11.2 { EXPLAIN SELECT * FROM t1 WHERE b IN (345, (SELECT 1 FROM t1 - WHERE b IN (345 NOT GLOB 510) + WHERE b IN (coalesce(1,random())) AND c GLOB 'abc*xyz')) AND c BETWEEN 'abc' AND 'xyz'; } {/ SeekScan /} diff --git a/test/in5.test b/test/in5.test index 6680641ff..933eb9002 100644 --- a/test/in5.test +++ b/test/in5.test @@ -266,4 +266,26 @@ do_execsql_test 9.2 { SELECT lower('1e500') FROM t0 WHERE rowid != lower('1e500'); } {1e500} +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 10.0 { + CREATE TABLE t1(a, b TEXT COLLATE NOCASE); + INSERT INTO t1 VALUES('abc', 'def'); + INSERT INTO t1 VALUES('ghi', 'jkl'); +} + +do_execsql_test 10.1 { + SELECT rowid FROM t1 WHERE (a, b) IN ( VALUES('abc', 'def'), ('ghi', 'JKL') ); +} {1 2} + +do_execsql_test 10.2 { + CREATE INDEX i1 ON t1(a, b COLLATE BINARY); +} + +do_execsql_test 10.3 { + SELECT rowid FROM t1 WHERE (a, b) IN ( VALUES('abc', 'def'), ('ghi', 'JKL') ); +} {1 2} + finish_test diff --git a/test/in7.test b/test/in7.test new file mode 100644 index 000000000..29013ff59 --- /dev/null +++ b/test/in7.test @@ -0,0 +1,197 @@ +# 2024-05-01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix in7 + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c PRIMARY KEY); + CREATE TABLE t2(x, y, z); +} + +foreach {tn nNext idx sql} { + 1 1 { + CREATE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE (a, b) IN (SELECT x, y FROM t2) + } + + 2 0 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE (a, b) IN (SELECT x, y FROM t2) + } + + 3 0 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a = ? AND b = ? + } + + 3 1 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a = ? AND b IS ? + } + + 4 0 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a = ? AND b IN (?, ?, ?); + } + + 5 1 { + CREATE UNIQUE INDEX i1 ON t1(a, b, c); + } { + SELECT * FROM t1 WHERE a = ? AND b = ? + } + + 6 0 { + } { + SELECT * FROM t1 WHERE c IN (SELECT z FROM t2) + } + + 7 0 { + } { + SELECT * FROM t1 WHERE (a, c) IN (SELECT z, x FROM t2) + } + + 8 1 { + } { + SELECT * FROM t1 WHERE a IN (SELECT z FROM t2) + } + + 9 1 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a IN (SELECT z FROM t2) AND b IS ? + } + 10 0 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a IN (SELECT z FROM t2) AND b = ? + } + 11 1 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a IS NULL AND b IN (SELECT z FROM t2) + } + 12 0 { + CREATE UNIQUE INDEX i1 ON t1(a, b); + } { + SELECT * FROM t1 WHERE a = ? AND b IN (SELECT z FROM t2) + } +} { + do_test 1.1.$tn { + execsql BEGIN + execsql $idx + + catch { array unset root_to_tbl } + catch { array unset csr_to_root } + + db eval {SELECT rootpage, tbl_name FROM sqlite_schema} { + set root_to_tbl($rootpage) $tbl_name + } + + set nSeen 0 + db eval "explain $sql" { + if {$opcode=="OpenRead"} { + set csr_to_root($p1) $p2 + } + if {$opcode=="Next"} { + catch { + set root $csr_to_root($p1) + set tbl $root_to_tbl($root) + if {$tbl=="t1"} {incr nSeen} + } + } + } + + execsql ROLLBACK + + set nSeen + } $nNext +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a TEXT PRIMARY KEY, b TEXT) WITHOUT ROWID; + INSERT INTO t1 VALUES('1', 'one'); + INSERT INTO t1 VALUES('2', NULL); + INSERT INTO t1 VALUES('3', 'three'); +} + +do_execsql_test 2.1 { + SELECT b FROM t1 WHERE a IN (1,2,3) ORDER BY b ASC NULLS LAST; +} {one three {}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1), (2), (3); + + CREATE TABLE x2(b); + INSERT INTO x2 VALUES(4), (5), (6); + + CREATE TABLE t1(u); + INSERT INTO t1 VALUES(1), (2), (3), (4), (5), (6); + + CREATE VIEW v1 AS SELECT u FROM t1 WHERE u IN ( + SELECT a FROM x1 + ); + CREATE VIEW v2 AS SELECT u FROM t1 WHERE u IN ( + SELECT b FROM x2 + ); +} + +do_execsql_test 3.1 { + SELECT * FROM v1 +} { + 1 2 3 +} + +do_execsql_test 3.2 { + SELECT * FROM v2 +} { + 4 5 6 +} + +do_execsql_test 3.3 { + SELECT * FROM v2 + UNION ALL + SELECT * FROM v1 +} { + 4 5 6 + 1 2 3 +} + +do_execsql_test 3.4 { + WITH w1 AS ( + SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 + ), + w2 AS ( + SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 + ) + SELECT * FROM v1 WHERE u IN w1 + UNION ALL + SELECT * FROM v2 WHERE u IN w2 +} { + 1 2 3 4 5 6 +} + + + +finish_test diff --git a/test/incrvacuum3.test b/test/incrvacuum3.test index e901bfe1f..b4cbc8b0c 100644 --- a/test/incrvacuum3.test +++ b/test/incrvacuum3.test @@ -50,8 +50,8 @@ proc check_on_disk {} { set sz [file size test.db] set fd [open test.db] set fd2 [open test2.db w] - fconfigure $fd -encoding binary -translation binary - fconfigure $fd2 -encoding binary -translation binary + fconfigure $fd -translation binary + fconfigure $fd2 -translation binary if {$sz>$::sqlite_pending_byte} { puts -nonewline $fd2 [read $fd $::sqlite_pending_byte] seek $fd [expr $::sqlite_pending_byte+512] diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 0316ee9d4..3ba59449d 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -615,6 +615,58 @@ do_execsql_test indexexpr1-2200 { GROUP BY t2.type, t1.tag ) v ON v.type = 0 AND v.tag = u.tag; } {7 100 8 101} +do_execsql_test indexexpr1-2210 { + DROP TABLE t1; + CREATE TABLE t1(x INT, y TEXT); + INSERT INTO t1(x,y) VALUES(1,'{b:5}'); + SELECT json_insert('{}', '$.a', coalesce(null,json(y)))->>'$.a.b' FROM t1; +} {5} +db null NULL +do_execsql_test indexexpr1-2211 { + CREATE INDEX t1j ON t1(coalesce(null,json(y))); + SELECT json_insert('{}', '$.a', coalesce(null,json(y)))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2220 { + DROP INDEX t1j; + SELECT json_insert('{}', '$.a', iif(1,json(y),123))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2221 { + CREATE INDEX t1j ON t1(iif(1,json(y),123)); + SELECT json_insert('{}', '$.a', iif(1,json(y),123))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2230 { + DROP INDEX t1j; + SELECT json_insert('{}', '$.a', ifnull(NULL,json(y)))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2231 { + CREATE INDEX t1j ON t1(ifnull(NULL,json(y))); + SELECT json_insert('{}', '$.a', ifnull(NULL,json(y)))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2240 { + DROP INDEX t1j; + SELECT json_insert('{}', '$.a', nullif(json(y),8))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2241 { + CREATE INDEX t1j ON t1(nullif(json(y),8)); + SELECT json_insert('{}', '$.a', nullif(json(y),8))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2250 { + DROP INDEX t1j; + SELECT json_insert('{}', '$.a', min('~',json(y)))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2251 { + CREATE INDEX t1j ON t1(min('~',json(y))); + SELECT json_insert('{}', '$.a', min('~',json(y)))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2260 { + DROP INDEX t1j; + SELECT json_insert('{}', '$.a', max('...',json(y)))->>'$.a.b' FROM t1; +} {5} +do_execsql_test indexexpr1-2261 { + CREATE INDEX t1j ON t1(max('...',json(y))); + SELECT json_insert('{}', '$.a', max('...',json(y)))->>'$.a.b' FROM t1; +} {5} + # 2023-11-08 Forum post https://sqlite.org/forum/forumpost/68d284c86b082c3e # diff --git a/test/ioerr.test b/test/ioerr.test index 29a3dede8..fb54d8b38 100644 --- a/test/ioerr.test +++ b/test/ioerr.test @@ -225,7 +225,7 @@ if {$tcl_platform(platform)=="unix" && [atomic_batch_write test.db]==0} { } forcecopy test2.db-journal test.db-journal set f [open test.db-journal a] - fconfigure $f -encoding binary + fconfigure $f -translation binary puts -nonewline $f "hello" puts -nonewline $f "\x00\x00\x00\x05\x01\x02\x03\x04" puts -nonewline $f "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7" diff --git a/test/ioerr5.test b/test/ioerr5.test index a430f5340..a0a74bd95 100644 --- a/test/ioerr5.test +++ b/test/ioerr5.test @@ -117,7 +117,7 @@ foreach locking_mode {normal exclusive} { # Read the contents of the database file into a Tcl variable. # set fd [open test.db] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set zDatabase [read $fd] close $fd @@ -138,7 +138,7 @@ foreach locking_mode {normal exclusive} { # do_test ioerr5-1.$locking_mode-$iFail.4 { set fd [open test.db] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set zDatabase2 [read $fd] close $fd expr {$zDatabase eq $zDatabase2} diff --git a/test/join5.test b/test/join5.test index 44c8b71a8..703c256f8 100644 --- a/test/join5.test +++ b/test/join5.test @@ -370,8 +370,12 @@ do_execsql_test 9.1 { ANALYZE sqlite_schema; INSERT INTO sqlite_stat1 VALUES('t1','t1x1','648 324 81 81 81 81 81 81 81081 81 81 81'); ANALYZE sqlite_schema; - SELECT a FROM (SELECT a FROM t1 NATURAL LEFT JOIN t1) NATURAL LEFT JOIN t1 WHERE (rowid,1)<=(5,0); -} {1} +} +do_catchsql_test 9.2 { + SELECT a FROM + (SELECT a FROM t1 NATURAL LEFT JOIN t1) NATURAL LEFT JOIN t1 + WHERE (rowid,1)<=(5,0); +} {0 1} # 2022-03-02 https://sqlite.org/forum/info/50a1bbe08ce4c29c # Bloom-filter pulldown is incompatible with skip-scan. diff --git a/test/joinH.test b/test/joinH.test index 370226680..908b93dee 100644 --- a/test/joinH.test +++ b/test/joinH.test @@ -201,13 +201,13 @@ do_execsql_test 9.0 { do_catchsql_test 9.1 { SELECT rowid FROM wo1, x1, x2; -} {1 {no such column: rowid}} +} {1 {ambiguous column name: rowid}} do_catchsql_test 9.2 { SELECT rowid FROM wo1, (x1, x2); -} {1 {no such column: rowid}} +} {1 {ambiguous column name: rowid}} do_catchsql_test 9.3 { SELECT rowid FROM wo1 JOIN (x1 JOIN x2); -} {1 {no such column: rowid}} +} {1 {ambiguous column name: rowid}} do_catchsql_test 9.4 { SELECT a FROM wo1, x1, x2; } {1 {ambiguous column name: a}} @@ -309,4 +309,36 @@ do_execsql_test 12.3 { SELECT * FROM t1 LEFT JOIN t2 ON true RIGHT JOIN t3 ON d2=e3 WHERE c2 BETWEEN NULL AND a1; } +#------------------------------------------------------------------------- +# 2024-04-05 dbsqlfuzz b9e65e2f110df998f1306571fae7af6c01e4d92b +reset_db +do_execsql_test 13.1 { + CREATE TABLE t1(a INT AS (b), b INT); + INSERT INTO t1(b) VALUES(123); + CREATE TABLE t2(a INT, c INT); + SELECT a FROM t2 NATURAL RIGHT JOIN t1; +} {123} +do_execsql_test 13.2 { + CREATE INDEX t1a ON t1(a); + SELECT a FROM t2 NATURAL RIGHT JOIN t1; +} {123} +# Further tests of the same logic (indexes on expressions +# used by RIGHT JOIN) from check-in ffe23af73fcb324d and +# forum post https://sqlite.org/forum/forumpost/9b491e1debf0b67a. +db null NULL +do_execsql_test 13.3 { + CREATE TABLE t3(a INT, b INT); + CREATE UNIQUE INDEX t3x ON t3(a, a+b); + INSERT INTO t3(a,b) VALUES(1,2),(4,8),(16,32),(4,80),(1,-300); + CREATE TABLE t4(x INT, y INT); + INSERT INTO t4(x,y) SELECT a, b FROM t3; + INSERT INTO t4(x,y) VALUES(99,99); + SELECT a1.a, sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t4 ON a=x + GROUP BY a1.a ORDER BY 1; +} {NULL NULL 1 -592 4 192 16 48} +do_execsql_test 13.4 { + SELECT sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t3 ON true + GROUP BY a1.a ORDER BY 1; +} {-1480 240 480} + finish_test diff --git a/test/json101.test b/test/json101.test index bae68e734..3963ffbb6 100644 --- a/test/json101.test +++ b/test/json101.test @@ -378,6 +378,20 @@ do_execsql_test json101-5.8 { WHERE jx.value<>jx.atom AND type NOT IN ('array','object'); } {} +# 2024-02-16 https://sqlite.org/forum/forumpost/ecb94cd210 +# Regression in json_tree()/json_each(). The value column +# should have the "J" subtype if the value is an array or +# object. +# +do_execsql_test json101-5.10 { + SELECT json_insert('{}','$.a',value) FROM json_tree('[1,2,3]') WHERE atom IS NULL; +} {{{"a":[1,2,3]}}} +# ^^^^^^^--- In double-quotes, a string literal, prior to bug fix + +do_execsql_test json101-5.11 { + SELECT json_insert('{}','$.a',value) FROM json_tree('"[1,2,3]"'); +} {{{"a":"[1,2,3]"}}} + do_execsql_test json101-6.1 { SELECT json_valid('{"a":55,"b":72,}'); } {0} diff --git a/test/json102.test b/test/json102.test index 15a54b47c..1a00cb67a 100644 --- a/test/json102.test +++ b/test/json102.test @@ -764,4 +764,35 @@ do_execsql_test json102-1720 { SELECT * FROM t1; } {ok 2023-08-03 876 5 {{"x":77,"y":6}}} +# 2024-05-21 https://sqlite.org/forum/forumpost/9e52cdfe15c3926e +# What if the RHS of the -> or ->> operator is a string that looks +# like a number? PostgreSQL treats it as a string. +# +do_execsql_test json102-1800 { + SELECT '{"1":"one","2":"two","3":"three"}'->>'2'; +} two +db null NULL +do_execsql_test json102-1801 { + SELECT '{"1":"one","2":"two","3":"three"}'->>2; +} NULL +do_execsql_test json102-1810 { + SELECT '["zero","one","two"]'->>'1'; +} NULL +do_execsql_test json102-1811 { + SELECT '["zero","one","two"]'->>1; +} one +do_execsql_test json102-1820 { + SELECT '{"1":"one","2":"two","3":"three"}'->'2'; +} {{"two"}} +do_execsql_test json102-1821 { + SELECT '{"1":"one","2":"two","3":"three"}'->2; +} {NULL} +do_execsql_test json102-1830 { + SELECT '["zero","one","two"]'->'1'; +} {NULL} +do_execsql_test json102-1831 { + SELECT '["zero","one","two"]'->1; +} {{"one"}} + + finish_test diff --git a/test/json106.test b/test/json106.test index 23fa02843..06859a10b 100644 --- a/test/json106.test +++ b/test/json106.test @@ -67,6 +67,12 @@ for {set ii 1} {$ii<=5000} {incr ii} { FROM t1, kv WHERE p->>key IS NOT val } 0 + do_execsql_test $ii.8 { + SELECT j0 FROM t1 WHERE json(j0)!=json(json_pretty(j0)); + } {} + do_execsql_test $ii.9 { + SELECT j5 FROM t1 WHERE json(j5)!=json(json_pretty(j5)); + } {} } diff --git a/test/json107.test b/test/json107.test new file mode 100644 index 000000000..779b557fb --- /dev/null +++ b/test/json107.test @@ -0,0 +1,86 @@ +# 2024-01-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Legacy JSON bug: If the input is a BLOB that when cast into TEXT looks +# like valid JSON, then treat it as valid JSON. +# +# The original intent of the JSON functions was to raise an error on any +# BLOB input. That intent was clearly documented, but the code failed to +# to implement it. Subsequently, many applications began to depend on the +# incorrect behavior, especially apps that used readfile() to read JSON +# content, since readfile() returns a BLOB. So we need to support the +# bug moving forward. +# +# The tests in this fail verify that the original buggy behavior is +# preserved. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json107 + +if {[db one {PRAGMA encoding}]!="UTF-8"} { + # These tests only work for a UTF-8 encoding. + finish_test + return +} + +do_execsql_test 1.1 { + SELECT json_valid( CAST('{"a":1}' AS BLOB) ); +} 1 +do_execsql_test 1.1.1 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 1); +} 1 +do_execsql_test 1.1.2 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 2); +} 1 +do_execsql_test 1.1.4 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 4); +} 0 +do_execsql_test 1.1.8 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 8); +} 0 + +do_execsql_test 1.2.1 { + SELECT CAST('{"a":123}' AS blob) -> 'a'; +} 123 +do_execsql_test 1.2.2 { + SELECT CAST('{"a":123}' AS blob) ->> 'a'; +} 123 +do_execsql_test 1.2.3 { + SELECT json_extract(CAST('{"a":123}' AS blob), '$.a'); +} 123 +do_execsql_test 1.3 { + SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456); +} {{{"a":123,"b":456}}} +do_execsql_test 1.4 { + SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a'); +} {{{"b":456}}} +do_execsql_test 1.5 { + SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789); +} {{{"a":789,"b":456}}} +do_execsql_test 1.6 { + SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789); +} {{{"a":789,"b":456}}} +do_execsql_test 1.7 { + SELECT json_type(CAST('{"a":123,"b":456}' AS blob)); +} object +do_execsql_test 1.8 { + SELECT json(CAST('{"a":123,"b":456}' AS blob)); +} {{{"a":123,"b":456}}} + +ifcapable vtab { + do_execsql_test 2.1 { + SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) ) + WHERE atom; + } {a 123 b 456} +} +finish_test diff --git a/test/json108.test b/test/json108.test new file mode 100644 index 000000000..71f3814dc --- /dev/null +++ b/test/json108.test @@ -0,0 +1,45 @@ +# 2024-03-06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Invariant tests for JSON built around the randomjson extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json108 + +# These tests require virtual table "json_tree" to run. +ifcapable !vtab { finish_test ; return } + +load_static_extension db randomjson +db eval { + CREATE TEMP TABLE t1(j0,j5); + WITH RECURSIVE c(n) AS (VALUES(0) UNION ALL SELECT n+1 FROM c WHERE n<9) + INSERT INTO t1 SELECT random_json(n), random_json5(n) FROM c; +} + +do_execsql_test 1.1 { + SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,NULL)); +} 10 +do_execsql_test 1.2 { + SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,NULL)); +} 10 +do_execsql_test 1.3 { + SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,'')); +} 10 +do_execsql_test 1.4 { + SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,char(9))); +} 10 +do_execsql_test 1.5 { + SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,'/*hello*/')); +} 10 + + +finish_test diff --git a/test/json501.test b/test/json501.test index 40b3b563d..bfd21055a 100644 --- a/test/json501.test +++ b/test/json501.test @@ -306,4 +306,31 @@ do_execsql_test 13.1 { SELECT json('{x:''a "b" c''}'); } {{{"x":"a \"b\" c"}}} +# 2024-01-31 +# Allow control characters within JSON5 string literals. +# +for {set c 1} {$c<=0x1f} {incr c} { + do_execsql_test 14.$c.1 { + SELECT json_valid('"abc' || char($c) || 'xyz"'); + } {0} + do_execsql_test 14.$c.2 { + SELECT json_valid('"abc' || char($c) || 'xyz"', 2); + } {1} + switch $c { + 8 {set e "\\b"} + 9 {set e "\\t"} + 10 {set e "\\n"} + 12 {set e "\\f"} + 13 {set e "\\r"} + default {set e [format "\\u00%02x" $c]} + } + do_execsql_test 14.$c.3 { + SELECT json('{label:"abc' || char($c) || 'xyz"}'); + } "{{\"label\":\"abc${e}xyz\"}}" + do_execsql_test 14.$c.4 { + SELECT jsonb('{label:"abc' || char($c) || 'xyz"}') -> '$'; + } "{{\"label\":\"abc${e}xyz\"}}" +} + + finish_test diff --git a/test/jsonb01.test b/test/jsonb01.test index d1b53ae6c..8f16428dc 100644 --- a/test/jsonb01.test +++ b/test/jsonb01.test @@ -46,4 +46,8 @@ foreach {id path res} { } $res } +do_catchsql_test jsonb01-2.0 { + SELECT x'8ce6ffffffff171333' -> '$'; +} {1 {malformed JSON}} + finish_test diff --git a/test/lemon-test01.y b/test/lemon-test01.y index 0fd514ff3..67890c637 100644 --- a/test/lemon-test01.y +++ b/test/lemon-test01.y @@ -2,6 +2,11 @@ // // lemon lemon-test01.y && gcc -g lemon-test01.c && ./a.out // +// This testcase was made obsolete by check-in 7cca80808cef192f on +// 2021-08-17 (associated with Forum Thread +// https://sqlite.org/forum/forumpost/bd91fd965c9803c4) and no longer +// works. It is retained for historical reference only. +// %token_prefix TK_ %token_type int %default_type int @@ -28,7 +33,7 @@ all ::= error B. #include "lemon-test01.h" static int nTest = 0; static int nErr = 0; - static int testCase(int testId, int shouldBe, int actual){ + static void testCase(int testId, int shouldBe, int actual){ nTest++; if( shouldBe==actual ){ printf("test %d: ok\n", testId); diff --git a/test/literal.test b/test/literal.test new file mode 100644 index 000000000..5aa331e39 --- /dev/null +++ b/test/literal.test @@ -0,0 +1,103 @@ +# 2024-01-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for SQL literals + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix literal + +proc test_literal {tn lit type val} { + do_execsql_test $tn.1 "SELECT typeof( $lit ), $lit" [list $type $val] + + ifcapable altertable { + do_execsql_test $tn.2 " + DROP TABLE IF EXISTS x1; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(123); + ALTER TABLE x1 ADD COLUMN b DEFAULT $lit ; + SELECT typeof(b), b FROM x1; + " [list $type $val] + } + + do_execsql_test $tn.3 " + DROP TABLE IF EXISTS x1; + CREATE TABLE x1(a DEFAULT $lit); + INSERT INTO x1 DEFAULT VALUES; + SELECT typeof(a), a FROM x1; + " [list $type $val] +} + +proc test_literal_error {tn lit unrec} { + do_catchsql_test $tn "SELECT $lit" "1 {unrecognized token: \"$unrec\"}" +} + + +test_literal 1.0 45 integer 45 +test_literal 1.1 0xFF integer 255 +test_literal 1.2 0xFFFFFFFF integer [expr 0xFFFFFFFF] +test_literal 1.3 0x123FFFFFFFF integer [expr 0x123FFFFFFFF] +test_literal 1.4 -0x123FFFFFFFF integer [expr -1 * 0x123FFFFFFFF] +test_literal 1.5 0xFFFFFFFFFFFFFFFF integer -1 +test_literal 1.7 0x7FFFFFFFFFFFFFFF integer [expr 0x7FFFFFFFFFFFFFFF] +test_literal 1.8 -0x7FFFFFFFFFFFFFFF integer [expr -0x7FFFFFFFFFFFFFFF] +test_literal 1.9 +0x7FFFFFFFFFFFFFFF integer [expr +0x7FFFFFFFFFFFFFFF] +test_literal 1.10 -45 integer -45 +test_literal 1.11 '0xFF' text 0xFF +test_literal 1.12 '-0xFF' text -0xFF +test_literal 1.13 -'0xFF' integer 0 +test_literal 1.14 -9223372036854775808 integer -9223372036854775808 + +test_literal 2.1 1e12 real 1000000000000.0 +test_literal 2.2 1.0 real 1.0 +test_literal 2.3 1e1000 real Inf +test_literal 2.4 -1e1000 real -Inf + +test_literal 3.1 1_000 integer 1000 +test_literal 3.2 1.1_1 real 1.11 +test_literal 3.3 1_0.1_1 real 10.11 +test_literal 3.4 1e1_000 real Inf +test_literal 3.5 12_3_456.7_8_9 real 123456.789 +test_literal 3.6 9_223_372_036_854_775_807 integer 9223372036854775807 +test_literal 3.7 9_223_372_036_854_775_808 real 9.22337203685478e+18 +test_literal 3.8 -9_223_372_036_854_775_808 integer -9223372036854775808 + +foreach {tn lit unrec} { + 0 123a456 123a456 + 1 1_ 1_ + 2 1_.4 1_.4 + 3 1e_4 1e_4 + 4 1_e4 1_e4 + 5 1.4_e4 1.4_e4 + 6 1.4e+_4 1.4e + 7 1.4e-_4 1.4e + 8 1.4e4_ 1.4e4_ + 9 1.4_e4 1.4_e4 + 10 1.4e_4 1.4e_4 + 11 12__34 12__34 + 12 1234_ 1234_ + 13 12._34 12._34 + 14 12_.34 12_.34 + 15 12.34_ 12.34_ + 16 1.0e1_______2 1.0e1_______2 +} { + test_literal_error 4.$tn $lit $unrec +} + +# dbsqlfuzz e3186a9e7826e9cd7f4085aa4452f8696485f9e1 +# See tag-20240224-a and -b +# +do_catchsql_test 5.1 { + SELECT 1 ORDER BY 2_3; +} {1 {1st ORDER BY term out of range - should be between 1 and 1}} + +finish_test diff --git a/test/literal2.tcl b/test/literal2.tcl new file mode 100644 index 000000000..e14a03587 --- /dev/null +++ b/test/literal2.tcl @@ -0,0 +1,40 @@ +# 2018 May 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname $argv0] pg_common.tcl] + +#========================================================================= + + +start_test literal2 "2024 Jan 23" + +execsql_test 1.0 { SELECT 123_456 } +errorsql_test 1.1 { SELECT 123__456 } + +execsql_float_test 2.1 { SELECT 1.0e1_2 } + + +execsql_test 3.0.0 { SELECT 0xFF_FF } +execsql_test 3.0.1 { SELECT 0xFF_EF } +errorsql_test 3.0.2 { SELECT 0xFF__EF } +# errorsql_test 3.0.3 { SELECT 0x_FFEF } +errorsql_test 3.0.4 { SELECT 0xFFEF_ } + +execsql_test 3.1.0 { SELECT 0XFF_FF } +execsql_test 3.1.1 { SELECT 0XFF_EF } +errorsql_test 3.1.2 { SELECT 0XFF__EF } +# errorsql_test 3.1.3 { SELECT 0X_FFEF } +errorsql_test 3.1.4 { SELECT 0XFFEF_ } + +finish_test + + diff --git a/test/literal2.test b/test/literal2.test new file mode 100644 index 000000000..ed177ca26 --- /dev/null +++ b/test/literal2.test @@ -0,0 +1,84 @@ +# 2024 Jan 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +#################################################### +# DO NOT EDIT! THIS FILE IS AUTOMATICALLY GENERATED! +#################################################### + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix literal2 + +do_execsql_test 1.0 { + SELECT 123_456 +} {123456} + +# PG says ERROR: trailing junk after numeric literal at or near "123_" +do_test 1.1 { catch { execsql { + SELECT 123__456 +} } } 1 + + +do_test 2.1 { + set myres {} + foreach r [db eval {SELECT 1.0e1_2}] { + lappend myres [format %.4f [set r]] + } + set res2 {1000000000000.0000} + set i 0 + foreach r [set myres] r2 [set res2] { + if {[set r]<([set r2]-0.0001) || [set r]>([set r2]+0.0001)} { + error "list element [set i] does not match: got=[set r] expected=[set r2]" + } + incr i + } + set {} {} +} {} + +do_execsql_test 3.0.0 { + SELECT 0xFF_FF +} {65535} + +do_execsql_test 3.0.1 { + SELECT 0xFF_EF +} {65519} + +# PG says ERROR: trailing junk after numeric literal at or near "0xFF_" +do_test 3.0.2 { catch { execsql { + SELECT 0xFF__EF +} } } 1 + +# PG says ERROR: trailing junk after numeric literal at or near "0xFFEF_" +do_test 3.0.4 { catch { execsql { + SELECT 0xFFEF_ +} } } 1 + +do_execsql_test 3.1.0 { + SELECT 0XFF_FF +} {65535} + +do_execsql_test 3.1.1 { + SELECT 0XFF_EF +} {65519} + +# PG says ERROR: trailing junk after numeric literal at or near "0XFF_" +do_test 3.1.2 { catch { execsql { + SELECT 0XFF__EF +} } } 1 + +# PG says ERROR: trailing junk after numeric literal at or near "0XFFEF_" +do_test 3.1.4 { catch { execsql { + SELECT 0XFFEF_ +} } } 1 + +finish_test diff --git a/test/lock5.test b/test/lock5.test index 99214afb1..e0c88aedc 100644 --- a/test/lock5.test +++ b/test/lock5.test @@ -11,10 +11,10 @@ # This file implements regression tests for SQLite library. The # focus of this script is database locks. # -# $Id: lock5.test,v 1.6 2008/12/04 12:34:16 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix lock5 # This file is only run if using the unix backend compiled with the # SQLITE_ENABLE_LOCKING_STYLE macro. @@ -101,10 +101,7 @@ do_test lock5-dotfile.X { ##################################################################### forcedelete test.db -if {[catch {sqlite3 db test.db -vfs unix-flock} msg]} { - finish_test - return -} +if {0==[catch {sqlite3 db test.db -vfs unix-flock} msg]} { do_test lock5-flock.1 { sqlite3 db test.db -vfs unix-flock @@ -149,13 +146,19 @@ do_test lock5-flock.8 { db2 close } {} +} + ##################################################################### +reset_db + do_test lock5-none.1 { sqlite3 db test.db -vfs unix-none sqlite3 db2 test.db -vfs unix-none execsql { PRAGMA mmap_size = 0 } db2 execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); BEGIN; INSERT INTO t1 VALUES(3, 4); } @@ -178,10 +181,12 @@ do_test lock5-none.5 { } {1 2} ifcapable memorymanage { - do_test lock5-none.6 { - sqlite3_release_memory 1000000 - execsql {SELECT * FROM t1} db2 - } {1 2 3 4} + if {[permutation]!="memsubsys1" && [permutation]!="memsubsys2"} { + do_test lock5-none.6 { + sqlite3_release_memory 1000000 + execsql {SELECT * FROM t1} db2 + } {1 2 3 4} + } } do_test lock5-none.X { @@ -193,4 +198,74 @@ ifcapable lock_proxy_pragmas { set env(SQLITE_FORCE_PROXY_LOCKING) $::using_proxy } +##################################################################### +reset_db +if {[permutation]!="inmemory_journal"} { + + # 1. Create a large database using the unix-dotfile VFS + # 2. Write a large transaction to the db, so that the cache spills, but do + # not commit it. + # 3. Make a copy of the database files on disk. + # 4. Try to read from the copy using unix-dotfile VFS. This fails because + # the dotfile still exists, so SQLite thinks the database is locked. + # 5. Remove the dotfile. + # 6. Try to read the db again. This time, the old transaction is rolled + # back and the read permitted. + # + do_test 2.dotfile.1 { + sqlite3 db test.db -vfs unix-dotfile + execsql { + PRAGMA cache_size = 10; + CREATE TABLE t1(x, y, z); + CREATE INDEX t1x ON t1(x); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1 SELECT hex(randomblob(20)), hex(randomblob(500)), i FROM s; + } + } {} + + do_execsql_test 2.dotfile.2 { + BEGIN; + UPDATE t1 SET z=z+1, x=hex(randomblob(20)); + } + + do_test 2.dotfile.3 { + list \ + [file exists test.db] \ + [file exists test.db-journal] \ + [file exists test.db.lock] + } {1 1 1} + + do_test 2.dotfile.4 { + forcecopy test.db test.db2 + forcecopy test.db-journal test.db2-journal + file mkdir test.db2.lock + + sqlite3 db2 test.db2 -vfs unix-dotfile + catchsql { + SELECT count(*) FROM t1; + } db2 + } {1 {database is locked}} + + do_test 2.dotfile.5 { + file delete test.db2.lock + execsql { + PRAGMA integrity_check + } db2 + } {ok} + + db2 close + + do_test 2.dotfile.6 { + forcecopy test.db test.db2 + forcecopy test.db-journal test.db2-journal + + sqlite3 db2 file:test.db2?nolock=1 -vfs unix-dotfile -uri 1 + catchsql { + SELECT count(*) FROM t1; + } db2 + } {0 1000} +} + finish_test diff --git a/test/memdb1.test b/test/memdb1.test index 5e219a4c0..c0510abae 100644 --- a/test/memdb1.test +++ b/test/memdb1.test @@ -84,7 +84,6 @@ do_test 152 { catchsql {INSERT INTO t1 VALUES(3,4);} } {1 {attempt to write a readonly database}} -breakpoint do_test 160 { db deserialize -maxsize 32768 $db1 db eval {SELECT * FROM t1} @@ -246,8 +245,9 @@ if {[wal_is_capable]} { db close set fd [open test.db] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set data [read $fd [expr 20*1024]] + close $fd sqlite3 db "" db deserialize $data @@ -267,4 +267,17 @@ if {[wal_is_capable]} { } {1 {database disk image is malformed}} } +# 2024-01-20 +# https://sqlite.org/forum/forumpost/498777780e16880a +# +# Make sure a database is initialized before serializing it. +# +reset_db +sqlite3 dbempty :memory: +do_test 900 { + set len [string length [dbempty serialize]] + expr {$len>0} +} 1 +dbempty close + finish_test diff --git a/test/merge1.test b/test/merge1.test index 7ec4dab10..686271648 100644 --- a/test/merge1.test +++ b/test/merge1.test @@ -64,21 +64,21 @@ do_eqp_test 101 { | |--LEFT | | `--MERGE (UNION ALL) | | |--LEFT - | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | `--RIGHT - | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | `--RIGHT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT `--MERGE (UNION ALL) |--LEFT | `--MERGE (UNION ALL) | |--LEFT - | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | `--RIGHT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT - `--SCAN generate_series VIRTUAL TABLE INDEX 23: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: } # Same test with the blanced-merge optimization @@ -129,17 +129,17 @@ do_eqp_test 111 { | | | |--LEFT | | | | `--MERGE (UNION ALL) | | | | |--LEFT - | | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | | | `--RIGHT - | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | | `--RIGHT - | | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | | `--RIGHT - | | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: | `--RIGHT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT - `--SCAN generate_series VIRTUAL TABLE INDEX 23: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: } finish_test diff --git a/test/misc1.test b/test/misc1.test index 83acc752a..8110d3867 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -654,7 +654,7 @@ do_catchsql_test misc1-21.1 { } {1 {near "#0": syntax error}} do_catchsql_test misc1-21.2 { VALUES(0,0x0MATCH#0; -} {1 {near ";": syntax error}} +} {1 {unrecognized token: "0x0MATCH"}} # 2015-04-15 do_execsql_test misc1-22.1 { diff --git a/test/misc2.test b/test/misc2.test index 4796d5d37..607799ea2 100644 --- a/test/misc2.test +++ b/test/misc2.test @@ -54,19 +54,34 @@ do_test misc2-2.1 { } } {} ifcapable subquery { - do_catchsql_test misc2-2.2 { - SELECT rowid, * FROM (SELECT * FROM t1, t2); - } {1 {no such column: rowid}} + ifcapable allow_rowid_in_view { + do_catchsql_test misc2-2.2 { + SELECT rowid, * FROM (SELECT * FROM t1, t2); + } {0 {{} 1 2 3 7 8 9}} + } else { + do_catchsql_test misc2-2.2 { + SELECT rowid, * FROM (SELECT * FROM t1, t2); + } {1 {no such column: rowid}} + } do_catchsql_test misc2-2.2b { SELECT 'rowid', * FROM (SELECT * FROM t1, t2); } {0 {rowid 1 2 3 7 8 9}} } ifcapable view { - do_catchsql_test misc2-2.3 { - CREATE VIEW v1 AS SELECT * FROM t1, t2; - SELECT rowid, * FROM v1; - } {1 {no such column: rowid}} + ifcapable allow_rowid_in_view { + do_catchsql_test misc2-2.3 { + CREATE VIEW v1 AS SELECT * FROM t1, t2; + SELECT rowid, * FROM v1; + } {0 {{} 1 2 3 7 8 9}} + } else { + do_catchsql_test misc2-2.3 { + CREATE VIEW v1 AS SELECT * FROM t1, t2; + SELECT rowid, * FROM v1; + } {1 {no such column: rowid}} + } + + do_catchsql_test misc2-2.3b { SELECT 'rowid', * FROM v1; } {0 {rowid 1 2 3 7 8 9}} diff --git a/test/misc3.test b/test/misc3.test index bc1f0ff91..f64b0fe1d 100644 --- a/test/misc3.test +++ b/test/misc3.test @@ -88,8 +88,8 @@ do_test misc3-2.4 { execsql {SELECT 2e-25*0.5e250} } 1e+225 do_test misc3-2.5 { - execsql {SELECT 2.0e-250*0.5e25} -} 1e-225 + execsql {SELECT format('%.15e',2.0e-250*0.5e25)} +} {1.0000000000000e-225} do_test misc3-2.6 { execsql {SELECT '-2.0e-127' * '-0.5e27'} } 1e-100 diff --git a/test/misc5.test b/test/misc5.test index f7c6048d9..84aa9586d 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -569,11 +569,11 @@ ifcapable subquery&&compound { } # Overflow the lemon parser stack by providing an overly complex -# expression. Make sure that the overflow is detected and reported. +# expression. Make sure that the overflow is detected and the +# stack is grown automatically such that the application calling +# SQLite never notices. # -# This test fails when building with -DYYSTACKDEPTH=0 -# -do_test misc5-7.1 { +do_test misc5-7.1.1 { execsql {CREATE TABLE t1(x)} set sql "INSERT INTO t1 VALUES(" set tail "" @@ -581,9 +581,21 @@ do_test misc5-7.1 { append sql "(1+" append tail ")" } - append sql 2$tail + append sql "0$tail); SELECT * FROM t1;" + catchsql $sql +} {0 200} +do_test misc5-7.1.2 { + execsql {DELETE FROM t1} + set sql "INSERT INTO t1 VALUES(" + set tail "" + for {set i 0} {$i<900} {incr i} { + append sql "(1+" + append tail ")" + } + append sql "0$tail); SELECT * FROM t1;" catchsql $sql -} {1 {parser stack overflow}} +} {0 900} + # Parser stack overflow is silently ignored when it occurs while parsing the # schema and PRAGMA writable_schema is turned on. diff --git a/test/misc8.test b/test/misc8.test index 32b3a597d..60b44fe1c 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -100,6 +100,11 @@ do_execsql_test misc8-2.1 { # 2016-02-26: An assertion fault found by the libFuzzer project # +ifcapable allow_rowid_in_view { + set nosuch "1 {ambiguous column name: rowid}" +} else { + set nosuch "1 {no such column: rowid}" +} do_catchsql_test misc8-3.0 { SELECT * FROM @@ -110,7 +115,7 @@ do_catchsql_test misc8-3.0 { (SELECT 6 AS j UNION ALL SELECT 7) AS x4 WHERE i<rowid ORDER BY 1; -} {1 {no such column: rowid}} +} $nosuch # The SQLITE_DBCONFIG_MAINDBNAME interface # diff --git a/test/mmap1.test b/test/mmap1.test index 3362f7187..6a9625427 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -45,18 +45,18 @@ proc register_rblob_code {dbname seed} { } -# For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on -# unix and 9 on windows. The difference is that windows only ever maps +# For cases 1.1 and 1.4, the number of pages read using xRead() is 8 on +# unix and 12 on windows. The difference is that windows only ever maps # an integer number of OS pages (i.e. creates mappings that are a multiple # of 4KB in size). Whereas on unix any sized mapping may be created. # foreach {t mmap_size nRead c2init} { - 1.1 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 0} - 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} - 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} - 1.4 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 67108864 } - 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } - 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } + 1.1 { PRAGMA mmap_size = 67108864 } /8|12/ {PRAGMA mmap_size = 0} + 1.2 { PRAGMA mmap_size = 53248 } /15[34]/ {PRAGMA mmap_size = 0} + 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} + 1.4 { PRAGMA mmap_size = 67108864 } /12|8/ {PRAGMA mmap_size = 67108864 } + 1.5 { PRAGMA mmap_size = 53248 } /15[34]/ {PRAGMA mmap_size = 67108864 } + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } } { do_multiclient_test tn { diff --git a/test/mmapcorrupt.test b/test/mmapcorrupt.test new file mode 100644 index 000000000..d434ec183 --- /dev/null +++ b/test/mmapcorrupt.test @@ -0,0 +1,55 @@ +# 2024 January 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test special cases of corrupt database handling in mmap-mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix mmapcorrupt + +ifcapable !mmap { + finish_test + return +} +database_may_be_corrupt + +db close +sqlite3_shutdown +sqlite3_config_lookaside 0 0 +sqlite3_initialize + +reset_db +do_execsql_test 1.0 { + PRAGMA page_size = 16384; + CREATE TABLE tn1(a PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE t0(a PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE t1(a PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t1 VALUES('B'); +} +db close + +set sz [file size test.db] +hexio_write test.db [expr $sz-3] 800380 + +sqlite3 db test.db +do_execsql_test 2.1 { + PRAGMA mmap_size = 1000000; + SELECT sql FROM sqlite_schema LIMIT 1; + SELECT * FROM t0; +} {1000000 {CREATE TABLE tn1(a PRIMARY KEY) WITHOUT ROWID}} + +do_execsql_test 2.2 { + INSERT INTO t0 SELECT * FROM t1; +} + +finish_test + diff --git a/test/nan.test b/test/nan.test index 615a4ad22..8d8a98ab3 100644 --- a/test/nan.test +++ b/test/nan.test @@ -281,6 +281,7 @@ do_test nan-4.14 { # These tests test some really, really small floating point numbers. # +load_static_extension db decimal if {$tcl_platform(platform) != "symbian"} { # These two are not run on symbian because tcl has trouble converting # the very small numbers back to text form (probably due to a difference @@ -291,15 +292,15 @@ if {$tcl_platform(platform) != "symbian"} { set small \ [string repeat 0 10000].[string repeat 0 323][string repeat 9 10000] db eval "INSERT INTO t1 VALUES($small)" - db eval {SELECT x, typeof(x) FROM t1} - } {9.88131291682493e-324 real} + db eval {SELECT decimal_exp(x), typeof(x) FROM t1} + } {/9\.88131291682493\d*e-324 real/} do_test nan-4.16 { db eval {DELETE FROM t1} set small \ -[string repeat 0 10000].[string repeat 0 323][string repeat 9 10000] db eval "INSERT INTO t1 VALUES($small)" - db eval {SELECT x, typeof(x) FROM t1} - } {-9.88131291682493e-324 real} + db eval {SELECT decimal_exp(x), typeof(x) FROM t1} + } {/-9\.88131291682493\d*e-324 real/} } do_test nan-4.17 { db eval {DELETE FROM t1} diff --git a/test/notnull2.test b/test/notnull2.test index 7f6808681..09161efbd 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -59,14 +59,14 @@ do_vmstep_test 1.4.2 { do_vmstep_test 1.5.1 { SELECT count(*) FROM t2 WHERE EXISTS( - SELECT t2.d IS NULL FROM t1 WHERE t1.a=450 + SELECT 1 FROM t1 WHERE t1.a=450 AND t2.d IS NULL ) -} 10000 {1000} +} 7000 {0} do_vmstep_test 1.5.2 { SELECT count(*) FROM t2 WHERE EXISTS( - SELECT t2.c IS NULL FROM t1 WHERE t1.a=450 + SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL ) -} +100000 {1000} +} +8000 {0} #------------------------------------------------------------------------- reset_db @@ -111,4 +111,12 @@ do_execsql_test 4.1 { SELECT * FROM (SELECT 3 AS c FROM t1) AS t3 LEFT JOIN t2 ON c IS NULL; } {3 {}} +# 2024-03-08 https://sqlite.org/forum/forumpost/440f2a2f17 +# +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a INT NOT NULL); + SELECT a IS NULL, a IS NOT NULL, count(*) FROM t1; +} {1 0 0} + finish_test diff --git a/test/orderby1.test b/test/orderby1.test index 7432b5473..41444a44c 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -520,7 +520,7 @@ do_eqp_test 8.1 { } { QUERY PLAN |--SCAN t1 USING INDEX i1 - `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY } do_execsql_test 8.2 { diff --git a/test/orderbyB.test b/test/orderbyB.test new file mode 100644 index 000000000..42d2de798 --- /dev/null +++ b/test/orderbyB.test @@ -0,0 +1,94 @@ +# 2024-08-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# Specifically, it tests cases with order-by-subquery optimization in which +# an ORDER BY in a subquery is used to help resolve an ORDER BY in the +# outer query without having to do an extra sort. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderbyb + +db null NULL +do_execsql_test 1.0 { + CREATE TABLE t1(a TEXT, b TEXT, c INT); + INSERT INTO t1 VALUES(NULL,NULL,NULL); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + INSERT INTO t1(a,b,c) SELECT char(p,p), char(q,q), n FROM + (SELECT ((n-1)%4)+0x61 AS p, abs(n*2-9+(n>=5))+0x60 AS q, n FROM c); + UPDATE t1 SET b=upper(b) WHERE c=1; + CREATE TABLE t2(k TEXT PRIMARY KEY, v INT) WITHOUT ROWID; + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + INSERT INTO t2(k,v) SELECT char(0x60+n,0x60+n), n FROM c; + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + INSERT INTO t2(k,v) SELECT char(0x40+n,0x40+n), n FROM c; + SELECT a,b,c,tx.v AS 'v-a', ty.v AS 'v-b' + FROM t1 LEFT JOIN t2 AS tx ON tx.k=a + LEFT JOIN t2 AS ty ON ty.k=b + ORDER BY c; +} { + NULL NULL NULL NULL NULL + aa GG 1 1 7 + bb ee 2 2 5 + cc cc 3 3 3 + dd aa 4 4 1 + aa bb 5 1 2 + bb dd 6 2 4 + cc ff 7 3 6 +} + +do_eqp_execsql_test 1.1 { + WITH t3(x,y) AS (SELECT a, b FROM t1 ORDER BY a, b LIMIT 8) + SELECT x, y, v FROM t3 LEFT JOIN t2 ON k=t3.y ORDER BY x, y COLLATE nocase; +} { + QUERY PLAN + |--CO-ROUTINE t3 + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN t3 + |--SEARCH t2 USING PRIMARY KEY (k=?) LEFT-JOIN + `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY +} { + NULL NULL NULL + aa bb 2 + aa GG 7 + bb dd 4 + bb ee 5 + cc cc 3 + cc ff 6 + dd aa 1 +} + +do_eqp_execsql_test 1.2 { + WITH t3(x,y) AS MATERIALIZED (SELECT a, b COLLATE nocase FROM t1 ORDER BY 1,2) + SELECT x, y, v FROM t3 LEFT JOIN t2 ON k=t3.y ORDER BY x,y; +} { + QUERY PLAN + |--MATERIALIZE t3 + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN t3 + `--SEARCH t2 USING PRIMARY KEY (k=?) LEFT-JOIN +} { + NULL NULL NULL + aa bb 2 + aa GG 7 + bb dd 4 + bb ee 5 + cc cc 3 + cc ff 6 + dd aa 1 +} + + +finish_test diff --git a/test/pendingrace.test b/test/pendingrace.test index ef42578f2..80160149a 100644 --- a/test/pendingrace.test +++ b/test/pendingrace.test @@ -61,12 +61,12 @@ proc my_db_restore {} { forcecopy sv_test.db-journal test.db-journal set fd1 [open sv_test.db r] - fconfigure $fd1 -encoding binary -translation binary + fconfigure $fd1 -translation binary set data [read $fd1] close $fd1 set fd1 [open test.db w] - fconfigure $fd1 -encoding binary -translation binary + fconfigure $fd1 -translation binary puts -nonewline $fd1 $data close $fd1 } diff --git a/test/percentile.test b/test/percentile.test index b2bd061e8..ab5b4883f 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -38,6 +38,23 @@ foreach {in out} { execsql {SELECT percentile(x,$in) FROM t1} } $out } +do_execsql_test percentile-1.1.median { + SELECT median(x) FROM t1; +} 8.0 + +foreach {in out} { + 1.0 11.0 + 0.5 8.0 + 0.125 4.0 + 0.15 4.4 + 0.2 5.2 + 0.8 11.0 + 0.89 11.0 +} { + do_test percentile-1.1b-$in { + execsql {SELECT percentile_cont(x,$in) FROM t1} + } $out +} # Add some NULL values. # @@ -109,6 +126,9 @@ do_test percentile-1.11 { do_test percentile-1.12 { catchsql {SELECT percentile(x,x'3530') FROM t1} } {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}} +do_test percentile-1.12b { + catchsql {SELECT percentile_cont(x,x'3530') FROM t1} +} {1 {2nd argument to percentile_cont() is not a number between 0.0 and 1.0}} # Second argument is out of range # @@ -118,6 +138,9 @@ do_test percentile-1.13 { do_test percentile-1.14 { catchsql {SELECT percentile(x,100.0000001) FROM t1} } {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}} +do_test percentile-1.14b { + catchsql {SELECT percentile_cont(x,1.0000001) FROM t1} +} {1 {2nd argument to percentile_cont() is not a number between 0.0 and 1.0}} # First argument is not NULL and is not NUMERIC # diff --git a/test/permutations.test b/test/permutations.test index 25aa7de01..c26d6ead1 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -95,6 +95,7 @@ foreach f [glob -nocomplain \ $testdir/../ext/lsm1/test/*.test \ $testdir/../ext/recover/*.test \ $testdir/../ext/rbu/*.test \ + $testdir/../ext/intck/*.test \ ] { lappend alltests $f } diff --git a/test/pragma.test b/test/pragma.test index 5b45a7440..e823a6763 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -372,27 +372,30 @@ ifcapable attach { db close sqlite3 db test.db execsql {PRAGMA integrity_check} - } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} + } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}} do_test pragma-3.3 { execsql {PRAGMA integrity_check=1} - } {{row 1 missing from index i2}} + } {{wrong # of entries in index i2}} do_test pragma-3.4 { execsql { ATTACH DATABASE 'test.db' AS t2; PRAGMA integrity_check } - } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} + } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}} do_test pragma-3.5 { execsql { PRAGMA integrity_check=4 } - } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2}} + } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} + do_catchsql_test pragma-3.5.2 { + PRAGMA integrity_check='4' + } {1 {no such table: 4}} do_catchsql_test pragma-3.6 { PRAGMA integrity_check=xyz } {1 {no such table: xyz}} do_catchsql_test pragma-3.6b { PRAGMA integrity_check=t2 - } {0 {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}} + } {0 {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}} do_catchsql_test pragma-3.6c { PRAGMA integrity_check=sqlite_schema } {0 ok} @@ -400,7 +403,7 @@ ifcapable attach { execsql { PRAGMA integrity_check=0 } - } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} + } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}} # Add additional corruption by appending unused pages to the end of # the database file testerr.db @@ -435,10 +438,10 @@ ifcapable attach { } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} +Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}} do_execsql_test pragma-3.9b { PRAGMA t2.integrity_check=t2; - } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} + } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}} do_execsql_test pragma-3.9c { PRAGMA t2.integrity_check=sqlite_schema; } {ok} @@ -455,7 +458,7 @@ Page 4: never used}} } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}} +Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2}} do_test pragma-3.12 { execsql { PRAGMA integrity_check=4 @@ -463,7 +466,7 @@ Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}} } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2}} +Page 6: never used} {wrong # of entries in index i2}} do_test pragma-3.13 { execsql { PRAGMA integrity_check=3 @@ -487,10 +490,10 @@ Page 5: never used}} } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {*** in database t3 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} +Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}} do_test pragma-3.16 { execsql { PRAGMA integrity_check(10) @@ -498,10 +501,10 @@ Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {*** in database t3 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2}} +Page 6: never used} {wrong # of entries in index i2}} do_test pragma-3.17 { execsql { PRAGMA integrity_check=8 @@ -509,7 +512,7 @@ Page 6: never used} {row 1 missing from index i2}} } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {*** in database t3 *** Page 4: never used Page 5: never used}} do_test pragma-3.18 { @@ -519,7 +522,7 @@ Page 5: never used}} } {{*** in database t2 *** Page 4: never used Page 5: never used -Page 6: never used} {row 1 missing from index i2}} +Page 6: never used} {wrong # of entries in index i2}} } do_test pragma-3.19 { catch {db close} @@ -556,6 +559,21 @@ ifcapable altertable { do_execsql_test pragma-3.23 { PRAGMA integrity_check(1); } {{non-unique entry in index t1a}} + + # forum post https://sqlite.org/forum/forumpost/ee4f6fa5ab + do_execsql_test pragma-3.24 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES (1); + ALTER TABLE t1 ADD COLUMN b NOT NULL DEFAULT 0.25; + SELECT * FROM t1; + PRAGMA integrity_check(t1); + } {1 0.25 ok} + do_execsql_test pragma-3.25 { + ALTER TABLE t1 ADD COLUMN c CHECK (1); + SELECT * FROM t1; + PRAGMA integrity_check(t1); + } {1 0.25 {} ok} } # PRAGMA integrity check (or more specifically the sqlite3BtreeCount() diff --git a/test/pragma4.test b/test/pragma4.test index b82df81cb..5360fbac4 100644 --- a/test/pragma4.test +++ b/test/pragma4.test @@ -83,7 +83,7 @@ foreach {tn sql} { # Verify that that P4_INTARRAY argument to OP_IntegrityCk is rendered # correctly. # -db close +catch {db close} forcedelete test.db sqlite3 db test.db do_test pragma4-2.100 { @@ -97,7 +97,7 @@ do_test pragma4-2.100 { } string map {\[ x \] x \173 {} \175 {}} \ [db eval {EXPLAIN PRAGMA integrity_check}] -} {/ IntegrityCk 2 2 1 x[0-9]+,1x /} +} {/ IntegrityCk 1 2 8 x[0-9]+,1x /} #-------------------------------------------------------------------------- @@ -264,5 +264,51 @@ do_execsql_test 5.0 { 0 a {} 0 'abc' 0 1 b {} 0 -1 0 2 c {} 0 +4.0 0 } +# 2024-03-24 https://sqlite.org/forum/forumpost/85b6a8b6705fb77a +# +catch {db2 close} +catch {db3 close} +ifcapable vtab { + reset_db + do_execsql_test 6.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a INT PRIMARY KEY, b INT); + CREATE TABLE t2(c INT PRIMARY KEY, d INT REFERENCES t1); + SELECT t.name, f."table", f."from", i.name, i.pk + FROM pragma_table_list() AS t + JOIN pragma_foreign_key_list(t.name, t.schema) AS f + JOIN pragma_table_info(f."table", t.schema) AS i + WHERE i.pk; + } {t2 t1 d a 1} +} + +# 2024-05-08 https://sqlite.org/forum/forumpost/cf29a33e94 +# +ifcapable vtab { + do_execsql_test 7.0 { + CREATE TABLE t3 ("a" TEXT, "b" TEXT); + CREATE TABLE t4 ("a" TEXT, "b" TEXT, "c" TEXT); + } + + do_execsql_test 7.1 { + CREATE TABLE pragma_t3 AS SELECT * FROM pragma_table_info('t3'); + CREATE TABLE pragma_t4 AS SELECT * FROM pragma_table_info('t4'); + } + + do_execsql_test 7.2 { + SELECT pragma_t4.name, pragma_t3.name + FROM pragma_t4 RIGHT JOIN pragma_t3 ON (pragma_t4.name=pragma_t3.name); + } {a a b b} + + do_execsql_test 7.3 { + SELECT t4.name, t3.name + FROM pragma_table_info('t4') t4 + RIGHT JOIN pragma_table_info('t3') t3 ON (t4.name=t3.name); + } {a a b b} +} + + + finish_test diff --git a/test/pragma6.test b/test/pragma6.test new file mode 100644 index 000000000..fc5566af1 --- /dev/null +++ b/test/pragma6.test @@ -0,0 +1,74 @@ +# 2024 February 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for PRAGMAs quick_check and integrity_check. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix pragma6 + +database_may_be_corrupt + +#------------------------------------------------------------------------- +# +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { + .open --hexdb + | size 12288 pagesize 4096 filename crash-540f4c1eb1e7ac.db + | page 1 offset 0 + | 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + | 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 03 .....@ ........ + | 32: 00 bb 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + | 96: 00 00 00 00 0d 00 00 00 02 0f 7f 00 0f c3 0f 7f ................ + | 3952: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 42 ...............B + | 3968: 02 06 17 11 11 01 71 74 61 62 6c 65 74 32 74 32 ......qtablet2t2 + | 3984: 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 32 .CREATE TABLE t2 + | 4000: 28 61 20 49 4e 54 2c 20 62 20 41 53 20 28 61 2a (a INT, b AS (a* + | 4016: 32 29 20 53 54 4f 52 45 44 20 4e 4f 54 20 4e 55 2) STORED NOT NU + | 4032: 4c 4c 29 3b 01 06 17 11 11 01 63 74 61 62 6c 65 LL);......ctable + | 4048: 74 31 74 31 02 43 52 45 41 54 45 20 54 41 42 4c t1t1.CREATE TABL + | 4064: 45 20 74 31 28 61 20 49 4e 54 2c 20 62 20 41 53 E t1(a INT, b AS + | 4080: 20 28 61 2a 32 29 20 4e 4f 54 20 4e 55 4c 4c 29 (a*2) NOT NULL) + | page 2 offset 4096 + | 0: 0d 00 00 00 05 0f e7 00 00 00 00 00 00 00 00 00 ................ + | 4064: 00 00 00 00 00 00 00 00 03 05 02 01 05 03 04 02 ................ + | 4080: 01 04 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ + | page 3 offset 8192 + | 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ + | 4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 05 ................ + | 4064: 03 01 01 05 0a 05 04 03 01 01 04 08 05 03 03 01 ................ + | 4080: 01 03 06 05 02 03 00 00 00 00 00 00 00 00 00 00 ................ + | end crash-540f4c1eb1e7ac.db + }] +} {} + +do_test 1.1 { + execsql { + CREATE TEMP TABLE t2( + a t1 PRIMARY KEY default 27, + b default(current_timestamp), + d TEXT UNIQUE DEFAULT 'ch`arlie', + c TEXT UNIQUE DEFAULT 084, + UNIQUE(c,b,b,a,b) + ) WITHOUT ROWID; + } + catchsql { INSERT INTO t1(a) VALUES(zeroblob(40000)) } + set {} {} +} {} + +do_test 1.2 { + execsql { PRAGMA integrity_check; } + execsql { PRAGMA quick_check; } + set {} {} +} {} + +finish_test diff --git a/test/printf.test b/test/printf.test index 6d4ad71d2..cc439e617 100644 --- a/test/printf.test +++ b/test/printf.test @@ -3833,4 +3833,21 @@ do_execsql_test printf-18.1 { SELECT length( format('%,.249f', -5.0e-300) ); } {252} +# 2024-02-16 +# https://sqlite.org/forum/info/393708f4a8 +# +# The problem introduced by on 2023-07-03 by +# https://sqlite.org/src/info/32befb224b254639 +# +do_execsql_test printf-19.1 { + SELECT format('%0.0f %0.0g %0.0g', 0.9, 0.09, 1.9); +} {{1 0.09 2}} +do_execsql_test printf-19.2 { + SELECT format('%0.0f %#0.0f',0.0, 0.0); +} {{0 0.}} +do_execsql_test printf-19.3 { + SELECT format('%,.0f %,.0f',12345e+10, 12345e+11); +} {{123,450,000,000,000 1,234,500,000,000,000}} + + finish_test diff --git a/test/pushdown.test b/test/pushdown.test index 1fbe6f34c..271d412e7 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -1,4 +1,4 @@ -# 2017 April 29 +# 2017-04-29 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -8,6 +8,26 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# +# Test cases for the push-down optimizations. +# +# +# There are two different meanings for "push-down optimization". +# +# (1) "MySQL push-down" means that WHERE clause terms that can be +# evaluated using only the index and without reference to the +# table are run first, so that if they are false, unnecessary table +# seeks are avoided. See https://sqlite.org/src/info/d7bb79ed3a40419d +# from 2017-04-29. +# +# (2) "WHERE-clause pushdown" means to push WHERE clause terms in +# outer queries down into subqueries. See +# https://sqlite.org/src/info/6df18e949d367629 from 2015-06-02. +# +# This module started out as tests for MySQL push-down only. But because +# of naming ambiguity, it has picked up test cases for WHERE-clause push-down +# over the years. +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -87,8 +107,8 @@ do_test 2.2 { } {three} # 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f -# Disable the push-down optimization for compound subqueries if any -# arm of the compound has an incompatible affinity. +# Disable the WHERE-clause push-down optimization for compound subqueries +# if any arm of the compound has an incompatible affinity. # reset_db do_execsql_test 3.1 { @@ -185,7 +205,7 @@ do_eqp_test 3.8 { # SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) # 2023-05-09 https://sqlite.org/forum/forumpost/a7d4be7fb6 -# Restriction (9) on the push-down optimization. +# Restriction (9) on the WHERE-clause push-down optimization. # reset_db db null - @@ -227,4 +247,82 @@ do_execsql_test 5.0 { WHERE e>0; } {- - 3 4 5} + +# 2024-04-05 +# Allow push-down of operators of the form "expr IN table". +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t01(w,x,y,z); + CREATE TABLE t02(w,x,y,z); + CREATE VIEW t0(w,x,y,z) AS + SELECT w,x,y,z FROM t01 UNION ALL SELECT w,x,y,z FROM t02; + CREATE INDEX t01x ON t01(w,x,y); + CREATE INDEX t02x ON t02(w,x,y); + CREATE VIEW v1(k) AS VALUES(77),(88),(99); + CREATE TABLE k1(k); + INSERT INTO k1 SELECT * FROM v1; +} +do_eqp_test 6.1 { + WITH k(n) AS (VALUES(77),(88),(99)) + SELECT max(z) FROM t0 WHERE w=123 AND x IN k AND y BETWEEN 44 AND 55; +} { + QUERY PLAN + |--CO-ROUTINE t0 + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY + | | |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?) + | | `--LIST SUBQUERY xxxxxx + | | |--MATERIALIZE k + | | | `--SCAN 3 CONSTANT ROWS + | | |--SCAN k + | | `--CREATE BLOOM FILTER + | `--UNION ALL + | |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?) + | `--REUSE LIST SUBQUERY xxxxxx + |--SEARCH t0 + `--REUSE LIST SUBQUERY xxxxxx +} +# ^^^^--- The key feature above is that the SEARCH for each subquery +# uses all three fields of the index w, x, and y. Prior to the push-down +# of "expr IN table", only the w term of the index would be used. Similar +# for the following tests: +# +do_eqp_test 6.2 { + SELECT max(z) FROM t0 WHERE w=123 AND x IN v1 AND y BETWEEN 44 AND 55; +} { + QUERY PLAN + |--CO-ROUTINE t0 + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY + | | |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?) + | | `--LIST SUBQUERY xxxxxx + | | |--CO-ROUTINE v1 + | | | `--SCAN 3 CONSTANT ROWS + | | |--SCAN v1 + | | `--CREATE BLOOM FILTER + | `--UNION ALL + | |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?) + | `--REUSE LIST SUBQUERY xxxxxx + |--SEARCH t0 + `--REUSE LIST SUBQUERY xxxxxx +} +do_eqp_test 6.3 { + SELECT max(z) FROM t0 WHERE w=123 AND x IN k1 AND y BETWEEN 44 AND 55; +} { + QUERY PLAN + |--CO-ROUTINE t0 + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY + | | |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?) + | | `--LIST SUBQUERY xxxxxx + | | |--SCAN k1 + | | `--CREATE BLOOM FILTER + | `--UNION ALL + | |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?) + | `--REUSE LIST SUBQUERY xxxxxx + |--SEARCH t0 + `--REUSE LIST SUBQUERY xxxxxx +} + finish_test diff --git a/test/quote.test b/test/quote.test index 6d7b317ea..4e40a9e57 100644 --- a/test/quote.test +++ b/test/quote.test @@ -103,7 +103,7 @@ foreach {tn sql errname} { 3 { CREATE INDEX i3 ON t1("w") } w 4 { CREATE INDEX i4 ON t1(x) WHERE z="w" } w } { - do_catchsql_test 2.1.$tn $sql [list 1 "no such column: $errname"] + do_catchsql_test 2.1.$tn $sql [list 1 "no such column: \"$errname\" - should this be a string literal in single-quotes?"] } do_execsql_test 2.2 { @@ -147,19 +147,19 @@ ifcapable altertable { CREATE TABLE t1(a,b); CREATE INDEX x1 on t1("b"); ALTER TABLE t1 DROP COLUMN b; - } {1 {error in index x1 after drop column: no such column: b}} + } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}} do_catchsql_test 3.1 { DROP TABLE t1; CREATE TABLE t1(a,"b"); CREATE INDEX x1 on t1("b"); ALTER TABLE t1 DROP COLUMN b; - } {1 {error in index x1 after drop column: no such column: b}} + } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}} do_catchsql_test 3.2 { DROP TABLE t1; CREATE TABLE t1(a,'b'); CREATE INDEX x1 on t1("b"); ALTER TABLE t1 DROP COLUMN b; - } {1 {error in index x1 after drop column: no such column: b}} + } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}} do_catchsql_test 3.3 { DROP TABLE t1; CREATE TABLE t1(a,"b"); @@ -172,7 +172,7 @@ ifcapable altertable { CREATE INDEX x1 ON t1("a"||"b"); INSERT INTO t1 VALUES(1,2,3),(1,4,5); ALTER TABLE t1 DROP COLUMN b; - } {1 {error in index x1 after drop column: no such column: b}} + } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}} sqlite3_db_config db SQLITE_DBCONFIG_DQS_DDL 1 do_catchsql_test 3.5 { DROP TABLE t1; diff --git a/test/readonly.test b/test/readonly.test new file mode 100644 index 000000000..303300e6f --- /dev/null +++ b/test/readonly.test @@ -0,0 +1,58 @@ +# 2024 March 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using databases in read-only mode on +# unix. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$tcl_platform(platform)=="windows"} { + finish_test + return +} +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix readonly + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2), (3, 4), (5, 6); +} + +db close +file attributes test.db -permissions r--r--r-- + +sqlite3 db test.db + +do_catchsql_test 1.1 { + INSERT INTO t1 VALUES(7, 8); +} {1 {attempt to write a readonly database}} + +do_execsql_test 1.2 { + BEGIN; + SELECT * FROM t1; +} {1 2 3 4 5 6} + +# The following attempts to open a read/write fd on the database 20,000 +# times. And each time instead opens a read-only fd. At one point this +# was failing to reuse cached fds, causing a "too many open file-descriptors" +# error. +do_test 1.3 { + for {set ii 0} {$ii < 20000} {incr ii} { + sqlite3 db2 test.db + db2 close + } + set {} {} +} {} + + +finish_test diff --git a/test/recover.test b/test/recover.test index 8d9ad013c..268d3a55b 100644 --- a/test/recover.test +++ b/test/recover.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # Test the shell tool ".ar" command. # @@ -42,7 +43,6 @@ proc compare_dbs {db1 db2} { proc recover_with_opts {opts} { set cmd ".recover $opts" set fd [open [list |$::CLI test.db $cmd]] - fconfigure $fd -encoding binary fconfigure $fd -translation binary set sql [read $fd] close $fd diff --git a/test/returning1.test b/test/returning1.test index 6c098dc25..e7be7c65a 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -212,17 +212,38 @@ do_execsql_test 10.2 { END; } -do_catchsql_test 10.3a { - INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; -} {1 {no such column: new.rowid}} - -do_catchsql_test 10.3b { - UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; -} {1 {no such column: new.rowid}} - -do_execsql_test 10.4 { - SELECT * FROM log; -} {} +ifcapable !allow_rowid_in_view { + do_catchsql_test 10.3a { + INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; + } {1 {no such column: new.rowid}} + + do_catchsql_test 10.3b { + UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; + } {1 {no such column: new.rowid}} + + do_execsql_test 10.4 { + SELECT * FROM log; + } {} +} else { + # Note: The values returned by the RETURNING clauses of the following + # two statements are the rowid columns of views. These values are not + # well defined, so the INSERT returns -1, and the UPDATE returns NULL. + # These match the values used for new.rowid expressions, but not much + # else. + do_catchsql_test 10.3a { + INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; + } {0 -1} + + do_catchsql_test 10.3b { + UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; + } {0 {{} {} {}}} + + do_execsql_test 10.4 { + SELECT * FROM log; + } { + insert -1 1234 5678 update {} z y update {} z y update {} z y + } +} # 2021-04-27 dbsqlfuzz 78b9400770ef8cc7d9427dfba26f4fcf46ea7dc2 # Returning clauses on TEMP tables with triggers. @@ -439,4 +460,88 @@ do_catchsql_test 19.1 { END; } {0 {}} +# 2024-04-24 +# https://sqlite.org/forum/forumpost/2c83569ce8945d39 +# +# If the RETURNING clause includes subqueries that reference the +# table being modified, make sure that the subqueries are identified +# as correlated so that the results are recomputed after each step +# instead of being computed once and reused. +# +reset_db +db null N +do_execsql_test 20.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT); + INSERT INTO t1 VALUES(1,10),(2,20),(3,30),(4,40),(6,60),(8,80); + BEGIN; + DELETE FROM t1 WHERE a<>3 + RETURNING a, + (SELECT min(a) FROM t1), + (SELECT max(a) FROM t1), + (SELECT round(avg(a),2) FROM t1); + ROLLBACK; +} { + 1 2 8 4.6 + 2 3 8 5.25 + 4 3 8 5.67 + 6 3 8 5.5 + 8 3 3 3.0 +} +do_execsql_test 20.2 { + BEGIN; + DELETE FROM t1 + RETURNING a, + (SELECT min(a) FROM t1), + (SELECT max(a) FROM t1), + (SELECT round(avg(a),2) FROM t1); + ROLLBACK; +} { + 1 2 8 4.6 + 2 3 8 5.25 + 3 4 8 6.0 + 4 6 8 7.0 + 6 8 8 8.0 + 8 N N N +} +do_execsql_test 20.3 { + BEGIN; + DELETE FROM t1 + RETURNING a, + (SELECT min(t2.a)+t1.a*100 FROM t1 AS t2), + (SELECT max(t2.a)+t1.a*100 FROM t1 AS t2), + (SELECT round(avg(t2.a),2)+t1.a*100 FROM t1 AS t2); + ROLLBACK; +} { + 1 102 108 104.6 + 2 203 208 205.25 + 3 304 308 306.0 + 4 406 408 407.0 + 6 608 608 608.0 + 8 N N N +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 21.0 { + PRAGMA writable_schema=ON; + INSERT INTO sqlite_schema DEFAULT VALUES RETURNING sqlite_schema.name; +} {{}} + +do_execsql_test 21.1 { + INSERT INTO sqlite_temp_schema DEFAULT VALUES RETURNING sqlite_temp_schema.name; +} {{}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 22.0 { + PRAGMA writable_schema=ON; + CREATE TABLE xyz (a); +} +do_catchsql_test 22.1 { + INSERT INTO sqlite_temp_schema DEFAULT VALUES + RETURNING + (SELECT * FROM xyz AS sqlite_master WHERE a=sqlite_master.name); +} {1 {no such column: sqlite_master.name}} + + finish_test diff --git a/test/rollback.test b/test/rollback.test index 423bf20fc..f32ce523e 100644 --- a/test/rollback.test +++ b/test/rollback.test @@ -111,7 +111,7 @@ if {$tcl_platform(platform) == "unix" ] set iOffset [expr (([file size testA.db-journal] + 511)/512)*512] set fd [open testA.db-journal a+] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary seek $fd $iOffset puts -nonewline $fd $zAppend diff --git a/test/rowid.test b/test/rowid.test index 4327004d3..84f0e4d30 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -803,18 +803,31 @@ do_execsql_test 16.0 { INSERT INTO t3(rowid, z) VALUES(3, 3); } -do_execsql_test 16.1 { SELECT rowid FROM t1, t2; } {1} -do_execsql_test 16.2 { SELECT rowid FROM t1, v1; } {1} -do_execsql_test 16.3 { SELECT rowid FROM t3, v1; } {3} -do_execsql_test 16.4 { SELECT rowid FROM t3, (SELECT 123); } {3} - -do_execsql_test 16.5 { SELECT rowid FROM t2, t1; } {1} -do_execsql_test 16.6 { SELECT rowid FROM v1, t1; } {1} -do_execsql_test 16.7 { SELECT rowid FROM v1, t3; } {3} -do_execsql_test 16.8 { SELECT rowid FROM (SELECT 123), t3; } {3} - -do_catchsql_test 16.5 { SELECT rowid FROM t1, t3; } {1 {no such column: rowid}} - +ifcapable allow_rowid_in_view { + set nosuch "1 {ambiguous column name: rowid}" + do_execsql_test 16.1 { SELECT rowid FROM t1, t2; } {1} + do_catchsql_test 16.2 { SELECT rowid FROM t1, v1; } $nosuch + do_catchsql_test 16.3 { SELECT rowid FROM t3, v1; } $nosuch + do_catchsql_test 16.4 { SELECT rowid FROM t3, (SELECT 123); } $nosuch + + do_execsql_test 16.5 { SELECT rowid FROM t2, t1; } {1} + do_catchsql_test 16.6 { SELECT rowid FROM v1, t1; } $nosuch + do_catchsql_test 16.7 { SELECT rowid FROM v1, t3; } $nosuch + do_execsql_test 16.8 { SELECT rowid FROM (SELECT 123), t3; } {3} +} else { + do_execsql_test 16.1 { SELECT rowid FROM t1, t2; } {1} + do_execsql_test 16.2 { SELECT rowid FROM t1, v1; } {1} + do_execsql_test 16.3 { SELECT rowid FROM t3, v1; } {3} + do_execsql_test 16.4 { SELECT rowid FROM t3, (SELECT 123); } {3} + + do_execsql_test 16.5 { SELECT rowid FROM t2, t1; } {1} + do_execsql_test 16.6 { SELECT rowid FROM v1, t1; } {1} + do_execsql_test 16.7 { SELECT rowid FROM v1, t3; } {3} + do_execsql_test 16.8 { SELECT rowid FROM (SELECT 123), t3; } {3} +} +do_catchsql_test 16.9 { + SELECT rowid FROM t1, t3; +} {1 {ambiguous column name: rowid}} finish_test diff --git a/test/rowvalue4.test b/test/rowvalue4.test index 784859cb1..1ef5fc292 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -236,9 +236,11 @@ do_eqp_test 5.1 { QUERY PLAN |--SEARCH d2 USING INDEX d2ab (a=? AND b=?) |--LIST SUBQUERY xxxxxx - | `--SCAN d1 + | |--SCAN d1 + | `--CREATE BLOOM FILTER `--LIST SUBQUERY xxxxxx - `--SCAN d1 + |--SCAN d1 + `--CREATE BLOOM FILTER } do_execsql_test 6.0 { diff --git a/test/scanstatus2.test b/test/scanstatus2.test index e4b510d20..c94db88f2 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -265,7 +265,7 @@ ifcapable trace { } proc trace {stmt sql} { - array set A [sqlite3_stmt_scanstatus -flags complex [format %x $stmt] 0] + array set A [sqlite3_stmt_scanstatus -flags complex [format %llx $stmt] 0] lappend ::trace_explain $A(zExplain) } db trace_v2 trace @@ -328,6 +328,17 @@ QUERY (nCycle=nnn) --SCAN xy2 (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db + +# Check that an OOB parameter (45) does not cause asan or valgrind errors. +# +do_test 7.0 { + db eval {SELECT * FROM sqlite_schema} + set stmt [db version -last-stmt-ptr] + sqlite3_stmt_scanstatus -flags complex $stmt 1000000 +} {} + #explain_i { SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1 } #puts_debug_info { SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1 } diff --git a/test/select7.test b/test/select7.test index d705ebfaf..0c4051006 100644 --- a/test/select7.test +++ b/test/select7.test @@ -155,6 +155,38 @@ if {[clang_sanitize_address]==0} { } } +# https://issues.chromium.org/issues/358174302 +# Need to support an unlimited number of terms in a VALUES clause, even +# if some of those terms contain double-quoted string literals. +# +do_execsql_test select7-6.5 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b,c); +} +sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 10 +sqlite3_db_config db SQLITE_DBCONFIG_DQS_DML 0 +do_catchsql_test select7-6.6 { + INSERT INTO t1 VALUES + (NULL,0,""), (X'',0.0,0.0), (X'',X'',""), (0.0,0.0,""), (NULL,NULL,0.0), + (0,"",0), (0.0,X'',0), ("",X'',0.0), (0.0,X'',NULL), (0,NULL,""), + (0,"",NULL), (0.0,NULL,X''), ("",X'',NULL), (NULL,0,""), + (0,NULL,0), (X'',X'',0.0); +} {1 {no such column: "" - should this be a string literal in single-quotes?}} +do_execsql_test select7-6.7 { + SELECT count(*) FROM t1; +} {0} +sqlite3_db_config db SQLITE_DBCONFIG_DQS_DML 1 +do_catchsql_test select7-6.8 { + INSERT INTO t1 VALUES + (NULL,0,""), (X'',0.0,0.0), (X'',X'',""), (0.0,0.0,""), (NULL,NULL,0.0), + (0,"",0), (0.0,X'',0), ("",X'',0.0), (0.0,X'',NULL), (0,NULL,""), + (0,"",NULL), (0.0,NULL,X''), ("",X'',NULL), (NULL,0,""), + (0,NULL,0), (X'',X'',0.0); +} {0 {}} +do_execsql_test select7-6.9 { + SELECT count(*) FROM t1; +} {16} + # This block of tests verifies that bug aa92c76cd4 is fixed. # do_test select7-7.1 { diff --git a/test/selectA.test b/test/selectA.test index 7d72bb3fa..91b154848 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -1340,10 +1340,10 @@ do_eqp_test 4.1.2 { `--MERGE (UNION ALL) |--LEFT | |--SCAN t5 USING INDEX i2 - | `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + | `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY `--RIGHT |--SCAN t4 USING INDEX i1 - `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY + `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY } do_execsql_test 4.1.3 { diff --git a/test/sessionfuzz.c b/test/sessionfuzz.c index 093c2b043..c389a5e99 100644 --- a/test/sessionfuzz.c +++ b/test/sessionfuzz.c @@ -60,6 +60,7 @@ #define SQLITE_DEBUG 1 #define SQLITE_THREADSAFE 0 +#undef SQLITE_OMIT_LOAD_EXTENSION #define SQLITE_OMIT_LOAD_EXTENSION 0 #define SQLITE_ENABLE_SESSION 1 #define SQLITE_ENABLE_PREUPDATE_HOOK 1 diff --git a/test/shell1.test b/test/shell1.test index 5d4243f47..f355989c3 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -11,6 +11,7 @@ # # The focus of this file is testing the CLI shell tool. # +# TESTRUNNER: shell # # Test plan: @@ -1058,7 +1059,7 @@ do_test shell1-5.0 { continue } # Tcl 8.7 maps 0x80 through 0x9f into valid UTF8. So skip those tests. - if {$i>=0x80 && $i<=0x9f} continue + if {$i>=0x80 && ($i<=0x9F || $tcl_version>=9.0)} continue if {$i>=0xE0 && $tcl_platform(os)=="OpenBSD"} continue if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os)=="Linux"} continue set hex [format %02X $i] diff --git a/test/shell2.test b/test/shell2.test index 16ed33c44..c6c27d216 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. # diff --git a/test/shell3.test b/test/shell3.test index f8d69946e..ef3ea784f 100644 --- a/test/shell3.test +++ b/test/shell3.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. # @@ -135,4 +136,13 @@ a["b"] .*/ x 'y}} 1} +do_test shell3-3.2 { + catchcmd "" { +.open xyz.db +SELECT ; + } +} {1 {Parse error near line 3: near ";": syntax error + SELECT ; + ^--- error here}} + finish_test diff --git a/test/shell4.test b/test/shell4.test index 193dc8b69..4b7e9b8ee 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. # These tests are specific to the .stats command. @@ -117,7 +118,7 @@ do_test shell4-2.2 { } {0 {}} do_test shell4-2.3 { catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n" -} {/^0 {PRAGMA.*}$/} +} {/^0 {SELECT.*}$/} do_test shell4-2.4 { catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;" } {0 {CREATE TABLE t1(x); diff --git a/test/shell5.test b/test/shell5.test index 39018a0ce..31d5449fd 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. # These tests are specific to the .import command. @@ -409,7 +410,7 @@ CREATE TABLE t4(a, b); # do_test shell5-3.1 { set fd [open shell5.csv w] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary puts -nonewline $fd "\"test 1\"\x1F,test 2\r\n\x1E" puts -nonewline $fd "test 3\x1Ftest 4\n" close $fd @@ -570,4 +571,30 @@ SELECT * FROM t1;} } {0 { 1 = あい 2 = うえお}} +# 2024-03-11 https://sqlite.org/forum/forumpost/ca014d7358 +# Import into a table that contains computed columns. +# +do_test shell5-7.1 { + set out [open shell5.csv w] + fconfigure $out -translation lf + puts $out {aaa|bbb} + close $out + forcedelete test.db + catchcmd :memory: {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); +.import shell5.csv t1 +SELECT * FROM t1;} +} {0 aaa|bbb|aaabbb} + +#------------------------------------------------------------------------- + +do_test shell5-8.1 { + + set out [open shell5.csv w] + fconfigure $out -translation lf + puts $out x + close $out + + catchcmd :memory: {.import --csv shell5.csv '""""""""""""""""""""""""""""""""""""""""""""""'} +} {0 {}} + finish_test diff --git a/test/shell6.test b/test/shell6.test index 49b4cc334..4841d6c01 100644 --- a/test/shell6.test +++ b/test/shell6.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # Test the shell tool ".lint fkey-indexes" command. # diff --git a/test/shell7.test b/test/shell7.test index 898018d77..460789e54 100644 --- a/test/shell7.test +++ b/test/shell7.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # Test the readfile() function built into the shell tool. Specifically, # that it does not truncate the blob read at the first embedded 0x00 @@ -32,7 +33,6 @@ do_execsql_test 1.0 { foreach {tn l x} [db eval { SELECT tn, length(x) AS l, x FROM f1 }] { forcedelete shell7_test.bin set fd [open shell7_test.bin w] - fconfigure $fd -encoding binary fconfigure $fd -translation binary puts -nonewline $fd $x close $fd diff --git a/test/shell8.test b/test/shell8.test index bee603923..ca37598e9 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # Test the shell tool ".ar" command. # @@ -164,7 +165,7 @@ foreach {tn tcl} { # # Because it is slow, only do this for $tn==1. if {$tn==1} { - do_test 1.$tn.1 { + do_test 1.$tn.4 { catchcmd test_ar.db $c1 file delete -force ar1 after 2000 @@ -193,4 +194,29 @@ do_test 2.1.1 { regsub -all {ar4} [dir_content ar4] ar2 } {ar2/file1 ar2/file2 ar2/junk1} +# Test symbolic links. +# +if {$tcl_platform(platform)=="unix"} { + populate_dir ar2 { + file1 "1234" + file2 "3456" + } + file link ar2/link1 file1 + + forcedelete shell8.db + forcedelete link1 + + do_test 3.1 { + catchcmd shell8.db {.ar -C ar2 -c file2 link1 } + } {0 {}} + + do_test 3.2 { + catchcmd shell8.db {.ar -x} + } {0 {}} + + do_test 3.3 { + catchcmd shell8.db {.ar -x} + } {0 {}} +} + finish_test diff --git a/test/shell9.test b/test/shell9.test index d7759551f..869cb075d 100644 --- a/test/shell9.test +++ b/test/shell9.test @@ -8,11 +8,15 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. Specifically, # testing that it is possible to run a ".dump" script that creates # virtual tables without explicitly disabling defensive mode. # +# And, that it can process a ".dump" script that contains strings +# delimited using double-quotes in the schema (DQS_DDL setting). +# # Test plan: # @@ -124,4 +128,22 @@ do_test 2.2.2 { contains_warning [catchcmd test.db ".dump r1"] } 1 +#------------------------------------------------------------------------- +reset_db +sqlite3_db_config db DQS_DDL 1 +do_execsql_test 3.1.0 { + CREATE TABLE t4(hello, check( hello IS NOT "xyz") ); +} +db close + +# Create .dump file in "testdump.txt". +# +set out [open testdump.txt w] +puts $out [lindex [catchcmd test.db .dump] 1] +close $out +do_test 3.1.1 { + forcedelete test.db + catchcmd test.db ".read testdump.txt" +} {0 {}} + finish_test diff --git a/test/skipscan1.test b/test/skipscan1.test index bd5b83d34..379de1956 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -337,12 +337,12 @@ do_execsql_test skipscan1-9.2 { } {/USING INDEX t9a_ab .ANY.a. AND b=./} -optimization_control db skip-scan 0 +optimization_control db skip-scan off do_execsql_test skipscan1-9.3 { EXPLAIN QUERY PLAN SELECT * FROM t9a WHERE b IN (SELECT x FROM t9b WHERE y!=5); } {/{SCAN t9a}/} -optimization_control db skip-scan 1 +optimization_control db all on do_execsql_test skipscan1-2.1 { CREATE TABLE t6(a TEXT, b INT, c INT, d INT); diff --git a/test/speedtest1.c b/test/speedtest1.c index 4f3258098..570942381 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -2150,6 +2150,50 @@ void testset_debug1(void){ } } +/* +** This testset focuses on the speed of parsing numeric literals (integers +** and real numbers). This was added to test the impact of allowing "_" +** characters to appear in numeric SQL literals to make them easier to read. +** For example, "SELECT 1_000_000;" instead of "SELECT 1000000;". +*/ +void testset_parsenumber(void){ + const char *zSql1 = "SELECT 1, 12, 123, 1234, 12345, 123456"; + const char *zSql2 = "SELECT 8227256643844975616, 7932208612563860480, " + "2010730661871032832, 9138463067404021760, " + "2557616153664746496, 2557616153664746496"; + const char *zSql3 = "SELECT 1.0, 1.2, 1.23, 123.4, 1.2345, 1.23456"; + const char *zSql4 = "SELECT 8.227256643844975616, 7.932208612563860480, " + "2.010730661871032832, 9.138463067404021760, " + "2.557616153664746496, 2.557616153664746496"; + + const int NROW = 100*g.szTest; + int ii; + + speedtest1_begin_test(100, "parsing small integers"); + for(ii=0; ii<NROW; ii++){ + sqlite3_exec(g.db, zSql1, 0, 0, 0); + } + speedtest1_end_test(); + + speedtest1_begin_test(110, "parsing large integers"); + for(ii=0; ii<NROW; ii++){ + sqlite3_exec(g.db, zSql2, 0, 0, 0); + } + speedtest1_end_test(); + + speedtest1_begin_test(200, "parsing small reals"); + for(ii=0; ii<NROW; ii++){ + sqlite3_exec(g.db, zSql3, 0, 0, 0); + } + speedtest1_end_test(); + + speedtest1_begin_test(210, "parsing large reals"); + for(ii=0; ii<NROW; ii++){ + sqlite3_exec(g.db, zSql4, 0, 0, 0); + } + speedtest1_end_test(); +} + #ifdef __linux__ #include <sys/types.h> #include <unistd.h> @@ -2557,6 +2601,8 @@ int main(int argc, char **argv){ testset_fp(); }else if( strcmp(zThisTest,"trigger")==0 ){ testset_trigger(); + }else if( strcmp(zThisTest,"parsenumber")==0 ){ + testset_parsenumber(); }else if( strcmp(zThisTest,"rtree")==0 ){ #ifdef SQLITE_ENABLE_RTREE testset_rtree(6, 147); diff --git a/test/sqllimits1.test b/test/sqllimits1.test index f16208f23..14d39e691 100644 --- a/test/sqllimits1.test +++ b/test/sqllimits1.test @@ -707,6 +707,7 @@ if {$SQLITE_MAX_EXPR_DEPTH==0} { }] } "1 {Expression tree is too large (maximum depth $::SQLITE_MAX_EXPR_DEPTH)}" +if 0 { # Attempting to beat the expression depth limit using nested SELECT # queries causes a parser stack overflow. do_test sqllimits1-9.2 { @@ -718,7 +719,6 @@ if {$SQLITE_MAX_EXPR_DEPTH==0} { catchsql [subst { $expr }] } "1 {parser stack overflow}" -if 0 { do_test sqllimits1-9.3 { execsql { PRAGMA max_page_count = 1000000; -- 1 GB @@ -922,7 +922,7 @@ do_catchsql_test sqllimits1-18.1 { do_catchsql_test sqllimits1-18.2 { INSERT INTO b1 VALUES(1), (2), (3), (4), (5), (6), (7), (8), (9), (10) UNION VALUES(11); -} {1 {too many terms in compound SELECT}} +} {0 {}} #------------------------------------------------------------------------- # diff --git a/test/starschema1.test b/test/starschema1.test new file mode 100644 index 000000000..af8168b51 --- /dev/null +++ b/test/starschema1.test @@ -0,0 +1,584 @@ +# 2024-05-28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the ability of the query planner to cope with +# star-schema queries on databases with goofy indexes. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix starschema1 + +do_execsql_test 1.1 { +CREATE TABLE t1( + a01 INT, a02 INT, a03 INT, a04 INT, a05 INT, a06 INT, a07 INT, a08 INT, + a09 INT, a10 INT, a11 INT, a12 INT, a13 INT, a14 INT, a15 INT, a16 INT, + a17 INT, a18 INT, a19 INT, a20 INT, a21 INT, a22 INT, a23 INT, a24 INT, + a25 INT, a26 INT, a27 INT, a28 INT, a29 INT, a30 INT, a31 INT, a32 INT, + a33 INT, a34 INT, a35 INT, a36 INT, a37 INT, a38 INT, a39 INT, a40 INT, + a41 INT, a42 INT, a43 INT, a44 INT, a45 INT, a46 INT, a47 INT, a48 INT, + a49 INT, a50 INT, a51 INT, a52 INT, a53 INT, a54 INT, a55 INT, a56 INT, + a57 INT, a58 INT, a59 INT, a60 INT, a61 INT, a62 INT, a63 INT, d TEXT); +CREATE TABLE x01(b01 INT, c01 TEXT); +CREATE TABLE x02(b02 INT, c02 TEXT); +CREATE TABLE x03(b03 INT, c03 TEXT); +CREATE TABLE x04(b04 INT, c04 TEXT); +CREATE TABLE x05(b05 INT, c05 TEXT); +CREATE TABLE x06(b06 INT, c06 TEXT); +CREATE TABLE x07(b07 INT, c07 TEXT); +CREATE TABLE x08(b08 INT, c08 TEXT); +CREATE TABLE x09(b09 INT, c09 TEXT); +CREATE TABLE x10(b10 INT, c10 TEXT); +CREATE TABLE x11(b11 INT, c11 TEXT); +CREATE TABLE x12(b12 INT, c12 TEXT); +CREATE TABLE x13(b13 INT, c13 TEXT); +CREATE TABLE x14(b14 INT, c14 TEXT); +CREATE TABLE x15(b15 INT, c15 TEXT); +CREATE TABLE x16(b16 INT, c16 TEXT); +CREATE TABLE x17(b17 INT, c17 TEXT); +CREATE TABLE x18(b18 INT, c18 TEXT); +CREATE TABLE x19(b19 INT, c19 TEXT); +CREATE TABLE x20(b20 INT, c20 TEXT); +CREATE TABLE x21(b21 INT, c21 TEXT); +CREATE TABLE x22(b22 INT, c22 TEXT); +CREATE TABLE x23(b23 INT, c23 TEXT); +CREATE TABLE x24(b24 INT, c24 TEXT); +CREATE TABLE x25(b25 INT, c25 TEXT); +CREATE TABLE x26(b26 INT, c26 TEXT); +CREATE TABLE x27(b27 INT, c27 TEXT); +CREATE TABLE x28(b28 INT, c28 TEXT); +CREATE TABLE x29(b29 INT, c29 TEXT); +CREATE TABLE x30(b30 INT, c30 TEXT); +CREATE TABLE x31(b31 INT, c31 TEXT); +CREATE TABLE x32(b32 INT, c32 TEXT); +CREATE TABLE x33(b33 INT, c33 TEXT); +CREATE TABLE x34(b34 INT, c34 TEXT); +CREATE TABLE x35(b35 INT, c35 TEXT); +CREATE TABLE x36(b36 INT, c36 TEXT); +CREATE TABLE x37(b37 INT, c37 TEXT); +CREATE TABLE x38(b38 INT, c38 TEXT); +CREATE TABLE x39(b39 INT, c39 TEXT); +CREATE TABLE x40(b40 INT, c40 TEXT); +CREATE TABLE x41(b41 INT, c41 TEXT); +CREATE TABLE x42(b42 INT, c42 TEXT); +CREATE TABLE x43(b43 INT, c43 TEXT); +CREATE TABLE x44(b44 INT, c44 TEXT); +CREATE TABLE x45(b45 INT, c45 TEXT); +CREATE TABLE x46(b46 INT, c46 TEXT); +CREATE TABLE x47(b47 INT, c47 TEXT); +CREATE TABLE x48(b48 INT, c48 TEXT); +CREATE TABLE x49(b49 INT, c49 TEXT); +CREATE TABLE x50(b50 INT, c50 TEXT); +CREATE TABLE x51(b51 INT, c51 TEXT); +CREATE TABLE x52(b52 INT, c52 TEXT); +CREATE TABLE x53(b53 INT, c53 TEXT); +CREATE TABLE x54(b54 INT, c54 TEXT); +CREATE TABLE x55(b55 INT, c55 TEXT); +CREATE TABLE x56(b56 INT, c56 TEXT); +CREATE TABLE x57(b57 INT, c57 TEXT); +CREATE TABLE x58(b58 INT, c58 TEXT); +CREATE TABLE x59(b59 INT, c59 TEXT); +CREATE TABLE x60(b60 INT, c60 TEXT); +CREATE TABLE x61(b61 INT, c61 TEXT); +CREATE TABLE x62(b62 INT, c62 TEXT); +CREATE TABLE x63(b63 INT, c63 TEXT); +/**** Uncomment to generate actual data ************************************ +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<172800) + INSERT INTO t1 + SELECT stmtrand()%12, stmtrand()%13, stmtrand()%14, stmtrand()%15, + stmtrand()%16, stmtrand()%17, stmtrand()%18, stmtrand()%19, + stmtrand()%20, stmtrand()%21, stmtrand()%22, stmtrand()%23, + stmtrand()%24, stmtrand()%25, stmtrand()%26, stmtrand()%27, + stmtrand()%28, stmtrand()%29, stmtrand()%30, stmtrand()%31, + stmtrand()%32, stmtrand()%33, stmtrand()%34, stmtrand()%35, + stmtrand()%36, stmtrand()%37, stmtrand()%38, stmtrand()%39, + stmtrand()%40, stmtrand()%41, stmtrand()%42, stmtrand()%43, + stmtrand()%28, stmtrand()%29, stmtrand()%30, stmtrand()%31, + stmtrand()%32, stmtrand()%33, stmtrand()%34, stmtrand()%35, + stmtrand()%36, stmtrand()%37, stmtrand()%38, stmtrand()%39, + stmtrand()%40, stmtrand()%41, stmtrand()%42, stmtrand()%43, + stmtrand()%28, stmtrand()%29, stmtrand()%30, stmtrand()%31, + stmtrand()%32, stmtrand()%33, stmtrand()%34, stmtrand()%35, + stmtrand()%36, stmtrand()%37, stmtrand()%38, stmtrand()%39, + stmtrand()%40, stmtrand()%41, stmtrand()%42, stmtrand() FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8) + INSERT INTO x01 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12) + INSERT INTO x02 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16) + INSERT INTO x03 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20) + INSERT INTO x04 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO x05 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32) + INSERT INTO x06 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36) + INSERT INTO x07 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40) + INSERT INTO x08 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44) + INSERT INTO x09 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48) + INSERT INTO x10 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52) + INSERT INTO x11 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56) + INSERT INTO x12 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60) + INSERT INTO x13 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64) + INSERT INTO x14 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72) + INSERT INTO x15 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<80) + INSERT INTO x16 SELECT n%40, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8) + INSERT INTO x17 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12) + INSERT INTO x18 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16) + INSERT INTO x19 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20) + INSERT INTO x20 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO x21 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32) + INSERT INTO x22 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36) + INSERT INTO x23 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40) + INSERT INTO x24 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44) + INSERT INTO x25 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48) + INSERT INTO x26 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52) + INSERT INTO x27 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56) + INSERT INTO x28 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60) + INSERT INTO x29 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64) + INSERT INTO x30 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72) + INSERT INTO x31 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<80) + INSERT INTO x32 SELECT n%40, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8) + INSERT INTO x33 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12) + INSERT INTO x34 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16) + INSERT INTO x35 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20) + INSERT INTO x36 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO x37 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32) + INSERT INTO x38 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36) + INSERT INTO x39 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40) + INSERT INTO x40 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44) + INSERT INTO x41 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48) + INSERT INTO x42 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52) + INSERT INTO x43 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56) + INSERT INTO x44 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60) + INSERT INTO x45 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64) + INSERT INTO x46 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72) + INSERT INTO x47 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<80) + INSERT INTO x48 SELECT n%40, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8) + INSERT INTO x49 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12) + INSERT INTO x50 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16) + INSERT INTO x51 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20) + INSERT INTO x52 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO x53 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32) + INSERT INTO x54 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36) + INSERT INTO x55 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40) + INSERT INTO x56 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44) + INSERT INTO x57 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48) + INSERT INTO x58 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52) + INSERT INTO x59 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56) + INSERT INTO x60 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60) + INSERT INTO x61 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64) + INSERT INTO x62 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c; +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72) + INSERT INTO x63 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c; +****************************************************************************/ +CREATE INDEX t1a01 ON t1(a01); +CREATE INDEX t1a02 ON t1(a02); +CREATE INDEX t1a03 ON t1(a03); +CREATE INDEX t1a04 ON t1(a04); +CREATE INDEX t1a05 ON t1(a05); +CREATE INDEX t1a06 ON t1(a06); +CREATE INDEX t1a07 ON t1(a07); +CREATE INDEX t1a08 ON t1(a08); +CREATE INDEX t1a09 ON t1(a09); +CREATE INDEX t1a10 ON t1(a10); +CREATE INDEX t1a11 ON t1(a11); +CREATE INDEX t1a12 ON t1(a12); +CREATE INDEX t1a13 ON t1(a13); +CREATE INDEX t1a14 ON t1(a14); +CREATE INDEX t1a15 ON t1(a15); +CREATE INDEX t1a16 ON t1(a16); +CREATE INDEX t1a17 ON t1(a17); +CREATE INDEX t1a18 ON t1(a18); +CREATE INDEX t1a19 ON t1(a19); +CREATE INDEX t1a20 ON t1(a20); +CREATE INDEX t1a21 ON t1(a21); +CREATE INDEX t1a22 ON t1(a22); +CREATE INDEX t1a23 ON t1(a23); +CREATE INDEX t1a24 ON t1(a24); +CREATE INDEX t1a25 ON t1(a25); +CREATE INDEX t1a26 ON t1(a26); +CREATE INDEX t1a27 ON t1(a27); +CREATE INDEX t1a28 ON t1(a28); +CREATE INDEX t1a29 ON t1(a29); +CREATE INDEX t1a30 ON t1(a30); +CREATE INDEX t1a31 ON t1(a31); +CREATE INDEX t1a32 ON t1(a32); +CREATE INDEX t1a33 ON t1(a33); +CREATE INDEX t1a34 ON t1(a34); +CREATE INDEX t1a35 ON t1(a35); +CREATE INDEX t1a36 ON t1(a36); +CREATE INDEX t1a37 ON t1(a37); +CREATE INDEX t1a38 ON t1(a38); +CREATE INDEX t1a39 ON t1(a39); +CREATE INDEX t1a40 ON t1(a40); +CREATE INDEX t1a41 ON t1(a41); +CREATE INDEX t1a42 ON t1(a42); +CREATE INDEX t1a43 ON t1(a43); +CREATE INDEX t1a44 ON t1(a44); +CREATE INDEX t1a45 ON t1(a45); +CREATE INDEX t1a46 ON t1(a46); +CREATE INDEX t1a47 ON t1(a47); +CREATE INDEX t1a48 ON t1(a48); +CREATE INDEX t1a49 ON t1(a49); +CREATE INDEX t1a50 ON t1(a50); +CREATE INDEX t1a51 ON t1(a51); +CREATE INDEX t1a52 ON t1(a52); +CREATE INDEX t1a53 ON t1(a53); +CREATE INDEX t1a54 ON t1(a54); +CREATE INDEX t1a55 ON t1(a55); +CREATE INDEX t1a56 ON t1(a56); +CREATE INDEX t1a57 ON t1(a57); +CREATE INDEX t1a58 ON t1(a58); +CREATE INDEX t1a59 ON t1(a59); +CREATE INDEX t1a60 ON t1(a60); +CREATE INDEX t1a61 ON t1(a61); +CREATE INDEX t1a62 ON t1(a62); +CREATE INDEX t1a63 ON t1(a63); +CREATE INDEX x01x ON x01(b01); +CREATE INDEX x02x ON x02(b02); +CREATE INDEX x03x ON x03(b03); +CREATE INDEX x04x ON x04(b04); +CREATE INDEX x05x ON x05(b05); +CREATE INDEX x06x ON x06(b06); +CREATE INDEX x07x ON x07(b07); +CREATE INDEX x08x ON x08(b08); +CREATE INDEX x09x ON x09(b09); +CREATE INDEX x10x ON x10(b10); +CREATE INDEX x11x ON x11(b11); +CREATE INDEX x12x ON x12(b12); +CREATE INDEX x13x ON x13(b13); +CREATE INDEX x14x ON x14(b14); +CREATE INDEX x15x ON x15(b15); +CREATE INDEX x16x ON x16(b16); +CREATE INDEX x17x ON x17(b17); +CREATE INDEX x18x ON x18(b18); +CREATE INDEX x19x ON x19(b19); +CREATE INDEX x20x ON x20(b20); +CREATE INDEX x21x ON x21(b21); +CREATE INDEX x22x ON x22(b22); +CREATE INDEX x23x ON x23(b23); +CREATE INDEX x24x ON x24(b24); +CREATE INDEX x25x ON x25(b25); +CREATE INDEX x26x ON x26(b26); +CREATE INDEX x27x ON x27(b27); +CREATE INDEX x28x ON x28(b28); +CREATE INDEX x29x ON x29(b29); +CREATE INDEX x30x ON x30(b30); +CREATE INDEX x31x ON x31(b31); +CREATE INDEX x32x ON x32(b32); +CREATE INDEX x33x ON x33(b33); +CREATE INDEX x34x ON x34(b34); +CREATE INDEX x35x ON x35(b35); +CREATE INDEX x36x ON x36(b36); +CREATE INDEX x37x ON x37(b37); +CREATE INDEX x38x ON x38(b38); +CREATE INDEX x39x ON x39(b39); +CREATE INDEX x40x ON x40(b40); +CREATE INDEX x41x ON x41(b41); +CREATE INDEX x42x ON x42(b42); +CREATE INDEX x43x ON x43(b43); +CREATE INDEX x44x ON x44(b44); +CREATE INDEX x45x ON x45(b45); +CREATE INDEX x46x ON x46(b46); +CREATE INDEX x47x ON x47(b47); +CREATE INDEX x48x ON x48(b48); +CREATE INDEX x49x ON x49(b49); +CREATE INDEX x50x ON x50(b50); +CREATE INDEX x51x ON x51(b51); +CREATE INDEX x52x ON x52(b52); +CREATE INDEX x53x ON x53(b53); +CREATE INDEX x54x ON x54(b54); +CREATE INDEX x55x ON x55(b55); +CREATE INDEX x56x ON x56(b56); +CREATE INDEX x57x ON x57(b57); +CREATE INDEX x58x ON x58(b58); +CREATE INDEX x59x ON x59(b59); +CREATE INDEX x60x ON x60(b60); +CREATE INDEX x61x ON x61(b61); +CREATE INDEX x62x ON x62(b62); +CREATE INDEX x63x ON x63(b63); +ANALYZE sqlite_schema; +INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES + ('t1','t1a01','172800 14400'), + ('t1','t1a02','172800 13293'), + ('t1','t1a03','172800 12343'), + ('t1','t1a04','172800 11520'), + ('t1','t1a05','172800 10800'), + ('t1','t1a06','172800 10165'), + ('t1','t1a07','172800 9600'), + ('t1','t1a08','172800 9095'), + ('t1','t1a09','172800 8640'), + ('t1','t1a10','172800 8229'), + ('t1','t1a11','172800 7855'), + ('t1','t1a12','172800 7514'), + ('t1','t1a13','172800 7200'), + ('t1','t1a14','172800 6912'), + ('t1','t1a15','172800 6647'), + ('t1','t1a16','172800 6400'), + ('t1','t1a17','172800 6172'), + ('t1','t1a18','172800 5959'), + ('t1','t1a19','172800 5760'), + ('t1','t1a20','172800 5575'), + ('t1','t1a21','172800 5400'), + ('t1','t1a22','172800 5237'), + ('t1','t1a23','172800 5083'), + ('t1','t1a24','172800 4938'), + ('t1','t1a25','172800 4800'), + ('t1','t1a26','172800 4671'), + ('t1','t1a27','172800 4548'), + ('t1','t1a28','172800 4431'), + ('t1','t1a29','172800 4320'), + ('t1','t1a30','172800 4215'), + ('t1','t1a31','172800 4115'), + ('t1','t1a32','172800 4019'), + ('t1','t1a33','172800 6172'), + ('t1','t1a34','172800 5959'), + ('t1','t1a35','172800 5760'), + ('t1','t1a36','172800 5575'), + ('t1','t1a37','172800 5400'), + ('t1','t1a38','172800 5237'), + ('t1','t1a39','172800 5083'), + ('t1','t1a40','172800 4938'), + ('t1','t1a41','172800 4800'), + ('t1','t1a42','172800 4671'), + ('t1','t1a43','172800 4548'), + ('t1','t1a44','172800 4431'), + ('t1','t1a45','172800 4320'), + ('t1','t1a46','172800 4215'), + ('t1','t1a47','172800 4115'), + ('t1','t1a48','172800 4019'), + ('t1','t1a49','172800 6172'), + ('t1','t1a50','172800 5959'), + ('t1','t1a51','172800 5760'), + ('t1','t1a52','172800 5575'), + ('t1','t1a53','172800 5400'), + ('t1','t1a54','172800 5237'), + ('t1','t1a55','172800 5083'), + ('t1','t1a56','172800 4938'), + ('t1','t1a57','172800 4800'), + ('t1','t1a58','172800 4671'), + ('t1','t1a59','172800 4548'), + ('t1','t1a60','172800 4431'), + ('t1','t1a61','172800 4320'), + ('t1','t1a62','172800 4215'), + ('t1','t1a63','172800 4115'), + ('x01','x01x','80 2'), + ('x02','x02x','120 2'), + ('x03','x03x','160 2'), + ('x04','x04x','20 2'), + ('x05','x05x','24 2'), + ('x06','x06x','32 2'), + ('x07','x07x','36 2'), + ('x08','x08x','40 2'), + ('x09','x09x','44 2'), + ('x10','x10x','48 2'), + ('x11','x11x','52 2'), + ('x12','x12x','56 2'), + ('x13','x13x','60 2'), + ('x14','x14x','64 2'), + ('x15','x15x','72 2'), + ('x16','x16x','80 2'), + ('x17','x17x','80 2'), + ('x18','x18x','120 2'), + ('x19','x19x','160 2'), + ('x20','x20x','20 2'), + ('x21','x21x','24 2'), + ('x22','x22x','32 2'), + ('x23','x23x','36 2'), + ('x24','x24x','40 2'), + ('x25','x25x','44 2'), + ('x26','x26x','48 2'), + ('x27','x27x','52 2'), + ('x28','x28x','56 2'), + ('x29','x29x','60 2'), + ('x30','x30x','64 2'), + ('x31','x31x','72 2'), + ('x32','x32x','80 2'), + ('x33','x33x','80 2'), + ('x34','x34x','120 2'), + ('x35','x35x','160 2'), + ('x36','x36x','20 2'), + ('x37','x37x','24 2'), + ('x38','x38x','32 2'), + ('x39','x39x','36 2'), + ('x40','x40x','40 2'), + ('x41','x41x','44 2'), + ('x42','x42x','48 2'), + ('x43','x43x','52 2'), + ('x44','x44x','56 2'), + ('x45','x45x','60 2'), + ('x46','x46x','64 2'), + ('x47','x47x','72 2'), + ('x48','x48x','80 2'), + ('x49','x49x','80 2'), + ('x50','x50x','120 2'), + ('x51','x51x','160 2'), + ('x52','x52x','20 2'), + ('x53','x53x','24 2'), + ('x54','x54x','32 2'), + ('x55','x55x','36 2'), + ('x56','x56x','40 2'), + ('x57','x57x','44 2'), + ('x58','x58x','48 2'), + ('x59','x59x','52 2'), + ('x60','x60x','56 2'), + ('x61','x61x','60 2'), + ('x62','x62x','64 2'), + ('x63','x63x','72 2'); +ANALYZE sqlite_schema; +} +do_execsql_test 1.2 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03 + FROM t1, x01, x02, x03 + WHERE a01=b01 AND a02=b02 AND a03=b03; +} {/SCAN t1.*SEARCH.*SEARCH.*SEARCH/} +do_execsql_test 1.3 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04 + FROM t1, x01, x02, x03, x04 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04; +} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH /} +do_execsql_test 1.4 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04, c05 + FROM t1, x01, x02, x03, x04, x05 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05; +} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH/} +do_execsql_test 1.5 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04, c05, c06 + FROM t1, x01, x02, x03, x04, x05, x06 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 + AND a06=b06; +} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH/} +do_execsql_test 1.6 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04, c05, c06, c07 + FROM t1, x01, x02, x03, x04, x05, x06, x07 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 + AND a06=b06 AND a07=b07; +} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH/} +do_execsql_test 1.7 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04, c05, c06, c07, c08 + FROM t1, x01, x02, x03, x04, x05, x06, x07, x08 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 + AND a06=b06 AND a07=b07 AND a08=b08; +} {~/SCAN.*SCAN/} +do_execsql_test 1.8 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04, c05, c06, c07, c08, + c09, c10, c11, c12, c13, c14, c15, c16, + c17, c18, c19, c20, c21, c22, c23, c24, + c25, c26, c27, c28, c29, c30, c31, c32 + FROM t1, x01, x02, x03, x04, x05, x06, x07, x08, + x09, x10, x11, x12, x13, x14, x15, x16, + x17, x18, x19, x20, x21, x22, x23, x24, + x25, x26, x27, x28, x29, x30, x31, x32 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 AND a06=b06 + AND a07=b07 AND a08=b08 AND a09=b09 AND a10=b10 AND a11=b11 AND a12=b12 + AND a13=b13 AND a14=b14 AND a15=b15 AND a16=b16 AND a17=b17 AND a18=b18 + AND a19=b19 AND a20=b20 AND a21=b21 AND a22=b22 AND a23=b23 AND a24=b24 + AND a25=b25 AND a26=b26 AND a27=b27 AND a28=b28 AND a29=b29 AND a30=b30 + AND a31=b31 AND a32=b32; +} {~/SCAN.*SCAN/} +do_execsql_test 1.9 { + EXPLAIN QUERY PLAN + SELECT c01, c02, c03, c04, c05, c06, c07, c08, + c09, c10, c11, c12, c13, c14, c15, c16, + c17, c18, c19, c20, c21, c22, c23, c24, + c25, c26, c27, c28, c29, c30, c31, c32, + c33, c34, c35, c36, c37, c38, c39, c40, + c41, c42, c43, c44, c45, c46, c47, c48, + c49, c50, c51, c52, c53, c54, c55, c56, + c57, c58, c59, c60, c61, c62, c63 + FROM t1, x01, x02, x03, x04, x05, x06, x07, x08, + x09, x10, x11, x12, x13, x14, x15, x16, + x17, x18, x19, x20, x21, x22, x23, x24, + x25, x26, x27, x28, x29, x30, x31, x32, + x33, x34, x35, x36, x37, x38, x39, x40, + x41, x42, x43, x44, x45, x46, x47, x48, + x49, x50, x51, x52, x53, x54, x55, x56, + x57, x58, x59, x60, x61, x62, x63 + WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 AND a06=b06 + AND a07=b07 AND a08=b08 AND a09=b09 AND a10=b10 AND a11=b11 AND a12=b12 + AND a13=b13 AND a14=b14 AND a15=b15 AND a16=b16 AND a17=b17 AND a18=b18 + AND a19=b19 AND a20=b20 AND a21=b21 AND a22=b22 AND a23=b23 AND a24=b24 + AND a25=b25 AND a26=b26 AND a27=b27 AND a28=b28 AND a29=b29 AND a30=b30 + AND a31=b31 AND a32=b32 AND a33=b33 AND a34=b34 AND a35=b35 AND a36=b36 + AND a37=b37 AND a38=b38 AND a39=b39 AND a40=b40 AND a41=b41 AND a42=b42 + AND a43=b43 AND a44=b44 AND a45=b45 AND a46=b46 AND a47=b47 AND a48=b48 + AND a49=b49 AND a50=b50 AND a51=b51 AND a52=b52 AND a53=b53 AND a54=b54 + AND a55=b55 AND a56=b56 AND a57=b57 AND a58=b58 AND a59=b59 AND a60=b60 + AND a61=b61 AND a62=b62 AND a63=b63; +} {~/SCAN.*SCAN/} + + + +finish_test diff --git a/test/stmtrand.test b/test/stmtrand.test new file mode 100644 index 000000000..e2f6935b8 --- /dev/null +++ b/test/stmtrand.test @@ -0,0 +1,48 @@ +# 2024-05-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Verify that the stmtrand() extension function works. +# +# Stmtrand() is a pseudo-random number generator designed for testing. +# it has the property that it returns the same sequence of pseudo-random +# numbers for each SQL statement invocation. This makes it suitable for +# testing because the results are reproducible. +# +# The optional argument is the seed for the PRNG. The seed is only used +# on the first invocation. If the argument is omitted, a seed of 0 is +# used. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix stmtrand + +load_static_extension db stmtrand + +do_execsql_test 1.1 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30) + SELECT stmtrand()%10 FROM c +} {4 1 8 3 6 1 4 9 2 1 2 1 0 1 8 1 4 7 6 1 8 7 0 3 0 9 4 5 9 9} +do_execsql_test 1.2 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30) + SELECT stmtrand(0)%10 FROM c +} {4 1 8 3 6 1 4 9 2 1 2 1 0 1 8 1 4 7 6 1 8 7 0 3 0 9 4 5 9 9} +do_execsql_test 1.3 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30) + SELECT stmtrand(1)%10 FROM c +} {3 8 9 4 7 2 3 8 5 2 7 0 3 0 7 2 5 6 1 8 5 6 7 8 7 2 9 6 8 0} +do_execsql_test 1.4 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30) + SELECT stmtrand(x)%10 FROM c +} {3 8 9 4 7 2 3 8 5 2 7 0 3 0 7 2 5 6 1 8 5 6 7 8 7 2 9 6 8 0} + + +finish_test diff --git a/test/subquery.test b/test/subquery.test index c51edba04..17061d4b6 100644 --- a/test/subquery.test +++ b/test/subquery.test @@ -651,5 +651,64 @@ do_eqp_test subquery-10.2 { # `--SEARCH t1 USING INDEX x12 (aa=?) # +#----------------------------------------------------------------------------- +# 2024-04-25 Column affinities for columns of compound subqueries +# +reset_db +sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db +do_execsql_test subquery-11.1 { + CREATE TABLE t1(ix INT, rx REAL, bx BLOB, tx TEXT, ax); + INSERT INTO t1 VALUES(1,1.0,x'31','x',NULL); + WITH c(a) AS (SELECT 'y' UNION SELECT tx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT tx FROM t1 UNION SELECT 'y') SELECT affinity(a) FROM c; +} {text text text text} +do_execsql_test subquery-11.2 { + WITH c(a) AS (SELECT 2 UNION SELECT tx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT tx FROM t1 UNION SELECT 2) SELECT affinity(a) FROM c; +} {blob blob blob blob} +do_execsql_test subquery-11.3 { + WITH c(a) AS (SELECT 2.0 UNION SELECT tx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT tx FROM t1 UNION SELECT 2.0) SELECT affinity(a) FROM c; +} {blob blob blob blob} +do_execsql_test subquery-11.4 { + WITH c(a) AS (SELECT null UNION SELECT tx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT tx FROM t1 UNION SELECT null) SELECT affinity(a) FROM c; +} {text text text text} +do_execsql_test subquery-11.5 { + WITH c(a) AS (SELECT x'32' UNION SELECT tx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT tx FROM t1 UNION SELECT x'32') SELECT affinity(a) FROM c; +} {text text text text} +do_execsql_test subquery-11.6 { + WITH c(a) AS (SELECT 3 UNION SELECT ix FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT ix FROM t1 UNION SELECT 3) SELECT affinity(a) FROM c; +} {integer integer integer integer} +do_execsql_test subquery-11.7 { + WITH c(a) AS (SELECT 3.0 UNION SELECT ix FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT ix FROM t1 UNION SELECT 3.0) SELECT affinity(a) FROM c; +} {integer integer integer integer} +do_execsql_test subquery-11.8 { + WITH c(a) AS (SELECT '3' UNION SELECT ix FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT ix FROM t1 UNION SELECT '3') SELECT affinity(a) FROM c; +} {blob blob blob blob} +do_execsql_test subquery-11.10 { + WITH c(a) AS (SELECT x'32' UNION SELECT ix FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT ix FROM t1 UNION SELECT x'32') SELECT affinity(a) FROM c; +} {integer integer integer integer} +do_execsql_test subquery-11.11 { + WITH c(a) AS (SELECT 4 UNION SELECT rx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT rx FROM t1 UNION SELECT 4) SELECT affinity(a) FROM c; +} {real real real real} +do_execsql_test subquery-11.12 { + WITH c(a) AS (SELECT '4' UNION SELECT rx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT rx FROM t1 UNION SELECT '4') SELECT affinity(a) FROM c; +} {blob blob blob blob} +do_execsql_test subquery-11.13 { + WITH c(a) AS (SELECT null UNION SELECT rx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT rx FROM t1 UNION SELECT null) SELECT affinity(a) FROM c; +} {real real real real} +do_execsql_test subquery-11.14 { + WITH c(a) AS (SELECT x'b4' UNION SELECT rx FROM t1) SELECT affinity(a) FROM c; + WITH c(a) AS (SELECT rx FROM t1 UNION SELECT x'b4') SELECT affinity(a) FROM c; +} {real real real real} finish_test diff --git a/test/superlock.test b/test/superlock.test index 704b0677a..10e7caa29 100644 --- a/test/superlock.test +++ b/test/superlock.test @@ -166,7 +166,7 @@ do_multiclient_test tn { proc read_content {file} { if {[file exists $file]==0} {return ""} set fd [open $file] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary set content [read $fd] close $fd return $content @@ -174,7 +174,7 @@ proc read_content {file} { proc write_content {file content} { set fd [open $file w+] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary puts -nonewline $fd $content close $fd } diff --git a/test/syscall.test b/test/syscall.test index 19313a5e6..fe4a4177f 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -211,7 +211,7 @@ forcedelete test.db test.db2 proc create_db_file {nByte} { set fd [open test.db w] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary puts -nonewline $fd [string range "xSQLite" 1 $nByte] close $fd } diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 8e90c549f..8c24198e1 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -121,26 +121,26 @@ do_eqp_test tabfunc01-3.10 { SELECT value FROM generate_series(1,10) ORDER BY value; } { QUERY PLAN - `--SCAN generate_series VIRTUAL TABLE INDEX 19: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x13: } do_eqp_test tabfunc01-3.11 { SELECT value FROM generate_series(1,10) ORDER BY +value; } { QUERY PLAN - |--SCAN generate_series VIRTUAL TABLE INDEX 3: + |--SCAN generate_series VIRTUAL TABLE INDEX 0x3: `--USE TEMP B-TREE FOR ORDER BY } do_eqp_test tabfunc01-3.12 { SELECT value FROM generate_series(1,10) ORDER BY value, stop; } { QUERY PLAN - `--SCAN generate_series VIRTUAL TABLE INDEX 19: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x13: } do_eqp_test tabfunc01-3.13 { SELECT value FROM generate_series(1,10) ORDER BY stop, value; } { QUERY PLAN - |--SCAN generate_series VIRTUAL TABLE INDEX 3: + |--SCAN generate_series VIRTUAL TABLE INDEX 0x3: `--USE TEMP B-TREE FOR ORDER BY } @@ -156,9 +156,9 @@ do_eqp_test tabfunc01-3.20 { QUERY PLAN `--MERGE (UNION ALL) |--LEFT - | `--SCAN generate_series VIRTUAL TABLE INDEX 23: + | `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: `--RIGHT - `--SCAN generate_series VIRTUAL TABLE INDEX 23: + `--SCAN generate_series VIRTUAL TABLE INDEX 0x17: } @@ -322,6 +322,32 @@ ifcapable altertable { } {1 {table pragma_compile_options may not be altered}} } +#----------------------------------------------------------------------------- +# 2024-04-26 LIMIT and OFFSET passed into virtual tables +# https://sqlite.org/forum/forumpost/c243b8f856 +# +do_execsql_test tabfunc01-900 { + SELECT * FROM ( + SELECT * FROM generate_series(1,10) + UNION ALL + SELECT * FROM generate_series(101,104) + ) LIMIT 10 OFFSET 5; +} {6 7 8 9 10 101 102 103 104} +do_execsql_test tabfunc01-910 { + SELECT * FROM ( + SELECT * FROM generate_series(1,10) + UNION ALL + SELECT * FROM generate_series(101,104) + ) LIMIT -1 OFFSET 5; +} {6 7 8 9 10 101 102 103 104} +do_execsql_test tabfunc01-920 { + SELECT * FROM ( + SELECT * FROM generate_series(1,10) + UNION ALL + SELECT * FROM generate_series(101,104) + ) LIMIT -1 OFFSET 0; +} {1 2 3 4 5 6 7 8 9 10 101 102 103 104} + # Free up memory allocations intarray_addr diff --git a/test/tester.tcl b/test/tester.tcl index 5754d7037..164ee47f4 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -310,66 +310,6 @@ proc do_delete_file {force args} { } } -if {$::tcl_platform(platform) eq "windows"} { - proc do_remove_win32_dir {args} { - set nRetry [getFileRetries] ;# Maximum number of retries. - set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. - - foreach dirName $args { - # On windows, sometimes even a [remove_win32_dir] can fail just after - # a directory is emptied. The cause is usually "tag-alongs" - programs - # like anti-virus software, automatic backup tools and various explorer - # extensions that keep a file open a little longer than we expect, - # causing the delete to fail. - # - # The solution is to wait a short amount of time before retrying the - # removal. - # - if {$nRetry > 0} { - for {set i 0} {$i < $nRetry} {incr i} { - set rc [catch { - remove_win32_dir $dirName - } msg] - if {$rc == 0} break - if {$nDelay > 0} { after $nDelay } - } - if {$rc} { error $msg } - } else { - remove_win32_dir $dirName - } - } - } - - proc do_delete_win32_file {args} { - set nRetry [getFileRetries] ;# Maximum number of retries. - set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. - - foreach fileName $args { - # On windows, sometimes even a [delete_win32_file] can fail just after - # a file is closed. The cause is usually "tag-alongs" - programs like - # anti-virus software, automatic backup tools and various explorer - # extensions that keep a file open a little longer than we expect, - # causing the delete to fail. - # - # The solution is to wait a short amount of time before retrying the - # delete. - # - if {$nRetry > 0} { - for {set i 0} {$i < $nRetry} {incr i} { - set rc [catch { - delete_win32_file $fileName - } msg] - if {$rc == 0} break - if {$nDelay > 0} { after $nDelay } - } - if {$rc} { error $msg } - } else { - delete_win32_file $fileName - } - } - } -} - proc execpresql {handle args} { trace remove execution $handle enter [list execpresql $handle] if {[info exists ::G(perm:presql)]} { @@ -554,7 +494,7 @@ if {[info exists cmdlinearg]==0} { } unset -nocomplain a set testdir [file normalize $testdir] - set cmdlinearg(TESTFIXTURE_HOME) [pwd] + set cmdlinearg(TESTFIXTURE_HOME) [file dirname [info nameofexec]] set cmdlinearg(INFO_SCRIPT) [file normalize [info script]] set argv0 [file normalize $argv0] if {$cmdlinearg(testdir)!=""} { @@ -847,6 +787,9 @@ proc do_test {name cmd expected} { } } else { set ok [expr {[string compare $result $expected]==0}] + if {!$ok} { + set ok [fpnum_compare $result $expected] + } } if {!$ok} { # if {![info exists ::testprefix] || $::testprefix eq ""} { @@ -897,7 +840,7 @@ proc catchsafecmd {db {cmd ""}} { proc catchcmdex {db {cmd ""}} { global CLI set out [open cmds.txt w] - fconfigure $out -encoding binary -translation binary + fconfigure $out -translation binary puts -nonewline $out $cmd close $out set line "exec -keepnewline -- $CLI $db < cmds.txt" @@ -905,7 +848,7 @@ proc catchcmdex {db {cmd ""}} { foreach chan $chans { catch { set modes($chan) [fconfigure $chan] - fconfigure $chan -encoding binary -translation binary -buffering none + fconfigure $chan -translation binary -buffering none } } set rc [catch { eval $line } msg] @@ -1042,7 +985,7 @@ proc query_plan_graph {sql} { } set a "\n QUERY PLAN\n" append a [append_graph " " dx cx 0] - regsub -all { 0x[A-F0-9]+\y} $a { xxxxxx} a + regsub -all {SUBQUERY 0x[A-F0-9]+\y} $a {SUBQUERY xxxxxx} a regsub -all {(MATERIALIZE|CO-ROUTINE|SUBQUERY) \d+\y} $a {\1 xxxxxx} a regsub -all {\((join|subquery)-\d+\)} $a {(\1-xxxxxx)} a return $a @@ -1113,6 +1056,29 @@ proc do_eqp_test {name sql res} { } } +# Do both an eqp_test and an execsql_test on the same SQL. +# +proc do_eqp_execsql_test {name sql res1 res2} { + if {[regexp {^\s+QUERY PLAN\n} $res1]} { + + set query_plan [query_plan_graph $sql] + + if {[list {*}$query_plan]==[list {*}$res1]} { + uplevel [list do_test ${name}a [list set {} ok] ok] + } else { + uplevel [list \ + do_test ${name}a [list query_plan_graph $sql] $res1 + ] + } + } else { + if {[string index $res 0]!="/"} { + set res1 "/*$res1*/" + } + uplevel do_execsql_test ${name}a [list "EXPLAIN QUERY PLAN $sql"] [list $res1] + } + uplevel do_execsql_test ${name}b [list $sql] [list $res2] +} + #------------------------------------------------------------------------- # Usage: do_select_tests PREFIX ?SWITCHES? TESTLIST diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 0c704daf2..d3a2c1f4c 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -54,22 +54,37 @@ proc usage {} { Usage: $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS? $a0 PERMUTATION FILE + $a0 errors ?-v|--verbose? + $a0 help $a0 njob ?NJOB? - $a0 status + $a0 script ?-msvc? CONFIG + $a0 status ?-d SECS? ?--cls? where SWITCHES are: - --buildonly - --dryrun - --jobs NUMBER-OF-JOBS - --zipvfs ZIPVFS-SOURCE-DIR + --buildonly Build test exes but do not run tests + --config CONFIGS Only use configs on comma-separate list CONFIGS + --dryrun Write what would have happened to testrunner.log + --explain Write summary to stdout + --jobs NUM Run tests using NUM separate processes + --omit CONFIGS Omit configs on comma-separated list CONFIGS + --status Show the full "status" report while running + --stop-on-coredump Stop running if any test segfaults + --stop-on-error Stop running after any reported error + --zipvfs ZIPVFSDIR ZIPVFS source directory + +Special values for PERMUTATION that work with plain tclsh: + + list - show all allowed PERMUTATION arguments. + mdevtest - tests recommended prior to normal development check-ins. + release - full release test with various builds. + sdevtest - like mdevtest but using ASAN and UBSAN. -Interesting values for PERMUTATION are: +Other PERMUTATION arguments must be run using testfixture, not tclsh: - veryquick - a fast subset of the tcl test scripts. This is the default. - full - all tcl test scripts. all - all tcl test scripts, plus a subset of test scripts rerun with various permutations. - release - full release test with various builds. + full - all tcl test scripts. + veryquick - a fast subset of the tcl test scripts. This is the default. If no PATTERN arguments are present, all tests specified by the PERMUTATION are run. Otherwise, each pattern is interpreted as a glob pattern. Only @@ -87,8 +102,20 @@ with the specified permutation. The "status" and "njob" commands are designed to be run from the same directory as a running testrunner.tcl script that is running tests. The "status" command prints a report describing the current state and progress -of the tests. The "njob" command may be used to query or modify the number -of sub-processes the test script uses to run tests. +of the tests. Use the "-d N" option to have the status display clear the +screen and repeat every N seconds. The "njob" command may be used to query +or modify the number of sub-processes the test script uses to run tests. + +The "script" command outputs the script used to build a configuration. +Add the "-msvc" option for a Windows-compatible script. For a list of +available configurations enter "$a0 script help". + +The "errors" commands shows the output of all tests that failed in the +most recent run. Complete output is shown if the -v or --verbose options +are used. Otherwise, an attempt is made to minimize the output to show +only the parts that contain the error messages. + +Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md }]] exit 1 @@ -126,6 +153,10 @@ proc guess_number_of_cores {} { } proc default_njob {} { + global env + if {[info exists env(NJOB)] && $env(NJOB)>=1} { + return $env(NJOB) + } set nCore [guess_number_of_cores] if {$nCore<=2} { set nHelper 1 @@ -151,7 +182,13 @@ set TRG(reporttime) 2000 set TRG(fuzztest) 0 ;# is the fuzztest option present. set TRG(zipvfs) "" ;# -zipvfs option, if any set TRG(buildonly) 0 ;# True if --buildonly option +set TRG(config) {} ;# Only build the named configurations +set TRG(omitconfig) {} ;# Do not build these configurations set TRG(dryrun) 0 ;# True if --dryrun option +set TRG(explain) 0 ;# True for the --explain option +set TRG(stopOnError) 0 ;# Stop running at first failure +set TRG(stopOnCore) 0 ;# Stop on a core-dump +set TRG(fullstatus) 0 ;# Full "status" report while running switch -nocase -glob -- $tcl_platform(os) { *darwin* { @@ -159,6 +196,7 @@ switch -nocase -glob -- $tcl_platform(os) { set TRG(make) make.sh set TRG(makecmd) "bash make.sh" set TRG(testfixture) testfixture + set TRG(shell) sqlite3 set TRG(run) run.sh set TRG(runcmd) "bash run.sh" } @@ -167,14 +205,16 @@ switch -nocase -glob -- $tcl_platform(os) { set TRG(make) make.sh set TRG(makecmd) "bash make.sh" set TRG(testfixture) testfixture + set TRG(shell) sqlite3 set TRG(run) run.sh set TRG(runcmd) "bash run.sh" } *win* { set TRG(platform) win set TRG(make) make.bat - set TRG(makecmd) make.bat + set TRG(makecmd) "call make.bat" set TRG(testfixture) testfixture.exe + set TRG(shell) sqlite3.exe set TRG(run) run.bat set TRG(runcmd) "run.bat" } @@ -239,7 +279,7 @@ set TRG(schema) { /* Fields updated as jobs run */ starttime INTEGER, endtime INTEGER, - state TEXT CHECK( state IN ('', 'ready', 'running', 'done', 'failed') ), + state TEXT CHECK( state IN ('','ready','running','done','failed','omit') ), output TEXT ); @@ -327,6 +367,14 @@ if {([llength $argv]==2 || [llength $argv]==1) #-------------------------------------------------------------------------- #-------------------------------------------------------------------------- +# Check if this is the "help" command: +# +if {[string compare -nocase help [lindex $argv 0]]==0} { + usage +} +#-------------------------------------------------------------------------- + +#-------------------------------------------------------------------------- # Check if this is the "script" command: # if {[string compare -nocase script [lindex $argv 0]]==0} { @@ -340,34 +388,36 @@ if {[string compare -nocase script [lindex $argv 0]]==0} { puts [trd_buildscript $config [file dirname $testdir] $bMsvc] exit } - -#-------------------------------------------------------------------------- -# Check if this is the "status" command: +# Helper routine for show_status # -if {[llength $argv]==1 - && [string compare -nocase status [lindex $argv 0]]==0 -} { - - proc display_job {jobdict {tm ""}} { - array set job $jobdict - - set dfname [format %-60s $job(displayname)] - - set dtm "" - if {$tm!=""} { set dtm "\[[expr {$tm-$job(starttime)}]ms\]" } - puts " $dfname $dtm" +proc display_job {jobdict {tm ""}} { + array set job $jobdict + set dfname [format %-60s $job(displayname)] + set dtm "" + if {$tm!=""} { + set dtm [format %-10s "\[[expr {$tm-$job(starttime)}]ms\]"] } + puts " $dfname $dtm" +} - sqlite3 mydb $TRG(dbname) - mydb timeout 1000 - mydb eval BEGIN - - set cmdline [mydb one { SELECT value FROM config WHERE name='cmdline' }] - set nJob [mydb one { SELECT value FROM config WHERE name='njob' }] - +# This procedure shows the "status" page. It uses the database +# connect passed in as the "db" parameter. If the "cls" parameter +# is true, then VT100 escape codes are used to format the display. +# +proc show_status {db cls} { + global TRG + $db eval BEGIN + if {[catch { + set cmdline [$db one { SELECT value FROM config WHERE name='cmdline' }] + set nJob [$db one { SELECT value FROM config WHERE name='njob' }] + } msg]} { + if {$cls} {puts "\033\[H\033\[2J"} + puts "Cannot read database: $TRG(dbname)" + return + } set now [clock_milliseconds] - set tm [mydb one { + set tm [$db one { SELECT COALESCE((SELECT value FROM config WHERE name='end'), $now) - (SELECT value FROM config WHERE name='start') @@ -375,7 +425,7 @@ if {[llength $argv]==1 set total 0 foreach s {"" ready running done failed} { set S($s) 0 } - mydb eval { + $db eval { SELECT state, count(*) AS cnt FROM jobs GROUP BY 1 } { incr S($state) $cnt @@ -384,18 +434,28 @@ if {[llength $argv]==1 set fin [expr $S(done)+$S(failed)] if {$cmdline!=""} {set cmdline " $cmdline"} + if {$cls} { + # Move the cursor to the top-left corner. Each iteration will simply + # overwrite. + puts -nonewline "\033\[H" + flush stdout + set clreol "\033\[K" + } else { + set clreol "" + } set f "" if {$S(failed)>0} { set f "$S(failed) FAILED, " } - puts "Command line: \[testrunner.tcl$cmdline\]" - puts "Jobs: $nJob" - puts "Summary: ${tm}ms, ($fin/$total) finished, ${f}$S(running) running" + puts "Command line: \[testrunner.tcl$cmdline\]$clreol" + puts "Jobs: $nJob " + puts "Summary: ${tm}ms, ($fin/$total) finished,\ + ${f}$S(running) running " set srcdir [file dirname [file dirname $TRG(info_script)]] if {$S(running)>0} { puts "Running: " - mydb eval { + $db eval { SELECT * FROM jobs WHERE state='running' ORDER BY starttime } job { display_job [array get job] $now @@ -403,17 +463,137 @@ if {[llength $argv]==1 } if {$S(failed)>0} { puts "Failures: " - mydb eval { + $db eval { SELECT * FROM jobs WHERE state='failed' ORDER BY starttime } job { display_job [array get job] } + set nOmit [$db one {SELECT count(*) FROM jobs WHERE state='omit'}] + if {$nOmit} { + puts "$nOmit jobs omitted due to failures$clreol" + } + } + if {$cls} { + # Clear everything else to the bottom of the screen + puts -nonewline "\033\[0J" + flush stdout + } + $db eval COMMIT +} + + + +#-------------------------------------------------------------------------- +# Check if this is the "status" command: +# +if {[llength $argv]>=1 + && [string compare -nocase status [lindex $argv 0]]==0 +} { + set delay 0 + set cls 0 + for {set ii 1} {$ii<[llength $argv]} {incr ii} { + set a0 [lindex $argv $ii] + if {$a0=="-d" && $ii+1<[llength $argv]} { + incr ii + set delay [lindex $argv $ii] + if {![string is integer -strict $delay]} { + puts "Argument to -d should be an integer" + exit 1 + } + } elseif {$a0=="-cls" || $a0=="--cls"} { + set cls 1 + } else { + puts "unknown option: \"$a0\"" + exit 1 + } + } + + if {![file readable $TRG(dbname)]} { + puts "Database missing: $TRG(dbname)" + exit + } + sqlite3 mydb $TRG(dbname) + mydb timeout 2000 + + # Clear the whole screen initially. + # + if {$delay>0 || $cls} {puts -nonewline "\033\[2J"} + + while {1} { + show_status mydb [expr {$delay>0 || $cls}] + if {$delay<=0} break + after [expr {$delay*1000}] } - mydb close exit } +# Scan the output of all jobs looking for the summary lines that +# report the number of test cases and the number of errors. +# Aggregate these numbers and return them. +# +proc aggregate_test_counts {db} { + set ncase 0 + set nerr 0 + $db eval {SELECT output FROM jobs WHERE displaytype IN ('tcl','fuzz')} { + set n 0 + set m 0 + if {[regexp {(\d+) errors out of (\d+) tests} $output all n m] + && [string is integer -strict $n] + && [string is integer -strict $m]} { + incr ncase $m + incr nerr $n + } elseif {[regexp {sessionfuzz.*: *(\d+) cases, (\d+) crash} $output \ + all m n] + && [string is integer -strict $m] + && [string is integer -strict $n]} { + incr ncase $m + incr nerr $n + } + } + return [list $nerr $ncase] +} + +#-------------------------------------------------------------------------- +# Check if this is the "errors" command: +# +if {[llength $argv]>=1 && [llength $argv]<=2 + && ([string compare -nocase errors [lindex $argv 0]]==0 || + [string match err* [lindex $argv 0]]==1) +} { + set verbose 0 + for {set ii 1} {$ii<[llength $argv]} {incr ii} { + set a0 [lindex $argv $ii] + if {$a0=="-v" || $a0=="--verbose" || $a0=="-verbose"} { + set verbose 1 + } else { + puts "unknown option: \"$a0\"". Use --help for more info." + exit 1 + } + } + set cnt 0 + sqlite3 mydb $TRG(dbname) + mydb timeout 2000 + mydb eval {SELECT displaytype, displayname, output + FROM jobs WHERE state='failed'} { + puts "**** $displayname ****" + if {$verbose || $displaytype!="tcl"} { + puts $output + } else { + foreach line [split $output \n] { + if {[string match {!*} $line] || [string match *failed* $line]} { + puts $line + } + } + } + incr cnt + } + set summary [aggregate_test_counts mydb] + mydb close + puts "Total [lindex $summary 0] errors out of [lindex $summary 1] tests" + exit +} + #------------------------------------------------------------------------- # Parse the command line. # @@ -433,8 +613,30 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} { if {$isLast} { usage } } elseif {($n>2 && [string match "$a*" --buildonly]) || $a=="-b"} { set TRG(buildonly) 1 + } elseif {($n>2 && [string match "$a*" --config]) || $a=="-c"} { + incr ii + set TRG(config) [lindex $argv $ii] } elseif {($n>2 && [string match "$a*" --dryrun]) || $a=="-d"} { set TRG(dryrun) 1 + } elseif {($n>2 && [string match "$a*" --explain]) || $a=="-e"} { + set TRG(explain) 1 + } elseif {($n>2 && [string match "$a*" --omit]) || $a=="-c"} { + incr ii + set TRG(omitconfig) [lindex $argv $ii] + } elseif {[string match "$a*" --stop-on-error]} { + set TRG(stopOnError) 1 + } elseif {[string match "$a*" --stop-on-coredump]} { + set TRG(stopOnCore) 1 + } elseif {[string match "$a*" --status]} { + if {$tcl_platform(platform)=="windows"} { + puts stdout \ +"The --status option is not available on Windows. A suggested work-around" + puts stdout \ +"is to run the following command in a separate window:\n" + puts stdout " [info nameofexe] $argv0 status -d 2\n" + } else { + set TRG(fullstatus) 1 + } } else { usage } @@ -444,8 +646,6 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} { } set argv [list] - - # This script runs individual tests - tcl scripts or [make xyz] commands - # in directories named "testdir$N", where $N is an integer. This variable # contains a list of integers indicating the directories in use. @@ -550,12 +750,6 @@ proc r_get_next_job {iJob} { return $ret } -#rename r_get_next_job r_get_next_job_r -#proc r_get_next_job {iJob} { - #puts [time { set res [r_get_next_job_r $iJob] }] - #set res -#} - # Usage: # # add_job OPTION ARG OPTION ARG... @@ -617,7 +811,16 @@ proc add_job {args} { trdb last_insert_rowid } -proc add_tcl_jobs {build config patternlist} { +# Argument $build is either an empty string, or else a list of length 3 +# describing the job to build testfixture. In the usual form: +# +# {ID DIRNAME DISPLAYNAME} +# +# e.g +# +# {1 /home/user/sqlite/test/testrunner_bld_xyz All-Debug} +# +proc add_tcl_jobs {build config patternlist {shelldepid ""}} { global TRG set topdir [file dirname $::testdir] @@ -666,34 +869,59 @@ proc add_tcl_jobs {build config patternlist} { if {[lsearch $lProp slow]>=0} { set priority 2 } if {[lsearch $lProp superslow]>=0} { set priority 4 } + set depid [lindex $build 0] + if {$shelldepid!="" && [lsearch $lProp shell]>=0} { set depid $shelldepid } + add_job \ -displaytype tcl \ -displayname $displayname \ -cmd $cmd \ - -depid [lindex $build 0] \ + -depid $depid \ -priority $priority - } } -proc add_build_job {buildname target} { +proc add_build_job {buildname target {postcmd ""} {depid ""}} { global TRG set dirname "[string tolower [string map {- _} $buildname]]_$target" set dirname "testrunner_bld_$dirname" + set cmd "$TRG(makecmd) $target" + if {$postcmd!=""} { + append cmd "\n" + append cmd $postcmd + } + set id [add_job \ -displaytype bld \ -displayname "Build $buildname ($target)" \ -dirname $dirname \ -build $buildname \ - -cmd "$TRG(makecmd) $target" \ + -cmd $cmd \ + -depid $depid \ -priority 3 ] list $id [file normalize $dirname] $buildname } +proc add_shell_build_job {buildname dirname depid} { + global TRG + + if {$TRG(platform)=="win"} { + set path [string map {/ \\} "$dirname/"] + set copycmd "xcopy $TRG(shell) $path" + } else { + set copycmd "cp $TRG(shell) $dirname/" + } + + return [ + add_build_job $buildname $TRG(shell) $copycmd $depid + ] +} + + proc add_make_job {bld target} { global TRG @@ -767,10 +995,30 @@ proc add_devtest_jobs {lBld patternlist} { foreach b $lBld { set bld [add_build_job $b $TRG(testfixture)] - add_tcl_jobs $bld veryquick $patternlist + add_tcl_jobs $bld veryquick $patternlist SHELL if {$patternlist==""} { add_fuzztest_jobs $b } + + if {[trdb one "SELECT EXISTS (SELECT 1 FROM jobs WHERE depid='SHELL')"]} { + set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]] + set sbldid [lindex $sbld 0] + trdb eval { + UPDATE jobs SET depid=$sbldid WHERE depid='SHELL' + } + } + + } +} + +# Check to ensure that the interpreter is a full-blown "testfixture" +# build and not just a "tclsh". If this is not the case, issue an +# error message and exit. +# +proc must_be_testfixture {} { + if {[lsearch [info commands] sqlite3_soft_heap_limit]<0} { + puts "Use testfixture, not tclsh, for these arguments." + exit 1 } } @@ -789,6 +1037,7 @@ proc add_jobs_from_cmdline {patternlist} { set first [lindex $patternlist 0] switch -- $first { all { + must_be_testfixture set patternlist [lrange $patternlist 1 end] set clist [trd_all_configs] foreach c $clist { @@ -797,16 +1046,26 @@ proc add_jobs_from_cmdline {patternlist} { } mdevtest { - add_devtest_jobs {All-O0 All-Debug} [lrange $patternlist 1 end] + set config_set { + All-O0 + All-Debug + } + add_devtest_jobs $config_set [lrange $patternlist 1 end] } sdevtest { - add_devtest_jobs {All-Sanitize All-Debug} [lrange $patternlist 1 end] + set config_set { + All-Sanitize + All-Debug + } + add_devtest_jobs $config_set [lrange $patternlist 1 end] } release { set patternlist [lrange $patternlist 1 end] foreach b [trd_builds $TRG(platform)] { + if {$TRG(config)!="" && ![regexp "\\y$b\\y" $TRG(config)]} continue + if {[regexp "\\y$b\\y" $TRG(omitconfig)]} continue set bld [add_build_job $b $TRG(testfixture)] foreach c [trd_configs $TRG(platform) $b] { add_tcl_jobs $bld $c $patternlist @@ -824,7 +1083,15 @@ proc add_jobs_from_cmdline {patternlist} { } } + list { + set allperm [array names ::testspec] + lappend allperm all mdevtest sdevtest release list + puts "Allowed values for the PERMUTATION argument: [lsort $allperm]" + exit 0 + } + default { + must_be_testfixture if {[info exists ::testspec($first)]} { add_tcl_jobs "" $first [lrange $patternlist 1 end] } else { @@ -837,6 +1104,7 @@ proc add_jobs_from_cmdline {patternlist} { proc make_new_testset {} { global TRG + trdb eval {PRAGMA journal_mode=WAL;} r_write_db { trdb eval $TRG(schema) set nJob $TRG(nJob) @@ -853,11 +1121,16 @@ proc make_new_testset {} { proc mark_job_as_finished {jobid output state endtm} { r_write_db { + if {$state=="failed"} { + set childstate omit + } else { + set childstate ready + } trdb eval { UPDATE jobs SET output=$output, state=$state, endtime=$endtm WHERE jobid=$jobid; - UPDATE jobs SET state='ready' WHERE depid=$jobid; + UPDATE jobs SET state=$childstate WHERE depid=$jobid; } } } @@ -888,6 +1161,14 @@ proc script_input_ready {fd iJob jobid} { } puts "FAILED: $job(displayname) ($iJob)" set state "failed" + if {$TRG(stopOnError)} { + puts "OUTPUT: $O($iJob)" + exit 1 + } + if {$TRG(stopOnCore) && [string first {core dumped} $O($iJob)]>0} { + puts "OUTPUT: $O($iJob)" + exit 1 + } } set tm [clock_milliseconds] @@ -948,6 +1229,16 @@ proc launch_another_job {iJob} { close $fd } + # Add a batch/shell file command to set the directory used for temp + # files to the test's working directory. Otherwise, tests that use + # large numbers of temp files (e.g. zipvfs), might generate temp + # filename collisions. + if {$TRG(platform)=="win"} { + set set_tmp_dir "SET SQLITE_TMPDIR=[file normalize $dir]" + } else { + set set_tmp_dir "export SQLITE_TMPDIR=\"[file normalize $dir]\"" + } + if { $TRG(dryrun) } { mark_job_as_finished $job(jobid) "" done 0 @@ -962,61 +1253,71 @@ proc launch_another_job {iJob} { set pwd [pwd] cd $dir set fd [open $TRG(run) w] - puts $fd $job(cmd) + puts $fd $set_tmp_dir + puts $fd $job(cmd) close $fd set fd [open "|$TRG(runcmd) 2>@1" r] cd $pwd - fconfigure $fd -blocking false + fconfigure $fd -blocking false -translation binary fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)] } return 1 } -proc one_line_report {} { +# Show the testing progress report +# +proc progress_report {} { global TRG - set tm [expr [clock_milliseconds] - $TRG(starttime)] - set tm [format "%d" [expr int($tm/1000.0 + 0.5)]] - - r_write_db { - trdb eval { - SELECT displaytype, state, count(*) AS cnt - FROM jobs - GROUP BY 1, 2 - } { - set v($state,$displaytype) $cnt - incr t($displaytype) $cnt + if {$TRG(fullstatus)} { + if {$::tcl_platform(platform)=="windows"} { + exec [info nameofexe] $::argv0 status --cls + } else { + show_status trdb 1 } - } - - set text "" - foreach j [lsort [array names t]] { - foreach k {done failed running} { incr v($k,$j) 0 } - set fin [expr $v(done,$j) + $v(failed,$j)] - lappend text "${j}($fin/$t($j))" - if {$v(failed,$j)>0} { - lappend text "f$v(failed,$j)" + } else { + set tm [expr [clock_milliseconds] - $TRG(starttime)] + set tm [format "%d" [expr int($tm/1000.0 + 0.5)]] + + r_write_db { + trdb eval { + SELECT displaytype, state, count(*) AS cnt + FROM jobs + GROUP BY 1, 2 + } { + set v($state,$displaytype) $cnt + incr t($displaytype) $cnt + } } - if {$v(running,$j)>0} { - lappend text "r$v(running,$j)" + + set text "" + foreach j [lsort [array names t]] { + foreach k {done failed running} { incr v($k,$j) 0 } + set fin [expr $v(done,$j) + $v(failed,$j)] + lappend text "${j}($fin/$t($j))" + if {$v(failed,$j)>0} { + lappend text "f$v(failed,$j)" + } + if {$v(running,$j)>0} { + lappend text "r$v(running,$j)" + } + } + + if {[info exists TRG(reportlength)]} { + puts -nonewline "[string repeat " " $TRG(reportlength)]\r" + } + set report "${tm} [join $text { }]" + set TRG(reportlength) [string length $report] + if {[string length $report]<100} { + puts -nonewline "$report\r" + flush stdout + } else { + puts $report } } - - if {[info exists TRG(reportlength)]} { - puts -nonewline "[string repeat " " $TRG(reportlength)]\r" - } - set report "${tm} [join $text { }]" - set TRG(reportlength) [string length $report] - if {[string length $report]<100} { - puts -nonewline "$report\r" - flush stdout - } else { - puts $report - } - - after $TRG(reporttime) one_line_report + after $TRG(reporttime) progress_report } proc launch_some_jobs {} { @@ -1041,13 +1342,14 @@ proc run_testset {} { launch_some_jobs - one_line_report + if {$TRG(fullstatus)} {puts "\033\[2J"} + progress_report while {[dirs_nHelper]>0} { after 500 {incr ::wakeup} vwait ::wakeup } close $TRG(log) - one_line_report + progress_report r_write_db { set tm [clock_milliseconds] @@ -1061,6 +1363,10 @@ proc run_testset {} { puts "FAILED: $displayname" } } + set nOmit [trdb one {SELECT count(*) FROM jobs WHERE state='omit'}] + if {$nOmit>0} { + puts "$nOmit jobs skipped due to prior failures" + } } puts "\nTest database is $TRG(dbname)" @@ -1078,15 +1384,42 @@ proc handle_buildonly {} { } } +# Handle the --explain option. Provide a human-readable +# explanation of all the tests that are in the trdb database jobs +# table. +# +proc explain_layer {indent depid} { + global TRG + if {$TRG(buildonly)} { + set showtests 0 + } else { + set showtests 1 + } + trdb eval {SELECT jobid, displayname, displaytype, dirname + FROM jobs WHERE depid=$depid ORDER BY displayname} { + if {$displaytype=="bld"} { + puts "${indent}$displayname in $dirname" + explain_layer "${indent} " $jobid + } elseif {$showtests} { + puts "${indent}[lindex $displayname end]" + } + } +} +proc explain_tests {} { + explain_layer "" "" +} + sqlite3 trdb $TRG(dbname) trdb timeout $TRG(timeout) set tm [lindex [time { make_new_testset }] 0] -if {$TRG(nJob)>1} { - puts "splitting work across $TRG(nJob) jobs" +if {$TRG(explain)} { + explain_tests +} else { + if {$TRG(nJob)>1} { + puts "splitting work across $TRG(nJob) jobs" + } + puts "built testset in [expr $tm/1000]ms.." + handle_buildonly + run_testset } -puts "built testset in [expr $tm/1000]ms.." - -handle_buildonly -run_testset trdb close -#puts [pwd] diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 984c6d827..af480fc4c 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -23,6 +23,7 @@ namespace eval trd { set tcltest(linux.No-lookaside) veryquick set tcltest(linux.Devkit) veryquick set tcltest(linux.Apple) veryquick + set tcltest(linux.Android) veryquick set tcltest(linux.Sanitize) veryquick set tcltest(linux.Device-One) all set tcltest(linux.Default) all_plus_autovacuum_crash @@ -53,6 +54,7 @@ namespace eval trd { set extra(linux.Device-Two) {fuzztest sourcetest threadtest} set extra(linux.No-lookaside) {fuzztest sourcetest} set extra(linux.Devkit) {fuzztest sourcetest} + set extra(linux.Android) {fuzztest sourcetest} set extra(linux.Apple) {fuzztest sourcetest} set extra(linux.Sanitize) {fuzztest sourcetest} set extra(linux.Default) {fuzztest sourcetest threadtest} @@ -108,6 +110,7 @@ namespace eval trd { -DSQLITE_ENABLE_STAT4 -DSQLITE_OMIT_LOOKASIDE=1 -DCONFIG_SLOWDOWN_FACTOR=5.0 + -DSQLITE_ENABLE_RBU --enable-debug --enable-all } @@ -245,6 +248,40 @@ namespace eval trd { -O2 -DSQLITE_ENABLE_LOCKING_STYLE=1 } + set build(Android) { + -Os + -DHAVE_USLEEP=1 + -DSQLITE_HAVE_ISNAN + -DSQLITE_POWERSAFE_OVERWRITE=1 + -DSQLITE_DEFAULT_FILE_FORMAT=4 + -DSQLITE_DEFAULT_AUTOVACUUM=1 + -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 + -DSQLITE_ENABLE_FTS3 + -DSQLITE_ENABLE_FTS3_BACKWARDS + -DSQLITE_ENABLE_FTS4 + -DSQLITE_SECURE_DELETE + -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE + -DBIONIC_IOCTL_NO_SIGNEDNESS_OVERLOAD + -DSQLITE_ALLOW_ROWID_IN_VIEW + -DSQLITE_ENABLE_BYTECODE_VTAB + -Wno-unused-parameter + -Werror + -DUSE_PREAD64 + -Dfdatasync=fdatasync + -DHAVE_MALLOC_H=1 + -DHAVE_MALLOC_USABLE_SIZE + -DSQLITE_ENABLE_DBSTAT_VTAB + } + # Compile-options used by Android but omitted from these + # tests: + # -DNDEBUG=1 + # -DSQLITE_DEFAULT_LEGACY_ALTER_TABLE + # -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576 + # -DSQLITE_DEFAULT_FILE_PERMISSIONS=0600 + # -DSQLITE_OMIT_BUILTIN_TEST + # -DSQLITE_OMIT_LOAD_EXTENSION + # -DSQLITE_OMIT_COMPILEOPTION_DIAGS + # set build(Apple) { -Os -DHAVE_GMTIME_R=1 @@ -449,7 +486,7 @@ proc make_sh_script {srcdir opts cflags makeOpts configOpts} { TCLDIR="$tcldir" if [ ! -f Makefile ] ; then - \$SRCDIR/configure --with-tcl=\$TCL $configOpts + \$SRCDIR/configure --with-tcl=\$TCLDIR $configOpts fi $myopts @@ -598,7 +635,12 @@ proc trd_buildscript {config srcdir bMsvc} { # Ensure that the named configuration exists. if {![info exists build($config)]} { - error "No such build config: $config" + if {$config!="help"} { + puts "No such build config: $config" + } + puts "Available configurations: [lsort [array names build]]" + flush stdout + exit 1 } # Generate and return the script. @@ -637,4 +679,3 @@ proc trd_test_script_properties {path} { set trd_test_script_properties_cache($path) } - diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test index 1f797686b..5dad78169 100644 --- a/test/tkt-2d1a5c67d.test +++ b/test/tkt-2d1a5c67d.test @@ -102,6 +102,7 @@ do_test 3.4 { set blobs [list] for {set i 1} {$i<100} {incr i} { set b [db incrblob -readonly t3 b $i] + fconfigure $b -translation binary read $b lappend blobs $b } diff --git a/test/tkt-8454a207b9.test b/test/tkt-8454a207b9.test index 20e142057..42b95c8f5 100644 --- a/test/tkt-8454a207b9.test +++ b/test/tkt-8454a207b9.test @@ -49,7 +49,7 @@ do_test tkt-8454a207b9.4 { ALTER TABLE t1 ADD COLUMN e DEFAULT -123.0; SELECT e, typeof(e) FROM t1; } -} {-123 integer} +} {-123.0 real} do_test tkt-8454a207b9.5 { db eval { ALTER TABLE t1 ADD COLUMN f DEFAULT -123.5; diff --git a/test/tkt-bd484a090c.test b/test/tkt-bd484a090c.test index 3d2b59995..7867c8dc9 100644 --- a/test/tkt-bd484a090c.test +++ b/test/tkt-bd484a090c.test @@ -30,7 +30,7 @@ do_test 2.1 { catchsql { SELECT datetime('now', 'localtime') } } {1 {local time unavailable}} do_test 2.2 { - catchsql { SELECT datetime('now', 'utc') } + catchsql { SELECT datetime('2000-01-01', 'utc') } } {1 {local time unavailable}} sqlite3_test_control SQLITE_TESTCTRL_LOCALTIME_FAULT 0 diff --git a/test/tkt3457.test b/test/tkt3457.test index 24b4f0eac..027349463 100644 --- a/test/tkt3457.test +++ b/test/tkt3457.test @@ -58,7 +58,7 @@ do_test tkt3457-1.1 { # start of the first journal-header has not been written by SQLite. # So write it now. set fd [open bak.db-journal a+] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary seek $fd 0 puts -nonewline $fd "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7" close $fd diff --git a/test/trigger1.test b/test/trigger1.test index 6de121fa9..afeb7ddcc 100644 --- a/test/trigger1.test +++ b/test/trigger1.test @@ -838,4 +838,17 @@ do_catchsql_test trigger1-23.1 { END; } {1 {near "#1": syntax error}} +# 2024-05-08 Allow arbitrary expressions as the 2nd argument to RAISE(). +# +do_catchsql_test trigger1-24.1 { + CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN + SELECT raise(abort,format('attempt to insert %d where is not a power of 2',new.a)) + WHERE (new.a & (new.a-1))!=0; + END; + INSERT INTO t1 VALUES(0),(1),(2),(4),(8),(65536); +} {0 {}} +do_catchsql_test trigger1-24.2 { + INSERT INTO t1 VALUES(9876); +} {1 {attempt to insert 9876 where is not a power of 2}} + finish_test diff --git a/test/trigger9.test b/test/trigger9.test index 6e31d1af9..47940de57 100644 --- a/test/trigger9.test +++ b/test/trigger9.test @@ -242,12 +242,27 @@ do_execsql_test 4.1 { END; } -do_catchsql_test 4.2 { - DELETE FROM v1 WHERE rowid=1; -} {1 {no such column: rowid}} +ifcapable !allow_rowid_in_view { + do_catchsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; + } {1 {no such column: rowid}} -do_catchsql_test 4.3 { - UPDATE v1 SET a=b WHERE rowid=2; -} {1 {no such column: rowid}} + do_catchsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; + } {1 {no such column: rowid}} +} else { + do_execsql_test 4.2a { + DELETE FROM log; + } + do_catchsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; + } {0 {}} + do_catchsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; + } {0 {}} + do_execsql_test 4.3b { + SELECT * FROM log; + } +} finish_test diff --git a/test/types3.test b/test/types3.test index 807ae84f9..457ee6d68 100644 --- a/test/types3.test +++ b/test/types3.test @@ -12,18 +12,15 @@ # of this file is testing the interaction of SQLite manifest types # with Tcl dual-representations. # -# $Id: types3.test,v 1.8 2008/04/28 13:02:58 drh Exp $ -# set testdir [file dirname $argv0] source $testdir/tester.tcl # A variable with only a string representation comes in as TEXT do_test types3-1.1 { - set V {} - append V x + set V [format %s xxxxx] concat [tcl_variable_type V] [execsql {SELECT typeof(:V)}] -} {string text} +} {text} # A variable with an integer representation comes in as INTEGER do_test types3-1.2 { @@ -96,4 +93,32 @@ do_test types3-2.6 { tcl_variable_type V } {} +# See https://sqlite.org/forum/forumpost/3776b48e71 +# +# On a text-affinity comparison of two values where one of +# the values has both MEM_Str and a numeric type like MEM_Int, +# make sure that only the MEM_Str representation is used. +# +sqlite3_create_function db +do_execsql_test types3-3.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x TEXT PRIMARY KEY); + INSERT INTO t1 VALUES('1'); + SELECT * FROM t1 WHERE NOT x=upper(1); +} {} +do_execsql_test types3-3.2 { + SELECT * FROM t1 WHERE NOT x=add_text_type(1); +} {} +do_execsql_test types3-3.3 { + SELECT * FROM t1 WHERE NOT x=add_int_type('1'); +} {} +do_execsql_test types3-3.4 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1.25); + SELECT * FROM t1 WHERE NOT x=add_real_type('1.25'); +} {} +do_execsql_test types3-3.5 { + SELECT * FROM t1 WHERE NOT x=add_text_type(1.25); +} {} + finish_test diff --git a/test/unionall.test b/test/unionall.test index 99cb48a25..905719907 100644 --- a/test/unionall.test +++ b/test/unionall.test @@ -351,7 +351,7 @@ do_catchsql_test 5.30 { SELECT * FROM (t1 NATURAL JOIN pragma_table_xinfo('t1_a') NATURAL JOIN t3) t1 NATURAL JOIN t2 NATURAL JOIN t3 WHERE rowid ISNULL>0 AND 0%y; -} {1 {no such column: rowid}} +} {1 {ambiguous column name: rowid}} } reset_db diff --git a/test/upsert1.test b/test/upsert1.test index 781831133..8af273a89 100644 --- a/test/upsert1.test +++ b/test/upsert1.test @@ -268,4 +268,28 @@ do_catchsql_test upsert1-1210 { INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT(b+?1) DO NOTHING; } {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}} +# 2024-04-11 https://sqlite.org/forum/forumpost/284955a3cd454a15 +# Incorrect value passed into a trigger that fires as the result of +# an upsert. +# +reset_db +do_execsql_test upsert1-1300 { + CREATE TABLE t1(x INT, y TEXT); + INSERT INTO t1 VALUES + (11, printf('%.9000c','a')), + (11, printf('%.9000c','a')), + (33, printf('%.9000c','b')), + (33, printf('%.9000c','b')); + CREATE TABLE t2(x INT UNIQUE, y TEXT); + CREATE TRIGGER r1 BEFORE UPDATE ON t2 BEGIN + SELECT raise(ABORT,'Incorrect old.y value passed to trigger!') + WHERE old.y != new.y; + /* ^^^ This trigger will fire and cause the ABORT if the problem has + ** not been fixed, or if there is a regression. */ + END; + INSERT INTO t2(x, y) SELECT x, y FROM t1 + WHERE true + ON CONFLICT (x) DO UPDATE SET y = excluded.y; +} {} + finish_test diff --git a/test/upsert5.test b/test/upsert5.test index 3161abf15..e56e71d4b 100644 --- a/test/upsert5.test +++ b/test/upsert5.test @@ -408,4 +408,46 @@ do_catchsql_test 2.1 { } {1 {no such table: nosuchtable}} +# 2024-03-08 https://sqlite.org/forum/forumpost/919c6579c8 +# A redundant ON CONFLICT clause in an upsert can lead to +# index corruption. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(aa INTEGER PRIMARY KEY, bb INT); + INSERT INTO t1 VALUES(11,22); + CREATE UNIQUE INDEX t1bb ON t1(bb); + REPLACE INTO t1 VALUES(11,33) + ON CONFLICT(bb) DO UPDATE SET aa = 44 + ON CONFLICT(bb) DO UPDATE SET aa = 44; + PRAGMA integrity_check; +} {ok} +do_execsql_test 3.1 { + SELECT * FROM t1 NOT INDEXED; +} {11 33} +do_execsql_test 3.2 { + SELECT * FROM t1 INDEXED BY t1bb; +} {11 33} +do_execsql_test 3.3 { + DROP TABLE t1; + CREATE TABLE t1(aa INTEGER PRIMARY KEY, bb INT, cc INT); + INSERT INTO t1 VALUES(10,21,32),(11,22,33),(12,23,34); + CREATE UNIQUE INDEX t1bb ON t1(bb); + CREATE UNIQUE INDEX t1cc ON t1(cc); + REPLACE INTO t1 VALUES(11,44,55) + ON CONFLICT(bb) DO UPDATE SET aa = 99 + ON CONFLICT(cc) DO UPDATE SET aa = 99 + ON CONFLICT(bb) DO UPDATE SET aa = 99; + PRAGMA integrity_check; +} {ok} +do_execsql_test 3.4 { + SELECT * FROM t1 NOT INDEXED ORDER BY +aa; +} {10 21 32 11 44 55 12 23 34} +do_execsql_test 3.5 { + SELECT * FROM t1 INDEXED BY t1bb ORDER BY +aa; +} {10 21 32 11 44 55 12 23 34} +do_execsql_test 3.6 { + SELECT * FROM t1 INDEXED BY t1cc ORDER BY +aa; +} {10 21 32 11 44 55 12 23 34} + finish_test diff --git a/test/vacuum-into.test b/test/vacuum-into.test index 698d65f54..d559b7fb3 100644 --- a/test/vacuum-into.test +++ b/test/vacuum-into.test @@ -26,13 +26,36 @@ ifcapable {!vacuum} { forcedelete out.db do_execsql_test vacuum-into-100 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b ANY, + c INT AS (b+1), --- See "2024-04-09" block + CHECK( typeof(b)!='integer' OR b>a-5 ) --- comment below + ); WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c; CREATE INDEX t1b ON t1(b); DELETE FROM t1 WHERE a%2; SELECT count(*), sum(a), sum(length(b)) FROM t1; } {50 2550 30000} + +# Update 2024-04-09 for forum post eec177d68fe7fa2c. +# +# VACUUM INTO is sensitive to tables holding both generated columns +# and CHECK constraints. +# +# CHECK constraints are ignored for read-only databases in order to save +# memory (see check-in 34ddf02d3d21151b on 2014-05-21). But the xfer +# optimization normally only works if CHECK constraints match between the +# source and destination tables. So the xfer optimization was not +# working for VACUUM INTO when the source was a read-only database and the +# table held CHECK constraints. But if the table has generated columns, +# then the xfer optimization is required or else VACUUM will raise an +# error. +# +# Fix this by ignoring CHECK constraints when determining whether or not +# the xfer optimization can run while doing VACUUM. + do_execsql_test vacuum-into-110 { VACUUM main INTO 'out.db'; } {} @@ -88,11 +111,21 @@ do_catchsql_test vacuum-into-420 { # The ability to VACUUM INTO a read-only database db close +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 1 +} else { + file attributes test.db -permissions 292 ;# 292 == 0444 +} sqlite3 db test.db -readonly 1 forcedelete test.db2 do_execsql_test vacuum-into-500 { VACUUM INTO 'test.db2'; } +if {$tcl_platform(platform)=="windows"} { + file attributes test.db -readonly 0 +} else { + file attributes test.db -permissions 420 ;# 420 = 0644 +} sqlite3 db2 test.db2 do_test vacuum-into-510 { db2 eval {SELECT name FROM sqlite_master ORDER BY 1} diff --git a/test/values.test b/test/values.test new file mode 100644 index 000000000..c3c52ceb1 --- /dev/null +++ b/test/values.test @@ -0,0 +1,715 @@ +# 2024 March 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix values + + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, c); +} + + +explain_i { + INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); +} +do_execsql_test 1.1.1 { + INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); +} +do_execsql_test 1.1.2 { + SELECT * FROM x1; +} { + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_execsql_test 1.2.0 { + DELETE FROM x1 +} +do_execsql_test 1.2.1 { + INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3) UNION ALL SELECT 4, 4, 4; + SELECT * FROM x1; +} {1 1 1 2 2 2 3 3 3 4 4 4} + +sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 4 + +do_execsql_test 1.2.2 { + DELETE FROM x1; + INSERT INTO x1 + VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, 5) + UNION ALL SELECT 6, 6, 6; + SELECT * FROM x1; +} {1 1 1 2 2 2 3 3 3 4 4 4 5 5 5 6 6 6} + +do_execsql_test 1.2.3 { + DELETE FROM x1; + INSERT INTO x1 + VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4) + UNION ALL SELECT 6, 6, 6; + SELECT * FROM x1; +} {1 1 1 2 2 2 3 3 3 4 4 4 6 6 6} + +do_execsql_test 1.2.4 { + DELETE FROM x1; + INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3) UNION ALL SELECT 6, 6, 6; + SELECT * FROM x1; +} { + 1 1 1 + 2 2 2 + 3 3 3 + 6 6 6 +} + +set a 4 +set b 5 +set c 6 +do_execsql_test 1.2.5 { + DELETE FROM x1; + INSERT INTO x1 + VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), + (4, 4, $a), (5, 5, $b), (6, 6, $c) +} + +do_execsql_test 1.2.6 { + SELECT * FROM x1; +} { + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 + 5 5 5 + 6 6 6 +} + +#------------------------------------------------------------------------- +# SQLITE_LIMIT_COMPOUND_SELECT set to 0. +# +reset_db + +do_execsql_test 2.0 { + CREATE TABLE x1(a, b, c); +} + +sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 3 + +do_catchsql_test 2.1.1 { + INSERT INTO x1 VALUES + (1, 1, 1), + (2, 2, 2), + (3, 3, 3), + (4, 4, 4), + (5, 5, 5), + (6, 6, 6), + (7, 7, 7), + (8, 8, 8), + (9, 9, 9), + (10, 10, 10, 10) +} {1 {all VALUES must have the same number of terms}} + +do_catchsql_test 2.1.2 { + INSERT INTO x1 VALUES + (1, 1, 1), + (2, 2, 2, 2), + (3, 3, 3), + (4, 4, 4), + (5, 5, 5), + (6, 6, 6), + (7, 7, 7), + (8, 8, 8), + (9, 9, 9), + (10, 10, 10) +} {1 {all VALUES must have the same number of terms}} + +sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 0 + +do_execsql_test 2.2 { + INSERT INTO x1 VALUES + (1, 1, 1), + (2, 2, 2), + (3, 3, 3), + (4, 4, 4), + (5, 5, 5), + (6, 6, 6), + (7, 7, 7), + (8, 8, 8), + (9, 9, 9), + (10, 10, 10) +} {} +do_execsql_test 2.3 { + INSERT INTO x1 VALUES + (1, 1, 1), + (2, 2, 2), + (3, 3, 3), + (4, 4, 4), + (5, 5, 5), + (6, 6, 6), + (7, 7, 7), + (8, 8, 8), + (9, 9, 9), + (10, 10, 10) + UNION ALL + SELECT 5, 12, 12 + ORDER BY 1 +} {} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE TABLE y1(x, y); +} + +do_execsql_test 3.1.1 { + DELETE FROM y1; + INSERT INTO y1 VALUES(1, 2), (3, 4), (row_number() OVER (), 5); +} +do_execsql_test 3.1.2 { + SELECT * FROM y1; +} {1 2 3 4 1 5} +do_execsql_test 3.2.1 { + DELETE FROM y1; + INSERT INTO y1 VALUES(1, 2), (3, 4), (row_number() OVER (), 6) + , (row_number() OVER (), 7) +} +do_execsql_test 3.1.2 { + SELECT * FROM y1; +} {1 2 3 4 1 6 1 7} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 4.0 { + CREATE TABLE x1(a PRIMARY KEY, b) WITHOUT ROWID; +} + +foreach {tn iLimit} {1 0 2 3} { + sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT $iLimit + + do_execsql_test 4.1.1 { + DELETE FROM x1; + INSERT INTO x1 VALUES + (1, 1), + (2, (SELECT * FROM (VALUES('a'), ('b'), ('c'), ('d')) )) + } + do_execsql_test 4.1.2 { + SELECT * FROM x1 + } {1 1 2 a} + + do_execsql_test 4.2.1 { + DELETE FROM x1; + INSERT INTO x1 VALUES + (1, 1), + (2, 2), + (3, 3), + (4, 4), + (5, (SELECT * FROM (VALUES('a'), ('b'), ('c'), ('d')) )) + } + do_execsql_test 4.2.2 { + SELECT * FROM x1 + } {1 1 2 2 3 3 4 4 5 a} + + do_execsql_test 4.3.1 { + DELETE FROM x1; + INSERT INTO x1 VALUES + (1, (SELECT * FROM (VALUES('a'), ('b'), ('c'), ('d'), ('e')) )) + } + do_execsql_test 4.3.2 { + SELECT * FROM x1 + } {1 a} +} + +#------------------------------------------------------------------------ +reset_db + +do_execsql_test 5.0 { + CREATE VIEW v1 AS VALUES(1, 2, 3), (4, 5, 6), (7, 8, 9); +} +do_execsql_test 5.1 { + SELECT * FROM v1 +} {1 2 3 4 5 6 7 8 9} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1), (2); +} + +do_execsql_test 6.1 { + SELECT ( VALUES( x ), ( x ) ) FROM t1; +} {1 2} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('x'), ('y'); +} + +do_execsql_test 6.1 { + SELECT * FROM t1, (VALUES(1), (2)) +} {x 1 x 2 y 1 y 2} + +do_execsql_test 6.2 { + VALUES(CAST(44 AS REAL)),(55); +} {44.0 55} + +#------------------------------------------------------------------------ +do_execsql_test 7.1 { + WITH x1(a, b) AS ( + VALUES(1, 2), ('a', 'b') + ) + SELECT * FROM x1 one, x1 two +} { + 1 2 1 2 + 1 2 a b + a b 1 2 + a b a b +} + +#------------------------------------------------------------------------- +reset_db + +set VVV { + ( VALUES('a', 'b'), ('c', 'd'), (123, NULL) ) +} +set VVV2 { + ( + SELECT 'a' AS column1, 'b' AS column2 + UNION ALL SELECT 'c', 'd' UNION ALL SELECT 123, NULL + ) +} + +do_execsql_test 8.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('d'), (NULL), (123) +} +foreach {tn q res} { + 1 "SELECT * FROM t1 LEFT JOIN VVV" { + d a b d c d d 123 {} + {} a b {} c d {} 123 {} + 123 a b 123 c d 123 123 {} + } + + 2 "SELECT * FROM t1 LEFT JOIN VVV ON (column1=x)" { + d {} {} + {} {} {} + 123 123 {} + } + + 3 "SELECT * FROM t1 RIGHT JOIN VVV" { + d a b d c d d 123 {} + {} a b {} c d {} 123 {} + 123 a b 123 c d 123 123 {} + } + + 4 "SELECT * FROM t1 RIGHT JOIN VVV ON (column1=x)" { + 123 123 {} + {} a b + {} c d + } + + 5 "SELECT * FROM t1 FULL OUTER JOIN VVV ON (column1=x)" { + d {} {} + {} {} {} + 123 123 {} + {} a b + {} c d + } + + 6 "SELECT count(*) FROM VVV" { 3 } + + 7 "SELECT (SELECT column1 FROM VVV)" { a } + + 8 "SELECT * FROM VVV UNION ALL SELECT * FROM VVV" { + a b c d 123 {} + a b c d 123 {} + } + + 9 "SELECT * FROM VVV INTERSECT SELECT * FROM VVV" { + 123 {} a b c d + } + + 10 "SELECT * FROM VVV eXCEPT SELECT * FROM VVV" { } + + 11 "SELECT * FROM VVV eXCEPT SELECT 'a', 'b'" { 123 {} c d } + +} { + set q1 [string map [list VVV $VVV] $q] + set q2 [string map [list VVV $VVV2] $q] + set q3 "WITH VVV AS $VVV $q" + + do_execsql_test 8.1.$tn.1 $q1 $res + do_execsql_test 8.1.$tn.2 $q2 $res + do_execsql_test 8.1.$tn.3 $q3 $res +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 9.1 { + VALUES(456), (123), (NULL) UNION ALL SELECT 122 ORDER BY 1 +} { {} 122 123 456 } + +do_execsql_test 9.2 { + VALUES (1, 2), (3, 4), ( + ( SELECT column1 FROM ( VALUES (5, 6), (7, 8) ) ), + ( SELECT max(column2) FROM ( VALUES (5, 1), (7, 6) ) ) + ) +} { 1 2 3 4 5 6 } + +do_execsql_test 10.1 { + CREATE TABLE a2(a, b, c DEFAULT 'xyz'); +} +do_execsql_test 10.2 { + INSERT INTO a2(a) VALUES(3),(4); +} + +#------------------------------------------------------------------------- +reset_db +ifcapable fts3 { + do_execsql_test 11.0 { + CREATE VIRTUAL TABLE ft USING fts3(x); + } + do_execsql_test 11.1 { + INSERT INTO ft VALUES('one'), ('two'); + } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 12.0 { + CREATE TABLE t1(a, b); +} +do_execsql_test 12.1 { + INSERT INTO t1 SELECT 1, 2 UNION ALL VALUES(3, 4), (5, 6); +} +do_execsql_test 12.2 { + SELECT * FROM t1 +} {1 2 3 4 5 6} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 13.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('xyz'); + + SELECT ( + VALUES( (max(substr('abc', 1, 1), x)) ), + (123), + (456) + ) + FROM t1; +} {xyz} + +do_catchsql_test 13.1 { + VALUES(300), (zeroblob(300) OVER win); +} {1 {zeroblob() may not be used as a window function}} + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 14.1 { + PRAGMA encoding = utf16; + CREATE TABLE t1(a, b); +} {} + +db close +sqlite3 db test.db + +do_execsql_test 14.2 { + INSERT INTO t1 VALUES + (17, 'craft'), + (16, 'urtlek' IN(1,2,3)); +} + +#-------------------------------------------------------------------------- +# +reset_db +do_eqp_test 15.1 { + VALUES(1),(2),(3),(4),(5); +} { + QUERY PLAN + `--SCAN 5-ROW VALUES CLAUSE +} +do_execsql_test 15.2 { + CREATE TABLE t1(a,b); +} +do_eqp_test 15.3 { + INSERT INTO t1 VALUES + (1,2),(3,4),(7,8); +} { + QUERY PLAN + `--SCAN 3-ROW VALUES CLAUSE +} +do_eqp_test 15.4 { + INSERT INTO t1 VALUES + (1,2),(3,4),(7,8), + (5,row_number()OVER()); +} { + QUERY PLAN + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | `--SCAN 3-ROW VALUES CLAUSE + `--UNION ALL + |--CO-ROUTINE (subquery-xxxxxx) + | `--SCAN CONSTANT ROW + `--SCAN (subquery-xxxxxx) +} +do_eqp_test 15.5 { + SELECT * FROM (VALUES(1),(2),(3),(4),(5),(6)), (VALUES('a'),('b'),('c')); +} { + QUERY PLAN + |--SCAN 6-ROW VALUES CLAUSE + `--SCAN 3-ROW VALUES CLAUSE +} +do_execsql_test 15.6 { + CREATE TABLE t2(x,y); +} +do_eqp_test 15.7 { + SELECT * FROM t2 UNION ALL VALUES(1,2),(3,4),(5,6),(7,8); +} { + QUERY PLAN + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | `--SCAN t2 + `--UNION ALL + `--SCAN 4-ROW VALUES CLAUSE +} + +#-------------------------------------------------------------------------- +# The VALUES-as-coroutine optimization can be applied to later rows of +# a VALUES clause even if earlier rows do not qualify. +# +reset_db +do_execsql_test 16.1 { + CREATE TABLE t1(a,b); +} +do_execsql_test 16.2 { + BEGIN; + INSERT INTO t1 VALUES(1,2),(3,4),(5,6), + (7,row_number()OVER()), + (9,10), (11,12), (13,14), (15,16); + SELECT * FROM t1 ORDER BY a, b; + ROLLBACK; +} {1 2 3 4 5 6 7 1 9 10 11 12 13 14 15 16} +do_eqp_test 16.3 { + INSERT INTO t1 VALUES(1,2),(3,4),(5,6), + (7,row_number()OVER()), + (9,10), (11,12), (13,14), (15,16); +} { + QUERY PLAN + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | `--SCAN 3-ROW VALUES CLAUSE + |--UNION ALL + | |--CO-ROUTINE (subquery-xxxxxx) + | | `--SCAN CONSTANT ROW + | `--SCAN (subquery-xxxxxx) + `--UNION ALL + `--SCAN 4-ROW VALUES CLAUSE +} +do_execsql_test 16.4 { + BEGIN; + INSERT INTO t1 VALUES + (1,row_number()OVER()), + (2,3), (4,5), (6,7); + SELECT * FROM t1 ORDER BY a, b; + ROLLBACK; +} {1 1 2 3 4 5 6 7} +do_eqp_test 16.5 { + INSERT INTO t1 VALUES + (1,row_number()OVER()), + (2,3), (4,5), (6,7); +} { + QUERY PLAN + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | |--CO-ROUTINE (subquery-xxxxxx) + | | `--SCAN CONSTANT ROW + | `--SCAN (subquery-xxxxxx) + `--UNION ALL + `--SCAN 3-ROW VALUES CLAUSE +} +do_execsql_test 16.6 { + BEGIN; + INSERT INTO t1 VALUES + (1,2),(3,4), + (5,row_number()OVER()), + (7,8),(9,10),(11,12), + (13,row_number()OVER()), + (15,16),(17,18),(19,20),(21,22); + SELECT * FROM t1 ORDER BY a, b; + ROLLBACK; +} { 1 2 3 4 5 1 7 8 9 10 11 12 13 1 15 16 17 18 19 20 21 22} +do_eqp_test 16.7 { + INSERT INTO t1 VALUES + (1,2),(3,4), + (5,row_number()OVER()), + (7,8),(9,10),(11,12), + (13,row_number()OVER()), + (15,16),(17,18),(19,20),(21,22); +} { + QUERY PLAN + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | `--SCAN 2-ROW VALUES CLAUSE + |--UNION ALL + | |--CO-ROUTINE (subquery-xxxxxx) + | | `--SCAN CONSTANT ROW + | `--SCAN (subquery-xxxxxx) + |--UNION ALL + | `--SCAN 3-ROW VALUES CLAUSE + |--UNION ALL + | |--CO-ROUTINE (subquery-xxxxxx) + | | `--SCAN CONSTANT ROW + | `--SCAN (subquery-xxxxxx) + `--UNION ALL + `--SCAN 4-ROW VALUES CLAUSE +} + +#-------------------------------------------------------------------------- +# 2024-03-23 dbsqlfuzz crash-c2c5e7e08b7e489d270a26d895077a03f678c33b +# +do_execsql_test 17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 AS SELECT * FROM (VALUES(1,2), (3,4 IN (1,2,3))); +} + +do_execsql_test 17.2 { + SELECT * FROM t1 +} {1 2 3 0} + +# 2024-03-25 dbsqlfuzz crash-74cf7c9904360322a6c917e4934b127543d1cd51 +# +do_catchsql_test 18.1 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(RAISE(IGNORE)),(0); +} {1 {RAISE() may only be used within a trigger-program}} +do_catchsql_test 18.2 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN + INSERT INTO t1(y) VALUES(RAISE(IGNORE)),(0); + END; + INSERT INTO t1 VALUES(1,2,3); + SELECT * FROM t1; +} {0 {1 2 3}} +do_catchsql_test 18.3.1 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN + INSERT INTO t1(y) VALUES(RAISE(ABORT,'error 18.3')),(0); + END; + INSERT INTO t1 VALUES(1,2,3); +} {1 {error 18.3}} +do_execsql_test 18.3.2 { + SELECT * FROM t1; +} {} +do_catchsql_test 18.4.1 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN + INSERT INTO t1(y) VALUES(1),(RAISE(ABORT,'error 18.4')),(0); + END; + INSERT INTO t1 VALUES(1,2,3); +} {1 {error 18.4}} +do_execsql_test 18.4.2 { + SELECT * FROM t1; +} {} +do_catchsql_test 18.5.1 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN + INSERT INTO t1(y) VALUES(1), + (CASE WHEN new.z>7 THEN RAISE(ABORT,'error 18.5') ELSE 2 END); + END; + INSERT INTO t1 VALUES(1,2,3); + SELECT * FROM t1; +} {0 {1 2 3 2 1 {} 3 2 {}}} +do_catchsql_test 18.5.2 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,2,13); +} {1 {error 18.5}} +do_catchsql_test 18.5.3 { + SELECT * FROM t1; +} {0 {}} + +# 2024-04-18 dbsqlfuzz crash-bde3bf80aedf25afa56e2997a0545a314765d3f8 +# Verify that the VALUES expressions used as an argument to an outer +# join work correctly. +# +reset_db +db null NULL +do_execsql_test 19.1 { + CREATE TABLE t1(a INT, b INT); + INSERT INTO t1 VALUES(11,22); + SELECT * FROM t1 LEFT JOIN (VALUES(33,44),(55,66)) AS t2 ON a=b; +} {11 22 NULL NULL} +do_execsql_test 19.2 { + SELECT * FROM (VALUES(33,44),(55,66)) AS t2 RIGHT JOIN t1 ON a=b; +} {NULL NULL 11 22} +do_execsql_test 19.3 { + SELECT *, '|' FROM t1 FULL JOIN (VALUES(33,44),(55,66)) AS t2 ON a=b + ORDER BY +column1 +} {11 22 NULL NULL | NULL NULL 33 44 | NULL NULL 55 66 |} +do_execsql_test 19.4 { + SELECT *, '|' FROM (VALUES(33,44),(55,66)) AS t2 FULL JOIN t1 ON a=b + ORDER BY +column1 +} {NULL NULL 11 22 | 33 44 NULL NULL | 55 66 NULL NULL |} + +# 2024-04-21 dbsqlfuzz 6fd1ff3a64bef4a6c092e8d757548e95698b0df5 +# A continuation of the 2024-04-18 problem above. We have to create +# Pseudo-cursor that is always NULL on the viaCoroutine loop in case +# there are OP_Columns generated against it by the sub-WHERE clause. +# +db null N +do_execsql_test 19.5 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + CREATE TABLE t1(a,b); INSERT INTO t1 VALUES(1,2); + CREATE TABLE t2(column1,column2); INSERT INTO t2 VALUES(11,22),(33,44); + CREATE TABLE t3(d,e); INSERT INTO t3 VALUES(3,4); +} +do_execsql_test 19.6 { + -- output verify using PG 14.2 + SELECT * + FROM t1 CROSS JOIN t2 FULL JOIN t3 ON a=d + ORDER BY +d, +column1; +} {1 2 11 22 N N + 1 2 33 44 N N + N N N N 3 4} +do_execsql_test 19.7 { + SELECT * + FROM t1 CROSS JOIN (VALUES(11,22),(33,44)) FULL JOIN t3 ON a=d + ORDER BY +d, +column1; +} {1 2 11 22 N N + 1 2 33 44 N N + N N N N 3 4} +do_execsql_test 19.8 { + -- output verified using PG 14.2 + SELECT * + FROM t1 CROSS JOIN t2 FULL JOIN t3 ON a=d + WHERE column1 IS NULL; +} {N N N N 3 4} +do_execsql_test 19.9 { + SELECT * + FROM t1 CROSS JOIN (VALUES(11,22),(33,44)) FULL JOIN t3 ON a=d + WHERE column1 IS NULL; +} {N N N N 3 4} + +finish_test diff --git a/test/valuesfault.test b/test/valuesfault.test new file mode 100644 index 000000000..bc5dddfb0 --- /dev/null +++ b/test/valuesfault.test @@ -0,0 +1,37 @@ +# 2024 March 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix valuesfault +source $testdir/malloc_common.tcl + + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, c); +} +faultsim_save_and_close + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 2 +} -body { + execsql { + INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); + } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test diff --git a/test/view.test b/test/view.test index 90c9c9909..241742025 100644 --- a/test/view.test +++ b/test/view.test @@ -801,4 +801,27 @@ ifcapable schema_pragmas { } { 0 a INT 0 {} 0 1 b BLOB 0 {} 0 } } +#----------------------------------------------------------------------- +# 2024-04-25 Trying to make type information on compound subqueries +# more predictable and rational. +# +reset_db +do_execsql_test view-31.1 { + CREATE TABLE x2(b TEXT); + CREATE TABLE x1(a TEXT); + INSERT INTO x1 VALUES('123'); + -- Two queries get the same result even though the order of terms + -- in the CTE is reversed + WITH c(x) AS ( SELECT b FROM x2 UNION SELECT 123 ) + SELECT count(*) FROM x1 WHERE a IN c; + WITH c(x) AS ( SELECT 123 UNION SELECT b FROM x2 ) + SELECT count(*) FROM x1 WHERE a IN c; +} {0 0} +do_execsql_test view-31.2 { + CREATE TABLE t3(a INTEGER, b TEXT); + INSERT INTO t3 VALUES(123, 123); + WITH s AS ( VALUES(123), (456) ) SELECT * FROM t3 WHERE b IN s; +} {123 123} + + finish_test diff --git a/test/vtabH.test b/test/vtabH.test index f1a046655..cf3dcafd9 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -128,10 +128,10 @@ if {$tcl_platform(platform)=="windows"} { set drive [string range [pwd] 0 1] set ::env(fstreeDrive) $drive } +reset_db +register_fs_module db if {$tcl_platform(platform)!="windows" || \ [regexp -nocase -- {^[A-Z]:} $drive]} { - reset_db - register_fs_module db do_execsql_test 3.0 { SELECT name FROM fsdir WHERE dir = '.' AND name = 'test.db'; SELECT name FROM fsdir WHERE dir = '.' AND name = '.' diff --git a/test/vtabL.test b/test/vtabL.test new file mode 100644 index 000000000..45528edcb --- /dev/null +++ b/test/vtabL.test @@ -0,0 +1,75 @@ +# 2024-03-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix vtabL + +ifcapable !vtab { + finish_test + return +} + +register_tcl_module db + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return $::create_table_sql + } + } + + return {} +} + +foreach {tn cts} { + 1 {SELECT 123} + 2 {SELECT 123, 456} + 3 {INSERT INTO t1 VALUES(5, 6)} + 4 {CREATE INDEX i1 ON t1(a)} + 5 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END;} + 6 {DROP TABLE nosuchtable} + 7 {DROP TABLE x1} + 8 {DROP TABLE t1} +} { + set ::create_table_sql $cts + do_catchsql_test 1.$tn { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + } {1 {declare_vtab: syntax error}} +} + +foreach {tn cts} { + 9 {CREATE TABLE xyz AS SELECT * FROM sqlite_schema} + 10 {CREATE TABLE xyz AS SELECT 1 AS 'col'} +} { + set ::create_table_sql $cts + do_catchsql_test 1.$tn { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + } {1 {declare_vtab: SQL logic error}} +} + +foreach {tn cts} { + 1 {CREATE TABLE IF NOT EXISTS t1(a, b)} + 2 {CREATE TABLE ""(a, b PRIMARY KEY) WITHOUT ROWID} +} { + set ::create_table_sql $cts + execsql { DROP TABLE IF EXISTS x1 } + do_execsql_test 2.$tn.1 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + } + do_execsql_test 2.$tn.2 { + SELECT a, b FROM x1 + } +} + +finish_test + diff --git a/test/wal.test b/test/wal.test index 234668240..50988debe 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1219,7 +1219,7 @@ foreach {tn pgsz works} { set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2] set fd [open test.db-wal w] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary puts -nonewline $fd $walhdr puts -nonewline $fd $framehdr puts -nonewline $fd $framebody diff --git a/test/wal2.test b/test/wal2.test index ae6134d8b..5e4c4ebc2 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1098,7 +1098,11 @@ if {$::tcl_platform(platform) == "unix"} { 3 00600 4 00755 } { - set effective [format %.5o [expr $permissions & ~$umask]] + if {$tcl_version>=9.0} { + set effective [format %.5d [expr $permissions & ~$umask]] + } else { + set effective [format %.5o [expr $permissions & ~$umask]] + } do_test wal2-12.2.$tn.1 { file attributes test.db -permissions $permissions string map {o 0} [file attributes test.db -permissions] diff --git a/test/wal_common.tcl b/test/wal_common.tcl index 917ad598f..cdba53b5b 100644 --- a/test/wal_common.tcl +++ b/test/wal_common.tcl @@ -65,7 +65,6 @@ proc wal_set_walhdr {filename {intlist {}}} { set fd [open $filename r+] fconfigure $fd -translation binary - fconfigure $fd -encoding binary seek $fd 0 puts -nonewline $fd $blob close $fd @@ -73,7 +72,6 @@ proc wal_set_walhdr {filename {intlist {}}} { set fd [open $filename] fconfigure $fd -translation binary - fconfigure $fd -encoding binary set blob [read $fd 24] close $fd @@ -89,5 +87,3 @@ proc wal_fix_walindex_cksum {hdrvar} { lset hdr 10 $c1 lset hdr 11 $c2 } - - diff --git a/test/walcksum.test b/test/walcksum.test index f3fc42711..10329ba6c 100644 --- a/test/walcksum.test +++ b/test/walcksum.test @@ -22,7 +22,6 @@ ifcapable !wal {finish_test ; return } # proc readfile {filename} { set fd [open $filename] - fconfigure $fd -encoding binary fconfigure $fd -translation binary set data [read $fd] close $fd @@ -59,7 +58,6 @@ proc log_checksum_write {filename iFrame endian} { set bin [binary format II $c1 $c2] set fd [open $filename r+] - fconfigure $fd -encoding binary fconfigure $fd -translation binary seek $fd $offset puts -nonewline $fd $bin @@ -114,7 +112,6 @@ proc log_checksum_writemagic {filename endian} { set val [expr {0x377f0682 | ($endian == "big" ? 1 : 0)}] set bin [binary format I $val] set fd [open $filename r+] - fconfigure $fd -encoding binary fconfigure $fd -translation binary puts -nonewline $fd $bin diff --git a/test/walslow.test b/test/walslow.test index 2a52a225d..6a0f147a0 100644 --- a/test/walslow.test +++ b/test/walslow.test @@ -109,7 +109,6 @@ foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} { forcecopy test.db-wal test2.db-wal set fd [open test2.db-wal r+] - fconfigure $fd -encoding binary fconfigure $fd -translation binary seek $fd $iOff diff --git a/test/whereL.test b/test/whereL.test index c3bdcb8f3..2e9ae219e 100644 --- a/test/whereL.test +++ b/test/whereL.test @@ -49,6 +49,33 @@ do_eqp_test 120 { |--SEARCH t2 USING INDEX sqlite_autoindex_t2_1 (a=?) `--SCAN t3 } +do_eqp_test 121 { + SELECT * FROM t1, t2, t3 + WHERE t1.a=t2.a AND t2.a=t3.j AND t3.j=abs(5) + ORDER BY t1.a; +} { + QUERY PLAN + |--SEARCH t1 USING INDEX sqlite_autoindex_t1_1 (a=?) + |--SEARCH t2 USING INDEX sqlite_autoindex_t2_1 (a=?) + `--SCAN t3 +} + +# The sqlite3ExprIsConstant() routine does not believe that +# the expression "coalesce(5,random())" is constant. So the +# optimization does not apply in this case. +# +sqlite3_create_function db +do_eqp_test 122 { + SELECT * FROM t1, t2, t3 + WHERE t1.a=t2.a AND t2.a=t3.j AND t3.j=coalesce(5,random()) + ORDER BY t1.a; +} { + QUERY PLAN + |--SCAN t3 + |--SEARCH t1 USING INDEX sqlite_autoindex_t1_1 (a=?) + |--SEARCH t2 USING INDEX sqlite_autoindex_t2_1 (a=?) + `--USE TEMP B-TREE FOR ORDER BY +} # Constant propagation in the face of collating sequences: # @@ -209,4 +236,22 @@ do_eqp_test 710 { `--SEARCH t1 USING INDEX idx (<expr>=?) } +# 2024-03-07 https://sqlite.org/forum/forumpost/ecdfc02339 +# A refinement is needed to the enhancements tested by the prior test case +# to avoid another problem with indexes on constant expressions. +# +reset_db +db null NULL +do_execsql_test 800 { + CREATE TABLE t0(c0, c1); + CREATE TABLE t1(c2); + CREATE INDEX i0 ON t1(NULL); + INSERT INTO t1(c2) VALUES (0.2); + CREATE VIEW v0(c3) AS SELECT DISTINCT c2 FROM t1; + SELECT * FROM v0 LEFT JOIN t0 ON c3<NULL LEFT JOIN t1 ON 1; +} {0.2 NULL NULL 0.2} +do_execsql_test 810 { + SELECT * FROM v0 LEFT JOIN t0 ON c3<NULL LEFT JOIN t1 ON 1 WHERE c2/0.1; +} {0.2 NULL NULL 0.2} + finish_test diff --git a/test/whereN.test b/test/whereN.test new file mode 100644 index 000000000..b9b889fa2 --- /dev/null +++ b/test/whereN.test @@ -0,0 +1,103 @@ +# 2024-04-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Tests for the whereInterstageHeuristic() routine in the query planner. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix whereN + +# The following is a simplified and "sanitized" version of the original +# real-world query that brought the problem to light. +# +# The issue is a slow query. The answer is correct, but it was taking too +# much time, because it was doing a full table scan rather than an indexed +# lookup. +# +# The problem was that the query planner was overestimating the number of +# output rows. The estimated number of output rows is accurate if the +# DSNAME parameter is "ds-one". In that case, a large fraction of the rows +# in "violation" end up being output. The query planner correctly deduces +# that it is faster to do a full table scan of the large "violation" table +# to avoid the after-query sort that implements the ORDER BY clause. However, +# if the DSNAME is "ds-two", then only a few rows (about 6) are generated, +# and it is much much faster to do an indexed lookup of "violation" followed +# by a sort operation to implement ORDER BY +# +# The problem, of course, is that the query planner has no way of knowing +# in advance how many rows will be generated. The query planner tries to +# estimate a worst case, which is a large number of output rows, and it picks +# the best plan for that case. However, the plan choosen is very inefficient +# when the number of output rows is small. +# +# The whereInterstageHeuristic() routine in the query planner attempts to +# correct this by adjusting the query plan such that it avoids the very bad +# query plan for a small number of rows, at the expense of a slightly less +# efficient plan for a large number of rows. The large number of rows case +# is perhaps 5% slower with the revised plan, but the small number of +# rows case is around 100 times faster. That seems like a good tradeoff. +# +do_execsql_test 1.0 { + CREATE TABLE datasource(dsid INT, name TEXT); + INSERT INTO datasource VALUES(1,'ds-one'),(2,'ds-two'),(3,'ds-three'); + CREATE INDEX ds1 ON datasource(name, dsid); + + CREATE TABLE rule(rid INT, team_id INT, dsid INT); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<9) + INSERT INTO rule(rid,team_id,dsid) SELECT n, 1, 1 FROM c; + WITH RECURSIVE c(n) AS (VALUES(10) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO rule(rid,team_id,dsid) SELECT n, 2, 2 FROM c; + CREATE INDEX rule2 ON rule(dsid, rid); + + CREATE TABLE violation(vid INT, rid INT, vx BLOB); + /*** Uncomment to insert actual data + WITH src(rid, cnt) AS (VALUES(1,3586),(2,1343),(3,6505),(5,76230), + (6,740),(7,287794),(8,457),(12,1), + (14,1),(16,1),(17,1),(18,1),(19,1)) + INSERT INTO violation(vid, rid, vx) + SELECT rid*1000000+value, rid, randomblob(15) + FROM src, generate_series(1,cnt); + ***/ + CREATE INDEX v1 ON violation(rid, vid); + CREATE INDEX v2 ON violation(vid); + ANALYZE; + DELETE FROM sqlite_stat1; + DROP TABLE IF EXISTS sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES + ('violation','v2','376661 1'), + ('violation','v1','376661 28974 1'), + ('rule','rule2','24 12 1'), + ('datasource','ds1','3 1 1'); + ANALYZE sqlite_schema; +} +set DSNAME ds-two ;# Only a few rows. Change to "ds-one" for many rows. +do_eqp_test 1.1 { + SELECT count(*), length(group_concat(vx)) FROM ( + SELECT V.* + FROM datasource DS, rule R, violation V + WHERE V.rid=R.rid + AND R.dsid=DS.dsid + AND DS.name=$DSNAME + ORDER BY V.vid desc + ); +} { + QUERY PLAN + |--CO-ROUTINE (subquery-xxxxxx) + | |--SEARCH DS USING COVERING INDEX ds1 (name=?) + | |--SEARCH R USING COVERING INDEX rule2 (dsid=?) + | |--SEARCH V USING INDEX v1 (rid=?) + | `--USE TEMP B-TREE FOR ORDER BY + `--SCAN (subquery-xxxxxx) +} +# ^^^^---- We want to see three SEARCH terms. No SCAN terms. +# The ORDER BY is implemented by a separate sorter pass. + +finish_test diff --git a/test/wherelimit3.test b/test/wherelimit3.test new file mode 100644 index 000000000..dea3e97d8 --- /dev/null +++ b/test/wherelimit3.test @@ -0,0 +1,67 @@ +# 2024-06-06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for query plans using LIMIT +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix wherelimit3 + +do_execsql_test 1.0 { + CREATE TABLE t1(a INT, b INT); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1000) + INSERT INTO t1 SELECT n, n FROM c; + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + ANALYZE; +} + +do_eqp_test 1.1 { + SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT 5; +} { + QUERY PLAN + |--SEARCH t1 USING INDEX t1a (a>? AND a<?) + `--USE TEMP B-TREE FOR ORDER BY +} +ifcapable stat4 { + do_eqp_test 1.2 { + SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT -1; + } { + QUERY PLAN + `--SCAN t1 USING INDEX t1b + } +} + +set N [expr 5] +do_eqp_test 1.3 { + SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N; +} { + QUERY PLAN + |--SEARCH t1 USING INDEX t1a (a>? AND a<?) + `--USE TEMP B-TREE FOR ORDER BY +} + +ifcapable stat4 { + set N [expr -1] + do_eqp_test 1.4 { + SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N; + } { + QUERY PLAN + `--SCAN t1 USING INDEX t1b + } +} + + + + + +finish_test diff --git a/test/win32longpath.test b/test/win32longpath.test index 01b4af70a..c0eac8d9d 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -49,19 +49,19 @@ set longPath(1) \\\\?\\$path\\[pid] set uriPath(1a) %5C%5C%3F%5C$path\\[pid] set uriPath(1b) %5C%5C%3F%5C$rawPath/[pid] -make_win32_dir $longPath(1) +file mkdir $longPath(1) set longPath(2) $longPath(1)\\[string repeat X 255] set uriPath(2a) $uriPath(1a)\\[string repeat X 255] set uriPath(2b) $uriPath(1b)/[string repeat X 255] -make_win32_dir $longPath(2) +file mkdir $longPath(2) set longPath(3) $longPath(2)\\[string repeat Y 255] set uriPath(3a) $uriPath(2a)\\[string repeat Y 255] set uriPath(3b) $uriPath(2b)/[string repeat Y 255] -make_win32_dir $longPath(3) +file mkdir $longPath(3) set fileName $longPath(3)\\test.db @@ -92,7 +92,6 @@ do_test 1.4 { } {5 6 7 8} db3 close -# puts " Database exists \{[exists_win32_path $fileName]\}" sqlite3 db3 $fileName -vfs win32-longpath @@ -115,7 +114,6 @@ do_test 1.6 { } {5 6 7 8 9 10 11 12} db3 close -# puts " Database exists \{[exists_win32_path $fileName]\}" foreach tn {1a 1b 1c 1d 1e 1f} { sqlite3 db3 $uri($tn) -vfs win32-longpath -uri 1 -translatefilename 0 @@ -129,11 +127,9 @@ foreach tn {1a 1b 1c 1d 1e 1f} { db3 close } -do_delete_win32_file $fileName -# puts " Files remaining \{[find_win32_file $longPath(3)\\*]\}" - -do_remove_win32_dir $longPath(3) -do_remove_win32_dir $longPath(2) -do_remove_win32_dir $longPath(1) +file delete -force $fileName +file delete -force $longPath(3) +file delete -force $longPath(2) +file delete -force $longPath(1) finish_test diff --git a/test/window1.test b/test/window1.test index d8348a898..457852c14 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2375,5 +2375,23 @@ do_execsql_test 77.2 { SELECT max(~likely(x)) FILTER (WHERE true) FROM t1 INDEXED BY t1x GROUP BY x; } {-2 -3 -5 -9} +# 2024-05-23 https://sqlite.org/forum/forumpost/bf8f43aa522c2299 +# +# A bug in group_concat() when used as a window function, reported +# just hours after the 3.46.0 release, though first appearing +# in 3.28.0. +# +# When used as a window function, a group_concat() was not +# correctly distinguishing between NULL and empty-string for +# its return value. +# +do_execsql_test 78.1 { + SELECT quote(group_concat(x) OVER ()) FROM (SELECT '' AS x); +} '' +do_execsql_test 78.2 { + SELECT quote(group_concat(x) OVER ( + ORDER BY y RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING + )) FROM (SELECT 'abc' AS x, 1 AS y); +} NULL finish_test diff --git a/test/windowE.test b/test/windowE.test index f20bcdaaa..9128468b3 100644 --- a/test/windowE.test +++ b/test/windowE.test @@ -54,5 +54,25 @@ do_execsql_test 1.3 { 5 5,4 5,4,1 5,4,1,6 5,4,1,6,3 5,4,1,6,3,2 } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(x); +} + +sqlite3_create_aggregate db + +breakpoint +do_catchsql_test 2.1 { + SELECT min(x) OVER w1 FROM t1 + WINDOW w1 AS (PARTITION BY x_count(x) OVER w1); +} {1 {x_count() may not be used as a window function}} + +do_catchsql_test 2.2 { + SELECT min(x) FILTER (WHERE x_count(x) OVER w1) OVER w1 FROM t1 + WINDOW w1 AS (PARTITION BY x OVER w1); +} {1 {near "OVER": syntax error}} + + finish_test diff --git a/test/windowpushd.test b/test/windowpushd.test index f2e04d72f..4bfb2c523 100644 --- a/test/windowpushd.test +++ b/test/windowpushd.test @@ -97,7 +97,11 @@ do_execsql_test 2.0 { } foreach tn {0 1} { - optimization_control db push-down $tn + if {$tn} { + optimization_control db all on + } else { + optimization_control db push-down off + } do_execsql_test 2.$tn.1.1 { SELECT * FROM v1; diff --git a/test/with2.test b/test/with2.test index 660df52b7..68790fe86 100644 --- a/test/with2.test +++ b/test/with2.test @@ -156,6 +156,16 @@ do_execsql_test 1.15 { SELECT * FROM t4; } {4 5} +do_execsql_test 1.15.2 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM (SELECT * FROM main.t4) WHERE x<10 + ) + SELECT * FROM t4; +} {4 5} + do_catchsql_test 1.16 { WITH t4(x) AS ( diff --git a/test/with6.test b/test/with6.test index 2a4bfc646..95e630547 100644 --- a/test/with6.test +++ b/test/with6.test @@ -360,6 +360,50 @@ do_eqp_test 400 { | | |--SEARCH raw USING COVERING INDEX sqlite_autoindex_raw_1 (country=? AND date>?) | | `--USE TEMP B-TREE FOR ORDER BY | |--SCAN src + | `--SEARCH raw USING INDEX sqlite_autoindex_raw_1 (country=? AND date>? AND date<?) + |--SCAN sums + |--BLOOM FILTER ON sums (country=? AND date=?) + |--SEARCH sums USING AUTOMATIC COVERING INDEX (country=? AND date=?) + |--BLOOM FILTER ON sums (country=? AND date=?) + |--SEARCH sums USING AUTOMATIC COVERING INDEX (country=? AND date=?) + |--BLOOM FILTER ON sums (country=? AND date=?) + |--SEARCH sums USING AUTOMATIC COVERING INDEX (country=? AND date=?) + |--BLOOM FILTER ON i (country=?) + `--SEARCH i USING AUTOMATIC COVERING INDEX (country=?) +} +optimization_control db order-by-subquery off +db cache flush +do_eqp_test 410 { + with recursive + init(country, date, fin) AS (SELECT country, min(date), max(date) FROM raw WHERE total > 0 GROUP BY country), + src(country, date) AS (SELECT raw.country, raw.date + FROM raw JOIN init i on raw.country = i.country AND raw.date > i.date + ORDER BY raw.country, raw.date), + vals(country, date, x, y) AS (SELECT src.country, src.date, julianday(raw.date) - julianday(src.date), log(delta+1) + FROM src JOIN raw on raw.country = src.country AND raw.date > date(src.date,'-7 days') AND raw.date <= src.date AND delta >= 0), + sums(country, date, x2, x, n, xy, y) AS (SELECT country, date, sum(x*x*1.0), sum(x*1.0), sum(1.0), sum(x*y*1.0), sum(y*1.0) FROM vals GROUP BY 1, 2), + mult(country, date, m) AS (SELECT country, date, 1.0/(x2 * n - x * x) FROM sums), + inv(country, date, a,b,c,d) AS (SELECT mult.country, mult.date, n * m, -x * m, -x * m, x2 * m + FROM mult JOIN sums on sums.country=mult.country AND mult.date=sums.date), + fit(country, date, a, b) AS (SELECT inv.country, inv.date, a * xy + b * y, c * xy + d * y + FROM inv + JOIN mult on mult.country = inv.country AND mult.date = inv.date + JOIN sums on sums.country = mult.country AND sums.date = mult.date + ) + SELECT *, nFin/nPrev - 1 AS growth, log(2)/log(nFin/nPrev) AS doubling + FROM (SELECT f.*, exp(b) - 1 AS nFin, exp(a* (-1) + b) - 1 AS nPrev + FROM fit f JOIN init i on i.country = f.country AND f.date <= date(i.fin,'-3 days')) + WHERE nPrev > 0 AND nFin > 0; +} { + QUERY PLAN + |--MATERIALIZE sums + | |--MATERIALIZE src + | | |--MATERIALIZE init + | | | `--SCAN raw USING INDEX sqlite_autoindex_raw_1 + | | |--SCAN i + | | |--SEARCH raw USING COVERING INDEX sqlite_autoindex_raw_1 (country=? AND date>?) + | | `--USE TEMP B-TREE FOR ORDER BY + | |--SCAN src | |--SEARCH raw USING INDEX sqlite_autoindex_raw_1 (country=? AND date>? AND date<?) | `--USE TEMP B-TREE FOR GROUP BY |--SCAN sums diff --git a/test/zipfile.test b/test/zipfile.test index 4d09e08b1..0f854a676 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -10,7 +10,10 @@ #*********************************************************************** # -package require Tcl 8.6 +if {$tcl_version<8.6} { + puts "Requires TCL 8.6 or later" + return +} set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -30,7 +33,7 @@ if {[catch {load_static_extension db fileio} error]} { proc readfile {f} { set fd [open $f] - fconfigure $fd -translation binary -encoding binary + fconfigure $fd -translation binary set data [read $fd] close $fd set data diff --git a/test/zipfile2.test b/test/zipfile2.test index 987e4f0cf..5277cd58f 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -10,7 +10,10 @@ #*********************************************************************** # -package require Tcl 8.6 +if {$tcl_version<8.6} { + puts "Requires TCL 8.6 or later" + return +} set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/tool/buildtclext.tcl b/tool/buildtclext.tcl new file mode 100644 index 000000000..6d9e4c306 --- /dev/null +++ b/tool/buildtclext.tcl @@ -0,0 +1,254 @@ +#!/usr/bin/tclsh +# +set help \ +{Run this TCL script to build and install the TCL interface library for +SQLite. Run the script with the specific "tclsh" for which the installation +should occur. + +There must be a valid "tclsqlite3.c" file in the working directory prior +to running this script. Use "make tclsqlite3.c" to generate that file. + +Options: + + --build-only Only build the extension, don't install it + --cc COMPILER Build using this compiler + --info Show info on existing SQLite TCL extension installs + --install-only Install an extension previously build + --uninstall Uninstall the extension + +Other options are retained and passed through into the compiler.} + + +set build 1 +set install 1 +set uninstall 0 +set infoonly 0 +set CC {} +set OPTS {} +for {set ii 0} {$ii<[llength $argv]} {incr ii} { + set a0 [lindex $argv $ii] + if {$a0=="--install-only"} { + set build 0 + } elseif {$a0=="--build-only"} { + set install 0 + } elseif {$a0=="--uninstall"} { + set build 0 + set install 0 + set uninstall 1 + } elseif {$a0=="--info"} { + set build 0 + set install 0 + set infoonly 1 + } elseif {$a0=="--cc" && $ii+1<[llength $argv]} { + incr ii + set CC [lindex $argv $ii] + } elseif {[string match -* $a0]} { + append OPTS " $a0" + } else { + puts stderr "Unknown option: \"$a0\"\n" + puts stderr $help + exit 1 + } +} + +# Find the root of the SQLite source tree +# +set srcdir [file normalize [file dir $argv0]/..] + +# Get the SQLite version number into $VERSION +# +set fd [open $srcdir/VERSION] +set VERSION [string trim [read $fd]] +close $fd + +if {$tcl_platform(platform)=="windows"} { + # We are only able to install, uninstall, and list on Windows. + # The build process is handled by the Makefile.msc, specifically + # using "nmake /f Makefile.msc pkgIndex.tcl tclsqlite3.dll" + # + if {$build} { + puts "Unable to build on Windows using the builttclext.tcl script." + puts "To build, run\n" + puts " \"nmake /f Makefile.msc pkgIndex.tcl tclsqlite3.dll" + exit 1 + } + set OUT tclsqlite3.dll +} else { + # Figure out the location of the tclConfig.sh file used by the + # tclsh that is executing this script. + # + if {[catch { + set LIBDIR [tcl::pkgconfig get libdir,install] + }]} { + puts stderr "$argv0: tclsh does not support tcl::pkgconfig." + exit 1 + } + if {![file exists $LIBDIR]} { + puts stderr "$argv0: cannot find the tclConfig.sh file." + puts stderr "$argv0: tclsh reported library directory \"$LIBDIR\"\ + does not exist." + exit 1 + } + if {![file exists $LIBDIR/tclConfig.sh] + || [file size $LIBDIR/tclConfig.sh]<5000} { + set n1 $LIBDIR/tcl$::tcl_version + if {[file exists $n1/tclConfig.sh] + && [file size $n1/tclConfig.sh]>5000} { + set LIBDIR $n1 + } else { + puts stderr "$argv0: cannot find tclConfig.sh in either $LIBDIR or $n1" + exit 1 + } + } + + # Read the tclConfig.sh file into the $tclConfig variable + # + #puts "using $LIBDIR/tclConfig.sh" + set fd [open $LIBDIR/tclConfig.sh rb] + set tclConfig [read $fd] + close $fd + + # Extract parameter we will need from the tclConfig.sh file + # + set TCLMAJOR 8 + regexp {TCL_MAJOR_VERSION='(\d)'} $tclConfig all TCLMAJOR + set SUFFIX so + regexp {TCL_SHLIB_SUFFIX='\.([^']+)'} $tclConfig all SUFFIX + if {$CC==""} { + set cc {} + regexp {TCL_CC='([^']+)'} $tclConfig all cc + if {$cc!=""} { + set CC $cc + } + } + if {$CC==""} { + set CC gcc + } + set CFLAGS -fPIC + regexp {TCL_SHLIB_CFLAGS='([^']+)'} $tclConfig all CFLAGS + set LIBS {} + regexp {TCL_STUB_LIB_SPEC='([^']+)'} $tclConfig all LIBS + set INC "-I$srcdir/src" + set inc {} + regexp {TCL_INCLUDE_SPEC='([^']+)'} $tclConfig all inc + if {$inc!=""} { + append INC " $inc" + } + set cmd {${CC} ${CFLAGS} ${LDFLAGS} -shared} + regexp {TCL_SHLIB_LD='([^']+)'} $tclConfig all cmd + set LDFLAGS "$INC -DUSE_TCL_STUBS" + if {[string length $OPTS]>1} { + append LDFLAGS $OPTS + } + set CMD [subst $cmd] + if {$TCLMAJOR>8} { + set OUT libtcl9sqlite$VERSION.$SUFFIX + } else { + set OUT libsqlite$VERSION.$SUFFIX + } +} + +# Show information about prior installs +# +if {$infoonly} { + set cnt 0 + foreach dir $auto_path { + foreach subdir [glob -nocomplain -types d $dir/sqlite3*] { + if {[file exists $subdir/pkgIndex.tcl]} { + puts $subdir + incr cnt + } + } + } + if {$cnt==0} { + puts "no current installations of the SQLite TCL extension" + } + exit +} + +# Uninstall the extension +# +if {$uninstall} { + set cnt 0 + foreach dir $auto_path { + if {[file isdirectory $dir/sqlite$VERSION]} { + incr cnt + if {![file writable $dir] || ![file writable $dir/sqlite$VERSION]} { + puts "cannot uninstall $dir/sqlite$VERSION - permission denied" + } else { + puts "uninstalling $dir/sqlite$VERSION..." + file delete -force $dir/sqlite$VERSION + } + } + } + if {$cnt==0} { + puts "nothing to uninstall" + } + exit +} + +if {$install} { + # Figure out where the extension will be installed. Put the extension + # in the first writable directory on $auto_path. + # + set DEST {} + foreach dir $auto_path { + if {[file writable $dir]} { + set DEST $dir + break + } elseif {[glob -nocomplain $dir/sqlite3*/pkgIndex.tcl]!=""} { + set conflict [lindex [glob $dir/sqlite3*/pkgIndex.tcl] 0] + puts "Unable to install. There is already a conflicting version" + puts "of the SQLite TCL Extension that cannot be overwritten at\n" + puts " [file dirname $conflict]\n" + puts "Consider running using sudo to work around this problem." + exit 1 + } + } + if {$DEST==""} { + puts "None of the directories on \$auto_path are writable by this process," + puts "so the installation cannot take place. Consider running using sudo" + puts "to work around this problem.\n" + puts "These are the (unwritable) \$auto_path directories:\n" + foreach dir $auto_path { + puts " * $dir" + } + exit 1 + } +} + +if {$build} { + # Generate the pkgIndex.tcl file + # + puts "generating pkgIndex.tcl..." + set fd [open pkgIndex.tcl w] + puts $fd [subst -nocommands {# -*- tcl -*- +# Tcl package index file, version ??? +# +package ifneeded sqlite3 $VERSION \\ + [list load [file join \$dir $OUT] sqlite3] +}] + close $fd + + # Generate and execute the command with which to do the compilation. + # + set cmd "$CMD tclsqlite3.c -o $OUT $LIBS" + puts $cmd + file delete -force $OUT + catch {exec {*}$cmd} errmsg + if {$errmsg!="" && ![file exists $OUT]} { + puts $errmsg + exit 1 + } +} + + +if {$install} { + # Install the extension + set DEST2 $DEST/sqlite$VERSION + file mkdir $DEST2 + puts "installing $DEST2/pkgIndex.tcl" + file copy -force pkgIndex.tcl $DEST2 + puts "installing $DEST2/$OUT" + file copy -force $OUT $DEST2 +} diff --git a/tool/find_tclconfig.tcl b/tool/find_tclconfig.tcl new file mode 100644 index 000000000..c3d3df8ec --- /dev/null +++ b/tool/find_tclconfig.tcl @@ -0,0 +1,24 @@ +# +# Run this TCL script to find and print the pathname for the tclConfig.sh +# file. Used by ../configure +# +if {[catch { + set libdir [tcl::pkgconfig get libdir,install] +}]} { + puts stderr "tclsh too old: does not support tcl::pkgconfig" + exit 1 +} +if {![file exists $libdir]} { + puts stderr "tclsh reported library directory \"$libdir\" does not exist" + exit 1 +} +if {![file exists $libdir/tclConfig.sh]} { + set n1 $libdir/tcl$::tcl_version + if {[file exists $n1/tclConfig.sh]} { + set libdir $n1 + } else { + puts stderr "cannot find tclConfig.sh in either $libdir or $n1" + exit 1 + } +} +puts $libdir diff --git a/tool/lemon.c b/tool/lemon.c index 7804837a0..d92df2a1a 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -60,6 +60,82 @@ static char *msort(char*,char**,int(*)(const char*,const char*)); #define lemonStrlen(X) ((int)strlen(X)) /* +** Header on the linked list of memory allocations. +*/ +typedef struct MemChunk MemChunk; +struct MemChunk { + MemChunk *pNext; + size_t sz; + /* Actually memory follows */ +}; + +/* +** Global linked list of all memory allocations. +*/ +static MemChunk *memChunkList = 0; + +/* +** Wrappers around malloc(), calloc(), realloc() and free(). +** +** All memory allocations are kept on a doubly-linked list. The +** lemon_free_all() function can be called prior to exit to clean +** up any memory leaks. +** +** This is not necessary. But compilers and getting increasingly +** fussy about memory leaks, even in command-line programs like Lemon +** where they do not matter. So this code is provided to hush the +** warnings. +*/ +static void *lemon_malloc(size_t nByte){ + MemChunk *p; + if( nByte<0 ) return 0; + p = malloc( nByte + sizeof(MemChunk) ); + if( p==0 ){ + fprintf(stderr, "Out of memory. Failed to allocate %lld bytes.\n", + (long long int)nByte); + exit(1); + } + p->pNext = memChunkList; + p->sz = nByte; + memChunkList = p; + return (void*)&p[1]; +} +static void *lemon_calloc(size_t nElem, size_t sz){ + void *p = lemon_malloc(nElem*sz); + memset(p, 0, nElem*sz); + return p; +} +static void lemon_free(void *pOld){ + if( pOld ){ + MemChunk *p = (MemChunk*)pOld; + p--; + memset(pOld, 0, p->sz); + } +} +static void *lemon_realloc(void *pOld, size_t nNew){ + void *pNew; + MemChunk *p; + if( pOld==0 ) return lemon_malloc(nNew); + p = (MemChunk*)pOld; + p--; + if( p->sz>=nNew ) return pOld; + pNew = lemon_malloc( nNew ); + memcpy(pNew, pOld, p->sz); + return pNew; +} + +/* Free all outstanding memory allocations. +** Do this right before exiting. +*/ +static void lemon_free_all(void){ + while( memChunkList ){ + MemChunk *pNext = memChunkList->pNext; + free( memChunkList ); + memChunkList = pNext; + } +} + +/* ** Compilers are starting to complain about the use of sprintf() and strcpy(), ** saying they are unsafe. So we define our own versions of those routines too. ** @@ -418,6 +494,8 @@ struct lemon { char *filename; /* Name of the input file */ char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ + char *reallocFunc; /* Function to use to allocate stack space */ + char *freeFunc; /* Function to use to free stack space */ int nconflict; /* Number of parsing conflicts */ int nactiontab; /* Number of entries in the yy_action[] table */ int nlookaheadtab; /* Number of entries in yy_lookahead[] */ @@ -495,7 +573,7 @@ static struct action *Action_new(void){ if( actionfreelist==0 ){ int i; int amt = 100; - actionfreelist = (struct action *)calloc(amt, sizeof(struct action)); + actionfreelist = (struct action *)lemon_calloc(amt, sizeof(struct action)); if( actionfreelist==0 ){ fprintf(stderr,"Unable to allocate memory for a new parser action."); exit(1); @@ -614,14 +692,14 @@ struct acttab { /* Free all memory associated with the given acttab */ void acttab_free(acttab *p){ - free( p->aAction ); - free( p->aLookahead ); - free( p ); + lemon_free( p->aAction ); + lemon_free( p->aLookahead ); + lemon_free( p ); } /* Allocate a new acttab structure */ acttab *acttab_alloc(int nsymbol, int nterminal){ - acttab *p = (acttab *) calloc( 1, sizeof(*p) ); + acttab *p = (acttab *) lemon_calloc( 1, sizeof(*p) ); if( p==0 ){ fprintf(stderr,"Unable to allocate memory for a new acttab."); exit(1); @@ -640,7 +718,7 @@ acttab *acttab_alloc(int nsymbol, int nterminal){ void acttab_action(acttab *p, int lookahead, int action){ if( p->nLookahead>=p->nLookaheadAlloc ){ p->nLookaheadAlloc += 25; - p->aLookahead = (struct lookahead_action *) realloc( p->aLookahead, + p->aLookahead = (struct lookahead_action *) lemon_realloc( p->aLookahead, sizeof(p->aLookahead[0])*p->nLookaheadAlloc ); if( p->aLookahead==0 ){ fprintf(stderr,"malloc failed\n"); @@ -690,7 +768,7 @@ int acttab_insert(acttab *p, int makeItSafe){ if( p->nAction + n >= p->nActionAlloc ){ int oldAlloc = p->nActionAlloc; p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; - p->aAction = (struct lookahead_action *) realloc( p->aAction, + p->aAction = (struct lookahead_action *) lemon_realloc( p->aAction, sizeof(p->aAction[0])*p->nActionAlloc); if( p->aAction==0 ){ fprintf(stderr,"malloc failed\n"); @@ -1312,7 +1390,7 @@ static struct config **basisend = 0; /* End of list of basis configs */ /* Return a pointer to a new configuration */ PRIVATE struct config *newconfig(void){ - return (struct config*)calloc(1, sizeof(struct config)); + return (struct config*)lemon_calloc(1, sizeof(struct config)); } /* The configuration "old" is no longer used */ @@ -1528,19 +1606,19 @@ static char *bDefineUsed = 0; /* True for every -D macro actually used */ static void handle_D_option(char *z){ char **paz; nDefine++; - azDefine = (char **) realloc(azDefine, sizeof(azDefine[0])*nDefine); + azDefine = (char **) lemon_realloc(azDefine, sizeof(azDefine[0])*nDefine); if( azDefine==0 ){ fprintf(stderr,"out of memory\n"); exit(1); } - bDefineUsed = (char*)realloc(bDefineUsed, nDefine); + bDefineUsed = (char*)lemon_realloc(bDefineUsed, nDefine); if( bDefineUsed==0 ){ fprintf(stderr,"out of memory\n"); exit(1); } bDefineUsed[nDefine-1] = 0; paz = &azDefine[nDefine-1]; - *paz = (char *) malloc( lemonStrlen(z)+1 ); + *paz = (char *) lemon_malloc( lemonStrlen(z)+1 ); if( *paz==0 ){ fprintf(stderr,"out of memory\n"); exit(1); @@ -1554,7 +1632,7 @@ static void handle_D_option(char *z){ */ static char *outputDir = NULL; static void handle_d_option(char *z){ - outputDir = (char *) malloc( lemonStrlen(z)+1 ); + outputDir = (char *) lemon_malloc( lemonStrlen(z)+1 ); if( outputDir==0 ){ fprintf(stderr,"out of memory\n"); exit(1); @@ -1564,7 +1642,7 @@ static void handle_d_option(char *z){ static char *user_templatename = NULL; static void handle_T_option(char *z){ - user_templatename = (char *) malloc( lemonStrlen(z)+1 ); + user_templatename = (char *) lemon_malloc( lemonStrlen(z)+1 ); if( user_templatename==0 ){ memory_error(); } @@ -1631,6 +1709,15 @@ static void stats_line(const char *zLabel, int iValue){ iValue); } +/* +** Comparison function used by qsort() to sort the azDefine[] array. +*/ +static int defineCmp(const void *pA, const void *pB){ + const char *zA = *(const char**)pA; + const char *zB = *(const char**)pB; + return strcmp(zA,zB); +} + /* The main program. Parse the command line and do it... */ int main(int argc, char **argv){ static int version = 0; @@ -1686,6 +1773,7 @@ int main(int argc, char **argv){ } memset(&lem, 0, sizeof(lem)); lem.errorcnt = 0; + qsort(azDefine, nDefine, sizeof(azDefine[0]), defineCmp); /* Initialize the machine */ Strsafe_init(); @@ -1801,6 +1889,7 @@ int main(int argc, char **argv){ /* return 0 on success, 1 on failure. */ exitcode = ((lem.errorcnt > 0) || (lem.nconflict > 0)) ? 1 : 0; + lemon_free_all(); exit(exitcode); return (exitcode); } @@ -2389,7 +2478,7 @@ static void parseonetoken(struct pstate *psp) case IN_RHS: if( x[0]=='.' ){ struct rule *rp; - rp = (struct rule *)calloc( sizeof(struct rule) + + rp = (struct rule *)lemon_calloc( sizeof(struct rule) + sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1); if( rp==0 ){ ErrorMsg(psp->filename,psp->tokenlineno, @@ -2441,17 +2530,17 @@ static void parseonetoken(struct pstate *psp) struct symbol *msp = psp->rhs[psp->nrhs-1]; if( msp->type!=MULTITERMINAL ){ struct symbol *origsp = msp; - msp = (struct symbol *) calloc(1,sizeof(*msp)); + msp = (struct symbol *) lemon_calloc(1,sizeof(*msp)); memset(msp, 0, sizeof(*msp)); msp->type = MULTITERMINAL; msp->nsubsym = 1; - msp->subsym = (struct symbol **) calloc(1,sizeof(struct symbol*)); + msp->subsym = (struct symbol**)lemon_calloc(1,sizeof(struct symbol*)); msp->subsym[0] = origsp; msp->name = origsp->name; psp->rhs[psp->nrhs-1] = msp; } msp->nsubsym++; - msp->subsym = (struct symbol **) realloc(msp->subsym, + msp->subsym = (struct symbol **) lemon_realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]); if( ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ){ @@ -2531,6 +2620,12 @@ static void parseonetoken(struct pstate *psp) }else if( strcmp(x,"default_type")==0 ){ psp->declargslot = &(psp->gp->vartype); psp->insertLineMacro = 0; + }else if( strcmp(x,"realloc")==0 ){ + psp->declargslot = &(psp->gp->reallocFunc); + psp->insertLineMacro = 0; + }else if( strcmp(x,"free")==0 ){ + psp->declargslot = &(psp->gp->freeFunc); + psp->insertLineMacro = 0; }else if( strcmp(x,"stack_size")==0 ){ psp->declargslot = &(psp->gp->stacksize); psp->insertLineMacro = 0; @@ -2661,7 +2756,7 @@ static void parseonetoken(struct pstate *psp) nLine = lemonStrlen(zLine); n += nLine + lemonStrlen(psp->filename) + nBack; } - *psp->declargslot = (char *) realloc(*psp->declargslot, n); + *psp->declargslot = (char *) lemon_realloc(*psp->declargslot, n); zBuf = *psp->declargslot + nOld; if( addLineMacro ){ if( nOld && zBuf[-1]!='\n' ){ @@ -2775,7 +2870,7 @@ static void parseonetoken(struct pstate *psp) }else if( ISUPPER(x[0]) || ((x[0]=='|' || x[0]=='/') && ISUPPER(x[1])) ){ struct symbol *msp = psp->tkclass; msp->nsubsym++; - msp->subsym = (struct symbol **) realloc(msp->subsym, + msp->subsym = (struct symbol **) lemon_realloc(msp->subsym, sizeof(struct symbol*)*msp->nsubsym); if( !ISUPPER(x[0]) ) x++; msp->subsym[msp->nsubsym-1] = Symbol_new(x); @@ -2990,10 +3085,10 @@ void Parse(struct lemon *gp) fseek(fp,0,2); filesize = ftell(fp); rewind(fp); - filebuf = (char *)malloc( filesize+1 ); + filebuf = (char *)lemon_malloc( filesize+1 ); if( filesize>100000000 || filebuf==0 ){ ErrorMsg(ps.filename,0,"Input file too large."); - free(filebuf); + lemon_free(filebuf); gp->errorcnt++; fclose(fp); return; @@ -3001,7 +3096,7 @@ void Parse(struct lemon *gp) if( fread(filebuf,1,filesize,fp)!=filesize ){ ErrorMsg(ps.filename,0,"Can't read in all %d bytes of this file.", filesize); - free(filebuf); + lemon_free(filebuf); gp->errorcnt++; fclose(fp); return; @@ -3113,7 +3208,7 @@ void Parse(struct lemon *gp) *cp = (char)c; /* Restore the buffer */ cp = nextcp; } - free(filebuf); /* Release the buffer after parsing */ + lemon_free(filebuf); /* Release the buffer after parsing */ gp->rule = ps.firstrule; gp->errorcnt = ps.errorcnt; } @@ -3131,7 +3226,7 @@ struct plink *Plink_new(void){ if( plink_freelist==0 ){ int i; int amt = 100; - plink_freelist = (struct plink *)calloc( amt, sizeof(struct plink) ); + plink_freelist = (struct plink *)lemon_calloc( amt, sizeof(struct plink) ); if( plink_freelist==0 ){ fprintf(stderr, "Unable to allocate memory for a new follow-set propagation link.\n"); @@ -3184,9 +3279,7 @@ void Plink_delete(struct plink *plp) ** Procedures for generating reports and tables in the LEMON parser generator. */ -/* Generate a filename with the given suffix. Space to hold the -** name comes from malloc() and must be freed by the calling -** function. +/* Generate a filename with the given suffix. */ PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) { @@ -3203,7 +3296,7 @@ PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) sz += lemonStrlen(suffix); if( outputDir ) sz += lemonStrlen(outputDir) + 1; sz += 5; - name = (char*)malloc( sz ); + name = (char*)lemon_malloc( sz ); if( name==0 ){ fprintf(stderr,"Can't allocate space for a filename.\n"); exit(1); @@ -3230,7 +3323,7 @@ PRIVATE FILE *file_open( ){ FILE *fp; - if( lemp->outname ) free(lemp->outname); + if( lemp->outname ) lemon_free(lemp->outname); lemp->outname = file_makename(lemp, suffix); fp = fopen(lemp->outname,mode); if( fp==0 && *mode=='w' ){ @@ -3546,14 +3639,14 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask) if( cp ){ c = *cp; *cp = 0; - path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); + path = (char *)lemon_malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); if( path ) lemon_sprintf(path,"%s/%s",argv0,name); *cp = c; }else{ pathlist = getenv("PATH"); if( pathlist==0 ) pathlist = ".:/bin:/usr/bin"; - pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1 ); - path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); + pathbuf = (char *) lemon_malloc( lemonStrlen(pathlist) + 1 ); + path = (char *)lemon_malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); if( (pathbuf != 0) && (path!=0) ){ pathbufptr = pathbuf; lemon_strcpy(pathbuf, pathlist); @@ -3569,7 +3662,7 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask) if( access(path,modemask)==0 ) break; } } - free(pathbufptr); + lemon_free(pathbufptr); } return path; } @@ -3700,7 +3793,7 @@ PRIVATE FILE *tplt_open(struct lemon *lemp) fprintf(stderr,"Can't open the template file \"%s\".\n",tpltname); lemp->errorcnt++; } - free(toFree); + lemon_free(toFree); return in; } @@ -3829,7 +3922,7 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){ } if( (int) (n+sizeof(zInt)*2+used) >= alloced ){ alloced = n + sizeof(zInt)*2 + used + 200; - z = (char *) realloc(z, alloced); + z = (char *) lemon_realloc(z, alloced); } if( z==0 ) return empty; while( n-- > 0 ){ @@ -4119,7 +4212,7 @@ void print_stack_union( /* Allocate and initialize types[] and allocate stddt[] */ arraysize = lemp->nsymbol * 2; - types = (char**)calloc( arraysize, sizeof(char*) ); + types = (char**)lemon_calloc( arraysize, sizeof(char*) ); if( types==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); @@ -4136,7 +4229,7 @@ void print_stack_union( len = lemonStrlen(sp->datatype); if( len>maxdtlength ) maxdtlength = len; } - stddt = (char*)malloc( maxdtlength*2 + 1 ); + stddt = (char*)lemon_malloc( maxdtlength*2 + 1 ); if( stddt==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); @@ -4185,7 +4278,7 @@ void print_stack_union( } if( types[hash]==0 ){ sp->dtnum = hash + 1; - types[hash] = (char*)malloc( lemonStrlen(stddt)+1 ); + types[hash] = (char*)lemon_malloc( lemonStrlen(stddt)+1 ); if( types[hash]==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); @@ -4207,13 +4300,13 @@ void print_stack_union( for(i=0; i<arraysize; i++){ if( types[i]==0 ) continue; fprintf(out," %s yy%d;\n",types[i],i+1); lineno++; - free(types[i]); + lemon_free(types[i]); } if( lemp->errsym && lemp->errsym->useCnt ){ fprintf(out," int yy%d;\n",lemp->errsym->dtnum); lineno++; } - free(stddt); - free(types); + lemon_free(stddt); + lemon_free(types); fprintf(out,"} YYMINORTYPE;\n"); lineno++; *plineno = lineno; } @@ -4309,7 +4402,7 @@ void ReportTable( struct action *ap; struct rule *rp; struct acttab *pActtab; - int i, j, n, sz; + int i, j, n, sz, mn, mx; int nLookAhead; int szActionType; /* sizeof(YYACTIONTYPE) */ int szCodeType; /* sizeof(YYCODETYPE) */ @@ -4442,7 +4535,7 @@ void ReportTable( if( mhflag ){ char *incName = file_makename(lemp, ".h"); fprintf(out,"#include \"%s\"\n", incName); lineno++; - free(incName); + lemon_free(incName); } tplt_xfer(lemp->name,in,out,&lineno); @@ -4501,6 +4594,21 @@ void ReportTable( fprintf(out,"#define %sARG_FETCH\n",name); lineno++; fprintf(out,"#define %sARG_STORE\n",name); lineno++; } + if( lemp->reallocFunc ){ + fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++; + }else{ + fprintf(out,"#define YYREALLOC realloc\n"); lineno++; + } + if( lemp->freeFunc ){ + fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++; + }else{ + fprintf(out,"#define YYFREE free\n"); lineno++; + } + if( lemp->reallocFunc && lemp->freeFunc ){ + fprintf(out,"#define YYDYNSTACK 1\n"); lineno++; + }else{ + fprintf(out,"#define YYDYNSTACK 0\n"); lineno++; + } if( lemp->ctx && lemp->ctx[0] ){ i = lemonStrlen(lemp->ctx); while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--; @@ -4534,7 +4642,7 @@ void ReportTable( ** table must be computed before generating the YYNSTATE macro because ** we need to know how many states can be eliminated. */ - ax = (struct axset *) calloc(lemp->nxstate*2, sizeof(ax[0])); + ax = (struct axset *) lemon_calloc(lemp->nxstate*2, sizeof(ax[0])); if( ax==0 ){ fprintf(stderr,"malloc failed\n"); exit(1); @@ -4592,7 +4700,7 @@ void ReportTable( } #endif } - free(ax); + lemon_free(ax); /* Mark rules that are actually used for reduce actions after all ** optimizations have been applied @@ -4624,6 +4732,22 @@ void ReportTable( fprintf(out,"#define YY_MIN_REDUCE %d\n", lemp->minReduce); lineno++; i = lemp->minReduce + lemp->nrule; fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; + + /* Minimum and maximum token values that have a destructor */ + mn = mx = 0; + for(i=0; i<lemp->nsymbol; i++){ + struct symbol *sp = lemp->symbols[i]; + + if( sp && sp->type!=TERMINAL && sp->destructor ){ + if( mn==0 || sp->index<mn ) mn = sp->index; + if( sp->index>mx ) mx = sp->index; + } + } + if( lemp->tokendest ) mn = 0; + if( lemp->vardest ) mx = lemp->nsymbol-1; + fprintf(out,"#define YY_MIN_DSTRCTR %d\n", mn); lineno++; + fprintf(out,"#define YY_MAX_DSTRCTR %d\n", mx); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); /* Now output the action table and its associates: @@ -4767,7 +4891,7 @@ void ReportTable( /* Generate the table of fallback tokens. */ if( lemp->has_fallback ){ - int mx = lemp->nterminal - 1; + mx = lemp->nterminal - 1; /* 2019-08-28: Generate fallback entries for every token to avoid ** having to do a range check on the index */ /* while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } */ @@ -5202,7 +5326,7 @@ void SetSize(int n) /* Allocate a new set */ char *SetNew(void){ char *s; - s = (char*)calloc( size, 1); + s = (char*)lemon_calloc( size, 1); if( s==0 ){ memory_error(); } @@ -5212,7 +5336,7 @@ char *SetNew(void){ /* Deallocate a set */ void SetFree(char *s) { - free(s); + lemon_free(s); } /* Add a new element to the set. Return TRUE if the element was added @@ -5271,7 +5395,7 @@ const char *Strsafe(const char *y) if( y==0 ) return 0; z = Strsafe_find(y); - if( z==0 && (cpy=(char *)malloc( lemonStrlen(y)+1 ))!=0 ){ + if( z==0 && (cpy=(char *)lemon_malloc( lemonStrlen(y)+1 ))!=0 ){ lemon_strcpy(cpy,y); z = cpy; Strsafe_insert(z); @@ -5307,13 +5431,13 @@ static struct s_x1 *x1a; /* Allocate a new associative array */ void Strsafe_init(void){ if( x1a ) return; - x1a = (struct s_x1*)malloc( sizeof(struct s_x1) ); + x1a = (struct s_x1*)lemon_malloc( sizeof(struct s_x1) ); if( x1a ){ x1a->size = 1024; x1a->count = 0; - x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*)); + x1a->tbl = (x1node*)lemon_calloc(1024, sizeof(x1node) + sizeof(x1node*)); if( x1a->tbl==0 ){ - free(x1a); + lemon_free(x1a); x1a = 0; }else{ int i; @@ -5348,7 +5472,7 @@ int Strsafe_insert(const char *data) struct s_x1 array; array.size = arrSize = x1a->size*2; array.count = x1a->count; - array.tbl = (x1node*)calloc(arrSize, sizeof(x1node) + sizeof(x1node*)); + array.tbl = (x1node*)lemon_calloc(arrSize, sizeof(x1node)+sizeof(x1node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x1node**)&(array.tbl[arrSize]); for(i=0; i<arrSize; i++) array.ht[i] = 0; @@ -5363,7 +5487,7 @@ int Strsafe_insert(const char *data) newnp->from = &(array.ht[h]); array.ht[h] = newnp; } - /* free(x1a->tbl); // This program was originally for 16-bit machines. + /* lemon_free(x1a->tbl); // This program was originally for 16-bit machines. ** Don't worry about freeing memory on modern platforms. */ *x1a = array; } @@ -5404,7 +5528,7 @@ struct symbol *Symbol_new(const char *x) sp = Symbol_find(x); if( sp==0 ){ - sp = (struct symbol *)calloc(1, sizeof(struct symbol) ); + sp = (struct symbol *)lemon_calloc(1, sizeof(struct symbol) ); MemoryCheck(sp); sp->name = Strsafe(x); sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; @@ -5475,13 +5599,13 @@ static struct s_x2 *x2a; /* Allocate a new associative array */ void Symbol_init(void){ if( x2a ) return; - x2a = (struct s_x2*)malloc( sizeof(struct s_x2) ); + x2a = (struct s_x2*)lemon_malloc( sizeof(struct s_x2) ); if( x2a ){ x2a->size = 128; x2a->count = 0; - x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*)); + x2a->tbl = (x2node*)lemon_calloc(128, sizeof(x2node) + sizeof(x2node*)); if( x2a->tbl==0 ){ - free(x2a); + lemon_free(x2a); x2a = 0; }else{ int i; @@ -5516,7 +5640,7 @@ int Symbol_insert(struct symbol *data, const char *key) struct s_x2 array; array.size = arrSize = x2a->size*2; array.count = x2a->count; - array.tbl = (x2node*)calloc(arrSize, sizeof(x2node) + sizeof(x2node*)); + array.tbl = (x2node*)lemon_calloc(arrSize, sizeof(x2node)+sizeof(x2node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x2node**)&(array.tbl[arrSize]); for(i=0; i<arrSize; i++) array.ht[i] = 0; @@ -5532,7 +5656,7 @@ int Symbol_insert(struct symbol *data, const char *key) newnp->from = &(array.ht[h]); array.ht[h] = newnp; } - /* free(x2a->tbl); // This program was originally written for 16-bit + /* lemon_free(x2a->tbl); // This program was originally written for 16-bit ** machines. Don't worry about freeing this trivial amount of memory ** on modern platforms. Just leak it. */ *x2a = array; @@ -5593,7 +5717,7 @@ struct symbol **Symbol_arrayof() int i,arrSize; if( x2a==0 ) return 0; arrSize = x2a->count; - array = (struct symbol **)calloc(arrSize, sizeof(struct symbol *)); + array = (struct symbol **)lemon_calloc(arrSize, sizeof(struct symbol *)); if( array ){ for(i=0; i<arrSize; i++) array[i] = x2a->tbl[i].data; } @@ -5641,7 +5765,7 @@ PRIVATE unsigned statehash(struct config *a) struct state *State_new() { struct state *newstate; - newstate = (struct state *)calloc(1, sizeof(struct state) ); + newstate = (struct state *)lemon_calloc(1, sizeof(struct state) ); MemoryCheck(newstate); return newstate; } @@ -5674,13 +5798,13 @@ static struct s_x3 *x3a; /* Allocate a new associative array */ void State_init(void){ if( x3a ) return; - x3a = (struct s_x3*)malloc( sizeof(struct s_x3) ); + x3a = (struct s_x3*)lemon_malloc( sizeof(struct s_x3) ); if( x3a ){ x3a->size = 128; x3a->count = 0; - x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*)); + x3a->tbl = (x3node*)lemon_calloc(128, sizeof(x3node) + sizeof(x3node*)); if( x3a->tbl==0 ){ - free(x3a); + lemon_free(x3a); x3a = 0; }else{ int i; @@ -5715,7 +5839,7 @@ int State_insert(struct state *data, struct config *key) struct s_x3 array; array.size = arrSize = x3a->size*2; array.count = x3a->count; - array.tbl = (x3node*)calloc(arrSize, sizeof(x3node) + sizeof(x3node*)); + array.tbl = (x3node*)lemon_calloc(arrSize, sizeof(x3node)+sizeof(x3node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x3node**)&(array.tbl[arrSize]); for(i=0; i<arrSize; i++) array.ht[i] = 0; @@ -5731,7 +5855,7 @@ int State_insert(struct state *data, struct config *key) newnp->from = &(array.ht[h]); array.ht[h] = newnp; } - free(x3a->tbl); + lemon_free(x3a->tbl); *x3a = array; } /* Insert the new data */ @@ -5772,7 +5896,7 @@ struct state **State_arrayof(void) int i,arrSize; if( x3a==0 ) return 0; arrSize = x3a->count; - array = (struct state **)calloc(arrSize, sizeof(struct state *)); + array = (struct state **)lemon_calloc(arrSize, sizeof(struct state *)); if( array ){ for(i=0; i<arrSize; i++) array[i] = x3a->tbl[i].data; } @@ -5814,13 +5938,13 @@ static struct s_x4 *x4a; /* Allocate a new associative array */ void Configtable_init(void){ if( x4a ) return; - x4a = (struct s_x4*)malloc( sizeof(struct s_x4) ); + x4a = (struct s_x4*)lemon_malloc( sizeof(struct s_x4) ); if( x4a ){ x4a->size = 64; x4a->count = 0; - x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*)); + x4a->tbl = (x4node*)lemon_calloc(64, sizeof(x4node) + sizeof(x4node*)); if( x4a->tbl==0 ){ - free(x4a); + lemon_free(x4a); x4a = 0; }else{ int i; @@ -5855,7 +5979,8 @@ int Configtable_insert(struct config *data) struct s_x4 array; array.size = arrSize = x4a->size*2; array.count = x4a->count; - array.tbl = (x4node*)calloc(arrSize, sizeof(x4node) + sizeof(x4node*)); + array.tbl = (x4node*)lemon_calloc(arrSize, + sizeof(x4node) + sizeof(x4node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x4node**)&(array.tbl[arrSize]); for(i=0; i<arrSize; i++) array.ht[i] = 0; @@ -5870,9 +5995,6 @@ int Configtable_insert(struct config *data) newnp->from = &(array.ht[h]); array.ht[h] = newnp; } - /* free(x4a->tbl); // This code was originall written for 16-bit machines. - ** on modern machines, don't worry about freeing this trival amount of - ** memory. */ *x4a = array; } /* Insert the new data */ diff --git a/tool/lempar.c b/tool/lempar.c index 8cc57897d..851a0e2e5 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -67,6 +67,9 @@ ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser ** ParseCTX_* As ParseARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -80,6 +83,8 @@ ** YY_NO_ACTION The yy_action[] code for no-op ** YY_MIN_REDUCE Minimum value for reduce actions ** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 @@ -101,6 +106,22 @@ # define yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -212,14 +233,9 @@ struct yyParser { #endif ParseARG_SDECL /* A place to hold %extra_argument */ ParseCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct yyParser yyParser; @@ -273,37 +289,45 @@ static const char *const yyRuleName[] = { #endif /* NDEBUG */ -#if YYSTACKDEPTH<=0 +#if YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); int newSize; int idx; yyStackEntry *pNew; - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystack = pNew; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; } +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -323,24 +347,14 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){ #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; #endif yypParser->yytos = yypParser->yystack; yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif } #ifndef Parse_ENGINEALWAYSONSTACK @@ -426,9 +440,26 @@ static void yy_pop_parser_stack(yyParser *pParser){ */ void ParseFinalize(void *p){ yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); #endif } @@ -654,25 +685,19 @@ static void yy_shift( assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ if( yyGrowStack(yypParser) ){ yypParser->yytos--; yyStackOverflow(yypParser); return; } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); } -#endif if( yyNewState > YY_MAX_SHIFT ){ yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } - yytos = yypParser->yytos; yytos->stateno = yyNewState; yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; @@ -911,19 +936,12 @@ void Parse( (int)(yypParser->yytos - yypParser->yystack)); } #endif -#if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - break; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); break; } } -#endif } yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index 9e425c0fd..135164e3d 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -105,6 +105,7 @@ set boolean_defnnz_options { set boolean_defnil_options { SQLITE_32BIT_ROWID SQLITE_4_BYTE_ALIGNED_MALLOC + SQLITE_ALLOW_ROWID_IN_VIEW SQLITE_ALLOW_URI_AUTHORITY SQLITE_BUG_COMPATIBLE_20160819 SQLITE_CASE_SENSITIVE_LIKE diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl index 6cb145db8..792695255 100644 --- a/tool/mkmsvcmin.tcl +++ b/tool/mkmsvcmin.tcl @@ -23,7 +23,7 @@ if {$argc==0} { proc readFile { fileName } { set file_id [open $fileName RDONLY] - fconfigure $file_id -encoding binary -translation binary + fconfigure $file_id -translation binary set result [read $file_id] close $file_id return $result @@ -31,7 +31,7 @@ proc readFile { fileName } { proc writeFile { fileName data } { set file_id [open $fileName {WRONLY CREAT TRUNC}] - fconfigure $file_id -encoding binary -translation binary + fconfigure $file_id -translation binary puts -nonewline $file_id $data close $file_id return "" diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl index 6fb3b7594..18fe1a265 100644 --- a/tool/mkopcodeh.tcl +++ b/tool/mkopcodeh.tcl @@ -81,6 +81,7 @@ while {![eof $in]} { set op($name) -1 set group($name) 0 set jump($name) 0 + set jump0($name) 0 set in1($name) 0 set in2($name) 0 set in3($name) 0 @@ -109,6 +110,7 @@ while {![eof $in]} { out2 {set out2($name) 1} out3 {set out3($name) 1} ncycle {set ncycle($name) 1} + jump0 {set jump($name) 1; set jump0($name) 1;} } } if {$group($name)} { @@ -137,6 +139,7 @@ puts "/* Automatically generated. Do not edit */" puts "/* See the tool/mkopcodeh.tcl script for details */" foreach name {OP_Noop OP_Explain OP_Abortable} { set jump($name) 0 + set jump0($name) 0 set in1($name) 0 set in2($name) 0 set in3($name) 0 @@ -256,7 +259,9 @@ for {set i 0} {$i<=$max} {incr i} { set name $def($i) puts -nonewline [format {#define %-16s %3d} $name $i] set com {} - if {[info exists jump($name)] && $jump($name)} { + if {[info exists jump0($name)] && $jump0($name)} { + lappend com "jump0" + } elseif {[info exists jump($name)] && $jump($name)} { lappend com "jump" } if {[info exists sameas($i)]} { @@ -289,6 +294,7 @@ for {set i 0} {$i<=$max} {incr i} { if {$out2($name)} {incr x 16} if {$out3($name)} {incr x 32} if {$ncycle($name)} {incr x 64} + if {$jump0($name)} {incr x 128} } set bv($i) $x } @@ -304,6 +310,7 @@ puts "#define OPFLG_IN3 0x08 /* in3: P3 is an input */" puts "#define OPFLG_OUT2 0x10 /* out2: P2 is an output */" puts "#define OPFLG_OUT3 0x20 /* out3: P3 is an output */" puts "#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */" +puts "#define OPFLG_JUMP0 0x80 /* jump0: P2 might be zero */" puts "#define OPFLG_INITIALIZER \173\\" for {set i 0} {$i<=$max} {incr i} { if {$i%8==0} { diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 53fa59ac7..ef8353df4 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -17,7 +17,7 @@ # After the "tsrc" directory has been created and populated, run # this script: # -# tclsh mksqlite3c.tcl +# tclsh mksqlite3c.tcl [flags] [extra source files] # # The amalgamated SQLite code will be written into sqlite3.c # @@ -42,6 +42,7 @@ set linemacros 0 set useapicall 0 set enable_recover 0 set srcdir tsrc +set extrasrc [list] for {set i 0} {$i<[llength $argv]} {incr i} { set x [lindex $argv $i] @@ -63,8 +64,10 @@ for {set i 0} {$i<[llength $argv]} {incr i} { } elseif {[regexp {^-?-((help)|\?)$} $x]} { puts $help exit 0 - } else { + } elseif {[regexp {^-?-} $x]} { error "unknown command-line option: $x" + } else { + lappend extrasrc $x } } set in [open $srcdir/sqlite3.h] @@ -349,6 +352,21 @@ proc copy_file {filename} { section_comment "End of $tail" } +# Read the source file named $filename and write it into the +# sqlite3.c output file. The only transformation is the trimming +# of EOL whitespace. +# +proc copy_file_verbatim {filename} { + global out + set in [open $filename r] + set tail [file tail $filename] + section_comment "Begin EXTRA_SRC file $tail" + while {![eof $in]} { + set line [string trimright [gets $in]] + puts $out $line + } + section_comment "End of EXTRA_SRC $tail" +} # Process the source files. Process files containing commonly # used subroutines first in order to help the compiler find @@ -470,13 +488,16 @@ set flist { sqlite3session.c fts5.c stmt.c -} +} if {$enable_recover} { lappend flist sqlite3recover.c dbdata.c } foreach file $flist { copy_file $srcdir/$file } +foreach file $extrasrc { + copy_file_verbatim $file +} puts $out \ "/* Return the source-id for this library */ diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl index c874d538f..066321363 100644 --- a/tool/mkvsix.tcl +++ b/tool/mkvsix.tcl @@ -156,7 +156,7 @@ proc readFile { fileName } { # may contain binary data. # set file_id [open $fileName RDONLY] - fconfigure $file_id -encoding binary -translation binary + fconfigure $file_id -translation binary set result [read $file_id] close $file_id return $result @@ -168,7 +168,7 @@ proc writeFile { fileName data } { # binary data. # set file_id [open $fileName {WRONLY CREAT TRUNC}] - fconfigure $file_id -encoding binary -translation binary + fconfigure $file_id -translation binary puts -nonewline $file_id $data close $file_id return "" diff --git a/tool/replace.tcl b/tool/replace.tcl index e87cb922f..6a462d95f 100644 --- a/tool/replace.tcl +++ b/tool/replace.tcl @@ -4,8 +4,8 @@ # only lines successfully modified with a regular # expression. # -fconfigure stdout -translation binary -encoding binary -fconfigure stderr -translation binary -encoding binary +fconfigure stdout -translation binary +fconfigure stderr -translation binary set mode [string tolower [lindex $argv 0]] set from [lindex $argv 1] set to [lindex $argv 2] diff --git a/tool/restore_jrnl.tcl b/tool/restore_jrnl.tcl index 05af4f9a2..200f9b1d2 100644 --- a/tool/restore_jrnl.tcl +++ b/tool/restore_jrnl.tcl @@ -114,40 +114,40 @@ proc dump_jrnl_page {jrnl_pgno} { set db_pgno [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset] 4]] set chksum [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset+4+$db_pgsz] 4]] set nonce [calc_nonce $jrnl_pgno] - puts [ format {jrnl_pg_offset: %08x (%d) jrnl_pgno: %d db_pgno: %d} \
- $jrnl_pg_offset $jrnl_pg_offset \
- $jrnl_pgno $db_pgno]
- puts [ format {nonce: %08x chksum: %08x} \
- $nonce $chksum]
+ puts [ format {jrnl_pg_offset: %08x (%d) jrnl_pgno: %d db_pgno: %d} \ + $jrnl_pg_offset $jrnl_pg_offset \ + $jrnl_pgno $db_pgno] + puts [ format {nonce: %08x chksum: %08x} \ + $nonce $chksum] # now hex dump the data - # This is derived from the Tcler's WIKI
- set fid [open $jrnl_name r]
- fconfigure $fid -translation binary -encoding binary
- seek $fid [expr $jrnl_pg_offset+4]
- set data [read $fid $db_pgsz]
- close $fid
+ # This is derived from the Tcler's WIKI + set fid [open $jrnl_name r] + fconfigure $fid -translation binary + seek $fid [expr $jrnl_pg_offset+4] + set data [read $fid $db_pgsz] + close $fid for {set addr 0} {$addr<$db_pgsz} {set addr [expr $addr+16]} { - # get 16 bytes of data
- set s [string range $data $addr [expr $addr+16]]
-
- # Convert the data to hex and to characters.
- binary scan $s H*@0a* hex ascii
-
- # Replace non-printing characters in the data.
- regsub -all -- {[^[:graph:] ]} $ascii {.} ascii
-
- # Split the 16 bytes into two 8-byte chunks
- regexp -- {(.{16})(.{0,16})} $hex -> hex1 hex2
-
- # Convert the hex to pairs of hex digits
- regsub -all -- {..} $hex1 {& } hex1
- regsub -all -- {..} $hex2 {& } hex2
-
- # Print the hex and ascii data
- puts [ format {%08x %-24s %-24s %-16s} \
- $addr $hex1 $hex2 $ascii ]
- }
+ # get 16 bytes of data + set s [string range $data $addr [expr $addr+16]] + + # Convert the data to hex and to characters. + binary scan $s H*@0a* hex ascii + + # Replace non-printing characters in the data. + regsub -all -- {[^[:graph:] ]} $ascii {.} ascii + + # Split the 16 bytes into two 8-byte chunks + regexp -- {(.{16})(.{0,16})} $hex -> hex1 hex2 + + # Convert the hex to pairs of hex digits + regsub -all -- {..} $hex1 {& } hex1 + regsub -all -- {..} $hex2 {& } hex2 + + # Print the hex and ascii data + puts [ format {%08x %-24s %-24s %-16s} \ + $addr $hex1 $hex2 $ascii ] + } } # Setup for the tests. Make a backup copy of the files. @@ -230,4 +230,3 @@ do_test restore_jrnl-1.0 { catchsql {PRAGMA integrity_check} } {0 ok} db close - diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index 8fe72b99b..abc1e7e98 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -596,6 +596,7 @@ set nindex [db eval {SELECT count(*) FROM sqlite_schema WHERE type='index'}] set sql {SELECT count(*) FROM sqlite_schema WHERE name LIKE 'sqlite_autoindex%'} set nautoindex [db eval $sql] set nmanindex [expr {$nindex-$nautoindex}] +set nwithoutrowid [db eval {SELECT count(*) FROM pragma_table_list WHERE wr}] # set total_payload [mem eval "SELECT sum(payload) FROM space_used"] set user_payload [mem one {SELECT int(sum(payload)) FROM space_used @@ -614,6 +615,7 @@ statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2 statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent statline {Number of tables in the database} $ntable +statline {Number of WITHOUT ROWID tables} $nwithoutrowid statline {Number of indices} $nindex statline {Number of defined indices} $nmanindex statline {Number of implied indices} $nautoindex @@ -672,6 +674,14 @@ if {$nindex>0} { subreport {All tables and indices} 1 0 } subreport {All tables} {NOT is_index} 0 +if {$nwithoutrowid>0} { + subreport {All WITHOUT ROWID tables} {is_without_rowid} 0 + set nrowidtab [db eval {SELECT count(*) FROM pragma_table_list + WHERE type='table' AND NOT wr}] + if {$nrowidtab>0} { + subreport {ALL rowid tables} {NOT is_without_rowid AND NOT is_index} 0 + } +} if {$nindex>0} { subreport {All indices} {is_index} 0 } diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 4cc25794a..5d425c3b3 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -161,6 +161,9 @@ while test "$1" != ""; do --fp) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" ;; + --parsenumber) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset parsenumber" + ;; --stmtscanstatus) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" ;; diff --git a/tool/sqldiff.c b/tool/sqldiff.c index cbdfc35cd..8b2293caf 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -31,7 +31,7 @@ ** correctly on Windows: ** ** fprintf() -> Wfprintf() -** +** */ #if defined(_WIN32) # include "console_io.h" @@ -65,7 +65,7 @@ struct GlobalVars { static void strFree(sqlite3_str *pStr){ sqlite3_free(sqlite3_str_finish(pStr)); } - + /* ** Print an error resulting from faulting command-line arguments and ** abort the program. @@ -383,7 +383,7 @@ static void printQuoted(FILE *out, sqlite3_value *X){ fprintf(out, "'"); for(i=j=0; zArg[i]; i++){ char c = zArg[i]; - int ctl = iscntrl(c); + int ctl = iscntrl((unsigned char)c); if( ctl>inctl ){ inctl = ctl; fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c); @@ -1998,7 +1998,7 @@ int main(int argc, char **argv){ if( g.bSchemaOnly && g.bSchemaCompare ){ cmdlineError("The --schema option is useless with --table %s .", zTab); } - rc = sqlite3_open(zDb1, &g.db); + rc = sqlite3_open_v2(zDb1, &g.db, SQLITE_OPEN_READONLY, 0); if( rc ){ cmdlineError("cannot open database file \"%s\"", zDb1); } @@ -2006,6 +2006,13 @@ int main(int argc, char **argv){ if( rc || zErrMsg ){ cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1); } + { + sqlite3 *db2 = 0; + if( sqlite3_open_v2(zDb2, &db2, SQLITE_OPEN_READONLY, 0) ){ + cmdlineError("cannot open database file \"%s\"", zDb2); + } + sqlite3_close(db2); + } #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(g.db, 1); for(i=0; i<nExt; i++){ diff --git a/vsixtest/vsixtest.tcl b/vsixtest/vsixtest.tcl index 5dce821dc..8b2288be3 100644 --- a/vsixtest/vsixtest.tcl +++ b/vsixtest/vsixtest.tcl @@ -132,7 +132,7 @@ proc readFile { fileName } { # may contain binary data. # set file_id [open $fileName RDONLY] - fconfigure $file_id -encoding binary -translation binary + fconfigure $file_id -translation binary set result [read $file_id] close $file_id return $result @@ -144,7 +144,7 @@ proc writeFile { fileName data } { # binary data. # set file_id [open $fileName {WRONLY CREAT TRUNC}] - fconfigure $file_id -encoding binary -translation binary + fconfigure $file_id -translation binary puts -nonewline $file_id $data close $file_id return "" |
